wip

ecml_tools/create/config.py

@@ -9,7 +9,6 @@
 import datetime
 import logging
 import os
-import warnings
 from copy import deepcopy
 
 import yaml
@@ -143,11 +142,21 @@ def __init__(self, config, *args, **kwargs):
 
         # deprecated/obsolete
         if "order" in self.output:
-            raise ValueError(f"Do not use 'order'. Use order_by in {self}")
+            raise ValueError(
+                f"Do not use 'order'. Use order_by instead. {list(self.keys())}"
+            )
         if "loops" in self:
-            assert "loop" not in self
-            warnings.warn("Should use loop instead of loops in config")
-            self.loop = self.pop("loops")
+            raise ValueError(
+                f"Do not use 'loops'. Use dates instead. {list(self.keys())}"
+            )
+        if "loop" in self:
+            print(f"Do not use 'loop'. Use 'dates' instead. {list(self.keys())}")
+            self.dates = self.pop("loop")
+
+        if isinstance(self.dates, dict):
+            self.dates = [self.dates]
+
+        self.loop = self.pop("dates")
 
         self.normalise()
 

ecml_tools/create/input.py

@@ -356,7 +356,7 @@ def __init__(self, context, dates, action, previous_sibling=None):
     @cached_property
     def datasource(self):
         print(f"loading source with {self.args} {self.kwargs}")
-        return self.action.function(*self.args, **self.kwargs)
+        return self.action.function(self.dates, *self.args, **self.kwargs)
 
     def __repr__(self):
         content = " ".join([f"{v}" for v in self.args])

ecml_tools/create/writer.py

@@ -7,6 +7,7 @@
 # nor does it submit to any jurisdiction.
 #
 
+import datetime
 import logging
 import time
 import warnings
@@ -177,8 +178,13 @@ def __init__(self, parts, full_array, parent, print=print):
 
     def write(self, result, igroup, dates):
         cube = result.get_cube()
-        assert cube.shape[0] == len(dates), (cube.shape[0], len(dates))
-        assert cube.coords["dates"] == dates, (cube.coords["dates"], dates)
+        assert cube.extended_user_shape[0] == len(dates), (
+            cube.extended_user_shape[0],
+            dates,
+        )
+        dates_in_data = cube.user_coords["valid_datetime"]
+        dates_in_data = [datetime.datetime.fromisoformat(_) for _ in dates_in_data]
+        assert dates_in_data == list(dates), (dates_in_data, list(dates))
         self.write_cube(cube, igroup)
 
     @property

tests/create-join.yaml

@@ -20,36 +20,27 @@ dates:
 
 input:
   join:
-    - pipe:
-      - join:
-        - mars:
-            <<: *mars_request
-            param: [2t]
-            levtype: sfc
-
-        - mars:
-            <<: *mars_request
-            param: [q, t]
-            levtype: pl
-            level: [50, 100]
-
-        - accumulations:
-            <<: *mars_request
-            param: [cp, tp]
-            accumulation_period: 6h
-
-        - constants:
-            <<: *mars_request
-            param:
-              - cos_latitude
-      - filter:
-          param: [2t, q, t, cp, tp, cos_latitude]
-          level: [sfc, 50, 100]
     - mars:
         <<: *mars_request
         param: [2t]
         levtype: sfc
 
+    - mars:
+        <<: *mars_request
+        param: [q, t]
+        levtype: pl
+        level: [50, 100]
+
+    - accumulations:
+        <<: *mars_request
+        param: [cp, tp]
+        accumulation_period: 6h
+
+    - constants:
+        <<: *mars_request
+        param:
+          - cos_latitude
+
 output:
   chunking: {dates: 1, ensembles: 1}
   dtype: float32