adding missing dates

ecml_tools/create/config.py

@@ -153,9 +153,6 @@ def __init__(self, config, *args, **kwargs):
             raise ValueError(f"Dates must be a dict. Got {self.dates}")
 
     def normalise(self):
-        if isinstance(self.input, (tuple, list)):
-            self.input = dict(concat=self.input)
-
         if "order_by" in self.output:
             self.output.order_by = normalize_order_by(self.output.order_by)
 

ecml_tools/utils/dates/__init__.py

@@ -7,6 +7,7 @@
 
 
 import datetime
+import warnings
 
 
 def frequency_to_hours(frequency):
@@ -19,6 +20,12 @@ def frequency_to_hours(frequency):
     return {"h": v, "d": v * 24}[unit]
 
 
+def normalize_date(x):
+    if isinstance(x, str):
+        return datetime.datetime.fromisoformat(x)
+    return x
+
+
 class Dates:
     """Base class for date generation.
 
@@ -33,17 +40,33 @@ class Dates:
 
     >>> len(Dates.from_config(start="2023-01-01 00:00", end="2023-01-02 00:00", frequency=12))
     3
+    >>> len(Dates.from_config(start="2023-01-01 00:00",
+    ...                   end="2023-01-02 00:00",
+    ...                   frequency=12,
+    ...                   missing=["2023-01-01 12:00"]))
+    3
+    >>> len(Dates.from_config(start="2023-01-01 00:00",
+    ...                   end="2023-01-02 00:00",
+    ...                   frequency=12,
+    ...                   missing=["2099-01-01 12:00"]))
+    3
     """
 
+    def __init__(self, missing=None):
+        if not missing:
+            missing = []
+        self.missing = [normalize_date(x) for x in missing]
+        if set(self.missing) - set(self.values):
+            warnings.warn(f"Missing dates {self.missing} not in list.")
+
     @classmethod
     def from_config(cls, **kwargs):
         if "values" in kwargs:
             return ValuesDates(**kwargs)
         return StartEndDates(**kwargs)
 
     def __iter__(self):
-        for v in self.values:
-            yield v
+        yield from self.values
 
     def __getitem__(self, i):
         return self.values[i]
@@ -59,7 +82,7 @@ def summary(self):
 class ValuesDates(Dates):
     def __init__(self, values, **kwargs):
         self.values = sorted(values)
-        assert not kwargs, f"Unexpected arguments {kwargs}"
+        super().__init__(**kwargs)
 
     def __repr__(self):
         return f"{self.__class__.__name__}({self.values[0]}..{self.values[-1]})"
@@ -70,8 +93,6 @@ def as_dict(self):
 
 class StartEndDates(Dates):
     def __init__(self, start, end, frequency=1, **kwargs):
-        assert not kwargs, f"Unexpected arguments {kwargs}"
-
         frequency = frequency_to_hours(frequency)
 
         def _(x):
@@ -82,9 +103,7 @@ def _(x):
         start = _(start)
         end = _(end)
 
-        if isinstance(start, datetime.date) and not isinstance(
-            start, datetime.datetime
-        ):
+        if isinstance(start, datetime.date) and not isinstance(start, datetime.datetime):
             start = datetime.datetime(start.year, start.month, start.day)
 
         if isinstance(end, datetime.date) and not isinstance(end, datetime.datetime):
@@ -105,6 +124,8 @@ def _(x):
             self.values.append(date)
             date += increment
 
+        super().__init__(**kwargs)
+
     def as_dict(self):
         return {
             "start": self.start.isoformat(),

ecml_tools/utils/dates/groups.py

@@ -26,18 +26,48 @@ class Groups:
     3
     >>> len(list(g)[1])
     2
+    >>> g = Groups(group_by=3,
+    ...            start="2023-01-01 00:00",
+    ...            end="2023-01-05 00:00",
+    ...            frequency=24,
+    ...            missing=["2023-01-02 00:00"])
+    >>> len(list(g))
+    2
+    >>> len(list(g)[0])
+    2
+    >>> len(list(g)[1])
+    2
     """
 
     def __init__(self, **kwargs):
         group_by = kwargs.pop("group_by")
         self.dates = Dates.from_config(**kwargs)
         self.grouper = Grouper.from_config(group_by)
+        self.filter = Filter(self.dates.missing)
 
     def __iter__(self):
-        return self.grouper(self.dates)
+        for dates in self.grouper(self.dates):
+            dates = self.filter(dates)
+            if not dates:
+                continue
+            yield dates
 
     def __len__(self):
-        return len(list(self.grouper(self.dates)))
+        count = 0
+        for dates in self.grouper(self.dates):
+            dates = self.filter(dates)
+            if not dates:
+                continue
+            count += 1
+        return count
+
+
+class Filter:
+    def __init__(self, missing):
+        self.missing = missing
+
+    def __call__(self, dates):
+        return [d for d in dates if d not in self.missing]
 
 
 class Grouper: