From a94279b937f5491d5a9b22ba6e69ee38e5aa917d Mon Sep 17 00:00:00 2001 From: Juniper Tyree <50025784+juntyr@users.noreply.github.com> Date: Wed, 18 Dec 2024 19:49:55 +0200 Subject: [PATCH 1/7] Relax pandas version pin to support 2.2.x --- requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index bb20775..f04636d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,6 +3,6 @@ orjson datetime #numpy=1.26.4 xarray -pandas==2.2.0 +pandas~=2.2.0 covjson-pydantic -conflator \ No newline at end of file +conflator From 5cb9d1bdfab2b42ed1ffca640f36db392d8c4b0c Mon Sep 17 00:00:00 2001 From: awarde96 Date: Thu, 19 Dec 2024 10:40:37 +0000 Subject: [PATCH 2/7] Bump version --- covjsonkit/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/covjsonkit/version.py b/covjsonkit/version.py index 4d04613..a3024cf 100644 --- a/covjsonkit/version.py +++ b/covjsonkit/version.py @@ -1 +1 @@ -__version__ = "0.0.28" +__version__ = "0.0.29" From e7afe22700430f77b93940efef294ba9b8e99fef Mon Sep 17 00:00:00 2001 From: awarde96 Date: Wed, 15 Jan 2025 08:20:48 +0000 Subject: [PATCH 3/7] Fix minor bug when converting to xarray, where steps were unsorted --- covjsonkit/decoder/BoundingBox.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/covjsonkit/decoder/BoundingBox.py b/covjsonkit/decoder/BoundingBox.py index 6dafcb7..9e0e3e6 100644 --- a/covjsonkit/decoder/BoundingBox.py +++ b/covjsonkit/decoder/BoundingBox.py @@ -84,9 +84,9 @@ def to_xarray(self): coverage["mars:metadata"]["step"] ] = coverage["ranges"][parameter]["values"] - datetimes = list(set(datetimes)) - numbers = list(set(numbers)) - steps = list(set(steps)) + datetimes = sorted(list(set(datetimes))) + numbers = sorted(list(set(numbers))) + steps = sorted(list(set(steps))) new_values = {} for parameter in values.keys(): From 6b7d79af50b61e93138c6348839cff22c688a304 Mon Sep 17 00:00:00 2001 From: awarde96 Date: Wed, 15 Jan 2025 13:03:16 +0000 Subject: [PATCH 4/7] Fix bounding box for climate-dt --- covjsonkit/encoder/BoundingBox.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/covjsonkit/encoder/BoundingBox.py b/covjsonkit/encoder/BoundingBox.py index 786b558..2b06a5a 100644 --- a/covjsonkit/encoder/BoundingBox.py +++ b/covjsonkit/encoder/BoundingBox.py @@ -89,7 +89,7 @@ def from_polytope(self, result): fields["lat"] = 0 fields["param"] = 0 fields["number"] = [0] - fields["step"] = 0 + fields["step"] = [0] fields["dates"] = [] fields["levels"] = [0] From 2713e9441d4955bc484224d1b4209b3dc1e0cfcd Mon Sep 17 00:00:00 2001 From: awarde96 Date: Thu, 16 Jan 2025 09:52:48 +0000 Subject: [PATCH 5/7] Fix timeseries for date as time_axis --- covjsonkit/encoder/TimeSeries.py | 125 ++++++++++++++++++++----------- 1 file changed, 83 insertions(+), 42 deletions(-) diff --git a/covjsonkit/encoder/TimeSeries.py b/covjsonkit/encoder/TimeSeries.py index 48fb9d5..6c70e07 100644 --- a/covjsonkit/encoder/TimeSeries.py +++ b/covjsonkit/encoder/TimeSeries.py @@ -190,14 +190,29 @@ def from_polytope_step(self, result): coords = {} mars_metadata = {} range_dict = {} - lat = 0 - param = 0 - number = [0] - step = 0 - levels = [0] - dates = [0] + fields = {} + fields["lat"] = 0 + fields["param"] = 0 + fields["number"] = [0] + fields["step"] = [0] + fields["dates"] = [] + fields["levels"] = [0] + + start = time.time() + logging.debug("Tree walking starts at: %s", start) # noqa: E501 + self.walk_tree(result, fields, coords, mars_metadata, range_dict) + end = time.time() + delta = end - start + logging.debug("Tree walking ends at: %s", end) # noqa: E501 + logging.debug("Tree walking takes: %s", delta) # noqa: E501 + + start = time.time() + logging.debug("Coords creation: %s", start) # noqa: E501 - self.walk_tree(result, lat, coords, mars_metadata, param, range_dict, number, step, dates, levels) + print("fields", fields) + print("coords", coords) + print("mars_metadata", mars_metadata) + print("range_dict", range_dict) self.add_reference( { @@ -210,43 +225,69 @@ def from_polytope_step(self, result): ) coordinates = {} - for date in range_dict.keys(): - for num in range_dict[date].keys(): - for para in range_dict[date][num].keys(): - for step in range_dict[date][num][para].keys(): - for dt in range_dict.keys(): - date_format = "%Y%m%dT%H%M%S" - new_date = pd.Timestamp(dt).strftime(date_format) - start_time = datetime.strptime(new_date, date_format) + + levels = fields["levels"] + if fields["param"] == 0: + raise ValueError("No parameters were returned, date requested may be out of range") + for para in fields["param"]: + self.add_parameter(para) + + logging.debug("The parameters added were: %s", self.parameters) # noqa: E501 + + for step in fields["step"]: + coordinates[fields["dates"][0]] = { + "x": [coords[fields["dates"][0]]["composite"][0][0]], + "y": [coords[fields["dates"][0]]["composite"][0][1]], + "z": [levels[0]], + } + coordinates[fields["dates"][0]]["t"] = [] + for level in fields["levels"]: + for num in fields["number"]: + for para in fields["param"]: + for date in fields["dates"]: + #date_format = "%Y%m%dT%H%M%S" + #new_date = pd.Timestamp(date).strftime(date_format) + #start_time = datetime.strptime(new_date, date_format) # add current date to list by converting it to iso format - stamp = start_time + timedelta(hours=int(step)) - if step not in coordinates: - coordinates[step] = { - "x": [coords[date]["composite"][0][0]], - "y": [coords[date]["composite"][0][1]], - "z": "sfc", - } - if "t" not in coordinates[step]: - coordinates[step]["t"] = [stamp.isoformat() + "Z"] - else: - coordinates[step]["t"].append(stamp.isoformat() + "Z") + #stamp = start_time + timedelta(hours=int(step)) + coordinates[fields["dates"][0]]["t"].append(date) + break break break - break - - for date in range_dict.keys(): - for num in range_dict[date].keys(): - for param in range_dict[date][num].keys(): - step_dict = {} - for step in range_dict[date][num][param].keys(): - if step not in step_dict: - step_dict[step] = [] - step_dict[step].append(range_dict[date][num][param][step]) - for step in range_dict[date][num][param].keys(): - mm = mars_metadata.copy() - mm["number"] = num - mm["Forecast date"] = date - mm["step"] = step - self.add_coverage(mm, coordinates[step], range_dict[date][num]) + print("coordinates", coordinates) + + end = time.time() + delta = end - start + logging.debug("Coords creation: %s", end) # noqa: E501 + logging.debug("Coords creation: %s", delta) # noqa: E501 + + # logging.debug("The values returned from walking tree: %s", range_dict) # noqa: E501 + # logging.debug("The coordinates returned from walking tree: %s", coordinates) # noqa: E501 + + start = time.time() + logging.debug("Coverage creation: %s", start) # noqa: E501 + + for step in fields["step"]: + for level in fields["levels"]: + for num in fields["number"]: + val_dict = {} + for para in fields["param"]: + val_dict[para] = [] + for date in fields["dates"]: + key = (date, level, num, para, step) + # for k, v in range_dict.items(): + # if k == key: + # val_dict[para].append(v[0]) + val_dict[para].append(range_dict[key][0]) + mm = mars_metadata.copy() + mm["number"] = num + mm["Forecast date"] = date + #del mm["step"] + self.add_coverage(mm, coordinates[fields["dates"][0]], val_dict) + + end = time.time() + delta = end - start + logging.debug("Coverage creation: %s", end) # noqa: E501 + logging.debug("Coverage creation: %s", delta) # noqa: E501 return self.covjson From 8b4c017f3696904d5c6e6becfac13dd8e7388aef Mon Sep 17 00:00:00 2001 From: awarde96 Date: Thu, 16 Jan 2025 09:55:03 +0000 Subject: [PATCH 6/7] Black --- covjsonkit/encoder/TimeSeries.py | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/covjsonkit/encoder/TimeSeries.py b/covjsonkit/encoder/TimeSeries.py index 6c70e07..3b56582 100644 --- a/covjsonkit/encoder/TimeSeries.py +++ b/covjsonkit/encoder/TimeSeries.py @@ -209,11 +209,6 @@ def from_polytope_step(self, result): start = time.time() logging.debug("Coords creation: %s", start) # noqa: E501 - print("fields", fields) - print("coords", coords) - print("mars_metadata", mars_metadata) - print("range_dict", range_dict) - self.add_reference( { "coordinates": ["x", "y", "z"], @@ -245,16 +240,15 @@ def from_polytope_step(self, result): for num in fields["number"]: for para in fields["param"]: for date in fields["dates"]: - #date_format = "%Y%m%dT%H%M%S" - #new_date = pd.Timestamp(date).strftime(date_format) - #start_time = datetime.strptime(new_date, date_format) + # date_format = "%Y%m%dT%H%M%S" + # new_date = pd.Timestamp(date).strftime(date_format) + # start_time = datetime.strptime(new_date, date_format) # add current date to list by converting it to iso format - #stamp = start_time + timedelta(hours=int(step)) + # stamp = start_time + timedelta(hours=int(step)) coordinates[fields["dates"][0]]["t"].append(date) break break break - print("coordinates", coordinates) end = time.time() delta = end - start @@ -282,7 +276,7 @@ def from_polytope_step(self, result): mm = mars_metadata.copy() mm["number"] = num mm["Forecast date"] = date - #del mm["step"] + # del mm["step"] self.add_coverage(mm, coordinates[fields["dates"][0]], val_dict) end = time.time() From 4fe1ddf043388bd8969744fc48bb25a31abe3108 Mon Sep 17 00:00:00 2001 From: awarde96 Date: Thu, 16 Jan 2025 12:18:21 +0000 Subject: [PATCH 7/7] Bump version --- covjsonkit/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/covjsonkit/version.py b/covjsonkit/version.py index 3dc1f76..485f44a 100644 --- a/covjsonkit/version.py +++ b/covjsonkit/version.py @@ -1 +1 @@ -__version__ = "0.1.0" +__version__ = "0.1.1"