Feature/try to merge irregular grid develop

.gitignore

@@ -13,10 +13,10 @@ site
 *.grib
 *.gif
 *.html
+*.req
+.mypy_cache
 example_eo
 example_mri
-.mypy_cache
-*.req
 polytope_python.egg-info
 new_test_venv_polytope
 new_venv_polytope
@@ -26,3 +26,9 @@ new_updated_numpy_venv
 newest-polytope-venv
 serializedTree
 new_polytope_venv
+
+newest-polytope-venv
+*.pstats
+*.profile
+new_polytope_venv
+*.json

examples/3D_shipping_route.py

@@ -6,9 +6,9 @@
 import numpy as np
 from earthkit import data
 
-from polytope.engine.hullslicer import HullSlicer
-from polytope.polytope import Polytope, Request
-from polytope.shapes import Ellipsoid, Path, Select
+from polytope_feature.engine.hullslicer import HullSlicer
+from polytope_feature.polytope import Polytope, Request
+from polytope_feature.shapes import Ellipsoid, Path, Select
 
 
 class Test:

examples/3D_shipping_route_wave_model.py

@@ -4,10 +4,10 @@
 import pytest
 from eccodes import codes_grib_find_nearest, codes_grib_new_from_file
 
-from polytope.datacube.backends.fdb import FDBDatacube
-from polytope.engine.hullslicer import HullSlicer
-from polytope.polytope import Polytope, Request
-from polytope.shapes import Ellipsoid, Path, Select
+from polytope_feature.datacube.backends.fdb import FDBDatacube
+from polytope_feature.engine.hullslicer import HullSlicer
+from polytope_feature.polytope import Polytope, Request
+from polytope_feature.shapes import Ellipsoid, Path, Select
 from tests.helper_functions import download_test_data
 
 

examples/4D_flight_path.py

@@ -4,10 +4,10 @@
 from earthkit import data
 from PIL import Image
 
-from polytope.datacube.backends.xarray import XArrayDatacube
-from polytope.engine.hullslicer import HullSlicer
-from polytope.polytope import Polytope, Request
-from polytope.shapes import Box, Path, Select
+from polytope_feature.datacube.backends.xarray import XArrayDatacube
+from polytope_feature.engine.hullslicer import HullSlicer
+from polytope_feature.polytope import Polytope, Request
+from polytope_feature.shapes import Box, Path, Select
 
 
 class Test:

examples/country_slicing.py

@@ -4,10 +4,10 @@
 from earthkit import data
 from shapely.geometry import shape
 
-from polytope.datacube.backends.xarray import XArrayDatacube
-from polytope.engine.hullslicer import HullSlicer
-from polytope.polytope import Polytope, Request
-from polytope.shapes import Polygon, Select, Union
+from polytope_feature.datacube.backends.xarray import XArrayDatacube
+from polytope_feature.engine.hullslicer import HullSlicer
+from polytope_feature.polytope import Polytope, Request
+from polytope_feature.shapes import Polygon, Select, Union
 
 
 class Test:

examples/cyclic_route_around_earth.py

@@ -3,10 +3,10 @@
 import numpy as np
 from earthkit import data
 
-from polytope.datacube.backends.xarray import XArrayDatacube
-from polytope.engine.hullslicer import HullSlicer
-from polytope.polytope import Polytope, Request
-from polytope.shapes import Box, PathSegment, Select
+from polytope_feature.datacube.backends.xarray import XArrayDatacube
+from polytope_feature.engine.hullslicer import HullSlicer
+from polytope_feature.polytope import Polytope, Request
+from polytope_feature.shapes import Box, PathSegment, Select
 
 
 class Test:

examples/healpix_grid_box_example.py

@@ -3,10 +3,10 @@
 from earthkit import data
 from eccodes import codes_grib_find_nearest, codes_grib_new_from_file
 
-from polytope.datacube.backends.xarray import XArrayDatacube
-from polytope.engine.hullslicer import HullSlicer
-from polytope.polytope import Polytope, Request
-from polytope.shapes import Box, Select
+from polytope_feature.datacube.backends.xarray import XArrayDatacube
+from polytope_feature.engine.hullslicer import HullSlicer
+from polytope_feature.polytope import Polytope, Request
+from polytope_feature.shapes import Box, Select
 
 
 class TestOctahedralGrid:

examples/octahedral_grid_box_example.py

@@ -6,10 +6,10 @@
 from eccodes import codes_grib_find_nearest, codes_grib_new_from_file
 from matplotlib import markers
 
-from polytope.datacube.backends.xarray import XArrayDatacube
-from polytope.engine.hullslicer import HullSlicer
-from polytope.polytope import Polytope, Request
-from polytope.shapes import Box, Select
+from polytope_feature.datacube.backends.xarray import XArrayDatacube
+from polytope_feature.engine.hullslicer import HullSlicer
+from polytope_feature.polytope import Polytope, Request
+from polytope_feature.shapes import Box, Select
 
 
 def find_nearest_latlon(grib_file, target_lat, target_lon):

examples/octahedral_grid_country_example.py

@@ -7,10 +7,10 @@
 from matplotlib import markers
 from shapely.geometry import shape
 
-from polytope.datacube.backends.xarray import XArrayDatacube
-from polytope.engine.hullslicer import HullSlicer
-from polytope.polytope import Polytope, Request
-from polytope.shapes import Polygon, Select, Union
+from polytope_feature.datacube.backends.xarray import XArrayDatacube
+from polytope_feature.engine.hullslicer import HullSlicer
+from polytope_feature.polytope import Polytope, Request
+from polytope_feature.shapes import Polygon, Select, Union
 
 
 def find_nearest_latlon(grib_file, target_lat, target_lon):

examples/read_me_example.py

@@ -1,8 +1,8 @@
 import numpy as np
 from earthkit import data
 
-from polytope.polytope import Polytope, Request
-from polytope.shapes import Box, Select
+from polytope_feature.polytope import Polytope, Request
+from polytope_feature.shapes import Box, Select
 
 ds = data.from_source("file", "./examples/data/winds.grib")
 array = ds.to_xarray()

examples/slicing_all_ecmwf_countries.py

@@ -4,10 +4,10 @@
 from earthkit import data
 from shapely.geometry import shape
 
-from polytope.datacube.backends.xarray import XArrayDatacube
-from polytope.engine.hullslicer import HullSlicer
-from polytope.polytope import Polytope, Request
-from polytope.shapes import Polygon, Select, Union
+from polytope_feature.datacube.backends.xarray import XArrayDatacube
+from polytope_feature.engine.hullslicer import HullSlicer
+from polytope_feature.polytope import Polytope, Request
+from polytope_feature.shapes import Polygon, Select, Union
 
 
 class Test:

examples/timeseries_example.py

@@ -3,10 +3,10 @@
 from earthkit import data
 from shapely.geometry import shape
 
-from polytope.datacube.backends.xarray import XArrayDatacube
-from polytope.engine.hullslicer import HullSlicer
-from polytope.polytope import Polytope, Request
-from polytope.shapes import Polygon, Select, Union
+from polytope_feature.datacube.backends.xarray import XArrayDatacube
+from polytope_feature.engine.hullslicer import HullSlicer
+from polytope_feature.polytope import Polytope, Request
+from polytope_feature.shapes import Polygon, Select, Union
 
 
 class Test:

examples/wind_farms.py

@@ -6,10 +6,10 @@
 from osgeo import gdal
 from shapely.geometry import shape
 
-from polytope.datacube.backends.xarray import XArrayDatacube
-from polytope.engine.hullslicer import HullSlicer
-from polytope.polytope import Polytope, Request
-from polytope.shapes import Polygon, Select, Union
+from polytope_feature.datacube.backends.xarray import XArrayDatacube
+from polytope_feature.engine.hullslicer import HullSlicer
+from polytope_feature.polytope import Polytope, Request
+from polytope_feature.shapes import Polygon, Select, Union
 
 
 class Test:

performance/fdb_performance.py

@@ -2,10 +2,10 @@
 
 import pandas as pd
 
-from polytope.datacube.backends.fdb import FDBDatacube
-from polytope.engine.hullslicer import HullSlicer
-from polytope.polytope import Polytope, Request
-from polytope.shapes import Box, Select
+from polytope_feature.datacube.backends.fdb import FDBDatacube
+from polytope_feature.engine.hullslicer import HullSlicer
+from polytope_feature.polytope import Polytope, Request
+from polytope_feature.shapes import Box, Select
 
 
 class TestSlicingFDBDatacube:

performance/fdb_performance_3D.py

@@ -2,10 +2,10 @@
 
 import pandas as pd
 
-from polytope.datacube.backends.fdb import FDBDatacube
-from polytope.engine.hullslicer import HullSlicer
-from polytope.polytope import Polytope, Request
-from polytope.shapes import Box, Select, Span
+from polytope_feature.datacube.backends.fdb import FDBDatacube
+from polytope_feature.engine.hullslicer import HullSlicer
+from polytope_feature.polytope import Polytope, Request
+from polytope_feature.shapes import Box, Select, Span
 
 
 class TestSlicingFDBDatacube:

performance/fdb_slice_many_numbers_timeseries.py

@@ -3,13 +3,12 @@
 import pandas as pd
 import pygribjump as gj
 
-from polytope.polytope import Polytope, Request
-from polytope.shapes import All, Point, Select
+from polytope_feature.polytope import Polytope, Request
+from polytope_feature.shapes import All, Point, Select
 
 time1 = time.time()
 # Create a dataarray with 3 labelled axes using different index types
 
-# config = {"class": "od", "expver": "0001", "levtype": "sfc", "type": "pf"}
 options = {
     "axis_config": [
         {"axis_name": "step", "transformations": [{"name": "type_change", "type": "int"}]},
@@ -62,7 +61,6 @@
     Select("class", ["od"]),
     Select("stream", ["enfo"]),
     Select("type", ["pf"]),
-    # Select("latitude", [0.035149384216], method="surrounding"),
     Point(["latitude", "longitude"], [[0.04, 0]], method="surrounding"),
     All("number"),
 )

performance/performance_many_num_steps.py

@@ -4,9 +4,9 @@
 import pandas as pd
 from earthkit import data
 
-from polytope.datacube.backends.fdb import FDBDatacube
-from polytope.polytope import Polytope, Request
-from polytope.shapes import All, Point, Select, Span
+from polytope_feature.datacube.backends.fdb import FDBDatacube
+from polytope_feature.polytope import Polytope, Request
+from polytope_feature.shapes import All, Point, Select, Span
 
 time1 = time.time()
 # Create a dataarray with 3 labelled axes using different index types

performance/scalability_test.py

@@ -3,10 +3,10 @@
 import numpy as np
 import xarray as xr
 
-from polytope.datacube.backends.xarray import XArrayDatacube
-from polytope.engine.hullslicer import HullSlicer
-from polytope.polytope import Polytope, Request
-from polytope.shapes import Box, Disk, Ellipsoid, Select
+from polytope_feature.datacube.backends.xarray import XArrayDatacube
+from polytope_feature.engine.hullslicer import HullSlicer
+from polytope_feature.polytope import Polytope, Request
+from polytope_feature.shapes import Box, Disk, Ellipsoid, Select
 
 
 class Test:

performance/scalability_test_2.py

@@ -3,10 +3,10 @@
 import numpy as np
 import xarray as xr
 
-from polytope.datacube.backends.xarray import XArrayDatacube
-from polytope.engine.hullslicer import HullSlicer
-from polytope.polytope import Polytope, Request
-from polytope.shapes import Box, Select, Union
+from polytope_feature.datacube.backends.xarray import XArrayDatacube
+from polytope_feature.engine.hullslicer import HullSlicer
+from polytope_feature.polytope import Polytope, Request
+from polytope_feature.shapes import Box, Select, Union
 
 
 class Test:

polytope_feature/datacube/backends/datacube.py

@@ -152,20 +152,40 @@ def remap_path(self, path: DatacubePath):
         return path
 
     @staticmethod
-    def create(datacube, config={}, axis_options={}, compressed_axes_options=[], alternative_axes=[], context=None):
+    def create(
+        datacube,
+        config={},
+        axis_options={},
+        compressed_axes_options=[],
+        point_cloud_options=None,
+        alternative_axes=[],
+        context=None,
+    ):
         # TODO: get the configs as None for pre-determined value and change them to empty dictionary inside the function
         if type(datacube).__name__ == "DataArray":
             from .xarray import XArrayDatacube
 
-            xadatacube = XArrayDatacube(datacube, axis_options, compressed_axes_options, context)
+            xadatacube = XArrayDatacube(datacube, axis_options, compressed_axes_options, point_cloud_options, context)
             return xadatacube
         if type(datacube).__name__ == "GribJump":
             from .fdb import FDBDatacube
 
             fdbdatacube = FDBDatacube(
-                datacube, config, axis_options, compressed_axes_options, alternative_axes, context
+                datacube,
+                config,
+                axis_options,
+                compressed_axes_options,
+                point_cloud_options,
+                alternative_axes,
+                context,
             )
             return fdbdatacube
+        if type(datacube).__name__ == "MockDatacube":
+            return datacube
 
     def check_branching_axes(self, request):
         pass
+
+    @abstractmethod
+    def find_point_cloud(self):
+        pass

polytope_feature/datacube/backends/fdb.py

@@ -10,7 +10,14 @@
 
 class FDBDatacube(Datacube):
     def __init__(
-        self, gj, config=None, axis_options=None, compressed_axes_options=[], alternative_axes=[], context=None
+        self,
+        gj,
+        config=None,
+        axis_options=None,
+        compressed_axes_options=[],
+        point_cloud_options=None,
+        alternative_axes=[],
+        context=None,
     ):
         if config is None:
             config = {}
@@ -23,6 +30,7 @@ def __init__(
 
         self.unwanted_path = {}
         self.axis_options = axis_options
+        self.has_point_cloud = point_cloud_options  # NOTE: here, will be True/False
 
         partial_request = config
         # Find values in the level 3 FDB datacube
@@ -70,7 +78,12 @@ def __init__(
                 val = self._axes[name].type
                 self._check_and_add_axes(options, name, val)
 
-        logging.info("Polytope created axes for %s", self._axes.keys())
+        logging.info("Polytope created axes for: " + str(self._axes.keys()))
+
+    def find_point_cloud(self):
+        # TODO: somehow, find the point cloud of irregular grid if it exists
+        if self.has_point_cloud:
+            return self.has_point_cloud
 
     def check_branching_axes(self, request):
         polytopes = request.polytopes()
@@ -282,6 +295,7 @@ def get_2nd_last_values(self, requests, leaf_path=None):
 
         leaf_path_copy = deepcopy(leaf_path)
         leaf_path_copy.pop("values", None)
+        leaf_path_copy.pop("index")
         return (leaf_path_copy, current_start_idxs, fdb_node_ranges, lat_length)
 
     def get_last_layer_before_leaf(self, requests, leaf_path, current_idx, fdb_range_n):
@@ -290,6 +304,7 @@ def get_last_layer_before_leaf(self, requests, leaf_path, current_idx, fdb_range
         for i, c in enumerate(requests.children):
             # now c are the leaves of the initial tree
             key_value_path = {c.axis.name: c.values}
+            leaf_path["index"] = c.indexes
             ax = c.axis
             (key_value_path, leaf_path, self.unwanted_path) = ax.unmap_path_key(
                 key_value_path, leaf_path, self.unwanted_path