Remove pyarrow required dependency (#64)

* WIP: array classes * Upgrade to geoarrow 0.4.0-beta.1 * Cleanup * lint * Upgrade dependencies * revert ndarray bump * fix pyo3 signature * Single ndarray version * Revamp `__init__` to remove pyarrow * lint * fix warning * remove more pyarrow dep * start fixing tests * progress on fixing tests * updated polars api * fmt * Use git tag * Fix h3 v4 names * Progress * relax numpy version * fix h3 test version * fix tests * fix tests * lint * Fix tests * check for bare exception * skip empty point test * Move up one level * rasterio bound * fix test imports
nmandery · Nov 15, 2024 · 1876aa5 · 1876aa5
1 parent dc1de07
commit 1876aa5
Show file tree

Hide file tree

Showing 48 changed files with 910 additions and 925 deletions.
diff --git a/Cargo.toml b/Cargo.toml
@@ -1,17 +1,14 @@
 [workspace]
 resolver = "2"
 
-members = [
-    "h3ronpy",
-    "crates/h3arrow"
-]
+members = ["h3ronpy", "crates/h3arrow"]
 
 [workspace.dependencies]
 geo = "0.28"
 geo-types = "0.7"
 h3o = { version = "0.6" }
 rayon = "^1"
-arrow = { version = "52" }
+arrow = { version = "53" }
 
 [profile.release]
 lto = "thin"

diff --git a/crates/h3arrow/Cargo.toml b/crates/h3arrow/Cargo.toml
@@ -21,10 +21,13 @@ spatial_index = ["dep:rstar"]
 [dependencies]
 ahash = "0.8"
 arrow = { workspace = true }
-geoarrow = { version = "0.3", optional = true, features = ["geozero"] }
+geoarrow = { git = "https://github.com/geoarrow/geoarrow-rs", rev = "49fd4cbdc4bc08a2f1e0341ec7df700df18d2bdb", optional = true }
 geo-types = { workspace = true }
 geo = { workspace = true }
-geozero = { version = "^0.13", default-features = false, features = ["with-geo", "with-wkb"], optional = true }
+geozero = { version = "^0.14", default-features = false, features = [
+    "with-geo",
+    "with-wkb",
+], optional = true }
 h3o = { workspace = true, features = ["geo"] }
 nom = "7"
 rayon = { workspace = true, optional = true }

diff --git a/crates/h3arrow/src/algorithm/string.rs b/crates/h3arrow/src/algorithm/string.rs
@@ -246,15 +246,13 @@ mod test {
 
     #[test]
     fn parse_utf8_array_cells_invalid_fail() {
-        let stringarray =
-            GenericStringArray::<i32>::from_iter(vec![Some("invalid".to_string())].into_iter());
+        let stringarray = GenericStringArray::<i32>::from_iter(vec![Some("invalid".to_string())]);
         assert!(CellIndexArray::parse_genericstringarray(&stringarray, false).is_err());
     }
 
     #[test]
     fn parse_utf8_array_cells_invalid_to_invalid() {
-        let utf8_array =
-            GenericStringArray::<i32>::from_iter(vec![Some("invalid".to_string())].into_iter());
+        let utf8_array = GenericStringArray::<i32>::from_iter(vec![Some("invalid".to_string())]);
         let cell_array = CellIndexArray::parse_genericstringarray(&utf8_array, true).unwrap();
         assert_eq!(1, cell_array.len());
         assert!(cell_array.iter().all(|v| v.is_none()))
@@ -268,9 +266,9 @@ mod test {
         let stringarray: GenericStringArray<i64> = cellindexarray.to_genericstringarray().unwrap();
 
         assert_eq!(cellindexarray.len(), stringarray.len());
-        assert_eq!(stringarray.is_valid(0), true);
+        assert!(stringarray.is_valid(0));
         assert_eq!(stringarray.value(0), "89283080ddbffff");
-        assert_eq!(stringarray.is_valid(1), false);
+        assert!(!stringarray.is_valid(1));
     }
 
     #[test]

diff --git a/crates/h3arrow/src/array/from_geoarrow.rs b/crates/h3arrow/src/array/from_geoarrow.rs
@@ -9,33 +9,13 @@ use crate::error::Error;
 use arrow::array::OffsetSizeTrait;
 use geo_types::Geometry;
 use geoarrow::array::WKBArray;
-use geoarrow::trait_::GeometryArrayAccessor;
-use geoarrow::GeometryArrayTrait;
+use geoarrow::trait_::ArrayAccessor;
+use geoarrow::ArrayBase;
 use h3o::CellIndex;
 #[cfg(feature = "rayon")]
 use rayon::prelude::{IntoParallelIterator, ParallelIterator};
 
 macro_rules! impl_to_cells {
-    ($array_type:ty, $offset:tt) => {
-        impl<$offset: OffsetSizeTrait> ToCellListArray<$offset> for $array_type {
-            fn to_celllistarray(
-                &self,
-                options: &ToCellsOptions,
-            ) -> Result<H3ListArray<CellIndex, $offset>, Error> {
-                self.iter_geo()
-                    .map(|v| v.map(Geometry::from))
-                    .to_celllistarray(options)
-            }
-        }
-
-        impl<$offset: OffsetSizeTrait> ToCellIndexArray for $array_type {
-            fn to_cellindexarray(&self, options: &ToCellsOptions) -> Result<CellIndexArray, Error> {
-                self.iter_geo()
-                    .map(|v| v.map(Geometry::from))
-                    .to_cellindexarray(options)
-            }
-        }
-    };
     ($array_type:ty) => {
         impl<O: OffsetSizeTrait> ToCellListArray<O> for $array_type {
             fn to_celllistarray(
@@ -58,12 +38,12 @@ macro_rules! impl_to_cells {
     };
 }
 
-impl_to_cells!(geoarrow::array::LineStringArray<O, 2>, O);
-impl_to_cells!(geoarrow::array::MultiLineStringArray<O, 2>, O);
-impl_to_cells!(geoarrow::array::MultiPointArray<O, 2>, O);
-impl_to_cells!(geoarrow::array::MultiPolygonArray<O, 2>, O);
+impl_to_cells!(geoarrow::array::LineStringArray<2>);
+impl_to_cells!(geoarrow::array::MultiLineStringArray<2>);
+impl_to_cells!(geoarrow::array::MultiPointArray<2>);
+impl_to_cells!(geoarrow::array::MultiPolygonArray<2>);
 impl_to_cells!(geoarrow::array::PointArray<2>);
-impl_to_cells!(geoarrow::array::PolygonArray<O, 2>, O);
+impl_to_cells!(geoarrow::array::PolygonArray<2>);
 
 impl<O: OffsetSizeTrait> ToCellListArray<O> for WKBArray<O> {
     fn to_celllistarray(

diff --git a/crates/h3arrow/src/array/to_geoarrow.rs b/crates/h3arrow/src/array/to_geoarrow.rs
@@ -14,7 +14,7 @@ pub trait ToGeoArrowPolygons {
     fn to_geoarrow_polygons<O: OffsetSizeTrait>(
         &self,
         use_degrees: bool,
-    ) -> Result<PolygonArray<O, 2>, Self::Error>;
+    ) -> Result<PolygonArray<2>, Self::Error>;
 }
 
 impl<T> ToGeoArrowPolygons for T
@@ -26,7 +26,7 @@ where
     fn to_geoarrow_polygons<O: OffsetSizeTrait>(
         &self,
         use_degrees: bool,
-    ) -> Result<PolygonArray<O, 2>, Self::Error> {
+    ) -> Result<PolygonArray<2>, Self::Error> {
         Ok(self.to_polygons(use_degrees)?.into())
     }
 }
@@ -51,7 +51,7 @@ pub trait ToGeoArrowLineStrings {
     fn to_geoarrow_lines<O: OffsetSizeTrait>(
         &self,
         use_degrees: bool,
-    ) -> Result<LineStringArray<O, 2>, Self::Error>;
+    ) -> Result<LineStringArray<2>, Self::Error>;
 }
 
 impl<T> ToGeoArrowLineStrings for T
@@ -62,7 +62,7 @@ where
     fn to_geoarrow_lines<O: OffsetSizeTrait>(
         &self,
         use_degrees: bool,
-    ) -> Result<LineStringArray<O, 2>, Self::Error> {
+    ) -> Result<LineStringArray<2>, Self::Error> {
         Ok(self.to_linestrings(use_degrees)?.into())
     }
 }

diff --git a/crates/h3arrow/src/array/validity.rs b/crates/h3arrow/src/array/validity.rs
@@ -3,6 +3,7 @@ use arrow::array::UInt64Array;
 /// Conversion corresponding to `From` with the difference that the validity mask
 /// is set accordingly to the validity to the contained values.
 pub trait FromWithValidity<T> {
+    #[allow(dead_code)]
     fn from_with_validity(value: T) -> Self;
 }
 

diff --git a/crates/h3arrow/src/spatial_index.rs b/crates/h3arrow/src/spatial_index.rs
@@ -212,13 +212,13 @@ mod tests {
         assert_eq!(mask.len(), 4);
 
         assert!(mask.is_valid(0));
-        assert_eq!(mask.value(0), false);
+        assert!(!mask.value(0));
 
         assert!(mask.is_valid(1));
-        assert_eq!(mask.value(1), true);
+        assert!(mask.value(1));
 
         assert!(mask.is_valid(2));
-        assert_eq!(mask.value(2), false);
+        assert!(!mask.value(2));
 
         assert!(!mask.is_valid(3));
     }
@@ -237,13 +237,13 @@ mod tests {
         assert_eq!(mask.len(), 4);
 
         assert!(mask.is_valid(0));
-        assert_eq!(mask.value(0), true);
+        assert!(mask.value(0));
 
         assert!(mask.is_valid(1));
-        assert_eq!(mask.value(1), false);
+        assert!(!mask.value(1));
 
         assert!(mask.is_valid(2));
-        assert_eq!(mask.value(2), false);
+        assert!(!mask.value(2));
 
         assert!(!mask.is_valid(3));
     }

diff --git a/h3ronpy/Cargo.toml b/h3ronpy/Cargo.toml
@@ -14,17 +14,27 @@ name = "h3ronpy"
 crate-type = ["cdylib"]
 
 [dependencies]
-arrow = { workspace = true, features = ["pyarrow"] }
+arrow = { workspace = true }
 env_logger = "^0.11"
 geo-types = { workspace = true }
 geo = { workspace = true }
 h3arrow = { path = "../crates/h3arrow", features = ["geoarrow", "rayon"] }
 hashbrown = "0.14"
 itertools = "0.13"
-ndarray = { version = "0.15", features = ["rayon"] }
-numpy = "0.21"
+ndarray = { version = "0.16", features = ["rayon"] }
+numpy = "0.22"
 ordered-float = ">=2.0.1"
-py_geo_interface = { version = "0.8", features = ["f64", "wkb"] }
-pyo3 = { version = "^0.21", features = ["extension-module", "abi3", "abi3-py39"] }
-rasterh3 = { version = "^0.8", features = ["rayon"] }
+py_geo_interface = { git = "https://github.com/nmandery/py_geo_interface", rev = "36723cdbabc2a7aad1746a8c06db17b4e39ce3b9", features = [
+    "f64",
+    "wkb",
+] }
+pyo3 = { version = "^0.22", features = [
+    "extension-module",
+    "abi3",
+    "abi3-py39",
+] }
+pyo3-arrow = { version = "0.5.1", default-features = false }
+rasterh3 = { git = "https://github.com/kylebarron/rasterh3", branch = "kyle/bump-ndarray", features = [
+    "rayon",
+] }
 rayon = { workspace = true }
diff --git a/h3ronpy/pyproject.toml b/h3ronpy/pyproject.toml
@@ -1,28 +1,30 @@
 [build-system]
-requires = [
-    "maturin>=1.7",
-]
+requires = ["maturin>=1.7"]
 build-backend = "maturin"
 
 [tool.pytest.ini_options]
 minversion = "6.0"
 addopts = "--doctest-modules -v -s"
-testpaths = [
-    "tests"
-]
+testpaths = ["tests"]
 
 [tool.ruff]
 # Never enforce `E501` (line length violations).
 ignore = ["E501"]
+select = [
+    # Pyflakes
+    "F",
+    # Pycodestyle
+    # "E",
+    "W",
+    # isort
+    "I",
+]
 
 [project]
 name = "h3ronpy"
 readme = "../README.rst"
 
-dependencies = [
-    "numpy<2",
-    "pyarrow>=17.0"
-]
+dependencies = ["numpy", "arro3-core>=0.4"]
 classifiers = [
     "Programming Language :: Python :: 3",
     "Topic :: Scientific/Engineering :: GIS",
@@ -31,20 +33,17 @@ classifiers = [
 
 
 [project.optional-dependencies]
-polars = [
-    "polars>=1"
-]
-pandas = [
-    "geopandas>=1"
-]
+polars = ["polars>=1"]
+pandas = ["geopandas>=1"]
 test = [
-    "rasterio",
+    "rasterio>=1.4",
     "Shapely>=1.7",
     "pytest>=6",
-    "h3>=3.7",
-    "pytest-benchmark"
+    "h3<4",
+    "pytest-benchmark",
+    "pyarrow>=15",
 ]
 
 [tool.maturin]
 python-source = "python"
-module-name = "h3ronpy.h3ronpyrs"
+module-name = "h3ronpy.h3ronpyrs"
diff --git a/h3ronpy/python/h3ronpy/__init__.py b/h3ronpy/python/h3ronpy/__init__.py
@@ -8,7 +8,10 @@
 if not _native.is_release_build():
     import warnings
 
-    warnings.warn("h3ronpy has not been compiled in release mode. Performance will be degraded.", RuntimeWarning)
+    warnings.warn(
+        "h3ronpy has not been compiled in release mode. Performance will be degraded.",
+        RuntimeWarning,
+    )
 
 
 __all__ = [