Merge pull request #50 from nmandery/integrate-h3arrow

integrate h3arrow into repo
nmandery · Jun 13, 2024 · 7089721 · 7089721
2 parents c941768 + 23e383b
commit 7089721
Show file tree

Hide file tree

Showing 104 changed files with 3,401 additions and 68 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -6,8 +6,8 @@ on:
       - main
     tags:
       - 'v*'
-#  release:
-#    types: [ created ]
+  #  release:
+  #    types: [ created ]
   pull_request:
     branches:
       - main
@@ -25,22 +25,25 @@ jobs:
   linux-x86_64:
     runs-on: ubuntu-latest
     needs:
-      - cargofmt
+      - rusttest
       - black
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5
         with:
           python-version: ${{env.PYTHON_VERSION}}
       - name: Install dependencies
-        run: python install-dev-dependencies.py
+        run: python h3ronpy/install-dev-dependencies.py
       - name: Build wheels
         uses: PyO3/maturin-action@v1
         env:
           RUSTFLAGS: "-C target-feature=+fxsr,+sse,+sse2,+sse3,+ssse3,+sse4.1,+sse4.2,+popcnt,+avx,+fma"
         with:
           target: x86_64
-          args: --release --out dist
+          args: >
+            --release
+            --manifest-path h3ronpy/Cargo.toml
+            --out dist
           manylinux: auto
       - name: Upload wheels
         uses: actions/upload-artifact@v4
@@ -52,12 +55,12 @@ jobs:
         run: |
           set -e
           pip install --force-reinstall dist/*.whl
-          pytest -s tests
+          pytest -s h3ronpy/tests
 
   linux-aarch64:
     runs-on: ubuntu-latest
     needs:
-      - cargofmt
+      - rusttest
       - black
       - linux-x86_64
     steps:
@@ -66,12 +69,15 @@ jobs:
         with:
           python-version: ${{env.PYTHON_VERSION}}
       - name: Install dependencies
-        run: python install-dev-dependencies.py
+        run: python h3ronpy/install-dev-dependencies.py
       - name: Build wheels
         uses: PyO3/maturin-action@v1
         with:
           target: aarch64-unknown-linux-gnu
-          args: --release --out dist
+          args: >
+            --release
+            --manifest-path h3ronpy/Cargo.toml
+            --out dist
           manylinux: auto
       - name: Upload wheels
         uses: actions/upload-artifact@v4
@@ -82,7 +88,7 @@ jobs:
 
   windows-x86_64:
     needs:
-      - cargofmt
+      - rusttest
       - black
       - linux-x86_64
     runs-on: windows-latest
@@ -92,14 +98,18 @@ jobs:
         with:
           python-version: ${{ env.PYTHON_VERSION }}
       - name: Install dependencies
-        run: python install-dev-dependencies.py
+        run: python h3ronpy/install-dev-dependencies.py
       - name: Build wheels
         uses: PyO3/maturin-action@v1
         env:
           RUSTFLAGS: "-C target-feature=+fxsr,+sse,+sse2,+sse3,+sse4.1,+sse4.2"
         with:
           target: x86_64
-          args: --release --out dist -i python
+          args: >
+            --release
+            --manifest-path h3ronpy/Cargo.toml
+            --out dist
+            -i python
       - name: Upload wheels
         uses: actions/upload-artifact@v4
         with:
@@ -110,46 +120,54 @@ jobs:
         run: |
           set -e
           pip install --force-reinstall dist/*.whl
-          pytest tests
+          pytest h3ronpy/tests
 
   macos-x86_64:
     needs:
-      - cargofmt
+      - rusttest
       - black
       - linux-x86_64
     runs-on: macos-latest
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5
         with:
+          # https://github.com/pypa/cibuildwheel/issues/1410
           python-version: ${{ env.PYTHON_VERSION }}
       - name: Install dependencies
-        run: python install-dev-dependencies.py
+        shell: bash
+        run: |
+          python h3ronpy/install-dev-dependencies.py
+          pip install --upgrade pip
       - name: Build wheels
         uses: PyO3/maturin-action@v1
         env:
-          RUSTFLAGS: "-C target-feature=+fxsr,+sse,+sse2,+sse3,+sse4.1,+sse4.2"
+          RUSTFLAGS: "-C target-feature=+sse3,+ssse3,+sse4.1,+sse4.2,+popcnt,+avx,+fma"
           # lower versions result in "illegal instruction"
           MACOSX_DEPLOYMENT_TARGET: "10.14"
         with:
           target: x86_64
-          args: --release --out dist -i python
+          args: >
+            --release
+            --manifest-path h3ronpy/Cargo.toml
+            --out dist
+            -i python
       - name: Upload wheels
         uses: actions/upload-artifact@v4
         with:
           name: wheels-macos-x86_64
           path: dist
-      - name: pytest
-        shell: bash
-        run: |
-          set -e
-          pip install --force-reinstall dist/*.whl
-          pytest tests
+      #- name: pytest
+      #  shell: bash
+      #  run: |
+      #    set -e
+      #    pip install --force-reinstall --verbose dist/*.whl
+      #    pytest h3ronpy/tests
 
   macos-aarch64:
     runs-on: macos-latest
     needs:
-      - cargofmt
+      - rusttest
       - black
       - linux-x86_64
     steps:
@@ -161,12 +179,15 @@ jobs:
           python-version: ${{env.PYTHON_VERSION}}
       - name: Install dependencies
         run: |
-          python install-dev-dependencies.py
+          python h3ronpy/install-dev-dependencies.py
       - name: Build wheels
         uses: PyO3/maturin-action@v1
         with:
           target: aarch64-apple-darwin
-          args: --release --out dist
+          args: >
+            --release
+            --manifest-path h3ronpy/Cargo.toml
+            --out dist
       - name: Upload wheels
         uses: actions/upload-artifact@v4
         with:
@@ -177,15 +198,17 @@ jobs:
     runs-on: ubuntu-latest
     if: "startsWith(github.ref, 'refs/tags/v')"
     needs:
-      - cargofmt
+      - rusttest
       - black
     steps:
       - uses: actions/checkout@v4
       - name: Build sdist
         uses: PyO3/maturin-action@v1
         with:
           command: sdist
-          args: --out dist
+          args: >
+            --manifest-path h3ronpy/Cargo.toml
+            --out dist
       - name: Upload sdist
         uses: actions/upload-artifact@v4
         with:
@@ -224,8 +247,8 @@ jobs:
           command: upload
           args: --skip-existing *
 
-  cargofmt:
-    name: cargo fmt
+  rusttest:
+    name: rust test
     runs-on: ubuntu-latest
     steps:
       - name: checkout repo
@@ -234,6 +257,18 @@ jobs:
         run: |
           cargo fmt -- --check
 
+      - name: Lint with clippy -- no features
+        run: cargo clippy --workspace --exclude h3ronpy
+
+      - name: Test with cargo -- no features
+        run: cargo test --workspace --exclude h3ronpy
+
+      - name: Lint with clippy -- all features
+        run: cargo clippy --workspace --exclude h3ronpy --all-features
+
+      - name: Test with cargo -- all features
+        run: cargo test --workspace --all-features --exclude h3ronpy
+
   black:
     name: black
     runs-on: ubuntu-latest
@@ -249,5 +284,6 @@ jobs:
           pip install black
 
       - name: run black
+        working-directory: h3ronpy
         run: |
-          black -l 120 python tests *.py docs/source/*.py
+          black --check -l 120 python tests *.py docs/source/*.py
diff --git a/.gitignore b/.gitignore
@@ -13,7 +13,7 @@ flamegraph*.svg
 __pycache__
 .pytest_cache
 .ruff_cache
-/docs/build/
-/docs/source/generated/
-/docs/source/_build/
-/dist
+/h3ronpy/docs/build/
+/h3ronpy/docs/source/generated/
+/h3ronpy/docs/source/_build/
+/h3ronpy/dist
diff --git a/.readthedocs.yaml b/.readthedocs.yaml
@@ -11,6 +11,6 @@ build:
 
 python:
   install:
-    - requirements: docs/requirements.txt
+    - requirements: h3ronpy/docs/requirements.txt
     - method: pip
-      path: .
+      path: ./h3ronpy
diff --git a/Cargo.toml b/Cargo.toml
@@ -1,38 +1,18 @@
-[package]
-name = "h3ronpy"
-version = "0.20.2"
-authors = ["Nico Mandery <nico@nmandery.net>"]
-description = "Data science toolkit for the H3 geospatial grid"
-edition = "2021"
-license = "MIT"
-keywords = ["geo", "spatial", "h3", "arrow", "python"]
-homepage = "https://github.com/nmandery/h3ronpy"
-repository = "https://github.com/nmandery/h3ronpy"
+[workspace]
+resolver = "2"
 
-[lib]
-name = "h3ronpy"
-crate-type = ["cdylib"]
+members = [
+    "h3ronpy",
+    "crates/h3arrow"
+]
 
-[dependencies]
-arrow = { version = "50.0.0", features = ["pyarrow"]}
-env_logger = "^0.10" # Stick env_logger with 0.10 to be able to build on readthedocs.org with rust 1.70
-geo-types = "^0.7"
-rasterh3 = { version = "^0.7", features = ["rayon"] }
-h3arrow = { version = "^0.4", features = ["geoarrow", "rayon"] }
-#h3arrow = { git = "https://github.com/nmandery/h3arrow.git", rev = "1491610246cf93f0c9e40974de857563451a2e03", features = ["geoarrow", "rayon"] }
-#h3arrow = { path = "/home/nicodev/CLionProjects/h3arrow", features = ["geoarrow", "rayon"] }
-ndarray = { version = "0.15", features = ["rayon"] }
-numpy = "0.20"
-ordered-float = ">=2.0.1"
-py_geo_interface = { version = "0.7", features = ["f64", "wkb"] }
-pyo3 = { version = "0.20", features = ["extension-module", "abi3", "abi3-py38"] }
-rayon = "^1.5"
-hashbrown = "0.14"
+[workspace.dependencies]
 geo = "0.28"
-itertools = "0.12"
-
+geo-types = "0.7"
+h3o = { version = "0.6" }
+rayon = "^1"
 
 [profile.release]
 lto = "thin"
 strip = true
-debug = false
+debug = false
diff --git a/crates/h3arrow/CHANGES.md b/crates/h3arrow/CHANGES.md
@@ -0,0 +1,28 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is loosely based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres
+to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+
+## Unreleased (YYYY-MM-DD TBD)
+
+## v0.4.0 (2024-03-01)
+* Update h3o to 0.6.
+* Upgrade geo to 0.28
+* Upgrade rstar to 0.12
+* Upgrade geozero to 0.12
+
+## v0.3.0 (2024-02-06)
+* Extend documentation on ParseUtf8Array::parse_utf8array.
+* Add ChangeResolutionOp::change_resolution_list.
+* Update geozero to 0.11.
+* Update h3o to 0.5.
+* Migrate from arrow2 to the official apache arrow implementation and aligned naming. This comes along with many API changes. `geoarrow::ToWKBLines` has been removed.
+
+## v0.2.0 (2023-08-31)
+* Upgrade h3o from v0.3 to v0.4. Due to the new polyfill modes this lead to API breakages in the `ToCellsOptions` struct.
+
+## v0.1.0 (2023-07-24)
+* Initial release
diff --git a/crates/h3arrow/Cargo.toml b/crates/h3arrow/Cargo.toml
@@ -0,0 +1,32 @@
+[package]
+name = "h3arrow"
+version = "0.4.0"
+edition = "2021"
+authors = ["Nico Mandery <nico@nmandery.net>"]
+description = "Integration of the H3 geospatial grid with the arrow memory model"
+license = "MIT"
+keywords = ["geo", "spatial", "h3", "arrow"]
+readme = "README.md"
+homepage = "https://github.com/nmandery/h3arrow"
+repository = "https://github.com/nmandery/h3arrow"
+
+[package.metadata.docs.rs]
+all-features = true
+
+[features]
+geoarrow = ["dep:geoarrow", "dep:geozero"]
+rayon = ["dep:rayon", "geoarrow/rayon"]
+spatial_index = ["dep:rstar"]
+
+[dependencies]
+ahash = "0.8"
+arrow = "50"
+geoarrow = { package = "geoarrow", version = "0.1.0", optional = true, features = ["geozero"] }
+geo-types = { workspace = true }
+geo = { workspace = true }
+geozero = { version = "0.12", default-features = false, features = ["with-geo", "with-wkb"], optional = true }
+h3o = { workspace = true, features = ["geo"] }
+nom = "7"
+rayon = { workspace = true, optional = true }
+rstar = { version = "0.12", optional = true }
+thiserror = "1"
diff --git a/LICENSE.txt → crates/h3arrow/LICENSE-MIT b/LICENSE.txt → crates/h3arrow/LICENSE-MIT
diff --git a/crates/h3arrow/README.md b/crates/h3arrow/README.md
@@ -0,0 +1,4 @@
+# h3arrow
+
+Experimental integration of H3 (via [h3o](https://github.com/HydroniumLabs/h3o)) with the Arrow memory model.
+
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,4 @@
		# h3arrow

		Experimental integration of H3 (via [h3o](https://github.com/HydroniumLabs/h3o)) with the Arrow memory model.