-
Notifications
You must be signed in to change notification settings - Fork 1
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #9 from oreum-industries/update-to-python311
Update to python 3.11 and MacOS accelerate BLAS
- Loading branch information
Showing
7 changed files
with
228 additions
and
164 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,13 +1,18 @@ | ||
| name: ci | ||
| on: [pull_request] | ||
| on: pull_request | ||
| jobs: | ||
| qc_coding_standards: | ||
| runs-on: ubuntu-latest | ||
| code_qc: | ||
| runs-on: ${{ matrix.platform }} | ||
| strategy: | ||
| matrix: | ||
| platform: ["ubuntu-latest"] | ||
| python-version: ["3.11"] | ||
| steps: | ||
| - uses: actions/checkout@v3 | ||
| - name: Set Up Basic Python Env | ||
| uses: actions/setup-python@v4 | ||
| - uses: actions/checkout@v4 | ||
| - name: Setup Python ${{ matrix.python-version }} | ||
| uses: actions/setup-python@v5 | ||
| with: | ||
| python-version: '3.10' | ||
| - name: Run Linter & Static Security Checks | ||
| run: make lint | ||
| python-version: ${{ matrix.python-version }} | ||
| - name: Run Linting and Code Security Checks | ||
| run: | | ||
| make lint |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Large diffs are not rendered by default.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,61 +1,88 @@ | ||
| # Makefile | ||
| # Simplifies dev install on MacOS x64 (Intel) | ||
| .PHONY: create-env dev help lint mamba uninstall | ||
| .SILENT: create-env dev help lint mamba uninstall | ||
| SHELL := /bin/bash | ||
| MAMBADL = https://github.com/conda-forge/miniforge/releases/latest/download/ | ||
| MAMBAV = Mambaforge-MacOSX-x86_64.sh | ||
| MAMBARC = $(HOME)/.mambarc | ||
| MAMBARCMSG = Please create file $(MAMBARC), particularly to set platform: osx-64 | ||
| MAMBADIR = $(HOME)/.mamba | ||
| # NOTE: | ||
| # + Intended for install on MacOS Apple Silicon arm64 using Accelerate | ||
| # (NOT Intel x86 using MKL via Rosetta 2) | ||
| # + Uses sh by default: to confirm shell create a recipe with $(info $(SHELL)) | ||
| .PHONY: dev help install-env install-mamba lint uninstall-env uninstall-mamba | ||
| .SILENT: dev help install-env install-mamba lint uninstall-env uninstall-mamba | ||
| MAMBADL := https://github.com/conda-forge/miniforge/releases/download/23.3.1-1 | ||
| MAMBAV := Miniforge3-MacOSX-arm64.sh | ||
| MAMBARCMSG := Please create file $(MAMBARC), importantly set `platform: osx-arm64` | ||
| MAMBARC := $(HOME)/.mambarc | ||
| MAMBADIR := $(HOME)/miniforge | ||
| PYTHON_DEFAULT = $(or $(shell which python3), $(shell which python)) | ||
| PYTHON_ENV = $(MAMBADIR)/envs/squirrel_radar/bin/python | ||
| ifneq ("$(wildcard $(PYTHON_ENV))","") | ||
| PYTHON = $(PYTHON_ENV) | ||
| else | ||
| PYTHON = $(PYTHON_DEFAULT) | ||
| endif | ||
| VERSION := $(shell echo $(VVERSION) | sed 's/v//') | ||
|
|
||
| create-env: ## create mamba (conda) environment CONDA_SUBDIR=osx-64 | ||
| export PATH=$(MAMBADIR)/bin:$$PATH; \ | ||
| if which mamba; then echo "mamba ready"; else make mamba; fi | ||
| mamba env create --file condaenv_squirrel_radar.yml; | ||
|
|
||
| dev: # create env for local dev on any machine MacOS x64 (Intel) | ||
| make create-env | ||
| dev: ## create env for local dev | ||
| make install-env | ||
| export PATH=$(MAMBADIR)/envs/squirrel_radar/bin:$$PATH; \ | ||
| export CONDA_ENV_PATH=$(MAMBADIR)/envs/squirrel_radar/bin; \ | ||
| export CONDA_DEFAULT_ENV=squirrel_radar; \ | ||
| export CONDA_SUBDIR=osx-64; \ | ||
| $(PYTHON_ENV) -m pip index versions oreum_core; \ | ||
| $(PYTHON_ENV) -m pip install -e .[dev]; \ | ||
| export CONDA_SUBDIR=osx-arm64; \ | ||
| $(PYTHON_ENV) -m pip install -e ".[dev]"; \ | ||
| $(PYTHON_ENV) -c "import numpy as np; np.__config__.show()" > dev/install_log/blas_info.txt; \ | ||
| pipdeptree -a > dev/install_log/pipdeptree.txt; \ | ||
| pipdeptree -a -r > dev/install_log/pipdeptree_rev.txt; \ | ||
| pip-licenses -saud -f markdown --output-file LICENSES_THIRD_PARTY.md; \ | ||
| pre-commit install; \ | ||
| pre-commit autoupdate; | ||
|
|
||
| help: | ||
| @echo "Use \`make <target>' where <target> is:" | ||
| @echo " dev create local dev env" | ||
| @echo " lint run code lint & security checks" | ||
| @echo " uninstall remove local dev env (use from parent dir as `make -C squirrel_radar uninstall`)" | ||
| install-env: ## create mamba (conda) environment | ||
| export PATH=$(MAMBADIR)/bin:$$PATH; \ | ||
| if which mamba; then echo "mamba ready"; else make install-mamba; fi | ||
| export PATH=$(MAMBADIR)/bin:$$PATH; \ | ||
| export CONDA_SUBDIR=osx-arm64; \ | ||
| mamba update -n base mamba; \ | ||
| mamba env create --file condaenv_squirrel_radar.yml -y; | ||
|
|
||
| lint: ## run code linters and static security (checks only) | ||
| $(PYTHON) -m pip install black flake8 isort | ||
| install-mamba: ## get mamba via miniforge, explicitly use bash | ||
| test -f $(MAMBARC) || { echo $(MAMBARCMSG); exit 1; } | ||
| wget $(MAMBADL)/$(MAMBAV) -O $(HOME)/miniforge.sh | ||
| chmod 755 $(HOME)/miniforge.sh | ||
| bash $(HOME)/miniforge.sh -b -p $(MAMBADIR) | ||
| export PATH=$(MAMBADIR)/bin:$$PATH; \ | ||
| conda init zsh; | ||
| rm $(HOME)/miniforge.sh | ||
|
|
||
| lint: ## run code lint & security checks | ||
| $(PYTHON) -m pip install black flake8 interrogate isort bandit | ||
| black --check --diff --config pyproject.toml src/ | ||
| isort --check-only src/ | ||
| flake8 src/ | ||
| interrogate src/ | ||
| bandit --config pyproject.toml -r src/ | ||
|
|
||
| mamba: ## get mamba via mambaforge for MacOS x86_64 (Intel via Rosetta2) | ||
| test -f $(MAMBARC) || { echo $(MAMBARCMSG); exit 1; } | ||
| wget $(MAMBADL)$(MAMBAV) -O $$HOME/mambaforge.sh | ||
| bash $$HOME/mambaforge.sh -b -p $$HOME/.mamba | ||
| export PATH=$$HOME/.mamba/bin:$$PATH; \ | ||
| conda init zsh; | ||
| rm $$HOME/mambaforge.sh | ||
| help: | ||
| @echo "Use \make <target> where <target> is:" | ||
| @echo " dev create local dev env" | ||
| @echo " lint run code lint & security checks" | ||
| @echo " uninstall-env remove env (use from parent dir \make -C squirrel_radar ...)" | ||
|
|
||
|
|
||
| test-dev-env: ## test the dev machine install of critial numeric packages | ||
| export PATH=$(MAMBADIR)/bin:$$PATH; \ | ||
| export PATH=$(MAMBADIR)/envs/squirrel_radar/bin:$$PATH; \ | ||
| export CONDA_ENV_PATH=$(MAMBADIR)/envs/squirrel_radar/bin; \ | ||
| export CONDA_DEFAULT_ENV=squirrel_radar; \ | ||
| $(PYTHON_ENV) -c "import numpy as np; np.test()" > dev/install_log/tests_numpy.txt; \ | ||
|
|
||
|
|
||
| uninstall-env: ## remove mamba env | ||
| export PATH=$(MAMBADIR)/bin:$$PATH; \ | ||
| export CONDA_SUBDIR=osx-arm64; \ | ||
| mamba env remove --name squirrel_radar -y; \ | ||
| conda clean --all -afy; | ||
| # mamba clean -afy # 2023-12-10 see: https://github.com/mamba-org/mamba/issues/3044 | ||
|
|
||
| uninstall: ## remove mamba env (use from parent dir as `make -C squirrel_radar/ uninstall`) | ||
| mamba env remove --name squirrel_radar -y | ||
| mamba clean -ay | ||
| uninstall-mamba: ## last ditch per https://github.com/conda-forge/miniforge#uninstallation | ||
| conda init zsh --reverse | ||
| rm -rf $(MAMBADIR) | ||
| rm -rf $(HOME)/.conda | ||
| rm -f $(HOME)/.condarc |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,23 +1,23 @@ | ||
| # Manually created as-at 2022-02-15 | ||
| # Last updated as-at 2023-07-31 | ||
| # NOTE: | ||
| # + Creates a virtual env for project usage | ||
| # + Require running on Intel x86 AMD64 CPU (or Rosetta2 on MacOS) | ||
| # + Install with mamba via Makefile, must force MKL via libblas (note not "blas") | ||
| # + Install numpy alongside from defaults channel (the only one compiled against MKL) | ||
| # see https://github.com/conda-forge/numpy-feedstock/issues/84#issuecomment-385186685 | ||
| # + This is a minimal set of packages dependent upon forcing MKL and conda-forge | ||
| # + There's also a fuller set of deps to be installed by pip in the pyproject.toml | ||
| # Manually created as-at 2023-0601 | ||
| # Last updated as-at 2023-12-10 | ||
| # NOTES: | ||
| # + Creates a virtual env for project development | ||
| # + Require running on Apple Silicon ARM64 (NOT Intel x86 AMD64 CPU via Rosetta2) | ||
| # + Install with mamba via Makefile, there's also a fuller set of deps to be | ||
| # installed by pip in the pyproject.toml | ||
| # + numpy not strictly needed here but because we're explcit about BLAS/LAPACK | ||
| # from acccelerate, then also get specific numpy | ||
| # also see https://numpy.org/install/ and https://github.com/numpy/numpy/pull/24053 | ||
| # + libblas, libcblas, liblapack, not strictly needed here but we want to be | ||
| # explcit about forcing BLAS to use Accelerate | ||
| name: squirrel_radar | ||
| channels: | ||
| - defaults | ||
| # - conda-forge # commented out - we declare specifics per package and want numpy from defaults | ||
| - conda-forge | ||
| # - defaults | ||
| dependencies: | ||
| - numpy>=1.24.3 # NOTE: the MKL version is only available in defaults | ||
| - conda-forge::ipykernel>=6.23.1 | ||
| - conda-forge::libblas=*[build=*mkl] # force MKL | ||
| - conda-forge::libcblas=*[build=*mkl] # force MKL | ||
| - conda-forge::liblapack=*[build=*mkl] # force MKL | ||
| - conda-forge::mkl==2021.4.* # NOTE 2022 version(s) dont work on MacOS https://stackoverflow.com/a/71640311/1165112 | ||
| - conda-forge::mkl-service==2.4.* | ||
| - conda-forge::python==3.10.* | ||
| - conda-forge::numpy>=1.25.2 #>=1.26.2 # see NOTES | ||
| - conda-forge::libblas=*[build=*accelerate] # see NOTES | ||
| - conda-forge::libcblas=*[build=*accelerate] # see NOTES | ||
| - conda-forge::liblapack=*[build=*accelerate] # see NOTES | ||
| - conda-forge::python==3.11.* |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters