Merge branch 'main' into docs

optimagic-dev · Nov 5, 2024 · 5f12dcf · 5f12dcf
2 parents 1bfcb89 + 7cb7710
commit 5f12dcf
Show file tree

Hide file tree

Showing 151 changed files with 764 additions and 300 deletions.
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -70,9 +70,10 @@ jobs:
           micromamba activate optimagic
           pytest -m "not slow and not jax"
   run-tests-with-old-pandas:
-    # This job is only for testing if optimagic works with older pandas versions, as
-    # many pandas functions we use will be deprecated in pandas 3. optimagic's behavior
-    # for older verions is handled in src/optimagic/compat.py.
+    # This job is only for testing if optimagic works with pandas<2, as many pandas
+    # functions we use will be deprecated in pandas 3. optimagic's behavior for older
+    # verions is handled in src/optimagic/compat.py. For compatibility with we have to
+    # restrict numpy<2.
     name: Run tests for ${{ matrix.os}} on ${{ matrix.python-version }} with pandas 1
     runs-on: ${{ matrix.os }}
     strategy:
@@ -96,6 +97,32 @@ jobs:
         run: |
           micromamba activate optimagic
           pytest -m "not slow and not jax"
+  run-tests-with-old-numpy:
+    # This job is only for testing if optimagic works with numpy<2. Because we already
+    # test pandas<2 with numpy<2, in this environment we restrict pandas>=2.
+    name: Run tests for ${{ matrix.os}} on ${{ matrix.python-version }} with numpy 1
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os:
+          - ubuntu-latest
+        python-version:
+          - '3.10'
+    steps:
+      - uses: actions/checkout@v4
+      - name: create build environment
+        uses: mamba-org/setup-micromamba@v1
+        with:
+          environment-file: ./.tools/envs/testenv-numpy.yml
+          cache-environment: true
+          create-args: |
+            python=${{ matrix.python-version }}
+      - name: run pytest
+        shell: bash -l {0}
+        run: |
+          micromamba activate optimagic
+          pytest -m "not slow and not jax"
   code-in-docs:
     name: Run code snippets in documentation
     runs-on: ubuntu-latest

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -6,7 +6,7 @@ repos:
       - id: check-useless-excludes
         # - id: identity  # Prints all files passed to pre-commits. Debugging.
   - repo: https://github.com/lyz-code/yamlfix
-    rev: 1.16.0
+    rev: 1.17.0
     hooks:
       - id: yamlfix
         exclude: tests/optimagic/optimizers/_pounders/fixtures
@@ -19,7 +19,7 @@ repos:
         always_run: true
         require_serial: true
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.6.0
+    rev: v5.0.0
     hooks:
       - id: check-added-large-files
         args:
@@ -56,7 +56,7 @@ repos:
       - id: yamllint
         exclude: tests/optimagic/optimizers/_pounders/fixtures
   - repo: https://github.com/PyCQA/docformatter
-    rev: v1.7.5
+    rev: eb1df34
     hooks:
       - id: docformatter
         args:
@@ -68,7 +68,7 @@ repos:
           - --blank
         exclude: src/optimagic/optimization/algo_options.py
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.5.7
+    rev: v0.7.2
     hooks:
       # Run the linter.
       - id: ruff
@@ -85,7 +85,7 @@ repos:
           - pyi
           - jupyter
   - repo: https://github.com/executablebooks/mdformat
-    rev: 0.7.17
+    rev: 0.7.18
     hooks:
       - id: mdformat
         additional_dependencies:
@@ -97,7 +97,7 @@ repos:
           - '88'
         files: (README\.md)
   - repo: https://github.com/executablebooks/mdformat
-    rev: 0.7.17
+    rev: 0.7.18
     hooks:
       - id: mdformat
         additional_dependencies:
@@ -108,7 +108,7 @@ repos:
           - '88'
         files: (docs/.)
   - repo: https://github.com/kynan/nbstripout
-    rev: 0.7.1
+    rev: 0.8.0
     hooks:
       - id: nbstripout
         exclude: |
@@ -119,12 +119,12 @@ repos:
         args:
           - --drop-empty-cells
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.11.1
+    rev: v1.13.0
     hooks:
       - id: mypy
         files: src|tests
         additional_dependencies:
-          - numpy<2.0
+          - numpy
           - packaging
           - pandas-stubs
           - sqlalchemy-stubs

diff --git a/.tools/envs/testenv-linux.yml b/.tools/envs/testenv-linux.yml
@@ -8,27 +8,27 @@ dependencies:
   - jax
   - cyipopt>=1.4.0  # dev, tests
   - pygmo>=2.19.0  # dev, tests
-  - nlopt  # dev, tests
+  - nlopt  # dev, tests, docs
   - pip  # dev, tests, docs
   - pytest  # dev, tests
   - pytest-cov  # tests
   - pytest-xdist  # dev, tests
   - statsmodels  # dev, tests
   - cloudpickle  # run, tests
   - joblib  # run, tests
-  - numpy<2.0  # run, tests
+  - numpy >= 2  # run, tests
   - pandas  # run, tests
   - plotly  # run, tests
   - pybaum >= 0.1.2  # run, tests
   - scipy>=1.2.1  # run, tests
   - sqlalchemy  # run, tests
   - seaborn  # dev, tests
-  - mypy>=1.11  # dev, tests
+  - mypy=1.13  # dev, tests
   - pyyaml  # dev, tests
   - jinja2  # dev, tests
   - annotated-types  # dev, tests
   - pip:  # dev, tests, docs
-      - DFO-LS  # dev, tests
+      - DFO-LS>=1.5.3  # dev, tests
       - Py-BOBYQA  # dev, tests
       - fides==0.7.4  # dev, tests
       - kaleido  # dev, tests
@@ -37,4 +37,5 @@ dependencies:
       - types-openpyxl  # dev, tests
       - types-jinja2  # dev, tests
       - sqlalchemy-stubs  # dev, tests
+      - sphinxcontrib-mermaid  # dev, tests, docs
       - -e ../../
diff --git a/.tools/envs/testenv-numpy.yml b/.tools/envs/testenv-numpy.yml
@@ -0,0 +1,38 @@
+---
+name: optimagic
+channels:
+  - conda-forge
+  - nodefaults
+dependencies:
+  - pandas>=2
+  - numpy<2
+  - cyipopt>=1.4.0  # dev, tests
+  - pygmo>=2.19.0  # dev, tests
+  - nlopt  # dev, tests, docs
+  - pip  # dev, tests, docs
+  - pytest  # dev, tests
+  - pytest-cov  # tests
+  - pytest-xdist  # dev, tests
+  - statsmodels  # dev, tests
+  - cloudpickle  # run, tests
+  - joblib  # run, tests
+  - plotly  # run, tests
+  - pybaum >= 0.1.2  # run, tests
+  - scipy>=1.2.1  # run, tests
+  - sqlalchemy  # run, tests
+  - seaborn  # dev, tests
+  - mypy=1.13  # dev, tests
+  - pyyaml  # dev, tests
+  - jinja2  # dev, tests
+  - annotated-types  # dev, tests
+  - pip:  # dev, tests, docs
+      - DFO-LS>=1.5.3  # dev, tests
+      - Py-BOBYQA  # dev, tests
+      - fides==0.7.4  # dev, tests
+      - kaleido  # dev, tests
+      - types-cffi  # dev, tests
+      - types-openpyxl  # dev, tests
+      - types-jinja2  # dev, tests
+      - sqlalchemy-stubs  # dev, tests
+      - sphinxcontrib-mermaid  # dev, tests, docs
+      - -e ../../
diff --git a/.tools/envs/testenv-others.yml b/.tools/envs/testenv-others.yml
@@ -6,27 +6,27 @@ channels:
 dependencies:
   - cyipopt>=1.4.0  # dev, tests
   - pygmo>=2.19.0  # dev, tests
-  - nlopt  # dev, tests
+  - nlopt  # dev, tests, docs
   - pip  # dev, tests, docs
   - pytest  # dev, tests
   - pytest-cov  # tests
   - pytest-xdist  # dev, tests
   - statsmodels  # dev, tests
   - cloudpickle  # run, tests
   - joblib  # run, tests
-  - numpy<2.0  # run, tests
+  - numpy >= 2  # run, tests
   - pandas  # run, tests
   - plotly  # run, tests
   - pybaum >= 0.1.2  # run, tests
   - scipy>=1.2.1  # run, tests
   - sqlalchemy  # run, tests
   - seaborn  # dev, tests
-  - mypy>=1.11  # dev, tests
+  - mypy=1.13  # dev, tests
   - pyyaml  # dev, tests
   - jinja2  # dev, tests
   - annotated-types  # dev, tests
   - pip:  # dev, tests, docs
-      - DFO-LS  # dev, tests
+      - DFO-LS>=1.5.3  # dev, tests
       - Py-BOBYQA  # dev, tests
       - fides==0.7.4  # dev, tests
       - kaleido  # dev, tests
@@ -35,4 +35,5 @@ dependencies:
       - types-openpyxl  # dev, tests
       - types-jinja2  # dev, tests
       - sqlalchemy-stubs  # dev, tests
+      - sphinxcontrib-mermaid  # dev, tests, docs
       - -e ../../
diff --git a/.tools/envs/testenv-pandas.yml b/.tools/envs/testenv-pandas.yml
@@ -4,34 +4,35 @@ channels:
   - conda-forge
   - nodefaults
 dependencies:
-  - pandas<2.0.0
+  - pandas<2
+  - numpy<2
   - cyipopt>=1.4.0  # dev, tests
   - pygmo>=2.19.0  # dev, tests
-  - nlopt  # dev, tests
+  - nlopt  # dev, tests, docs
   - pip  # dev, tests, docs
   - pytest  # dev, tests
   - pytest-cov  # tests
   - pytest-xdist  # dev, tests
   - statsmodels  # dev, tests
   - cloudpickle  # run, tests
   - joblib  # run, tests
-  - numpy<2.0  # run, tests
   - plotly  # run, tests
   - pybaum >= 0.1.2  # run, tests
   - scipy>=1.2.1  # run, tests
   - sqlalchemy  # run, tests
   - seaborn  # dev, tests
-  - mypy>=1.11  # dev, tests
+  - mypy=1.13  # dev, tests
   - pyyaml  # dev, tests
   - jinja2  # dev, tests
   - annotated-types  # dev, tests
   - pip:  # dev, tests, docs
-      - DFO-LS  # dev, tests
+      - DFO-LS>=1.5.3  # dev, tests
       - Py-BOBYQA  # dev, tests
       - fides==0.7.4  # dev, tests
       - kaleido  # dev, tests
       - types-cffi  # dev, tests
       - types-openpyxl  # dev, tests
       - types-jinja2  # dev, tests
       - sqlalchemy-stubs  # dev, tests
+      - sphinxcontrib-mermaid  # dev, tests, docs
       - -e ../../
diff --git a/.tools/update_envs.py b/.tools/update_envs.py
@@ -33,20 +33,27 @@ def main():
     ## test environment others
     test_env_others = deepcopy(test_env)
 
-    ## test environment for pandas version 1
+    ## test environment for pandas version < 2 (requires numpy < 2)
     test_env_pandas = deepcopy(test_env)
-    test_env_pandas = [line for line in test_env_pandas if "pandas" not in line]
-    test_env_pandas.insert(_insert_idx, "  - pandas<2.0.0")
-
-    # create docs testing environment
-
+    for pkg in ["numpy", "pandas"]:
+        test_env_pandas = [line for line in test_env_pandas if pkg not in line]
+        test_env_pandas.insert(_insert_idx, f"  - {pkg}<2")
+
+    ## test environment for numpy version < 2 (with pandas >= 2)
+    test_env_numpy = deepcopy(test_env)
+    for pkg in ["numpy", "pandas"]:
+        test_env_numpy = [line for line in test_env_numpy if pkg not in line]
+    test_env_numpy.insert(_insert_idx, "  - numpy<2")
+    test_env_numpy.insert(_insert_idx, "  - pandas>=2")
+
+    # test environment for documentation
     docs_env = [line for line in lines if _keep_line(line, "docs")]
     docs_env.append("      - -e ../../")  # add local installation
 
     # write environments
     for name, env in zip(
-        ["linux", "others", "pandas"],
-        [test_env_linux, test_env_others, test_env_pandas],
+        ["linux", "others", "pandas", "numpy"],
+        [test_env_linux, test_env_others, test_env_pandas, test_env_numpy],
         strict=False,
     ):
         # Specify newline to avoid wrong line endings on Windows.

diff --git a/docs/rtd_environment.yml b/docs/rtd_environment.yml
@@ -27,6 +27,7 @@ dependencies:
   - patsy
   - joblib
   - plotly
+  - nlopt
   - annotated-types
   - pip:
       - ../
@@ -38,3 +39,5 @@ dependencies:
       - types-openpyxl  # dev, tests
       - types-jinja2  # dev, tests
       - sqlalchemy-stubs  # dev, tests
+      - sphinxcontrib-mermaid  # dev, tests, docs
+      - fides==0.7.4  # dev, tests
diff --git a/docs/source/algorithms.md b/docs/source/algorithms.md
@@ -1083,6 +1083,8 @@ install each of them separately:
 ```{eval-rst}
 .. dropdown::  nag_dfols
 
+    *Note*: We recommend to install `DFO-LS` version 1.5.3 or higher. Versions of 1.5.0 or lower also work but the versions `1.5.1` and `1.5.2` contain bugs that can lead to errors being raised.
+
     .. code-block::
 
         "nag_dfols"

diff --git a/docs/source/conf.py b/docs/source/conf.py
@@ -48,13 +48,17 @@
     "sphinxcontrib.bibtex",
     "sphinx_panels",
     "sphinx_design",
+    "sphinxcontrib.mermaid",
 ]
 
 myst_enable_extensions = [
     "colon_fence",
     "dollarmath",
     "html_image",
 ]
+myst_fence_as_directive = ["mermaid"]
+
+
 copybutton_prompt_text = ">>> "
 copybutton_only_copy_prompt_lines = False
 

diff --git a/docs/source/estimagic/explanation/bootstrap_montecarlo_comparison.ipynb b/docs/source/estimagic/explanation/bootstrap_montecarlo_comparison.ipynb
@@ -42,13 +42,14 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "import estimagic as em\n",
     "import matplotlib.pyplot as plt\n",
     "import numpy as np\n",
     "import pandas as pd\n",
     "import scipy\n",
     "import statsmodels.api as sm\n",
-    "from joblib import Parallel, delayed"
+    "from joblib import Parallel, delayed\n",
+    "\n",
+    "import estimagic as em"
    ]
   },
   {

diff --git a/docs/source/estimagic/tutorials/bootstrap_overview.ipynb b/docs/source/estimagic/tutorials/bootstrap_overview.ipynb
@@ -19,11 +19,12 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "import estimagic as em\n",
     "import numpy as np\n",
     "import pandas as pd\n",
     "import seaborn as sns\n",
-    "import statsmodels.api as sm"
+    "import statsmodels.api as sm\n",
+    "\n",
+    "import estimagic as em"
    ]
   },
   {