diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 8c901ed..61d63a6 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -83,6 +83,19 @@ jobs: run: | pixi run autotest-Windows + - name: Scheduled job test + if: ${{ github.event_name == 'schedule' }} + run: | + pixi run build-all + + - name: Upload failed test output + if: failure() + uses: actions/upload-artifact@v4 + with: + name: failed-${{ runner.os }}-intel + path: ./autotest/.failed + + - name: Print coverage report before upload run: | pixi run coverage-report diff --git a/.github/workflows/pymake-gcc.yml b/.github/workflows/pymake-gcc.yml index d514ba7..09f8940 100644 --- a/.github/workflows/pymake-gcc.yml +++ b/.github/workflows/pymake-gcc.yml @@ -81,6 +81,19 @@ jobs: run: | pixi run autotest-Windows + - name: Scheduled job test + if: ${{ github.event_name == 'schedule' }} + run: | + pixi run build-all + + - name: Upload failed test output + if: failure() + uses: actions/upload-artifact@v4 + with: + name: failed-${{ runner.os }}-gcc + path: ./autotest/.failed + + - name: Print coverage report before upload run: | pixi run coverage-report diff --git a/.gitignore b/.gitignore index f700a60..66ff0ef 100644 --- a/.gitignore +++ b/.gitignore @@ -66,6 +66,7 @@ target/ # files in the autotest directory autotest/temp*/ +autotest/.failed/ autotest/*.json autotest/code.md mod_temp/ diff --git a/README.md b/README.md index 47b236f..c2a73c3 100644 --- a/README.md +++ b/README.md @@ -173,7 +173,7 @@ python mymf6script.py -fc ifort -cc icc When pymake is installed, a `make-program` (or `make-program.exe` for Windows) program is installed. `make-program` can be used to build MODFLOW 6, MODFLOW-2005, MODFLOW-NWT, MODFLOW-USG, MODFLOW-LGR, MODFLOW-2000, MODPATH 6, MODPATH 7, -GSFLOW, VS2DT, MT3DMS, MT3D-USGS, SEAWAT, and SUTRA. Utility programs CRT, Triangle, and GRIDGEN can also +GSFLOW, VS2DT, MT3DMS, MT3D-USGS, and SEAWAT. Utility programs CRT, Triangle, and GRIDGEN can also be built. `make-program` downloads the distribution file (requires an internet connection), unzips the file, sets the pymake settings required to build the program, and compiles the program from the source files. Optional pymake command line arguments can be used to customize the build (`-fc`, `-cc`, `--fflags`, etc.). For example, MODFLOW 6 could be diff --git a/autotest/conftest.py b/autotest/conftest.py index 1a11328..92a48fc 100644 --- a/autotest/conftest.py +++ b/autotest/conftest.py @@ -6,8 +6,6 @@ import pytest -# import modflow-devtools fixtures - pytest_plugins = ["modflow_devtools.fixtures"] diff --git a/autotest/pytest.ini b/autotest/pytest.ini index 950aeb6..ffe4bb3 100644 --- a/autotest/pytest.ini +++ b/autotest/pytest.ini @@ -6,3 +6,4 @@ markers = regression: regression tests requests: usgsprograms requests tests schedule: tests to run if a scheduled job + slow: tests taking more than a few seconds to complete diff --git a/autotest/test_build.py b/autotest/test_build.py index 1e02c2e..edee58e 100644 --- a/autotest/test_build.py +++ b/autotest/test_build.py @@ -19,12 +19,13 @@ ] test_ostag = get_ostag() test_fc_env = os.environ.get("FC") -if "win" in test_ostag: - meson_exclude = ("mt3dms", "vs2dt", "triangle", "gridgen", "sutra") -elif "win" not in test_ostag and test_fc_env in ("ifort",): - meson_exclude = ("mf2000", "mf2005", "swtv4", "mflgr", "sutra") -else: - meson_exclude = ("sutra",) +meson_exclude = tuple() +# if "win" in test_ostag: +# meson_exclude = ("mt3dms", "vs2dt", "triangle", "gridgen") +# elif "win" not in test_ostag and test_fc_env in ("ifort",): +# meson_exclude = ("mf2000", "mf2005", "swtv4", "mflgr") +# else: +# meson_exclude = tuple() targets_meson = [t for t in targets if t not in meson_exclude] @@ -61,26 +62,6 @@ def build_with_makefile(target, path, fc): return success, errmsg -@pytest.mark.base -@flaky(max_runs=RERUNS) -@pytest.mark.parametrize("target", targets) -def test_build(function_tmpdir, target: str) -> None: - with set_dir(function_tmpdir): - pm = pymake.Pymake(verbose=True) - pm.target = target - pm.inplace = True - fc = os.environ.get("FC", "gfortran") - assert ( - pymake.build_apps( - target, - pm, - verbose=True, - clean=False, - ) - == 0 - ), f"could not compile {target}" - - @pytest.mark.base @flaky(max_runs=RERUNS) @pytest.mark.parametrize("target", targets_meson) @@ -121,3 +102,24 @@ def test_makefile_build(function_tmpdir, target: str) -> None: success, errmsg = build_with_makefile(target, function_tmpdir, pm.fc) assert success, errmsg + + +@pytest.mark.base +@flaky(max_runs=RERUNS) +@pytest.mark.parametrize("target", targets) +def test_build(function_tmpdir, target: str) -> None: + with set_dir(function_tmpdir): + pm = pymake.Pymake(verbose=True) + pm.target = target + pm.inplace = True + fc = os.environ.get("FC", "gfortran") + assert ( + pymake.build_apps( + target, + pm, + verbose=True, + clean=False, + meson=False, + ) + == 0 + ), f"could not compile {target}" diff --git a/autotest/test_cli_cmds.py b/autotest/test_cli_cmds.py index f31e0a3..4d87177 100644 --- a/autotest/test_cli_cmds.py +++ b/autotest/test_cli_cmds.py @@ -5,22 +5,24 @@ import pytest from flaky import flaky -from modflow_devtools.misc import set_dir, set_env +from modflow_devtools.misc import set_dir -from pymake import linker_update_environment +import pymake RERUNS = 3 -targets = ( +TARGETS = ( "triangle", "crt", ) -meson_parm = ( +MESON_PARM = ( True, False, ) +TARGETS_ALL = pymake.usgs_program_data.get_keys(current=True) + def run_cli_cmd(cmd: list) -> None: process = subprocess.Popen( @@ -44,51 +46,68 @@ def run_cli_cmd(cmd: list) -> None: @flaky(max_runs=RERUNS) @pytest.mark.dependency(name="make_program") @pytest.mark.base -@pytest.mark.parametrize("target", targets) +@pytest.mark.parametrize("target", TARGETS) def test_make_program(function_tmpdir, target: str) -> None: - cmd = [ - "make-program", - target, - "--appdir", - str(function_tmpdir), - "--verbose", - ] - run_cli_cmd(cmd) + with set_dir(function_tmpdir): + cmd = [ + "make-program", + target, + "--appdir", + str(function_tmpdir), + "--verbose", + ] + run_cli_cmd(cmd) @flaky(max_runs=RERUNS) -@pytest.mark.dependency(name="make_program") +@pytest.mark.dependency(name="make_program_mf2005") @pytest.mark.base def test_make_program_double(function_tmpdir) -> None: - cmd = [ - "make-program", - "mf2005", - "--double", - "--verbose", - "--appdir", - str(function_tmpdir), - ] - run_cli_cmd(cmd) + with set_dir(function_tmpdir): + cmd = [ + "make-program", + "mf2005", + "--double", + "--verbose", + "--appdir", + str(function_tmpdir), + ] + run_cli_cmd(cmd) @pytest.mark.dependency(name="make_program_all") @pytest.mark.schedule -def test_make_program_all(module_tmpdir) -> None: - cmd = [ - "make-program", - ":", - "--appdir", - str(module_tmpdir / "all"), - "--verbose", - "--dryrun", - ] - run_cli_cmd(cmd) +def test_make_program_all(function_tmpdir) -> None: + with set_dir(function_tmpdir): + cmd = [ + "make-program", + ":", + "--appdir", + str(function_tmpdir), + "--verbose", + ] + run_cli_cmd(cmd) + + +@pytest.mark.dependency(name="make_program_all_parametrize") +@pytest.mark.schedule +@pytest.mark.parametrize("target", TARGETS_ALL) +def test_make_program_all_parametrize(function_tmpdir, target: str) -> None: + with set_dir(function_tmpdir): + cmd = [ + "make-program", + target, + "--appdir", + str(function_tmpdir), + "--verbose", + ] + run_cli_cmd(cmd) @flaky(max_runs=RERUNS) @pytest.mark.dependency(name="mfpymake") @pytest.mark.base -@pytest.mark.parametrize("meson", meson_parm) +@pytest.mark.parametrize("meson", MESON_PARM) def test_mfpymake(function_tmpdir, meson: bool) -> None: with set_dir(function_tmpdir): src = ( @@ -116,7 +135,7 @@ def test_mfpymake(function_tmpdir, meson: bool) -> None: fc = os.environ.get("FC") cmd.append(fc) - linker_update_environment(fc=fc) + pymake.linker_update_environment(fc=fc) if meson: cmd.append("--meson") diff --git a/autotest/test_gridgen.py b/autotest/test_gridgen.py index 1c69312..6ce30c5 100644 --- a/autotest/test_gridgen.py +++ b/autotest/test_gridgen.py @@ -1,9 +1,10 @@ +import os import pathlib as pl import subprocess -from os import environ from platform import system import pytest +from modflow_devtools.misc import set_dir import pymake @@ -22,20 +23,7 @@ def prog_data(target) -> dict: @pytest.fixture(scope="module") def workspace(module_tmpdir, prog_data) -> pl.Path: - return module_tmpdir / prog_data.dirname - - -@pytest.fixture(scope="module") -def pm(module_tmpdir, target) -> pymake.Pymake: - pm = pymake.Pymake(verbose=True) - pm.target = str(target) - pm.appdir = str(module_tmpdir) - pm.cc = environ.get("CXX", "g++") - pm.fc = None - pm.inplace = True - pm.makeclean = True - yield pm - pm.finalize() + return module_tmpdir / f"temp/{prog_data.dirname}" def run_command(args, cwd): @@ -58,17 +46,22 @@ def run_gridgen(cmd, ws, exe): return run_command(args, ws) == 0 -@pytest.mark.dependency(name="download") -@pytest.mark.base -def test_download(pm, module_tmpdir, target): - pm.download_target(target, download_path=module_tmpdir) - assert pm.download, f"could not download {target} distribution" - - -@pytest.mark.dependency(name="build", depends=["download"]) -@pytest.mark.base -def test_compile(pm, target): - assert pm.build() == 0, f"could not compile {target}" +@pytest.mark.dependency(name="build") +@pytest.mark.regression +def test_compile(module_tmpdir, target): + cc = os.environ.get("CC", "g++") + fc = None + pymake.linker_update_environment(cc=cc, fc=fc) + with set_dir(module_tmpdir): + assert ( + pymake.build_apps( + target.stem, + verbose=True, + clean=False, + meson=True, + ) + == 0 + ), f"could not compile {target.stem}" @pytest.mark.dependency(name="test", depends=["build"]) @@ -89,7 +82,7 @@ def test_compile(pm, target): "grid02qtg-to-vtkfilesv action05_vtkfile.dfn", ], ) -def test_gridgen(cmd, workspace, target): +def test_gridgen(module_tmpdir, cmd, workspace, target): assert run_gridgen( - cmd, workspace / "examples" / "biscayne", target + cmd, workspace / "examples" / "biscayne", module_tmpdir / target ), f"could not run {cmd}" diff --git a/autotest/test_mf2005.py b/autotest/test_mf2005.py index dbf361e..af7e04a 100644 --- a/autotest/test_mf2005.py +++ b/autotest/test_mf2005.py @@ -1,8 +1,10 @@ +import os import sys from pathlib import Path import flopy import pytest +from modflow_devtools.misc import set_dir import pymake @@ -22,18 +24,7 @@ def prog_data(target) -> dict: @pytest.fixture(scope="module") def workspace(module_tmpdir, prog_data) -> Path: - return module_tmpdir / prog_data.dirname - - -@pytest.fixture(scope="module") -def pm(module_tmpdir, target) -> pymake.Pymake: - pm = pymake.Pymake(verbose=True) - pm.target = str(target) - pm.appdir = str(module_tmpdir) - pm.fflags = "-O3" - pm.cflags = "-O3" - yield pm - pm.finalize() + return module_tmpdir / f"temp/{prog_data.dirname}" def run_mf2005(namefile, ws, exe): @@ -42,17 +33,22 @@ def run_mf2005(namefile, ws, exe): return success -@pytest.mark.dependency(name="download") -@pytest.mark.base -def test_download(pm, module_tmpdir, target): - pm.download_target(target, download_path=module_tmpdir) - assert pm.download, f"could not download {target}" - - -@pytest.mark.dependency(name="build", depends=["download"]) -@pytest.mark.base -def test_compile(pm, target): - assert pm.build() == 0, f"could not compile {target}" +@pytest.mark.dependency(name="build") +@pytest.mark.regression +def test_compile(module_tmpdir, target): + cc = os.environ.get("CC", "gcc") + fc = os.environ.get("FC", "gfortran") + pymake.linker_update_environment(cc=cc, fc=fc) + with set_dir(module_tmpdir): + assert ( + pymake.build_apps( + target.stem, + verbose=True, + clean=False, + meson=True, + ) + == 0 + ), f"could not compile {target.stem}" @pytest.mark.dependency(name="test", depends=["build"]) diff --git a/autotest/test_mf6.py b/autotest/test_mf6.py index b9470f6..f74f9ed 100644 --- a/autotest/test_mf6.py +++ b/autotest/test_mf6.py @@ -31,18 +31,6 @@ def target(module_tmpdir) -> Path: return module_tmpdir / f"{name}{ext}" -@pytest.fixture(scope="module") -def target_so(module_tmpdir) -> Path: - sharedobject_target = "libmf6" - if sys.platform.lower() == "win32": - sharedobject_target += ".dll" - elif sys.platform.lower() == "darwin": - sharedobject_target += ".dylib" - else: - sharedobject_target += ".so" - return module_tmpdir / sharedobject_target - - @pytest.fixture(scope="module") def prog_data(target) -> dict: return pymake.usgs_program_data.get_target(target.name) @@ -50,66 +38,25 @@ def prog_data(target) -> dict: @pytest.fixture(scope="module") def workspace(module_tmpdir, prog_data) -> Path: - return module_tmpdir / prog_data.dirname - - -@pytest.fixture(scope="module") -def pm(module_tmpdir, target) -> pymake.Pymake: - pm = pymake.Pymake(verbose=True) - pm.target = str(target) - pm.appdir = module_tmpdir - pm.makefile = True - pm.makeclean = True - pm.makefiledir = module_tmpdir - pm.inplace = True - pm.networkx = True - pm.verbose = True - yield pm - pm.finalize() - - -def build_with_makefile(pm, workspace, exe): - exe_path = Path(exe) - success = False - with set_dir(workspace): - if os.path.isfile("makefile"): - # wait to delete on windows - if sys.platform.lower() == "win32": - time.sleep(6) - - # clean prior to make - print(f"clean {exe} with makefile") - os.system("make clean") - - # build MODFLOW 6 with makefile - print(f"build {exe} with makefile") - return_code = os.system("make") - - # test if running on Windows with ifort, if True the makefile - # should fail - if sys.platform.lower() == "win32" and pm.fc == "ifort": - if return_code != 0: - success = True - else: - success = False - # verify that target was made - else: - success = exe_path.is_file() + return module_tmpdir / f"temp/{prog_data.dirname}" - return success - -@pytest.mark.dependency(name="download") -@pytest.mark.base -def test_download(pm, module_tmpdir, target): - pm.download_target(target, download_path=module_tmpdir) - assert pm.download, f"could not download {target} distribution" - - -@pytest.mark.dependency(name="build", depends=["download"]) -@pytest.mark.base -def test_compile(pm, target): - assert pm.build() == 0, f"could not compile {target}" +@pytest.mark.dependency(name="build") +@pytest.mark.regression +def test_compile(module_tmpdir, target): + cc = os.environ.get("CC", "gcc") + fc = os.environ.get("FC", "gfortran") + pymake.linker_update_environment(cc=cc, fc=fc) + with set_dir(module_tmpdir): + assert ( + pymake.build_apps( + target.stem, + verbose=True, + clean=False, + meson=True, + ) + == 0 + ), f"could not compile {target.stem}" @pytest.mark.dependency(name="test", depends=["build"]) @@ -118,39 +65,3 @@ def test_compile(pm, target): def test_mf6(ws, target): success, _ = flopy.run_model(target, None, model_ws=ws, silent=False) assert success, f"could not run {ws}" - - -@pytest.mark.dependency(name="makefile", depends=["build"]) -@pytest.mark.base -def test_makefile(pm, module_tmpdir, target): - assert build_with_makefile( - pm, module_tmpdir, target - ), f"could not compile {target} with makefile" - - -@pytest.mark.dependency(name="shared", depends=["makefile"]) -@pytest.mark.base -def test_sharedobject(pm, module_tmpdir, workspace, target_so, prog_data): - # reconfigure pymake object - pm.target = str(target_so) - pm.appdir = module_tmpdir - pm.srcdir = workspace / prog_data.srcdir - pm.srcdir2 = workspace / "src" - pm.excludefiles = [os.path.join(pm.srcdir2, "mf6.f90")] - pm.makefile = True - pm.makeclean = True - pm.sharedobject = True - pm.inplace = True - pm.dryrun = False - - # build the target - assert pm.build() == 0, f"could not compile {pm.target}" - assert target_so.is_file() - - -@pytest.mark.dependency(name="shared_makefile", depends=["shared", "makefile"]) -@pytest.mark.base -def test_sharedobject_makefile(pm, module_tmpdir, target_so): - assert build_with_makefile( - pm, module_tmpdir, target_so - ), f"could not compile {target_so} with makefile" diff --git a/autotest/test_mflgr.py b/autotest/test_mflgr.py deleted file mode 100644 index 1de8c96..0000000 --- a/autotest/test_mflgr.py +++ /dev/null @@ -1,34 +0,0 @@ -from pathlib import Path - -import pytest - -import pymake - - -@pytest.fixture(scope="module") -def target(module_tmpdir) -> Path: - target = "mflgr" - return module_tmpdir / target - - -@pytest.fixture(scope="module") -def prog_dict(target) -> dict: - return pymake.usgs_program_data.get_target(target) - - -@pytest.fixture(scope="module") -def workspace(module_tmpdir, prog_dict) -> Path: - return module_tmpdir / prog_dict.dirname - - -def compile_code(ws, exe): - return pymake.build_apps( - str(exe), download_dir=ws, appdir=ws, verbose=True - ) - - -@pytest.mark.base -def test_compile(module_tmpdir, target): - assert ( - compile_code(module_tmpdir, target) == 0 - ), f"could not compile {target}" diff --git a/autotest/test_mfnwt.py b/autotest/test_mfnwt.py deleted file mode 100644 index f7cb801..0000000 --- a/autotest/test_mfnwt.py +++ /dev/null @@ -1,91 +0,0 @@ -import os -import sys -import time -from pathlib import Path - -import pytest -from modflow_devtools.misc import set_dir - -import pymake - - -@pytest.fixture(scope="module") -def target(module_tmpdir) -> str: - target = "mfnwt" - return module_tmpdir / target - - -@pytest.fixture(scope="module") -def prog_data(target) -> dict: - return pymake.usgs_program_data.get_target(target.name) - - -@pytest.fixture(scope="module") -def workspace(module_tmpdir, prog_data) -> Path: - return module_tmpdir / prog_data.dirname - - -@pytest.fixture(scope="module") -def pm(module_tmpdir, target) -> pymake.Pymake: - pm = pymake.Pymake(verbose=True) - pm.target = str(target) - pm.appdir = str(module_tmpdir) - pm.makefile = True - pm.makefiledir = str(module_tmpdir) - pm.inplace = True - pm.dryrun = False - pm.verbose = True - yield pm - pm.finalize() - - -def build_with_makefile(ws): - success = True - with set_dir(ws): - if os.path.isfile("makefile"): - # wait to delete on windows - if sys.platform.lower() == "win32": - time.sleep(6) - - # clean prior to make - print(f"clean {target} with makefile") - os.system("make clean") - - # build MODFLOW-NWT with makefile - print(f"build {target} with makefile") - return_code = os.system("make") - - # test if running on Windows with ifort, if True the makefile - # should fail - errmsg = f"{target} created by makefile does not exist." - if sys.platform.lower() == "win32" and pm.fc == "ifort": - if return_code != 0: - success = True - else: - success = False - # verify that MODFLOW-NWT was made - else: - success = os.path.isfile(target) - else: - errmsg = "makefile does not exist" - - assert success, errmsg - - -@pytest.mark.dependency(name="download") -@pytest.mark.base -def test_download(pm, module_tmpdir, target): - pm.download_target(target, download_path=module_tmpdir) - assert pm.download, f"could not download {target} distribution" - - -@pytest.mark.dependency(name="build", depends=["download"]) -@pytest.mark.base -def test_compile(pm, target): - assert pm.build() == 0, f"could not compile {target}" - - -@pytest.mark.dependency(name="makefile", depends=["build"]) -@pytest.mark.base -def test_makefile(workspace): - build_with_makefile(workspace) diff --git a/autotest/test_mfusg.py b/autotest/test_mfusg.py index 0f13c3e..381bf45 100644 --- a/autotest/test_mfusg.py +++ b/autotest/test_mfusg.py @@ -4,9 +4,13 @@ import flopy import pytest +from modflow_devtools.misc import set_dir import pymake +APPS = ["mfusg", "mfusg_gsi"] +EXT = ".exe" if system() == "Windows" else "" + @pytest.fixture(scope="module") def targets(module_tmpdir): @@ -21,25 +25,7 @@ def prog_data(targets) -> dict: @pytest.fixture(scope="module") def workspace(module_tmpdir, prog_data) -> Path: - return module_tmpdir / prog_data.dirname - - -@pytest.fixture(scope="module") -def pm(module_tmpdir, targets) -> pymake.Pymake: - pm = pymake.Pymake(verbose=True) - pm.target = str(targets[0]) - pm.appdir = str(module_tmpdir) - yield pm - pm.finalize() - - -@pytest.fixture(scope="module") -def pm_gsi(module_tmpdir, targets) -> pymake.Pymake: - pm_gsi = pymake.Pymake(verbose=True) - pm_gsi.target = str(targets[1]) - pm_gsi.appdir = str(module_tmpdir) - yield pm_gsi - pm_gsi.finalize() + return module_tmpdir / f"temp/{prog_data.dirname}" def edit_namefile(namefile): @@ -67,24 +53,27 @@ def run_mfusg(fn, exe): assert success, errmsg -@pytest.mark.dependency(name="download") -@pytest.mark.base -def test_download(pm, pm_gsi, module_tmpdir, targets): - pm.download_target(targets[0], download_path=module_tmpdir) - assert pm.download, f"could not download {targets[0]}" - - pm_gsi.download_target(targets[1], download_path=module_tmpdir) - assert pm_gsi.download, f"could not download {targets[1]}" - - -@pytest.mark.dependency(name="build", depends=["download"]) -@pytest.mark.base -def test_compile(pm, pm_gsi, targets): - assert pm.build() == 0, f"could not compile {targets[0]}" - assert (targets[0]).is_file() - - assert pm_gsi.build() == 0, f"could not compile {targets[1]}" - assert targets[1].is_file() +@pytest.mark.dependency(name="build") +@pytest.mark.regression +@pytest.mark.parametrize( + "target", + APPS, +) +def test_compile(module_tmpdir, target): + target_path = module_tmpdir / f"{target}{EXT}" + cc = os.environ.get("CC", "gcc") + fc = os.environ.get("FC", "gfortran") + pymake.linker_update_environment(cc=cc, fc=fc) + with set_dir(module_tmpdir): + assert ( + pymake.build_apps( + target, + verbose=True, + clean=False, + meson=True, + ) + == 0 + ), f"could not compile {target}" @pytest.mark.dependency(name="test", depends=["build"]) @@ -101,8 +90,12 @@ def test_compile(pm, pm_gsi, targets): "03_conduit_confined/ex3.nam", ], ) -def test_mfusg(workspace, namefile, targets): +@pytest.mark.parametrize( + "target", + APPS, +) +def test_mfusg(module_tmpdir, workspace, namefile, target): + target_path = module_tmpdir / f"{target}{EXT}" namefile_path = workspace / "test" / namefile edit_namefile(namefile_path) - run_mfusg(namefile_path, targets[0]) - run_mfusg(namefile_path, targets[1]) + run_mfusg(namefile_path, target_path) diff --git a/autotest/test_misc_programs.py b/autotest/test_misc_programs.py deleted file mode 100644 index 93d4e30..0000000 --- a/autotest/test_misc_programs.py +++ /dev/null @@ -1,24 +0,0 @@ -import pytest - -import pymake - -targets = [ - "crt", - "vs2dt", - "zonbud3", -] - - -@pytest.mark.base -@pytest.mark.parametrize("target", targets) -def test_compile(module_tmpdir, target): - bin_dir = module_tmpdir / "bin" - assert ( - pymake.build_apps( - str(bin_dir / target), - download_dir=str(module_tmpdir), - appdir=str(bin_dir), - verbose=True, - ) - == 0 - ), f"could not compile {target}" diff --git a/autotest/test_mp6.py b/autotest/test_mp6.py index 51c1452..099585c 100644 --- a/autotest/test_mp6.py +++ b/autotest/test_mp6.py @@ -5,6 +5,7 @@ import flopy import pytest +from modflow_devtools.misc import set_dir import pymake @@ -23,16 +24,7 @@ def prog_data(target) -> dict: @pytest.fixture(scope="module") def workspace(module_tmpdir, prog_data) -> Path: - return module_tmpdir / prog_data.dirname - - -@pytest.fixture(scope="module") -def pm(module_tmpdir, target) -> pymake.Pymake: - pm = pymake.Pymake(verbose=True) - pm.target = str(target) - pm.appdir = module_tmpdir - yield pm - pm.finalize() + return module_tmpdir / f"temp/{prog_data.dirname}" def update_files(fn, workspace): @@ -59,17 +51,22 @@ def update_files(fn, workspace): os.rename(fname2, fname1) -@pytest.mark.dependency(name="download") -@pytest.mark.base -def test_download(pm, module_tmpdir, target): - pm.download_target(target, download_path=module_tmpdir) - assert pm.download, f"could not download {target} distribution" - - -@pytest.mark.dependency(name="build", depends=["download"]) -@pytest.mark.base -def test_compile(pm, target): - assert pm.build() == 0, f"could not compile {target}" +@pytest.mark.dependency(name="build") +@pytest.mark.regression +def test_compile(module_tmpdir, target): + cc = os.environ.get("CC", "gcc") + fc = os.environ.get("FC", "gfortran") + pymake.linker_update_environment(cc=cc, fc=fc) + with set_dir(module_tmpdir): + assert ( + pymake.build_apps( + target.stem, + verbose=True, + clean=False, + meson=True, + ) + == 0 + ), f"could not compile {target.stem}" @pytest.mark.dependency(name="test", depends=["build"]) diff --git a/autotest/test_mp7.py b/autotest/test_mp7.py index 24b1c66..320a1f2 100644 --- a/autotest/test_mp7.py +++ b/autotest/test_mp7.py @@ -5,6 +5,7 @@ import flopy import pytest +from modflow_devtools.misc import set_dir import pymake @@ -24,16 +25,7 @@ def prog_data(target) -> dict: @pytest.fixture(scope="module") def workspace(module_tmpdir, prog_data) -> Path: - return module_tmpdir / prog_data.dirname - - -@pytest.fixture(scope="module") -def pm(module_tmpdir, target) -> pymake.Pymake: - pm = pymake.Pymake(verbose=True) - pm.target = str(target) - pm.appdir = str(module_tmpdir) - yield pm - pm.finalize() + return module_tmpdir / f"temp/{prog_data.dirname}" def replace_data(dpth): @@ -142,20 +134,30 @@ def run_modpath7(namefile, mp7_exe, mf2005_exe, mfusg_exe, mf6_exe): return success -@pytest.mark.dependency(name="download") -@pytest.mark.base -def test_download(pm, module_tmpdir, target): - pm.download_target(target, download_path=module_tmpdir) - assert pm.download, f"could not download {target} distribution" - - -@pytest.mark.dependency(name="build", depends=["download"]) -@pytest.mark.base -def test_compile(pm, target): - assert pm.build() == 0, f"could not compile {target}" +@pytest.mark.dependency(name="build") +@pytest.mark.regression +def test_compile(module_tmpdir, target): + cc = os.environ.get("CC", "gcc") + fc = os.environ.get("FC", "gfortran") + pymake.linker_update_environment(cc=cc, fc=fc) + with set_dir(module_tmpdir): + assert ( + pymake.build_apps( + target.stem, + verbose=True, + clean=False, + meson=True, + ) + == 0 + ), f"could not compile {target.stem}" -@pytest.mark.dependency(name="download_exes") +@pytest.mark.dependency( + name="download_exes", + depends=[ + "build", + ], +) @pytest.mark.regression def test_download_exes(module_tmpdir): pymake.getmfexes( @@ -163,9 +165,7 @@ def test_download_exes(module_tmpdir): ) -@pytest.mark.dependency( - name="test", depends=["download", "download_exes", "build"] -) +@pytest.mark.dependency(name="test", depends=["build", "download_exes"]) @pytest.mark.regression @pytest.mark.parametrize( "namefile", diff --git a/autotest/test_mt3d.py b/autotest/test_mt3d.py index f9c2ea0..71af565 100644 --- a/autotest/test_mt3d.py +++ b/autotest/test_mt3d.py @@ -1,35 +1,26 @@ import os import sys from pathlib import Path +from platform import system import flopy import pytest +from modflow_devtools.misc import set_dir import pymake - -@pytest.fixture(scope="module") -def target(module_tmpdir) -> Path: - return module_tmpdir / "mt3dusgs" +APPS = ["mt3dms", "mt3dusgs"] +EXT = ".exe" if system() == "Windows" else "" @pytest.fixture(scope="module") -def prog_data(target) -> dict: - return pymake.usgs_program_data.get_target(target.name) +def prog_data() -> dict: + return pymake.usgs_program_data.get_target("mt3dusgs") @pytest.fixture(scope="module") def workspace(module_tmpdir, prog_data) -> Path: - return module_tmpdir / prog_data.dirname - - -@pytest.fixture(scope="module") -def pm(module_tmpdir) -> pymake.Pymake: - pm = pymake.Pymake(verbose=True) - pm.appdir = str(module_tmpdir) - pm.makeclean = True - yield pm - pm.finalize() + return module_tmpdir / f"temp/{prog_data.dirname}" def run_mt3dusgs(workspace, mt3dms_exe, mfnwt_exe, mf6_exe): @@ -87,42 +78,44 @@ def run_mt3dusgs(workspace, mt3dms_exe, mfnwt_exe, mf6_exe): return success -@pytest.mark.dependency(name="download_mt3dms") -@pytest.mark.base -def test_download_mt3dms(pm, module_tmpdir): - pm.target = "mt3dms" - pm.download_target(pm.target, download_path=module_tmpdir) - assert pm.download, f"could not download {pm.target} distribution" - - -@pytest.mark.dependency(name="build_mt3dms", depends=["download_mt3dms"]) -@pytest.mark.base -def test_compile_mt3dms(pm): - assert pm.build() == 0, f"could not compile {pm.target}" - - -@pytest.mark.dependency(name="download") -@pytest.mark.base -def test_download(pm, module_tmpdir, target): - pm.reset(str(target)) - pm.download_target(target, download_path=module_tmpdir) - assert pm.download, f"could not download {target} distribution" - - -@pytest.mark.dependency(name="build", depends=["download"]) -@pytest.mark.base -def test_compile(pm, target): - assert pm.build() == 0, f"could not compile {target}" - - +@pytest.mark.dependency(name="build") +@pytest.mark.regression +@pytest.mark.parametrize( + "target", + APPS, +) +def test_compile(module_tmpdir, target): + target_path = module_tmpdir / f"{target}{EXT}" + cc = os.environ.get("CC", "gcc") + fc = os.environ.get("FC", "gfortran") + pymake.linker_update_environment(cc=cc, fc=fc) + with set_dir(module_tmpdir): + assert ( + pymake.build_apps( + target, + verbose=True, + clean=False, + meson=True, + ) + == 0 + ), f"could not compile {target}" + + +@pytest.mark.dependency( + name="download_exes", + depends=[ + "build", + ], +) @pytest.mark.regression -@pytest.mark.skipif(sys.platform == "darwin", reason="do not run on OSX") +# @pytest.mark.skipif(sys.platform == "darwin", reason="do not run on OSX") def test_download_exes(module_tmpdir): pymake.getmfexes(module_tmpdir, exes=("mfnwt", "mf6"), verbose=True) +@pytest.mark.dependency(name="test", depends=["build", "download_exes"]) @pytest.mark.regression -@pytest.mark.skipif(sys.platform == "darwin", reason="do not run on OSX") +# @pytest.mark.skipif(sys.platform == "darwin", reason="do not run on OSX") @pytest.mark.skipif(sys.platform == "win32", reason="do not run on Windows") @pytest.mark.parametrize( "ws", @@ -144,7 +137,8 @@ def test_download_exes(module_tmpdir): "p01SpatialStresses(mf6)", ], ) -def test_mt3dusgs(module_tmpdir, workspace, ws, target): +def test_mt3dusgs(module_tmpdir, workspace, ws): + target = module_tmpdir / f"{APPS[1]}{EXT}" mfnwt_exe = module_tmpdir / "mfnwt" if pymake.usgs_program_data().get_version(mfnwt_exe) == "1.2.0": exclude = [ diff --git a/autotest/test_seawat.py b/autotest/test_seawat.py index 1dfbd43..515c230 100644 --- a/autotest/test_seawat.py +++ b/autotest/test_seawat.py @@ -5,7 +5,7 @@ import flopy import pytest -from modflow_devtools.misc import is_in_ci +from modflow_devtools.misc import is_in_ci, set_dir import pymake @@ -24,17 +24,7 @@ def prog_data(target) -> dict: @pytest.fixture(scope="module") def workspace(module_tmpdir, prog_data) -> Path: - return module_tmpdir / prog_data.dirname - - -@pytest.fixture(scope="module") -def pm(module_tmpdir, target) -> pymake.Pymake: - pm = pymake.Pymake(verbose=True) - pm.target = str(target) - pm.appdir = str(module_tmpdir) - pm.double = True - yield pm - # pm.finalize() + return module_tmpdir / f"temp/{prog_data.dirname}" def edit_namefile(namefile): @@ -75,17 +65,22 @@ def build_seawat_dependency_graphs(src_path, dep_path): assert success, "could not build dependency graphs" -@pytest.mark.dependency(name="download") -@pytest.mark.base -def test_download(pm, module_tmpdir, target): - pm.download_target(target, download_path=module_tmpdir) - assert pm.download, f"could not download {target}" - - -@pytest.mark.dependency(name="build", depends=["download"]) -@pytest.mark.base -def test_compile(pm, target): - assert pm.build() == 0, f"could not compile {target}" +@pytest.mark.dependency(name="build") +@pytest.mark.regression +def test_compile(module_tmpdir, target): + cc = os.environ.get("CC", "gcc") + fc = os.environ.get("FC", "gfortran") + pymake.linker_update_environment(cc=cc, fc=fc) + with set_dir(module_tmpdir): + assert ( + pymake.build_apps( + target.stem, + verbose=True, + clean=False, + meson=True, + ) + == 0 + ), f"could not compile {target.stem}" @pytest.mark.dependency(name="test", depends=["build"]) diff --git a/autotest/test_triangle_makefile.py b/autotest/test_triangle_makefile.py deleted file mode 100644 index 45579d8..0000000 --- a/autotest/test_triangle_makefile.py +++ /dev/null @@ -1,69 +0,0 @@ -import os -import sys - -import flopy -import pytest - -import pymake - - -@pytest.mark.base -def test_pymake_makefile(module_tmpdir): - target = "triangle" - pm = pymake.Pymake(verbose=True) - pm.makefile = True - pm.makeclean = True - # pm.cc = "gcc" - - if sys.platform.lower() == "win32": - if pm.cc == "icl": - return - target += ".exe" - - # get current directory - cwd = os.getcwd() - - # change to working directory so triangle download directory is - # a subdirectory in the working directory - os.chdir(module_tmpdir) - - # build triangle and makefile - assert ( - pymake.build_apps(target, clean=False, pymake_object=pm) == 0 - ), f"could not build {target}" - - if os.path.isfile(os.path.join(module_tmpdir, "makefile")): - print("cleaning with GNU make") - # clean prior to make - print(f"clean {target} with makefile") - success, _ = flopy.run_model( - "make", - None, - cargs="clean", - model_ws=module_tmpdir, - report=True, - normal_msg="rm -rf ./triangle", - silent=False, - ) - - # build triangle with makefile - if success: - print(f"build {target} with makefile") - success, _ = flopy.run_model( - "make", - None, - model_ws=module_tmpdir, - report=True, - normal_msg="cc -O2 -o triangle ./obj_temp/triangle.o", - silent=False, - ) - - # finalize Pymake object - pm.finalize() - - # return to starting directory - os.chdir(cwd) - - assert os.path.isfile( - os.path.join(module_tmpdir, target) - ), f"could not build {target} with makefile" diff --git a/docs/build_apps.md b/docs/build_apps.md index f5c329a..442f921 100644 --- a/docs/build_apps.md +++ b/docs/build_apps.md @@ -13,7 +13,7 @@ usage: make-program [-h] [-fc {ifort,mpiifort,gfortran,none}] [-cc {gcc,clang,cl Download and build USGS MODFLOW and related programs. positional arguments: - targets Program(s) to build. Options: crt, gridgen, libmf6, mf2000, mf2005, mf6, mflgr, mfnwt, mfusg, mfusg_gsi, mp6, mp7, mt3dms, mt3dusgs, sutra, swtv4, triangle, vs2dt, zbud6, zonbud3, + targets Program(s) to build. Options: crt, gridgen, libmf6, mf2000, mf2005, mf6, mflgr, mfnwt, mfusg, mfusg_gsi, mp6, mp7, mt3dms, mt3dusgs, swtv4, triangle, vs2dt, zbud6, zonbud3, zonbudusg, :. Specifying the target to be ':' will build all of the programs. Multiple targets can be specified by separating individual targets by a comma (i.e., mf6,zbud6). options: @@ -54,7 +54,7 @@ Examples: ``` `make-program` can be used to build MODFLOW 6, MODFLOW-2005, MODFLOW-NWT, MODFLOW-USG, MODFLOW-LGR, MODFLOW-2000, -MODPATH 6, MODPATH 7, GSFLOW, VS2DT, MT3DMS, MT3D-USGS, SEAWAT, GSFLOW, PRMS, and SUTRA. Utility programs CRT, Triangle, +MODPATH 6, MODPATH 7, GSFLOW, VS2DT, MT3DMS, MT3D-USGS, and SEAWAT. Utility programs CRT, Triangle, and GRIDGEN can also be built. `make-program` downloads the distribution file from the USGS (requires internet connection), unzips the distribution file, sets the pymake settings required to build the program, and compiles the source files to build the program. MT3DMS will be downloaded from the University of Alabama and Triangle will be diff --git a/docs/index.rst b/docs/index.rst index 9fa2d6d..68e7f45 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -17,7 +17,7 @@ programs released by the USGS. pymake includes example scripts for building MODFLOW 6, MODFLOW-2005, MODFLOW-NWT, MODFLOW-USG, MODFLOW-LGR, MODFLOW-2000, MODPATH 6, MODPATH 7, -VS2DT, MT3DMS, MT3D-USGS, SEAWAT, and SUTRA. Example scripts for +VS2DT, MT3DMS, MT3D-USGS, and SEAWAT. Example scripts for creating the utility programs CRT, Triangle, and GRIDGEN are also included. The scripts download the distribution file from the USGS (and other organizations) and compile the source into a binary executable. diff --git a/pixi.toml b/pixi.toml index 589bea5..6a5021a 100644 --- a/pixi.toml +++ b/pixi.toml @@ -34,18 +34,19 @@ postinstall = "pip install --no-build-isolation --no-deps --disable-pip-version- # format check-lint = "ruff check ." -fix-lint = "ruff check . --fix" check-format = "ruff format . --check" -fix-format = "ruff format ." +fix-style = "ruff check . --fix; ruff format ." # build test = "meson test --verbose --no-rebuild -C" # test download-examples = {cmd = "python ci_setup.py", cwd = "autotest"} -autotest = { cmd = "pytest -v -n auto --dist=loadfile -m='base or regression' --durations 0 --cov=pymake --cov-report=xml --keep-failed .failed", cwd = "autotest" } -autotest-base = { cmd = "pytest -v -n auto --dist=loadfile -m='base' --durations 0 --cov=pymake --cov-report=xml --keep-failed .failed", cwd = "autotest" } -autotest-Windows = { cmd = "pytest -v -m='base' --durations 0 --cov=pymake --cov-report=xml --basetemp=$RUNNER_TEMP/pytest_temp --keep-failed .failed", cwd = "autotest" } +autotest = { cmd = "pytest -v -n auto --dist=loadgroup -m='base or regression' --durations 0 --cov=pymake --cov-report=xml --keep-failed .failed", cwd = "autotest" } +autotest-base = { cmd = "pytest -v -n auto --dist=loadgroup -m='base' --durations 0 --cov=pymake --cov-report=xml --keep-failed .failed", cwd = "autotest" } +autotest-Windows = { cmd = "pytest -v -n auto --dist=loadgroup -m='base' --durations 0 --cov=pymake --cov-report=xml --basetemp=$RUNNER_TEMP/pytest_temp --keep-failed .failed", cwd = "autotest" } +build-all = { cmd = "pytest -v -m='schedule' -n auto --keep-failed .failed test_cli_cmds.py", cwd = "autotest" } +autotest-slow = { cmd = "pytest -v -n auto -m='slow' --durations 0 --cov=pymake --cov-report=xml --keep-failed .failed", cwd = "autotest" } # coverage report coverage-report = { cmd = "coverage report", cwd = "autotest"} diff --git a/pymake/pymake_base.py b/pymake/pymake_base.py index 08da481..b16c212 100644 --- a/pymake/pymake_base.py +++ b/pymake/pymake_base.py @@ -96,9 +96,9 @@ def main( sharedobject=False, appdir=None, verbose=False, - inplace=False, + inplace=True, networkx=False, - meson=False, + meson=True, mesondir=".", ): """Main pymake function. @@ -160,7 +160,7 @@ def main( boolean indicating that the source files in srcdir, srcdir2, and defined in extrafiles will be used directly. If inplace is False, source files will be copied to a directory named srcdir_temp. - (default is False) + (default is True) networkx : bool boolean indicating that the NetworkX python package will be used to create the Directed Acyclic Graph (DAG) used to determine the order @@ -183,9 +183,11 @@ def main( if meson: if not inplace: inplace = True + makeclean = False print( f"Using meson to build {os.path.basename(target)}, " - + "resetting inplace to True" + + "resetting inplace to True and " + + "makeclean to False." ) if srcdir is not None and target is not None: diff --git a/pymake/pymake_build_apps.py b/pymake/pymake_build_apps.py index 2feb231..bac0a4d 100644 --- a/pymake/pymake_build_apps.py +++ b/pymake/pymake_build_apps.py @@ -42,7 +42,7 @@ def build_apps( appdir=None, verbose=None, double=False, - meson=False, + meson=True, mesondir=".", clean=True, ): @@ -63,9 +63,9 @@ def build_apps( force double precision. (default is False) meson : bool boolean indicating that the executable should be built using the - meson build system. (default is False) + meson build system. (default is True) mesondir : str - Main meson.build file path + Main meson.build file path. (default is the current directory ".") clean : bool boolean determining of final download should be removed @@ -127,7 +127,8 @@ def build_apps( shutil.rmtree(pth) # set object to clean after each build - pmobj.makeclean = True + if not pmobj.meson: + pmobj.makeclean = True # reset variables based on passed args if download_dir is not None: diff --git a/pymake/pymake_parser.py b/pymake/pymake_parser.py index 33f1b12..9ea38bf 100644 --- a/pymake/pymake_parser.py +++ b/pymake/pymake_parser.py @@ -256,8 +256,8 @@ def _get_standard_arg_dict(): }, "meson": { "tag": ("--meson",), - "help": """Use meson to build executable. (default is False)""", - "default": False, + "help": """Use meson to build executable. (default is True)""", + "default": True, "choices": None, "action": "store_true", }, diff --git a/pymake/utils/_meson_build.py b/pymake/utils/_meson_build.py index 0a18d45..d64c931 100644 --- a/pymake/utils/_meson_build.py +++ b/pymake/utils/_meson_build.py @@ -1,4 +1,5 @@ import os +import shutil from contextlib import contextmanager from pathlib import Path @@ -197,7 +198,8 @@ def meson_setup( command_list.append(f"--bindir={libdir}") if os.path.isdir(build_dir): - command_list.append("--wipe") + shutil.rmtree(build_dir) + # command_list.append("--wipe") command = " ".join(command_list) print(f"\n{command}\n") @@ -544,22 +546,19 @@ def _create_main_meson_build( line += f"\tversion: '{target_version}',\n" line += "\tmeson_version: '>= 1.1.0',\n" line += "\tdefault_options: [\n\t\t'b_vscrt=static_from_buildtype',\n" - line += f"\t\t'optimization={optlevel_int}',\n" - line += "\t\t'debug=" + line += "\t\t'buildtype=" if debug: - line += "true',\n" + line += "debug',\n" else: - line += "false',\n" - if target in ("mf6", "libmf6", "zbud6"): - line += "\t\t'fortran_std=f2008'\n" + line += "release',\n" + if fc is not None: + if target in ("mf6", "libmf6", "zbud6"): + line += "\t\t'fortran_std=f2008'\n" line += "\t])\n\n" f.write(line) - line = "if get_option('optimization') >= '2'\n" - line += "\tprofile = 'release'\n" - line += "else\n" - line += "\tprofile = 'develop'\n" - line += "endif\n\n" + line = "build_type = get_option('buildtype')\n" + line += "message('The build type is:', build_type)\n\n" f.write(line) line = "" diff --git a/pymake/utils/usgsprograms.txt b/pymake/utils/usgsprograms.txt index cef494a..fe41c06 100644 --- a/pymake/utils/usgsprograms.txt +++ b/pymake/utils/usgsprograms.txt @@ -9,7 +9,6 @@ vs2dt , 3.3 , True , https://water.usgs.gov/water-resources/software triangle , 1.6 , True , https://github.com/MODFLOW-USGS/triangle/releases/download/1.0/triangle-1.0.zip , triangle-1.0 , src , True , False , False gridgen , 1.0.02 , True , https://water.usgs.gov/water-resources/software/GRIDGEN/gridgen.1.0.02.zip , gridgen.1.0.02 , src , True , False , False crt , 1.3.1 , True , https://water.usgs.gov/ogw/CRT/CRT_1.3.1.zip , CRT_1.3.1 , SOURCE , True , False , False -sutra , 3.0 , True , https://water.usgs.gov/water-resources/software/sutra/SUTRA_3_0_0.zip , SutraSuite , SUTRA_3_0/source , True , False , False mf2000 , 1.19.01, True , https://water.usgs.gov/nrp/gwsoftware/modflow2000/mf2k1_19_01.tar.gz , mf2k.1_19 , src , True , False , False mf2005 , 1.12.00, True , https://github.com/MODFLOW-USGS/mf2005/releases/download/v.1.12.00/MF2005.1_12u.zip , MF2005.1_12u , src , True , False , False mfusg , 1.5 , True , https://water.usgs.gov/water-resources/software/MODFLOW-USG/mfusg1_5.zip , mfusg1_5 , src , True , False , False