From b34f154362320059d92050b54014ec1fc9c9f09f Mon Sep 17 00:00:00 2001 From: jdhughes-usgs Date: Tue, 14 Nov 2023 14:25:33 -0600 Subject: [PATCH] feat(MfSimulationList): add functionality to parse the mfsim.lst file (#2005) * add testing for MfSimulationList --- autotest/test_mfsimlist.py | 108 ++++++++++++ flopy/mf6/utils/__init__.py | 1 + flopy/mf6/utils/mfsimlistfile.py | 288 +++++++++++++++++++++++++++++++ 3 files changed, 397 insertions(+) create mode 100644 autotest/test_mfsimlist.py create mode 100644 flopy/mf6/utils/mfsimlistfile.py diff --git a/autotest/test_mfsimlist.py b/autotest/test_mfsimlist.py new file mode 100644 index 0000000000..e45a046e89 --- /dev/null +++ b/autotest/test_mfsimlist.py @@ -0,0 +1,108 @@ +import numpy as np +import pytest +from autotest.conftest import get_example_data_path +from modflow_devtools.markers import requires_exe + +import flopy +from flopy.mf6 import MFSimulation + + +def base_model(sim_path): + load_path = get_example_data_path() / "mf6-freyberg" + + sim = MFSimulation.load(sim_ws=load_path) + sim.set_sim_path(sim_path) + sim.write_simulation() + sim.run_simulation() + + return sim + + +@pytest.mark.xfail +def test_mfsimlist_nofile(function_tmpdir): + mfsimlst = flopy.mf6.utils.MfSimulationList(function_tmpdir / "fail.lst") + + +@requires_exe("mf6") +def test_mfsimlist_normal(function_tmpdir): + sim = base_model(function_tmpdir) + mfsimlst = flopy.mf6.utils.MfSimulationList(function_tmpdir / "mfsim.lst") + assert mfsimlst.is_normal_termination, "model did not terminate normally" + + +@pytest.mark.xfail +def test_mfsimlist_runtime_fail(function_tmpdir): + sim = base_model(function_tmpdir) + mfsimlst = flopy.mf6.utils.MfSimulationList(function_tmpdir / "mfsim.lst") + runtime_sec = mfsimlst.get_runtime(units="abc") + + +@requires_exe("mf6") +def test_mfsimlist_runtime(function_tmpdir): + sim = base_model(function_tmpdir) + mfsimlst = flopy.mf6.utils.MfSimulationList(function_tmpdir / "mfsim.lst") + for sim_timer in ("elapsed", "formulate", "solution"): + runtime_sec = mfsimlst.get_runtime(simulation_timer=sim_timer) + if not np.isnan(runtime_sec): + runtime_min = mfsimlst.get_runtime( + units="minutes", simulation_timer=sim_timer + ) + assert runtime_sec / 60.0 == runtime_min, ( + f"model {sim_timer} time conversion from " + + "sec to minutes does not match" + ) + + runtime_hrs = mfsimlst.get_runtime( + units="hours", simulation_timer=sim_timer + ) + assert runtime_min / 60.0 == runtime_hrs, ( + f"model {sim_timer} time conversion from " + + "minutes to hours does not match" + ) + + +@requires_exe("mf6") +def test_mfsimlist_iterations(function_tmpdir): + it_outer_answer = 13 + it_total_answer = 413 + + sim = base_model(function_tmpdir) + mfsimlst = flopy.mf6.utils.MfSimulationList(function_tmpdir / "mfsim.lst") + + it_outer = mfsimlst.get_outer_iterations() + assert it_outer == it_outer_answer, ( + f"outer iterations is not equal to {it_outer_answer} " + + f"({it_outer})" + ) + + it_total = mfsimlst.get_total_iterations() + assert it_total == it_total_answer, ( + f"total iterations is not equal to {it_total_answer} " + + f"({it_total})" + ) + + +@requires_exe("mf6") +def test_mfsimlist_memory(function_tmpdir): + virtual_answer = 0.0 + + sim = base_model(function_tmpdir) + mfsimlst = flopy.mf6.utils.MfSimulationList(function_tmpdir / "mfsim.lst") + + total_memory = mfsimlst.get_memory_usage() + assert total_memory > 0.0, ( + f"total memory is not greater than 0.0 " + f"({total_memory})" + ) + + virtual_memory = mfsimlst.get_memory_usage(virtual=True) + if not np.isnan(virtual_memory): + assert virtual_memory == virtual_answer, ( + f"virtual memory is not equal to {virtual_answer} " + + f"({virtual_memory})" + ) + + non_virtual_memory = mfsimlst.get_non_virtual_memory_usage() + assert total_memory == non_virtual_memory, ( + f"total memory ({total_memory}) " + + f"does not equal non-virtual memory ({non_virtual_memory})" + ) diff --git a/flopy/mf6/utils/__init__.py b/flopy/mf6/utils/__init__.py index 8abfe1ddc0..dd19d3a5d2 100644 --- a/flopy/mf6/utils/__init__.py +++ b/flopy/mf6/utils/__init__.py @@ -2,5 +2,6 @@ from .binarygrid_util import MfGrdFile from .generate_classes import generate_classes from .lakpak_utils import get_lak_connections +from .mfsimlistfile import MfSimulationList from .model_splitter import Mf6Splitter from .postprocessing import get_residuals, get_structured_faceflows diff --git a/flopy/mf6/utils/mfsimlistfile.py b/flopy/mf6/utils/mfsimlistfile.py new file mode 100644 index 0000000000..387da10be7 --- /dev/null +++ b/flopy/mf6/utils/mfsimlistfile.py @@ -0,0 +1,288 @@ +import os +import pathlib as pl +import re + +import numpy as np + + +class MfSimulationList: + def __init__(self, file_name: os.PathLike): + # Set up file reading + if isinstance(file_name, str): + file_name = pl.Path(file_name) + if not file_name.is_file(): + raise FileNotFoundError(f"file_name `{file_name}` not found") + self.file_name = file_name + self.f = open(file_name, "r", encoding="ascii", errors="replace") + + @property + def is_normal_termination(self) -> bool: + """ + Determine if the simulation terminated normally + + Returns + ------- + success: bool + Boolean indicating if the simulation terminated normally + + """ + # rewind the file + self._rewind_file() + + seekpoint = self._seek_to_string("Normal termination of simulation.") + self.f.seek(seekpoint) + line = self.f.readline() + if line == "": + success = False + else: + success = True + return success + + def get_runtime( + self, units: str = "seconds", simulation_timer: str = "elapsed" + ) -> float: + """ + Get model runtimes from the simulation list file. + + Parameters + ---------- + units : str + Units in which to return the timer. Acceptable values are + 'seconds', 'minutes', 'hours' (default is 'seconds') + simulation_timer : str + Timer to return. Acceptable values are 'elapsed', 'formulate', + 'solution' (default is 'elapsed') + + Returns + ------- + out : float + Floating point value with the runtime in requested units. Returns + NaN if runtime not found in list file + + """ + UNITS = ( + "seconds", + "minutes", + "hours", + ) + TIMERS = ( + "elapsed", + "formulate", + "solution", + ) + TIMERS_DICT = { + "elapsed": "Elapsed run time:", + "formulate": "Total formulate time:", + "solution": "Total solution time:", + } + + simulation_timer = simulation_timer.lower() + if simulation_timer not in TIMERS: + msg = ( + "simulation_timers input variable must be " + + " ,".join(TIMERS) + + f": {simulation_timer} was specified." + ) + raise ValueError(msg) + + units = units.lower() + if units not in UNITS: + msg = ( + "units input variable must be " + + " ,".join(UNITS) + + f": {units} was specified." + ) + raise ValueError(msg) + + # rewind the file + self._rewind_file() + + if simulation_timer == "elapsed": + seekpoint = self._seek_to_string(TIMERS_DICT[simulation_timer]) + self.f.seek(seekpoint) + line = self.f.readline().strip() + if line == "": + return np.nan + + # yank out the floating point values from the Elapsed run time string + times = list(map(float, re.findall(r"[+-]?[0-9.]+", line))) + # pad an array with zeros and times with + # [days, hours, minutes, seconds] + times = np.array([0 for _ in range(4 - len(times))] + times) + # convert all to seconds + time2sec = np.array([24 * 60 * 60, 60 * 60, 60, 1]) + times_sec = np.sum(times * time2sec) + else: + seekpoint = self._seek_to_string(TIMERS_DICT[simulation_timer]) + line = self.f.readline().strip() + if line == "": + return np.nan + times_sec = float(line.split()[3]) + # return in the requested units + if units == "seconds": + return times_sec + elif units == "minutes": + return times_sec / 60.0 + elif units == "hours": + return times_sec / 60.0 / 60.0 + + def get_outer_iterations(self) -> int: + """ + Get the total outer iterations from the simulation list file. + + Parameters + ---------- + + Returns + ------- + outer_iterations : float + Sum of all TOTAL ITERATIONS found in the list file + + """ + # initialize total_iterations + outer_iterations = 0 + + # rewind the file + self._rewind_file() + + while True: + seekpoint = self._seek_to_string("CALLS TO NUMERICAL SOLUTION IN") + self.f.seek(seekpoint) + line = self.f.readline() + if line == "": + break + outer_iterations += int(line.split()[0]) + + return outer_iterations + + def get_total_iterations(self) -> int: + """ + Get the total number of iterations from the simulation list file. + + Parameters + ---------- + + Returns + ------- + total_iterations : float + Sum of all TOTAL ITERATIONS found in the list file + + """ + # initialize total_iterations + total_iterations = 0 + + # rewind the file + self._rewind_file() + + while True: + seekpoint = self._seek_to_string("TOTAL ITERATIONS") + self.f.seek(seekpoint) + line = self.f.readline() + if line == "": + break + total_iterations += int(line.split()[0]) + + return total_iterations + + def get_memory_usage(self, virtual=False) -> float: + """ + Get the simulation memory usage from the simulation list file. + + Parameters + ---------- + virtual : bool + Return total or virtual memory usage (default is total) + + Returns + ------- + memory_usage : float + Total memory usage for a simulation (in Gigabytes) + + """ + # initialize total_iterations + memory_usage = 0.0 + + # rewind the file + self._rewind_file() + + tags = ( + "MEMORY MANAGER TOTAL STORAGE BY DATA TYPE", + "Total", + "Virtual", + ) + + while True: + seekpoint = self._seek_to_string(tags[0]) + self.f.seek(seekpoint) + line = self.f.readline() + if line == "": + break + units = line.split()[-1] + if units == "GIGABYTES": + conversion = 1.0 + elif units == "MEGABYTES": + conversion = 1e-3 + elif units == "KILOBYTES": + conversion = 1e-6 + elif units == "BYTES": + conversion = 1e-9 + else: + raise ValueError(f"Unknown memory unit '{units}'") + + if virtual: + tag = tags[2] + else: + tag = tags[1] + seekpoint = self._seek_to_string(tag) + self.f.seek(seekpoint) + line = self.f.readline() + if line == "": + break + memory_usage = float(line.split()[-1]) * conversion + + return memory_usage + + def get_non_virtual_memory_usage(self): + """ + + Returns + ------- + non_virtual: float + Non-virtual memory usage, which is the difference between the + total and virtual memory usage + + """ + return self.get_memory_usage() - self.get_memory_usage(virtual=True) + + def _seek_to_string(self, s): + """ + Parameters + ---------- + s : str + Seek through the file to the next occurrence of s. Return the + seek location when found. + + Returns + ------- + seekpoint : int + Next location of the string + + """ + while True: + seekpoint = self.f.tell() + line = self.f.readline() + if line == "": + break + if s in line: + break + return seekpoint + + def _rewind_file(self): + """ + Rewind the simulation list file + + Returns + ------- + + """ + self.f.seek(0)