diff --git a/autotest/test_mfsimlist.py b/autotest/test_mfsimlist.py index 5a4f1661ab..150d07204d 100644 --- a/autotest/test_mfsimlist.py +++ b/autotest/test_mfsimlist.py @@ -1,4 +1,5 @@ import numpy as np +import pandas as pd import pytest from autotest.conftest import get_example_data_path from modflow_devtools.markers import requires_exe @@ -6,11 +7,22 @@ import flopy from flopy.mf6 import MFSimulation +MEMORY_UNITS = ("gigabytes", "megabytes", "kilobytes", "bytes") + + +def base_model(sim_path, memory_print_option=None): + MEMORY_PRINT_OPTIONS = ("summary", "all") + if memory_print_option is not None: + if memory_print_option.lower() not in MEMORY_PRINT_OPTIONS: + raise ValueError( + f"invalid memory_print option ({memory_print_option.lower()})" + ) -def base_model(sim_path): load_path = get_example_data_path() / "mf6-freyberg" sim = MFSimulation.load(sim_ws=load_path) + if memory_print_option is not None: + sim.memory_print_option = memory_print_option sim.set_sim_path(sim_path) sim.write_simulation() sim.run_simulation() @@ -27,7 +39,7 @@ def test_mfsimlist_nofile(function_tmpdir): def test_mfsimlist_normal(function_tmpdir): sim = base_model(function_tmpdir) mfsimlst = flopy.mf6.utils.MfSimulationList(function_tmpdir / "mfsim.lst") - assert mfsimlst.is_normal_termination, "model did not terminate normally" + assert mfsimlst.normal_termination, "model did not terminate normally" @pytest.mark.xfail @@ -95,6 +107,13 @@ def test_mfsimlist_memory(function_tmpdir): f"total memory is not greater than 0.0 " + f"({total_memory})" ) + total_memory_kb = mfsimlst.get_memory_usage(units="kilobytes") + assert np.allclose(total_memory_kb, total_memory * 1e6), ( + f"total memory in kilobytes ({total_memory_kb}) is not equal to " + + f"the total memory converted to kilobytes " + + f"({total_memory * 1e6})" + ) + virtual_memory = mfsimlst.get_memory_usage(virtual=True) if not np.isnan(virtual_memory): assert virtual_memory == virtual_answer, ( @@ -107,3 +126,41 @@ def test_mfsimlist_memory(function_tmpdir): f"total memory ({total_memory}) " + f"does not equal non-virtual memory ({non_virtual_memory})" ) + + +@requires_exe("mf6") +@pytest.mark.parametrize("mem_option", (None, "summary")) +def test_mfsimlist_memory_summary(mem_option, function_tmpdir): + KEYS = ("TDIS", "FREYBERG", "SLN_1") + sim = base_model(function_tmpdir, memory_print_option=mem_option) + mfsimlst = flopy.mf6.utils.MfSimulationList(function_tmpdir / "mfsim.lst") + + if mem_option is None: + mem_dict = mfsimlst.get_memory_summary() + assert mem_dict is None, "Expected None to be returned" + else: + for units in MEMORY_UNITS: + mem_dict = mfsimlst.get_memory_summary(units=units) + for key in KEYS: + assert key in KEYS, f"memory summary key ({key}) not in KEYS" + + +@requires_exe("mf6") +@pytest.mark.parametrize("mem_option", (None, "all")) +def test_mfsimlist_memory_all(mem_option, function_tmpdir): + sim = base_model(function_tmpdir, memory_print_option=mem_option) + mfsimlst = flopy.mf6.utils.MfSimulationList(function_tmpdir / "mfsim.lst") + + if mem_option is None: + mem_dict = mfsimlst.get_memory_all() + assert mem_dict is None, "Expected None to be returned" + else: + for units in MEMORY_UNITS: + mem_dict = mfsimlst.get_memory_all(units=units) + total = 0.0 + for key, value in mem_dict.items(): + total += value["MEMORYSIZE"] + # total_ = mfsimlst.get_memory_usage(units=units) + # diff = total_ - total + # percent_diff = 100.0 * diff / total_ + assert total > 0.0, "memory is not greater than zero" diff --git a/flopy/mf6/utils/mfsimlistfile.py b/flopy/mf6/utils/mfsimlistfile.py index 713d244d80..c7270711b3 100644 --- a/flopy/mf6/utils/mfsimlistfile.py +++ b/flopy/mf6/utils/mfsimlistfile.py @@ -1,6 +1,7 @@ import os import pathlib as pl import re +import warnings import numpy as np @@ -15,10 +16,13 @@ def __init__(self, file_name: os.PathLike): self.file_name = file_name self.f = open(file_name, "r", encoding="ascii", errors="replace") + self.normal_termination = self._get_termination_message() + self.memory_print_option = self._memory_print_option() + @property def is_normal_termination(self) -> bool: """ - Determine if the simulation terminated normally + Return boolean indicating if the simulation terminated normally Returns ------- @@ -26,17 +30,7 @@ def is_normal_termination(self) -> bool: Boolean indicating if the simulation terminated normally """ - # rewind the file - self._rewind_file() - - seekpoint = self._seek_to_string("Normal termination of simulation.") - self.f.seek(seekpoint) - line = self.f.readline() - if line == "": - success = False - else: - success = True - return success + return self.normal_termination def get_runtime( self, units: str = "seconds", simulation_timer: str = "elapsed" @@ -104,11 +98,13 @@ def get_runtime( if line == "": return np.nan - # yank out the floating point values from the Elapsed run time string + # parse floating point values from the Elapsed run time string times = list(map(float, re.findall(r"[+-]?[0-9.]+", line))) + # pad an array with zeros and times with # [days, hours, minutes, seconds] times = np.array([0 for _ in range(4 - len(times))] + times) + # convert all to seconds time2sec = np.array([24 * 60 * 60, 60 * 60, 60, 1]) times_sec = np.sum(times * time2sec) @@ -119,7 +115,8 @@ def get_runtime( if line == "": return np.nan times_sec = float(line.split()[3]) - # return in the requested units + + # return time in the requested units if units == "seconds": return times_sec elif units == "minutes": @@ -185,7 +182,11 @@ def get_total_iterations(self) -> int: return total_iterations - def get_memory_usage(self, virtual=False) -> float: + def get_memory_usage( + self, + virtual: bool = False, + units: str = "gigabytes", + ) -> float: """ Get the simulation memory usage from the simulation list file. @@ -193,6 +194,9 @@ def get_memory_usage(self, virtual=False) -> float: ---------- virtual : bool Return total or virtual memory usage (default is total) + units : str + Memory units for return results. Valid values are 'gigabytes', + 'megabytes', 'kilobytes', and 'bytes' (default is 'gigabytes'). Returns ------- @@ -200,7 +204,7 @@ def get_memory_usage(self, virtual=False) -> float: Total memory usage for a simulation (in Gigabytes) """ - # initialize total_iterations + # initialize memory_usage memory_usage = 0.0 # rewind the file @@ -214,21 +218,16 @@ def get_memory_usage(self, virtual=False) -> float: while True: seekpoint = self._seek_to_string(tags[0]) + self.f.seek(seekpoint) - line = self.f.readline() + line = self.f.readline().strip() if line == "": break - units = line.split()[-1] - if units == "GIGABYTES": - conversion = 1.0 - elif units == "MEGABYTES": - conversion = 1e-3 - elif units == "KILOBYTES": - conversion = 1e-6 - elif units == "BYTES": - conversion = 1e-9 - else: - raise ValueError(f"Unknown memory unit '{units}'") + sim_units = line.split()[-1] + unit_conversion = self._get_memory_unit_conversion( + sim_units, + return_units_str=units.upper(), + ) if virtual: tag = tags[2] @@ -239,7 +238,7 @@ def get_memory_usage(self, virtual=False) -> float: line = self.f.readline() if line == "": break - memory_usage = float(line.split()[-1]) * conversion + memory_usage = float(line.split()[-1]) * unit_conversion return memory_usage @@ -255,6 +254,170 @@ def get_non_virtual_memory_usage(self): """ return self.get_memory_usage() - self.get_memory_usage(virtual=True) + def get_memory_summary(self, units: str = "gigabytes") -> dict: + """ + Get the summary memory information if it is available in the + simulation list file. Summary memory information is only available + if the memory_print_option is set to 'summary' in the simulation + name file options block. + + Parameters + ---------- + units : str + Memory units for return results. Valid values are 'gigabytes', + 'megabytes', 'kilobytes', and 'bytes' (default is 'gigabytes'). + + Returns + ------- + memory_summary : dict + dictionary with the total memory for each simulation component. + None is returned if summary memory data is not present in the + simulation listing file. + + + """ + # initialize the return variable + memory_summary = None + + if self.memory_print_option != "summary": + msg = ( + "Cannot retrieve memory data using get_memory_summary() " + + "since memory_print_option is not set to 'SUMMARY'. " + + "Returning None." + ) + warnings.warn(msg, category=Warning) + + else: + # rewind the file + self._rewind_file() + + seekpoint = self._seek_to_string( + "SUMMARY INFORMATION ON VARIABLES " + + "STORED IN THE MEMORY MANAGER" + ) + self.f.seek(seekpoint) + line = self.f.readline().strip() + + if line != "": + sim_units = line.split()[-1] + unit_conversion = self._get_memory_unit_conversion( + sim_units, + return_units_str=units.upper(), + ) + # read the header + for k in range(3): + _ = self.f.readline() + terminator = 100 * "-" + memory_summary = {} + while True: + line = self.f.readline().strip() + if line == terminator: + break + data = line.split() + memory_summary[data[0]] = float(data[-1]) * unit_conversion + + return memory_summary + + def get_memory_all(self, units: str = "gigabytes") -> dict: + """ + Get a dictionary of the memory table written if it is available in the + simulation list file. The memory table is only available + if the memory_print_option is set to 'all' in the simulation + name file options block. + + Parameters + ---------- + units : str + Memory units for return results. Valid values are 'gigabytes', + 'megabytes', 'kilobytes', and 'bytes' (default is 'gigabytes'). + + Returns + ------- + memory_all : dict + dictionary with the memory information for each variable in the + MODFLOW 6 memory manager. The dictionary keys are the full memory + path for a variable (the memory path and variable name). The + dictionary entry for each key includes the memory path, the + variable name, data type, size, and memory used for each variable. + None is returned if the memory table is not present in the + simulation listing file. + + + """ + # initialize the return variable + memory_all = None + + TYPE_SIZE = { + "INTEGER": 4.0, + "DOUBLE": 8.0, + "LOGICAL": 4.0, + "STRING": 1.0, + } + if self.memory_print_option != "all": + msg = ( + "Cannot retrieve memory data using get_memory_all() since " + + "memory_print_option is not set to 'ALL'. Returning None." + ) + warnings.warn(msg, category=Warning) + else: + # rewind the file + self._rewind_file() + + seekpoint = self._seek_to_string( + "DETAILED INFORMATION ON VARIABLES " + + "STORED IN THE MEMORY MANAGER" + ) + self.f.seek(seekpoint) + line = self.f.readline().strip() + + if line != "": + sim_units = "BYTES" + unit_conversion = self._get_memory_unit_conversion( + sim_units, + return_units_str=units.upper(), + ) + # read the header + for k in range(3): + _ = self.f.readline() + terminator = 173 * "-" + memory_all = {} + # read the data + while True: + line = self.f.readline().strip() + if line == terminator: + break + if "STRING LEN=" in line: + mempath = line[0:50].strip() + varname = line[51:67].strip() + data_type = line[68:84].strip() + no_items = float(line[84:105].strip()) + assoc_var = line[106:].strip() + variable_bytes = ( + TYPE_SIZE["STRING"] + * float(data_type.replace("STRING LEN=", "")) + * no_items + ) + else: + data = line.split() + mempath = data[0] + varname = data[1] + data_type = data[2] + no_items = float(data[3]) + assoc_var = data[4] + variable_bytes = TYPE_SIZE[data_type] * no_items + + if assoc_var == "--": + size_bytes = variable_bytes * unit_conversion + memory_all[f"{mempath}/{varname}"] = { + "MEMPATH": mempath, + "VARIABLE": varname, + "DATATYPE": data_type, + "SIZE": no_items, + "MEMORYSIZE": size_bytes, + } + + return memory_all + def _seek_to_string(self, s): """ Parameters @@ -287,3 +450,95 @@ def _rewind_file(self): """ self.f.seek(0) + + def _get_termination_message(self) -> bool: + """ + Determine if the simulation terminated normally + + Returns + ------- + success: bool + Boolean indicating if the simulation terminated normally + + """ + # rewind the file + self._rewind_file() + + seekpoint = self._seek_to_string("Normal termination of simulation.") + self.f.seek(seekpoint) + line = self.f.readline().strip() + if line == "": + success = False + else: + success = True + return success + + def _get_memory_unit_conversion( + self, + sim_units_str: str, + return_units_str, + ) -> float: + """ + Calculate memory unit conversion factor that converts from reported + units to gigabytes + + Parameters + ---------- + sim_units_str : str + Memory Units in the simulation listing file. Valid values are + 'GIGABYTES', 'MEGABYTES', 'KILOBYTES', or 'BYTES'. + + Returns + ------- + unit_conversion : float + Unit conversion factor + + """ + valid_units = ( + "GIGABYTES", + "MEGABYTES", + "KILOBYTES", + "BYTES", + ) + if sim_units_str not in valid_units: + raise ValueError(f"Unknown memory unit '{sim_units_str}'") + + factor = [1.0, 1e-3, 1e-6, 1e-9] + if return_units_str == "MEGABYTES": + factor = [v * 1e3 for v in factor] + elif return_units_str == "KILOBYTES": + factor = [v * 1e6 for v in factor] + elif return_units_str == "BYTES": + factor = [v * 1e9 for v in factor] + factor_dict = {tag: factor[idx] for idx, tag in enumerate(valid_units)} + + return factor_dict[sim_units_str] + + def _memory_print_option(self) -> str: + """ + Determine the memory print option selected + + Returns + ------- + option: str + memory_print_option ('summary', 'all', or None) + + """ + # rewind the file + self._rewind_file() + + seekpoint = self._seek_to_string("MEMORY_PRINT_OPTION SET TO") + self.f.seek(seekpoint) + line = self.f.readline().strip() + if line == "": + option = None + else: + option_list = re.findall(r'"([^"]*)"', line) + if len(option_list) < 1: + raise LookupError( + "could not parse memory_print_option from" + f"'{line}'" + ) + option = option_list[-1].lower() + if option not in ("all", "summary"): + raise ValueError(f"unknown memory print option {option}") + return option