feat(mfsimlist): add functionality to parse memory_print_options

autotest/test_mfsimlist.py

@@ -1,16 +1,28 @@
 import numpy as np
+import pandas as pd
 import pytest
 from autotest.conftest import get_example_data_path
 from modflow_devtools.markers import requires_exe
 
 import flopy
 from flopy.mf6 import MFSimulation
 
+MEMORY_UNITS = ("gigabytes", "megabytes", "kilobytes", "bytes")
+
+
+def base_model(sim_path, memory_print_option=None):
+    MEMORY_PRINT_OPTIONS = ("summary", "all")
+    if memory_print_option is not None:
+        if memory_print_option.lower() not in MEMORY_PRINT_OPTIONS:
+            raise ValueError(
+                f"invalid memory_print option ({memory_print_option.lower()})"
+            )
 
-def base_model(sim_path):
     load_path = get_example_data_path() / "mf6-freyberg"
 
     sim = MFSimulation.load(sim_ws=load_path)
+    if memory_print_option is not None:
+        sim.memory_print_option = memory_print_option
     sim.set_sim_path(sim_path)
     sim.write_simulation()
     sim.run_simulation()
@@ -95,6 +107,13 @@ def test_mfsimlist_memory(function_tmpdir):
         f"total memory is not greater than 0.0 " + f"({total_memory})"
     )
 
+    total_memory_kb = mfsimlst.get_memory_usage(units="kilobytes")
+    assert np.allclose(total_memory_kb, total_memory * 1e6), (
+        f"total memory in kilobytes ({total_memory_kb}) is not equal to "
+        + f"the total memory converted to kilobytes "
+        + f"({total_memory * 1e6})"
+    )
+
     virtual_memory = mfsimlst.get_memory_usage(virtual=True)
     if not np.isnan(virtual_memory):
         assert virtual_memory == virtual_answer, (
@@ -107,3 +126,41 @@ def test_mfsimlist_memory(function_tmpdir):
             f"total memory ({total_memory}) "
             + f"does not equal non-virtual memory ({non_virtual_memory})"
         )
+
+
+@requires_exe("mf6")
+@pytest.mark.parametrize("mem_option", (None, "summary"))
+def test_mfsimlist_memory_summary(mem_option, function_tmpdir):
+    KEYS = ("TDIS", "FREYBERG", "SLN_1")
+    sim = base_model(function_tmpdir, memory_print_option=mem_option)
+    mfsimlst = flopy.mf6.utils.MfSimulationList(function_tmpdir / "mfsim.lst")
+
+    if mem_option is None:
+        mem_dict = mfsimlst.get_memory_summary()
+        assert mem_dict is None, "Expected None to be returned"
+    else:
+        for units in MEMORY_UNITS:
+            mem_dict = mfsimlst.get_memory_summary(units=units)
+            for key in KEYS:
+                assert key in KEYS, f"memory summary key ({key}) not in KEYS"
+
+
+@requires_exe("mf6")
+@pytest.mark.parametrize("mem_option", (None, "all"))
+def test_mfsimlist_memory_all(mem_option, function_tmpdir):
+    sim = base_model(function_tmpdir, memory_print_option=mem_option)
+    mfsimlst = flopy.mf6.utils.MfSimulationList(function_tmpdir / "mfsim.lst")
+
+    if mem_option is None:
+        mem_dict = mfsimlst.get_memory_all()
+        assert mem_dict is None, "Expected None to be returned"
+    else:
+        for units in MEMORY_UNITS:
+            mem_dict = mfsimlst.get_memory_all(units=units)
+            total = 0.0
+            for key, value in mem_dict.items():
+                total += value["MEMORYSIZE"]
+            # total_ = mfsimlst.get_memory_usage(units=units)
+            # diff = total_ - total
+            # percent_diff = 100.0 * diff / total_
+            assert total > 0.0, "memory is not greater than zero"

flopy/mf6/utils/mfsimlistfile.py

@@ -31,7 +31,7 @@ def is_normal_termination(self) -> bool:
 
         seekpoint = self._seek_to_string("Normal termination of simulation.")
         self.f.seek(seekpoint)
-        line = self.f.readline()
+        line = self.f.readline().strip()
         if line == "":
             success = False
         else:
@@ -185,22 +185,29 @@ def get_total_iterations(self) -> int:
 
         return total_iterations
 
-    def get_memory_usage(self, virtual=False) -> float:
+    def get_memory_usage(
+        self,
+        virtual: bool = False,
+        units: str = "gigabytes",
+    ) -> float:
         """
         Get the simulation memory usage from the simulation list file.
 
         Parameters
         ----------
         virtual : bool
             Return total or virtual memory usage (default is total)
+        units : str
+            Memory units for return results. Valid values are 'gigabytes',
+            'megabytes', 'kilobytes', and 'bytes' (default is 'gigabytes').
 
         Returns
         -------
         memory_usage : float
             Total memory usage for a simulation (in Gigabytes)
 
         """
-        # initialize total_iterations
+        # initialize memory_usage
         memory_usage = 0.0
 
         # rewind the file
@@ -214,21 +221,16 @@ def get_memory_usage(self, virtual=False) -> float:
 
         while True:
             seekpoint = self._seek_to_string(tags[0])
+
             self.f.seek(seekpoint)
-            line = self.f.readline()
+            line = self.f.readline().strip()
             if line == "":
                 break
-            units = line.split()[-1]
-            if units == "GIGABYTES":
-                conversion = 1.0
-            elif units == "MEGABYTES":
-                conversion = 1e-3
-            elif units == "KILOBYTES":
-                conversion = 1e-6
-            elif units == "BYTES":
-                conversion = 1e-9
-            else:
-                raise ValueError(f"Unknown memory unit '{units}'")
+            sim_units = line.split()[-1]
+            unit_conversion = self._get_memory_unit_conversion(
+                sim_units,
+                return_units_str=units.upper(),
+            )
 
             if virtual:
                 tag = tags[2]
@@ -239,7 +241,7 @@ def get_memory_usage(self, virtual=False) -> float:
             line = self.f.readline()
             if line == "":
                 break
-            memory_usage = float(line.split()[-1]) * conversion
+            memory_usage = float(line.split()[-1]) * unit_conversion
 
         return memory_usage
 
@@ -255,6 +257,149 @@ def get_non_virtual_memory_usage(self):
         """
         return self.get_memory_usage() - self.get_memory_usage(virtual=True)
 
+    def get_memory_summary(self, units: str = "gigabytes") -> dict:
+        """
+        Get the summary memory information if it is available in the
+        simulation list file. Summary memory information is only available
+        if the memory_print_option is set to 'summary' in the simulation
+        name file options block.
+
+        Parameters
+        ----------
+        units : str
+            Memory units for return results. Valid values are 'gigabytes',
+            'megabytes', 'kilobytes', and 'bytes' (default is 'gigabytes').
+
+        Returns
+        -------
+        memory_summary : dict
+            dictionary with the total memory for each simulation component.
+            None is returned if summary memory data is not present in the
+            simulation listing file.
+
+
+        """
+        # initialize the return variable
+        memory_summary = None
+
+        # rewind the file
+        self._rewind_file()
+
+        tag = "SUMMARY INFORMATION ON VARIABLES STORED IN THE MEMORY MANAGER"
+        seekpoint = self._seek_to_string(tag)
+        self.f.seek(seekpoint)
+        line = self.f.readline().strip()
+        if line != "":
+            sim_units = line.split()[-1]
+            unit_conversion = self._get_memory_unit_conversion(
+                sim_units,
+                return_units_str=units.upper(),
+            )
+            # read the header
+            for k in range(3):
+                _ = self.f.readline()
+            terminator = 100 * "-"
+            memory_summary = {}
+            while True:
+                line = self.f.readline().strip()
+                if line == terminator:
+                    break
+                data = line.split()
+                memory_summary[data[0]] = float(data[-1]) * unit_conversion
+
+        return memory_summary
+
+    def get_memory_all(self, units: str = "gigabytes") -> dict:
+        """
+        Get a dictionary of the memory table written if it is available in the
+        simulation list file. The memory table is only available
+        if the memory_print_option is set to 'all' in the simulation
+        name file options block.
+
+        Parameters
+        ----------
+        units : str
+            Memory units for return results. Valid values are 'gigabytes',
+            'megabytes', 'kilobytes', and 'bytes' (default is 'gigabytes').
+
+        Returns
+        -------
+        memory_all : dict
+            dictionary with the memory information for each variable in the
+            MODFLOW 6 memory manager. The dictionary keys are the full memory
+            path for a variable (the memory path and variable name). The
+            dictionary entry for each key includes the memory path, the
+            variable name, data type, size, and memory used for each variable.
+            None is returned if the memory table is not present in the
+            simulation listing file.
+
+
+        """
+        # initialize the return variable
+        memory_all = None
+
+        TYPE_SIZE = {
+            "INTEGER": 4.0,
+            "DOUBLE": 8.0,
+            "LOGICAL": 4.0,
+            "STRING": 1.0,
+        }
+
+        # rewind the file
+        self._rewind_file()
+
+        tag = "DETAILED INFORMATION ON VARIABLES STORED IN THE MEMORY MANAGER"
+        seekpoint = self._seek_to_string(tag)
+        self.f.seek(seekpoint)
+        line = self.f.readline().strip()
+        if line != "":
+            sim_units = "BYTES"
+            unit_conversion = self._get_memory_unit_conversion(
+                sim_units,
+                return_units_str=units.upper(),
+            )
+            # read the header
+            for k in range(3):
+                _ = self.f.readline()
+            terminator = 173 * "-"
+            memory_all = {}
+            # read the data
+            while True:
+                line = self.f.readline().strip()
+                if line == terminator:
+                    break
+                if "STRING LEN=" in line:
+                    mempath = line[0:50].strip()
+                    varname = line[51:67].strip()
+                    data_type = line[68:84].strip()
+                    no_items = float(line[84:105].strip())
+                    assoc_var = line[106:].strip()
+                    variable_bytes = (
+                        TYPE_SIZE["STRING"]
+                        * float(data_type.replace("STRING LEN=", ""))
+                        * no_items
+                    )
+                else:
+                    data = line.split()
+                    mempath = data[0]
+                    varname = data[1]
+                    data_type = data[2]
+                    no_items = float(data[3])
+                    assoc_var = data[4]
+                    variable_bytes = TYPE_SIZE[data_type] * no_items
+
+                if assoc_var == "--":
+                    size_bytes = variable_bytes * unit_conversion
+                    memory_all[f"{mempath}/{varname}"] = {
+                        "MEMPATH": mempath,
+                        "VARIABLE": varname,
+                        "DATATYPE": data_type,
+                        "SIZE": no_items,
+                        "MEMORYSIZE": size_bytes,
+                    }
+
+        return memory_all
+
     def _seek_to_string(self, s):
         """
         Parameters
@@ -287,3 +432,44 @@ def _rewind_file(self):
 
         """
         self.f.seek(0)
+
+    def _get_memory_unit_conversion(
+        self,
+        sim_units_str: str,
+        return_units_str,
+    ) -> float:
+        """
+        Calculate memory unit conversion factor that converts from reported
+        units to gigabytes
+
+        Parameters
+        ----------
+        sim_units_str : str
+            Memory Units in the simulation listing file. Valid values are
+            'GIGABYTES', 'MEGABYTES', 'KILOBYTES', or 'BYTES'.
+
+        Returns
+        -------
+        unit_conversion : float
+            Unit conversion factor
+
+        """
+        valid_units = (
+            "GIGABYTES",
+            "MEGABYTES",
+            "KILOBYTES",
+            "BYTES",
+        )
+        if sim_units_str not in valid_units:
+            raise ValueError(f"Unknown memory unit '{sim_units_str}'")
+
+        factor = [1.0, 1e-3, 1e-6, 1e-9]
+        if return_units_str == "MEGABYTES":
+            factor = [v * 1e3 for v in factor]
+        elif return_units_str == "KILOBYTES":
+            factor = [v * 1e6 for v in factor]
+        elif return_units_str == "BYTES":
+            factor = [v * 1e9 for v in factor]
+        factor_dict = {tag: factor[idx] for idx, tag in enumerate(valid_units)}
+
+        return factor_dict[sim_units_str]