From 2e21aab836a539ee2a50924597ba0c23c4a6906b Mon Sep 17 00:00:00 2001 From: David Young Date: Thu, 21 Oct 2021 11:46:30 +0800 Subject: [PATCH 1/4] Store the logging file path The logging file path is set by the user or to a default path, but either may change depending on the available path locations for the rotating file handler. Store this path in `config` so that the path can be displayed to the user. --- magmap/io/cli.py | 2 +- magmap/settings/config.py | 3 +++ magmap/settings/logs.py | 18 ++++++++++-------- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/magmap/io/cli.py b/magmap/io/cli.py index 7e2278999..993b86290 100644 --- a/magmap/io/cli.py +++ b/magmap/io/cli.py @@ -413,7 +413,7 @@ def process_cli_args(): log_path = os.path.join( config.user_app_dirs.user_data_dir, "out.log") # log to file - logs.add_file_handler(config.logger, log_path) + config.log_path = logs.add_file_handler(config.logger, log_path) # redirect standard out/error to logging sys.stdout = logs.LogWriter(config.logger.info) diff --git a/magmap/settings/config.py b/magmap/settings/config.py index 1167e328e..ba4504baf 100644 --- a/magmap/settings/config.py +++ b/magmap/settings/config.py @@ -65,6 +65,9 @@ class Verbosity(Enum): #: :class:`logging.Logger`: Root logger for the application. logger = logs.setup_logger() +#: Path to log file. +log_path: Optional[pathlib.Path] = None + # IMAGE FILES diff --git a/magmap/settings/logs.py b/magmap/settings/logs.py index e1642389c..c24ca55ad 100644 --- a/magmap/settings/logs.py +++ b/magmap/settings/logs.py @@ -98,7 +98,8 @@ def update_log_level(logger, level): return logger -def add_file_handler(logger, path, backups=5): +def add_file_handler( + logger: logging.Logger, path: str, backups: int = 5) -> pathlib.Path: """Add a rotating log file handler with a new log file. Rotates the file each time this function is called for the given number @@ -107,17 +108,18 @@ def add_file_handler(logger, path, backups=5): creating a log filed named with an incremented number (eg ``out1.log``). Args: - logger (:class:`logging.Logger`): Logger to update. - path (str): Path to log. Increments to ``path.`` if the + logger: Logger to update. + path: Path to log. Increments to ``path.`` if the file at ``path`` cannot be rotated. - backups (int): Number of backups to maintain; defaults to 5. + backups: Number of backups to maintain; defaults to 5. Returns: - :class:`logging.Logger`: The logger for chained calls. + The log output path. """ # check if log file already exists pathl = pathlib.Path(path) + path_log = pathl roll = pathl.is_file() # create a rotations file handler to manage number of backups while @@ -129,8 +131,8 @@ def add_file_handler(logger, path, backups=5): try: # if the existing file at path cannot be rotated, increment the # filename to create a new series of rotating log files - path_log = (pathl if i == 0 else - f"{pathl.parent / pathl.stem}{i}{pathl.suffix}") + path_log = pathl if i == 0 else pathlib.Path( + f"{pathl.parent / pathl.stem}{i}{pathl.suffix}") logger.debug(f"Trying logger path: {path_log}") handler_file = handlers.RotatingFileHandler( path_log, backupCount=backups) @@ -150,4 +152,4 @@ def add_file_handler(logger, path, backups=5): "%(asctime)s - %(name)s - %(levelname)s - %(message)s")) logger.addHandler(handler_file) - return logger + return path_log From f28d3ade33b160950e756996212766194f29e192 Mon Sep 17 00:00:00 2001 From: David Young Date: Thu, 21 Oct 2021 11:50:00 +0800 Subject: [PATCH 2/4] Catch exceptions during single file import errors Errors during single file import have been assumed to result from RAW files but may occur for other reasons such as unrecognized or incompatible file types. Catch any `ValueError`s, skipping the file and notifying the user. Also, change the notification for conversion to RGB to grayscale so that it appears in the GUI. --- magmap/io/importer.py | 81 +++++++++++++++++++++++-------------------- 1 file changed, 44 insertions(+), 37 deletions(-) diff --git a/magmap/io/importer.py b/magmap/io/importer.py index 043e4bc2a..28613cec9 100644 --- a/magmap/io/importer.py +++ b/magmap/io/importer.py @@ -1162,44 +1162,51 @@ def import_files(): for filei, file in enumerate(chl_files): libmag.printcb("importing {}".format(file), fn_feedback) try: - # load standard image types - img = io.imread(file) - except ValueError: - # load as a RAW image file - img = np.memmap( - file, dtype=import_md[config.MetaKeys.DTYPE], - shape=tuple(import_md[config.MetaKeys.SHAPE][2:4]), - mode="r") - - if rgb_to_grayscale and img.ndim >= 3 and img.shape[2] == 3: - # assume that 3-channel images are RGB - # TODO: remove rgb_to_grayscale since must give single channel? - print("converted from 3-channel (assuming RGB) to grayscale") - img = color.rgb2gray(img) - - if img5d is None: - # generate an array for all planes and channels based on - # dimensions of the first extracted plane and any channel keys - shape = [1, len(chl_files), *img.shape] + try: + # load standard image types + img = io.imread(file) + except ValueError: + # load as a RAW image file + img = np.memmap( + file, dtype=import_md[config.MetaKeys.DTYPE], + shape=tuple(import_md[config.MetaKeys.SHAPE][2:4]), + mode="r") + + if rgb_to_grayscale and img.ndim >= 3 and img.shape[2] == 3: + # assume that 3-channel images are RGB + # TODO: remove rgb_to_grayscale since must give single chl? + libmag.printcb( + "Converted from 3-channel (assuming RGB) to grayscale", + fn_feedback) + img = color.rgb2gray(img) + + if img5d is None: + # generate an array for all planes and channels based on + # dims of the first extracted plane and any channel keys + shape = [1, len(chl_files), *img.shape] + if num_chls > 1: + shape.append(num_chls) + os.makedirs( + os.path.dirname(filename_image5d_npz), exist_ok=True) + img5d = np.lib.format.open_memmap( + filename_image5d_npz, mode="w+", dtype=img.dtype, + shape=tuple(shape)) + + # insert plane, without using channel dimension if no channel + # designators were found in file names if num_chls > 1: - shape.append(num_chls) - os.makedirs( - os.path.dirname(filename_image5d_npz), exist_ok=True) - img5d = np.lib.format.open_memmap( - filename_image5d_npz, mode="w+", dtype=img.dtype, - shape=tuple(shape)) - - # insert plane, without using channel dimension if no channel - # designators were found in file names - if num_chls > 1: - img5d[0, filei, ..., chli] = img - else: - img5d[0, filei] = img - - # measure near low/high intensity values - low, high = np.percentile(img, (0.5, 99.5)) - lows.append(low) - highs.append(high) + img5d[0, filei, ..., chli] = img + else: + img5d[0, filei] = img + + # measure near low/high intensity values + low, high = np.percentile(img, (0.5, 99.5)) + lows.append(low) + highs.append(high) + except ValueError as e1: + libmag.printcb( + f"Could not load '{file}'; skipping it because of error: " + f"{e1}", fn_feedback) lows_chls.append(min(lows)) highs_chls.append(max(highs)) From 8c2a6d010a0c9bccb4f557808d99baaa7e7a6824 Mon Sep 17 00:00:00 2001 From: David Young Date: Thu, 21 Oct 2021 11:54:13 +0800 Subject: [PATCH 3/4] Notify the user of global exceptions during import through the GUI The GUI has simply notified the user when the import could not be completed but has not provided explanations for the import error. Catch global exceptions during import to notify the user of the error and point them to the log file. This notification contains the basic error, while the full stack trace is posted to the console and log file. This notification has not been added to import through the command-line since presumably the error message would be visible in the console. --- magmap/gui/import_threads.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/magmap/gui/import_threads.py b/magmap/gui/import_threads.py index c9d87e9b0..1cb9c544c 100644 --- a/magmap/gui/import_threads.py +++ b/magmap/gui/import_threads.py @@ -6,6 +6,8 @@ from magmap.io import importer from magmap.settings import config +_logger = config.logger.getChild(__name__) + class SetupImportThread(QtCore.QThread): """Thread for setting up file import by extracting image metadata. @@ -81,6 +83,14 @@ def run(self): img5d = importer.import_multiplane_images( self.chl_paths, self.prefix, self.import_md, config.series, fn_feedback=self.fn_feedback) + + except Exception as e: + # provide feedback for any errors during import + self.fn_feedback(f"Error during import:\n{e}") + if config.log_path: + self.fn_feedback(f"See log for more info: {config.log_path}\n") + _logger.exception(e) + finally: if img5d is not None: # set up the image for immediate use within MagellanMapper From 80ab209080f3ca364e85f19e64ac0ca8bbfb7420 Mon Sep 17 00:00:00 2001 From: David Young Date: Mon, 25 Oct 2021 11:05:18 +0800 Subject: [PATCH 4/4] Changelog update for import feedback for errors --- docs/release/release_v1.5.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/release/release_v1.5.md b/docs/release/release_v1.5.md index 7c5bbf1da..6b2ec5836 100644 --- a/docs/release/release_v1.5.md +++ b/docs/release/release_v1.5.md @@ -75,6 +75,8 @@ See the [table of CLI changes](../cli.md#changes-in-magellanmapper-v15) for a su - Plane index is only added when exporting multiple planes - Improvements to image import - Single plane RAW images can be loaded when importing files from a directory, in addition to multiplane RAW files + - Skips single plane files that give errors (eg non-image files in the input directory) + - Provides import error feedback in the GUI - The known parts of the import image shape are populated even if the full shape is not known - The Bio-Formats library has been updated to support more file formats (from Bio-Formats 5.1.8 to 6.6.0 via Python-Bioformats 1.1.0 to 4.0.5, respectively) - Fixed to disable the import directory button when metadata is insufficient