From 4b40d1e8425b5618312499ac801abfc4b24970f1 Mon Sep 17 00:00:00 2001 From: jtyoung84 <104453205+jtyoung84@users.noreply.github.com> Date: Sun, 15 Sep 2024 12:05:24 -0700 Subject: [PATCH 1/2] Fix 159 rig json (#160) * tests: update unit tests for rig issue * fix: use raw json file contents if pyd serialization fails --- src/aind_metadata_mapper/gather_metadata.py | 53 +- src/aind_metadata_mapper/models.py | 16 +- .../schema_files_with_issues/rig.json | 784 ++++++++++++++++++ tests/test_gather_metadata.py | 81 +- 4 files changed, 895 insertions(+), 39 deletions(-) create mode 100644 tests/resources/gather_metadata_job/schema_files_with_issues/rig.json diff --git a/src/aind_metadata_mapper/gather_metadata.py b/src/aind_metadata_mapper/gather_metadata.py index 79b4dc6d..46848f3d 100644 --- a/src/aind_metadata_mapper/gather_metadata.py +++ b/src/aind_metadata_mapper/gather_metadata.py @@ -23,6 +23,7 @@ from aind_data_schema.core.subject import Subject from aind_data_schema_models.pid_names import PIDName from pydantic import ValidationError +from pydantic_core import PydanticSerializationError from aind_metadata_mapper.bergamo.models import ( JobSettings as BergamoSessionJobSettings, @@ -343,12 +344,12 @@ def get_instrument_metadata(self) -> Optional[dict]: else: return None - def get_main_metadata(self) -> Metadata: - """Get main Metadata model""" + def get_main_metadata(self) -> dict: + """Get serialized main Metadata model""" def load_model( filepath: Optional[Path], model: Type[AindCoreModel] - ) -> Optional[AindCoreModel]: + ) -> Optional[dict]: """ Validates contents of file with an AindCoreModel Parameters @@ -358,16 +359,25 @@ def load_model( Returns ------- - Optional[AindCodeModel] + Optional[dict] """ if filepath is not None and filepath.is_file(): with open(filepath, "r") as f: contents = json.load(f) try: - output = model.model_validate_json(json.dumps(contents)) - except (ValidationError, AttributeError, ValueError, KeyError): - output = model.model_construct(**contents) + valid_model = model.model_validate_json( + json.dumps(contents) + ) + output = json.loads(valid_model.model_dump_json()) + except ( + ValidationError, + AttributeError, + ValueError, + KeyError, + PydanticSerializationError, + ): + output = contents return output else: @@ -410,22 +420,26 @@ def load_model( acquisition=acquisition, instrument=instrument, ) - return metadata + metadata_json = json.loads(metadata.model_dump_json(by_alias=True)) + return metadata_json except Exception as e: logging.warning(f"Issue with metadata construction! {e.args}") - metadata = Metadata.model_construct( + # Set basic parameters + metadata = Metadata( name=self.settings.metadata_settings.name, location=self.settings.metadata_settings.location, - subject=subject, - data_description=data_description, - procedures=procedures, - session=session, - rig=rig, - processing=processing, - acquisition=acquisition, - instrument=instrument, ) - return metadata + metadata_json = json.loads(metadata.model_dump_json(by_alias=True)) + # Attach dict objects + metadata_json["subject"] = subject + metadata_json["data_description"] = data_description + metadata_json["procedures"] = procedures + metadata_json["session"] = session + metadata_json["rig"] = rig + metadata_json["processing"] = processing + metadata_json["acquisition"] = acquisition + metadata_json["instrument"] = instrument + return metadata_json def _write_json_file(self, filename: str, contents: dict) -> None: """ @@ -504,14 +518,13 @@ def run_job(self) -> None: self._gather_automated_metadata() self._gather_non_automated_metadata() if self.settings.metadata_settings is not None: - metadata = self.get_main_metadata() + contents = self.get_main_metadata() # TODO: may need to update aind-data-schema write standard file # class output_path = ( self.settings.directory_to_write_to / Metadata.default_filename() ) - contents = json.loads(metadata.model_dump_json(by_alias=True)) with open(output_path, "w") as f: json.dump( contents, diff --git a/src/aind_metadata_mapper/models.py b/src/aind_metadata_mapper/models.py index 9bd8f0ff..ef6dc8d8 100644 --- a/src/aind_metadata_mapper/models.py +++ b/src/aind_metadata_mapper/models.py @@ -25,7 +25,7 @@ ) -class SessionSettings(BaseSettings, extra='allow'): +class SessionSettings(BaseSettings, extra="allow"): """Settings needed to retrieve session metadata""" job_settings: Annotated[ @@ -39,7 +39,7 @@ class SessionSettings(BaseSettings, extra='allow'): ] -class AcquisitionSettings(BaseSettings, extra='allow'): +class AcquisitionSettings(BaseSettings, extra="allow"): """Fields needed to retrieve acquisition metadata""" # TODO: we can change this to a tagged union once more acquisition settings @@ -47,21 +47,21 @@ class AcquisitionSettings(BaseSettings, extra='allow'): job_settings: SmartSpimAcquisitionJobSettings -class SubjectSettings(BaseSettings, extra='allow'): +class SubjectSettings(BaseSettings, extra="allow"): """Fields needed to retrieve subject metadata""" subject_id: str metadata_service_path: str = "subject" -class ProceduresSettings(BaseSettings, extra='allow'): +class ProceduresSettings(BaseSettings, extra="allow"): """Fields needed to retrieve procedures metadata""" subject_id: str metadata_service_path: str = "procedures" -class RawDataDescriptionSettings(BaseSettings, extra='allow'): +class RawDataDescriptionSettings(BaseSettings, extra="allow"): """Fields needed to retrieve data description metadata""" name: str @@ -71,7 +71,7 @@ class RawDataDescriptionSettings(BaseSettings, extra='allow'): metadata_service_path: str = "funding" -class ProcessingSettings(BaseSettings, extra='allow'): +class ProcessingSettings(BaseSettings, extra="allow"): """Fields needed to retrieve processing metadata""" pipeline_process: dict = Field( @@ -83,7 +83,7 @@ class ProcessingSettings(BaseSettings, extra='allow'): ) -class MetadataSettings(BaseSettings, extra='allow'): +class MetadataSettings(BaseSettings, extra="allow"): """Fields needed to retrieve main Metadata""" name: str @@ -98,7 +98,7 @@ class MetadataSettings(BaseSettings, extra='allow'): instrument_filepath: Optional[Path] = None -class JobSettings(BaseSettings, extra='allow'): +class JobSettings(BaseSettings, extra="allow"): """Fields needed to gather all metadata""" job_settings_name: Literal["GatherMetadata"] = "GatherMetadata" diff --git a/tests/resources/gather_metadata_job/schema_files_with_issues/rig.json b/tests/resources/gather_metadata_job/schema_files_with_issues/rig.json new file mode 100644 index 00000000..046edad0 --- /dev/null +++ b/tests/resources/gather_metadata_job/schema_files_with_issues/rig.json @@ -0,0 +1,784 @@ +{ + "describedBy": "https://raw.githubusercontent.com/AllenNeuralDynamics/aind-data-schema/main/src/aind_data_schema/core/rig.py", + "schema_version": "0.5.2", + "rig_id": "323_EPHYS1_OPTO_2024-07-03", + "modification_date": "2024-07-03", + "mouse_platform": { + "device_type": "Wheel", + "name": "Running wheel", + "serial_number": null, + "manufacturer": null, + "model": null, + "path_to_cad": null, + "port_index": null, + "additional_settings": {}, + "notes": null, + "surface_material": null, + "date_surface_replaced": null, + "radius": "8.5", + "width": "5", + "size_unit": "millimeter", + "encoder": { + "device_type": "Encoder", + "name": "CUI Devices Encoder", + "serial_number": null, + "manufacturer": null, + "model": "AMT102", + "path_to_cad": null, + "port_index": null, + "additional_settings": {}, + "notes": null + }, + "encoder_output": null, + "pulse_per_revolution": 32800, + "magnetic_brake": { + "device_type": "Brake", + "name": "Placid Industries magnetic brake", + "serial_number": null, + "manufacturer": null, + "model": "B1-2FM", + "path_to_cad": null, + "port_index": null, + "additional_settings": {}, + "notes": null + }, + "brake_output": null, + "torque_sensor": { + "device_type": "Torque sensor", + "name": "Transducer Techniques torque sensor", + "serial_number": null, + "manufacturer": null, + "model": "RTS-10", + "path_to_cad": null, + "port_index": null, + "additional_settings": {}, + "notes": null + }, + "torque_output": null + }, + "stimulus_devices": [], + "cameras": [ + { + "name": "Face Camera Assembly", + "camera_target": "Face side right", + "camera": { + "device_type": "Detector", + "name": "Face Camera", + "serial_number": "23022709", + "manufacturer": { + "name": "Teledyne FLIR", + "abbreviation": "FLIR", + "registry": { + "name": "Research Organization Registry", + "abbreviation": "ROR" + }, + "registry_identifier": "01j1gwp17" + }, + "model": "Blackfly S BFS-U3-04S2M", + "path_to_cad": null, + "port_index": null, + "additional_settings": {}, + "notes": null, + "detector_type": "Camera", + "data_interface": "USB", + "cooling": "None", + "computer_name": "W10DT713669", + "frame_rate": null, + "frame_rate_unit": "hertz", + "immersion": null, + "chroma": "Monochrome", + "sensor_width": 720, + "sensor_height": 540, + "size_unit": "pixel", + "sensor_format": "1/2.9", + "sensor_format_unit": "inches", + "bit_depth": null, + "bin_mode": "None", + "bin_width": null, + "bin_height": null, + "bin_unit": "pixel", + "gain": null, + "crop_width": null, + "crop_height": null, + "crop_unit": "pixel", + "recording_software": null, + "driver": null, + "driver_version": null + }, + "lens": { + "device_type": "Lens", + "name": "Camera lens", + "serial_number": null, + "manufacturer": { + "name": "Edmund Optics", + "abbreviation": null, + "registry": { + "name": "Research Organization Registry", + "abbreviation": "ROR" + }, + "registry_identifier": "01j1gwp17" + }, + "model": "M2514-MP2", + "path_to_cad": null, + "port_index": null, + "additional_settings": {}, + "notes": null, + "focal_length": "25", + "focal_length_unit": "millimeter", + "size": null, + "lens_size_unit": "inch", + "optimized_wavelength_range": null, + "wavelength_unit": "nanometer", + "max_aperture": "f/1.4" + }, + "filter": null, + "position": null + } + ], + "enclosure": null, + "ephys_assemblies": [ + { + "name": "Probe A ephys assembly", + "manipulator": { + "device_type": "Manipulator", + "name": "45880", + "serial_number": "33738", + "manufacturer": { + "name": "New Scale Technologies", + "abbreviation": null, + "registry": null, + "registry_identifier": null + }, + "model": null, + "path_to_cad": null, + "port_index": null, + "additional_settings": {}, + "notes": null + }, + "probes": [ + { + "device_type": "Ephys probe", + "name": "Probe A", + "serial_number": "22112109161", + "manufacturer": null, + "model": null, + "path_to_cad": null, + "port_index": null, + "additional_settings": {}, + "notes": null, + "probe_model": "Neuropixels Opto (Demonstrator)", + "lasers": [ + { + "device_type": "Laser", + "name": "Oxxius Blue Laser", + "serial_number": "LAS-08787", + "manufacturer": { + "name": "Oxxius", + "abbreviation": null, + "registry": null, + "registry_identifier": null + }, + "model": null, + "path_to_cad": null, + "port_index": null, + "additional_settings": {}, + "notes": null, + "wavelength": 473, + "wavelength_unit": "nanometer", + "maximum_power": null, + "power_unit": "milliwatt", + "coupling": "Single-mode fiber", + "coupling_efficiency": null, + "coupling_efficiency_unit": "percent", + "item_number": null + }, + { + "device_type": "Laser", + "name": "Oxxius Red Laser", + "serial_number": "LAS-08677", + "manufacturer": { + "name": "Oxxius", + "abbreviation": null, + "registry": null, + "registry_identifier": null + }, + "model": null, + "path_to_cad": null, + "port_index": null, + "additional_settings": {}, + "notes": null, + "wavelength": 638, + "wavelength_unit": "nanometer", + "maximum_power": null, + "power_unit": "milliwatt", + "coupling": "Single-mode fiber", + "coupling_efficiency": null, + "coupling_efficiency_unit": "percent", + "item_number": null + } + ], + "headstage": null + } + ] + } + ], + "fiber_assemblies": [], + "stick_microscopes": [ + { + "name": "Stick_assembly_1", + "camera_target": "Brain surface", + "camera": { + "device_type": "Detector", + "name": "Probe Camera 1", + "serial_number": "22438385", + "manufacturer": { + "name": "Teledyne FLIR", + "abbreviation": "FLIR", + "registry": { + "name": "Research Organization Registry", + "abbreviation": "ROR" + }, + "registry_identifier": "01j1gwp17" + }, + "model": "Blackfly S BFS-U3-120S4C", + "path_to_cad": null, + "port_index": null, + "additional_settings": {}, + "notes": null, + "detector_type": "Camera", + "data_interface": "USB", + "cooling": "None", + "computer_name": "W10DT713668", + "frame_rate": null, + "frame_rate_unit": "hertz", + "immersion": null, + "chroma": "Color", + "sensor_width": 4000, + "sensor_height": 3000, + "size_unit": "pixel", + "sensor_format": "1/1.7", + "sensor_format_unit": "inches", + "bit_depth": null, + "bin_mode": "None", + "bin_width": null, + "bin_height": null, + "bin_unit": "pixel", + "gain": null, + "crop_width": null, + "crop_height": null, + "crop_unit": "pixel", + "recording_software": null, + "driver": null, + "driver_version": null + }, + "lens": { + "device_type": "Lens", + "name": "Probe lens", + "serial_number": null, + "manufacturer": { + "name": "Edmund Optics", + "abbreviation": null, + "registry": { + "name": "Research Organization Registry", + "abbreviation": "ROR" + }, + "registry_identifier": "01j1gwp17" + }, + "model": "InfiniProbe S-25", + "path_to_cad": null, + "port_index": null, + "additional_settings": {}, + "notes": null, + "focal_length": null, + "focal_length_unit": "millimeter", + "size": null, + "lens_size_unit": "inch", + "optimized_wavelength_range": null, + "wavelength_unit": "nanometer", + "max_aperture": null + }, + "filter": null, + "position": null + }, + { + "name": "Stick_assembly_2", + "camera_target": "Brain surface", + "camera": { + "device_type": "Detector", + "name": "Probe Camera 2", + "serial_number": "20105611", + "manufacturer": { + "name": "Teledyne FLIR", + "abbreviation": "FLIR", + "registry": { + "name": "Research Organization Registry", + "abbreviation": "ROR" + }, + "registry_identifier": "01j1gwp17" + }, + "model": "Blackfly S BFS-U3-120S4C", + "path_to_cad": null, + "port_index": null, + "additional_settings": {}, + "notes": null, + "detector_type": "Camera", + "data_interface": "USB", + "cooling": "None", + "computer_name": "W10DT713668", + "frame_rate": null, + "frame_rate_unit": "hertz", + "immersion": null, + "chroma": "Color", + "sensor_width": 4000, + "sensor_height": 3000, + "size_unit": "pixel", + "sensor_format": "1/1.7", + "sensor_format_unit": "inches", + "bit_depth": null, + "bin_mode": "None", + "bin_width": null, + "bin_height": null, + "bin_unit": "pixel", + "gain": null, + "crop_width": null, + "crop_height": null, + "crop_unit": "pixel", + "recording_software": null, + "driver": null, + "driver_version": null + }, + "lens": { + "device_type": "Lens", + "name": "Probe lens", + "serial_number": null, + "manufacturer": { + "name": "Edmund Optics", + "abbreviation": null, + "registry": { + "name": "Research Organization Registry", + "abbreviation": "ROR" + }, + "registry_identifier": "01j1gwp17" + }, + "model": "InfiniProbe S-25", + "path_to_cad": null, + "port_index": null, + "additional_settings": {}, + "notes": null, + "focal_length": null, + "focal_length_unit": "millimeter", + "size": null, + "lens_size_unit": "inch", + "optimized_wavelength_range": null, + "wavelength_unit": "nanometer", + "max_aperture": null + }, + "filter": null, + "position": null + }, + { + "name": "Stick_assembly_3", + "camera_target": "Brain surface", + "camera": { + "device_type": "Detector", + "name": "Probe Camera 3", + "serial_number": "22438379", + "manufacturer": { + "name": "Teledyne FLIR", + "abbreviation": "FLIR", + "registry": { + "name": "Research Organization Registry", + "abbreviation": "ROR" + }, + "registry_identifier": "01j1gwp17" + }, + "model": "Blackfly S BFS-U3-120S4C", + "path_to_cad": null, + "port_index": null, + "additional_settings": {}, + "notes": null, + "detector_type": "Camera", + "data_interface": "USB", + "cooling": "None", + "computer_name": "W10DT713668", + "frame_rate": null, + "frame_rate_unit": "hertz", + "immersion": null, + "chroma": "Color", + "sensor_width": 4000, + "sensor_height": 3000, + "size_unit": "pixel", + "sensor_format": "1/1.7", + "sensor_format_unit": "inches", + "bit_depth": null, + "bin_mode": "None", + "bin_width": null, + "bin_height": null, + "bin_unit": "pixel", + "gain": null, + "crop_width": null, + "crop_height": null, + "crop_unit": "pixel", + "recording_software": null, + "driver": null, + "driver_version": null + }, + "lens": { + "device_type": "Lens", + "name": "Probe lens", + "serial_number": null, + "manufacturer": { + "name": "Edmund Optics", + "abbreviation": null, + "registry": { + "name": "Research Organization Registry", + "abbreviation": "ROR" + }, + "registry_identifier": "01j1gwp17" + }, + "model": "InfiniProbe S-25", + "path_to_cad": null, + "port_index": null, + "additional_settings": {}, + "notes": null, + "focal_length": null, + "focal_length_unit": "millimeter", + "size": null, + "lens_size_unit": "inch", + "optimized_wavelength_range": null, + "wavelength_unit": "nanometer", + "max_aperture": null + }, + "filter": null, + "position": null + } + ], + "laser_assemblies": [ + { + "name": "External 640 laser", + "manipulator": { + "device_type": "Manipulator", + "name": "45879", + "serial_number": "33711", + "manufacturer": { + "name": "New Scale Technologies", + "abbreviation": null, + "registry": null, + "registry_identifier": null + }, + "model": null, + "path_to_cad": null, + "port_index": null, + "additional_settings": {}, + "notes": null + }, + "lasers": [ + { + "device_type": "Laser", + "name": "Coherent Red Laser", + "serial_number": "M171024016", + "manufacturer": { + "name": "Coherent Scientific", + "abbreviation": null, + "registry": { + "name": "Research Organization Registry", + "abbreviation": "ROR" + }, + "registry_identifier": "031tysd23" + }, + "model": null, + "path_to_cad": null, + "port_index": null, + "additional_settings": {}, + "notes": null, + "wavelength": 640, + "wavelength_unit": "nanometer", + "maximum_power": null, + "power_unit": "milliwatt", + "coupling": null, + "coupling_efficiency": null, + "coupling_efficiency_unit": "percent", + "item_number": null + } + ], + "collimator": { + "device_type": "Collimator", + "name": "Collimator", + "serial_number": null, + "manufacturer": { + "name": "Thorlabs", + "abbreviation": null, + "registry": { + "name": "Research Organization Registry", + "abbreviation": "ROR" + }, + "registry_identifier": "04gsnvb07" + }, + "model": "CFC2A", + "path_to_cad": null, + "port_index": null, + "additional_settings": {}, + "notes": null + }, + "fiber": { + "device_type": "Patch", + "name": "Patch cable", + "serial_number": null, + "manufacturer": null, + "model": null, + "path_to_cad": null, + "port_index": null, + "additional_settings": {}, + "notes": null, + "core_diameter": "125", + "numerical_aperture": "0.12", + "photobleaching_date": null + } + } + ], + "patch_cords": [], + "light_sources": [], + "detectors": [], + "objectives": [], + "filters": [], + "lenses": [], + "digital_micromirror_devices": [], + "polygonal_scanners": [], + "pockels_cells": [], + "additional_devices": [], + "daqs": [ + { + "device_type": "Harp device", + "name": "Harp Behavior", + "serial_number": null, + "manufacturer": { + "name": "Open Ephys Production Site", + "abbreviation": "OEPS", + "registry": { + "name": "Research Organization Registry", + "abbreviation": "ROR" + }, + "registry_identifier": "007rkz355" + }, + "model": null, + "path_to_cad": null, + "port_index": null, + "additional_settings": {}, + "notes": null, + "data_interface": "USB", + "computer_name": "W10DT713669", + "channels": [ + { + "channel_name": "DO0", + "device_name": "Face Camera", + "channel_type": "Digital Output", + "port": null, + "channel_index": null, + "sample_rate": null, + "sample_rate_unit": "hertz", + "event_based_sampling": null + }, + { + "channel_name": "AI2", + "device_name": "Running wheel", + "channel_type": "Analog Input", + "port": null, + "channel_index": null, + "sample_rate": null, + "sample_rate_unit": "hertz", + "event_based_sampling": null + } + ], + "firmware_version": null, + "hardware_version": null, + "harp_device_type": { + "name": "Behavior", + "whoami": 1216 + }, + "core_version": "2.1", + "tag_version": null, + "is_clock_generator": false + }, + { + "device_type": "Neuropixels basestation", + "name": "NPopto Basestation", + "serial_number": null, + "manufacturer": { + "name": "Interuniversity Microelectronics Center", + "abbreviation": "IMEC", + "registry": { + "name": "Research Organization Registry", + "abbreviation": "ROR" + }, + "registry_identifier": "02kcbn207" + }, + "model": null, + "path_to_cad": null, + "port_index": null, + "additional_settings": {}, + "notes": null, + "data_interface": "PXI", + "computer_name": "W10DT713668", + "channels": [], + "firmware_version": null, + "hardware_version": null, + "basestation_firmware_version": "2.0168", + "bsc_firmware_version": "1.02", + "slot": 5, + "ports": [ + { + "index": 1, + "probes": [ + "Probe A" + ] + } + ] + } + ], + "calibrations": [ + { + "calibration_date": "2024-03-14T00:00:00-07:00", + "device_name": "Red Oxxius Laser", + "description": "Laser power calibration", + "input": { + "voltage (V)": [ + 0.108, + 0.175, + 0.243, + 0.311, + 0.376, + 0.447, + 0.506, + 0.579, + 0.639, + 0.706, + 1.021, + 1.34, + 1.66, + 1.99, + 2.62, + 3.21, + 4.24 + ] + }, + "output": { + "power (mW)": [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 15, + 20, + 25, + 30, + 40, + 50, + 60 + ] + }, + "notes": null + }, + { + "calibration_date": "2024-01-18T00:00:00-08:00", + "device_name": "Blue Oxxius Laser", + "description": "Laser power calibration", + "input": { + "voltage (V)": [ + 0.145, + 0.254, + 0.358, + 0.46, + 0.561, + 0.659, + 0.774, + 0.867, + 0.955, + 1.051, + 1.51, + 1.91, + 2.3, + 2.69, + 3.38, + 4.0, + 4.595 + ] + }, + "output": { + "power (mW)": [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 15, + 20, + 25, + 30, + 40, + 50, + 60 + ] + }, + "notes": null + }, + { + "calibration_date": "2024-02-12T00:00:00-08:00", + "device_name": "Red Coherent Laser", + "description": "Laser power calibration", + "input": { + "voltage (V)": [ + 0.032, + 0.045, + 0.135, + 0.313, + 0.401, + 0.837, + 1.7, + 2.588, + 3.405, + 5.0 + ] + }, + "output": { + "power (mW)": [ + 0.4, + 1, + 2, + 4, + 5, + 10, + 20, + 30, + 40, + 58.27 + ] + }, + "notes": null + } + ], + "ccf_coordinate_transform": null, + "origin": null, + "rig_axes": null, + "modalities": [ + { + "name": "Behavior videos", + "abbreviation": "behavior-videos" + }, + { + "name": "Extracellular electrophysiology", + "abbreviation": "ecephys" + } + ], + "notes": null +} diff --git a/tests/test_gather_metadata.py b/tests/test_gather_metadata.py index 29fbe599..8b4ab166 100644 --- a/tests/test_gather_metadata.py +++ b/tests/test_gather_metadata.py @@ -48,6 +48,7 @@ / "gather_metadata_job" ) METADATA_DIR = RESOURCES_DIR / "metadata_files" +METADATA_DIR_WITH_RIG_ISSUE = RESOURCES_DIR / "schema_files_with_issues" EXAMPLE_BERGAMO_CONFIGS = RESOURCES_DIR / "test_bergamo_configs.json" @@ -634,6 +635,19 @@ def test_get_rig_metadata_none(self): contents = metadata_job.get_rig_metadata() self.assertIsNone(contents) + def test_get_problematic_rig_metadata(self): + """Tests get_rig_metadata when there is a pydantic serialization + issue.""" + metadata_dir = METADATA_DIR_WITH_RIG_ISSUE + + job_settings = JobSettings( + directory_to_write_to=RESOURCES_DIR, + metadata_dir=metadata_dir, + ) + metadata_job = GatherMetadataJob(settings=job_settings) + contents = metadata_job.get_rig_metadata() + self.assertIsNotNone(contents) + def test_get_acquisition_metadata(self): """Tests get_acquisition_metadata""" metadata_dir = RESOURCES_DIR / "metadata_files" @@ -740,6 +754,25 @@ def test_gather_non_automated_metadata(self, mock_write_file: MagicMock): metadata_job._gather_non_automated_metadata() mock_write_file.assert_called() + @patch( + "aind_metadata_mapper.gather_metadata.GatherMetadataJob." + "_write_json_file" + ) + def test_gather_non_automated_metadata_with_ser_issues( + self, mock_write_file: MagicMock + ): + """Tests _gather_non_automated_metadata method when there are + serialization issues""" + metadata_dir = METADATA_DIR_WITH_RIG_ISSUE + + job_settings = JobSettings( + directory_to_write_to=RESOURCES_DIR, + metadata_dir=metadata_dir, + ) + metadata_job = GatherMetadataJob(settings=job_settings) + metadata_job._gather_non_automated_metadata() + mock_write_file.assert_called() + def test_get_main_metadata_with_warnings(self): """Tests get_main_metadata method raises validation warnings""" job_settings = JobSettings( @@ -779,10 +812,36 @@ def test_get_main_metadata_with_warnings(self): self.assertEqual(expected_warnings, str(w.warning)) self.assertEqual( "s3://some-bucket/ecephys_632269_2023-10-10_10-10-10", - main_metadata.location, + main_metadata["location"], ) - self.assertEqual("Invalid", main_metadata.metadata_status.value) - self.assertEqual("632269", main_metadata.subject.subject_id) + self.assertEqual("Invalid", main_metadata["metadata_status"]) + self.assertEqual("632269", main_metadata["subject"]["subject_id"]) + + @patch("logging.warning") + def test_get_main_metadata_with_ser_issues(self, mock_log: MagicMock): + """Tests get_main_metadata method when rig.json file has + serialization issues.""" + job_settings = JobSettings( + directory_to_write_to=RESOURCES_DIR, + metadata_settings=MetadataSettings( + name="ecephys_632269_2023-10-10_10-10-10", + location="s3://some-bucket/ecephys_632269_2023-10-10_10-10-10", + subject_filepath=(METADATA_DIR / "subject.json"), + data_description_filepath=( + METADATA_DIR / "data_description.json" + ), + procedures_filepath=(METADATA_DIR / "procedures.json"), + session_filepath=None, + rig_filepath=(METADATA_DIR_WITH_RIG_ISSUE / "rig.json"), + processing_filepath=(METADATA_DIR / "processing.json"), + acquisition_filepath=None, + instrument_filepath=None, + ), + ) + metadata_job = GatherMetadataJob(settings=job_settings) + main_metadata = metadata_job.get_main_metadata() + mock_log.assert_called_once() + self.assertIsNotNone(main_metadata["rig"]["schema_version"]) @patch("logging.warning") def test_get_main_metadata_with_validation_errors( @@ -808,14 +867,14 @@ def test_get_main_metadata_with_validation_errors( ) metadata_job = GatherMetadataJob(settings=job_settings) main_metadata = metadata_job.get_main_metadata() - self.assertIsNotNone(main_metadata.subject) - self.assertIsNotNone(main_metadata.procedures) - self.assertIsNotNone(main_metadata.data_description) - self.assertIsNotNone(main_metadata.session) - self.assertIsNotNone(main_metadata.rig) - self.assertIsNotNone(main_metadata.processing) - self.assertIsNotNone(main_metadata.acquisition) - self.assertIsNotNone(main_metadata.instrument) + self.assertIsNotNone(main_metadata["subject"]) + self.assertIsNotNone(main_metadata["procedures"]) + self.assertIsNotNone(main_metadata["data_description"]) + self.assertIsNotNone(main_metadata["session"]) + self.assertIsNotNone(main_metadata["rig"]) + self.assertIsNotNone(main_metadata["processing"]) + self.assertIsNotNone(main_metadata["acquisition"]) + self.assertIsNotNone(main_metadata["instrument"]) mock_warn.assert_called_once() @patch("builtins.open", new_callable=mock_open()) From ff50710b8cc2ff537c08137d6f63e8e0d6fd6144 Mon Sep 17 00:00:00 2001 From: jtyoung84 <104453205+jtyoung84@users.noreply.github.com> Date: Sun, 15 Sep 2024 12:43:57 -0700 Subject: [PATCH 2/2] release v0.18.2 --- src/aind_metadata_mapper/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aind_metadata_mapper/__init__.py b/src/aind_metadata_mapper/__init__.py index 8e53d5b5..63ce3dd3 100644 --- a/src/aind_metadata_mapper/__init__.py +++ b/src/aind_metadata_mapper/__init__.py @@ -1,3 +1,3 @@ """Init package""" -__version__ = "0.18.1" +__version__ = "0.18.2"