From 4b692cd5cfa0227d0dc9b53aa26998dc33799f8a Mon Sep 17 00:00:00 2001 From: jtyoung84 <104453205+jtyoung84@users.noreply.github.com> Date: Thu, 4 Apr 2024 15:50:32 -0700 Subject: [PATCH 1/8] feat: updating schema --- pyproject.toml | 2 +- src/aind_metadata_mapper/bergamo/session.py | 97 ++----------------- src/aind_metadata_mapper/ephys/session.py | 8 +- src/aind_metadata_mapper/fib/session.py | 16 +-- tests/resources/ephys/ephys_session.json | 2 +- tests/resources/fib/000000_ophys_session.json | 2 +- tests/test_ephys.py | 6 +- tests/test_fib.py | 9 +- 8 files changed, 29 insertions(+), 113 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index a22865f9..93fcf4e4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,7 +17,7 @@ readme = "README.md" dynamic = ["version"] dependencies = [ - "aind-data-schema==0.26.5", + "aind-data-schema==0.33.3", "scanimage-tiff-reader==1.4.1.4", "tifffile==2024.2.12", "pydantic-settings>=2.0", diff --git a/src/aind_metadata_mapper/bergamo/session.py b/src/aind_metadata_mapper/bergamo/session.py index ec424273..5ca80e07 100644 --- a/src/aind_metadata_mapper/bergamo/session.py +++ b/src/aind_metadata_mapper/bergamo/session.py @@ -23,8 +23,8 @@ from aind_data_schema.models.stimulus import ( PhotoStimulation, PhotoStimulationGroup, - StimulusEpoch, ) +from aind_data_schema.core.session import StimulusEpoch from aind_data_schema.models.units import PowerUnit, SizeUnit from pydantic import Field from pydantic_settings import BaseSettings @@ -401,8 +401,6 @@ def _transform(self, extracted_source: RawImageInfo) -> Session: ] data_stream = Stream( - mouse_platform_name=self.job_settings.mouse_platform_name, - active_mouse_platform=self.job_settings.active_mouse_platform, stream_start_time=self.job_settings.stream_start_time, stream_end_time=self.job_settings.stream_end_time, stream_modalities=[Modality.POPHYS], @@ -459,96 +457,17 @@ def _transform(self, extracted_source: RawImageInfo) -> Session: session_type=self.job_settings.session_type, iacuc_protocol=self.job_settings.iacuc_protocol, rig_id=self.job_settings.rig_id, + mouse_platform_name=self.job_settings.mouse_platform_name, + active_mouse_platform=self.job_settings.active_mouse_platform, data_streams=[data_stream], stimulus_epochs=[ StimulusEpoch( - stimulus=PhotoStimulation( - stimulus_name="PhotoStimulation", - number_groups=( - self.job_settings.num_of_photo_stim_groups - ), - groups=[ - PhotoStimulationGroup( - group_index=( - self.job_settings.photo_stim_groups[0][ - "group_index" - ] - ), - number_of_neurons=int( - np.array( - photostim_groups[0]["rois"][1][ - "scanfields" - ]["slmPattern"] - ).shape[0] - ), - stimulation_laser_power=int( - photostim_groups[0]["rois"][1][ - "scanfields" - ]["powers"] - ), - number_trials=( - self.job_settings.photo_stim_groups[0][ - "number_trials" - ] - ), - number_spirals=int( - photostim_groups[0]["rois"][1][ - "scanfields" - ]["repetitions"] - ), - spiral_duration=photostim_groups[0]["rois"][1][ - "scanfields" - ]["duration"], - inter_spiral_interval=photostim_groups[0][ - "rois" - ][2]["scanfields"]["duration"], - ), - PhotoStimulationGroup( - group_index=( - self.job_settings.photo_stim_groups[1][ - "group_index" - ] - ), - number_of_neurons=int( - np.array( - photostim_groups[0]["rois"][1][ - "scanfields" - ]["slmPattern"] - ).shape[0] - ), - stimulation_laser_power=int( - photostim_groups[0]["rois"][1][ - "scanfields" - ]["powers"] - ), - number_trials=( - self.job_settings.photo_stim_groups[1][ - "number_trials" - ] - ), - number_spirals=int( - photostim_groups[0]["rois"][1][ - "scanfields" - ]["repetitions"] - ), - spiral_duration=photostim_groups[0]["rois"][1][ - "scanfields" - ]["duration"], - inter_spiral_interval=photostim_groups[0][ - "rois" - ][2]["scanfields"]["duration"], - ), - ], - inter_trial_interval=( - self.job_settings.photo_stim_inter_trial_interval - ), - ), - stimulus_start_time=( - self.job_settings.stimulus_start_time - ), - stimulus_end_time=self.job_settings.stimulus_end_time, + stimulus_start_time=datetime(), + stimulus_end_time=datetime(), + stimulus_name="", + stimulus_modalities=List[StimulusModality.] ) - ], + ] ) def run_job(self) -> JobResponse: diff --git a/src/aind_metadata_mapper/ephys/session.py b/src/aind_metadata_mapper/ephys/session.py index bc744d7f..aab32d2e 100644 --- a/src/aind_metadata_mapper/ephys/session.py +++ b/src/aind_metadata_mapper/ephys/session.py @@ -89,6 +89,8 @@ def _transform(self, extracted_source: ParsedInformation) -> Session: ] ephys_session["maintenance"] = experiment_data["maintenance"] ephys_session["calibrations"] = experiment_data["calibrations"] + ephys_session["mouse_platform_name"] = "Running Wheel" + ephys_session["active_mouse_platform"] = False # Constant throughout data streams stick_microscopes = experiment_data["stick_microscopes"] @@ -106,12 +108,6 @@ def _transform(self, extracted_source: ParsedInformation) -> Session: session_stream["stream_end_time"] = datetime.strptime( stage[-1][0], "%Y/%m/%d %H:%M:%S.%f" ) - session_stream["mouse_platform_name"] = data_stream[ - "mouse_platform_name" - ] - session_stream["active_mouse_platform"] = data_stream[ - "active_mouse_platform" - ] session_stream["stream_modalities"] = [Modality.ECEPHYS] session_stream["stick_microscopes"] = stick_microscopes session_stream["camera_names"] = camera_names diff --git a/src/aind_metadata_mapper/fib/session.py b/src/aind_metadata_mapper/fib/session.py index 72823b26..44f1a3e2 100644 --- a/src/aind_metadata_mapper/fib/session.py +++ b/src/aind_metadata_mapper/fib/session.py @@ -11,13 +11,13 @@ FiberConnectionConfig, LightEmittingDiodeConfig, Session, - Stream, + StimulusEpoch, + Stream, StimulusModality, ) from aind_data_schema.models.modalities import Modality from aind_data_schema.models.stimulus import ( OptoStimulation, PulseShape, - StimulusEpoch, ) from pydantic import Field from pydantic_settings import BaseSettings @@ -148,10 +148,10 @@ def _transform(self, extracted_source: ParsedMetadata) -> Session: opto_stim = OptoStimulation( stimulus_name=stimulus_name, pulse_shape=PulseShape.SQUARE, - pulse_frequency=frequency, - number_pulse_trains=trial_num, - pulse_width=pulse_width, - pulse_train_duration=opto_duration, + pulse_frequency=[frequency], + number_pulse_trains=[trial_num], + pulse_width=[pulse_width], + pulse_train_duration=[opto_duration], pulse_train_interval=opto_interval, baseline_duration=opto_base, fixed_pulse_train_interval=True, # TODO: Check this is right @@ -165,9 +165,11 @@ def _transform(self, extracted_source: ParsedMetadata) -> Session: seconds=experiment_duration ) stimulus_epochs = StimulusEpoch( - stimulus=opto_stim, + stimulus_name=stimulus_name, stimulus_start_time=session_start_time, stimulus_end_time=end_datetime, + stimulus_modalities=[StimulusModality.NONE], + stimulus_parameters=[opto_stim] ) # create light source instance diff --git a/tests/resources/ephys/ephys_session.json b/tests/resources/ephys/ephys_session.json index d8caade0..58ccba3f 100644 --- a/tests/resources/ephys/ephys_session.json +++ b/tests/resources/ephys/ephys_session.json @@ -1,6 +1,6 @@ { "describedBy": "https://raw.githubusercontent.com/AllenNeuralDynamics/aind-data-schema/main/src/aind_data_schema/core/session.py", - "schema_version": "0.1.4", + "schema_version": "0.2.1", "experimenter_full_name": [ "Al Dente" ], diff --git a/tests/resources/fib/000000_ophys_session.json b/tests/resources/fib/000000_ophys_session.json index 46a12eb3..c6375853 100644 --- a/tests/resources/fib/000000_ophys_session.json +++ b/tests/resources/fib/000000_ophys_session.json @@ -1,6 +1,6 @@ { "describedBy": "https://raw.githubusercontent.com/AllenNeuralDynamics/aind-data-schema/main/src/aind_data_schema/core/session.py", - "schema_version": "0.1.4", + "schema_version": "0.2.1", "protocol_id": [], "experimenter_full_name": [ "Don Key" diff --git a/tests/test_ephys.py b/tests/test_ephys.py index 1800ec1e..4f150e13 100644 --- a/tests/test_ephys.py +++ b/tests/test_ephys.py @@ -41,6 +41,8 @@ def setUpClass(cls): "rig_id": "323_EPHYS2-RF_2024-01-18_01", "animal_weight_prior": None, "animal_weight_post": None, + "mouse_platform_name": "Running Wheel", + "active_mouse_platform": False, "calibrations": [], "maintenance": [], "camera_names": [], @@ -123,8 +125,6 @@ def setUpClass(cls): } ], }, - "mouse_platform_name": "Running Wheel", - "active_mouse_platform": False, "notes": "699889_2024-01-18_12-12-04", }, { @@ -168,8 +168,6 @@ def setUpClass(cls): } ], }, - "mouse_platform_name": "Running Wheel", - "active_mouse_platform": False, "notes": "699889_2024-01-18_12-24-55; Surface Finding", }, ], diff --git a/tests/test_fib.py b/tests/test_fib.py index 14a1efb5..06638af9 100644 --- a/tests/test_fib.py +++ b/tests/test_fib.py @@ -26,8 +26,8 @@ def setUpClass(cls): with open(EXAMPLE_MD_PATH, "r") as f: raw_md_contents = f.read() - with open(EXPECTED_SESSION, "r") as f: - expected_session_contents = Session(**json.load(f)) + # with open(EXPECTED_SESSION, "r") as f: + # expected_session_contents = Session(**json.load(f)) cls.example_job_settings = JobSettings( string_to_parse=raw_md_contents, @@ -74,7 +74,7 @@ def setUpClass(cls): active_mouse_platform=False, ) - cls.expected_session = expected_session_contents + # cls.expected_session = expected_session_contents def test_constructor_from_string(self) -> None: """Tests that the settings can be constructed from a json string""" @@ -106,7 +106,8 @@ def test_transform(self): etl_job1 = FIBEtl(job_settings=self.example_job_settings) parsed_info = etl_job1._extract() actual_session = etl_job1._transform(parsed_info) - self.assertEqual(self.expected_session, actual_session) + print(actual_session) + # self.assertEqual(self.expected_session, actual_session) def test_run_job(self): """Tests that the teensy response maps correctly to ophys session.""" From 806aed8db3d3bd49e34c0e580c74efe5b1ca4348 Mon Sep 17 00:00:00 2001 From: jtyoung84 <104453205+jtyoung84@users.noreply.github.com> Date: Fri, 5 Apr 2024 09:27:40 -0700 Subject: [PATCH 2/8] feat: updates bergamo session --- src/aind_metadata_mapper/bergamo/session.py | 106 ++++++++++++++++++-- 1 file changed, 98 insertions(+), 8 deletions(-) diff --git a/src/aind_metadata_mapper/bergamo/session.py b/src/aind_metadata_mapper/bergamo/session.py index 5ca80e07..1914875a 100644 --- a/src/aind_metadata_mapper/bergamo/session.py +++ b/src/aind_metadata_mapper/bergamo/session.py @@ -18,13 +18,14 @@ LaserConfig, Modality, Session, + StimulusEpoch, + StimulusModality, Stream, ) from aind_data_schema.models.stimulus import ( PhotoStimulation, PhotoStimulationGroup, ) -from aind_data_schema.core.session import StimulusEpoch from aind_data_schema.models.units import PowerUnit, SizeUnit from pydantic import Field from pydantic_settings import BaseSettings @@ -449,6 +450,7 @@ def _transform(self, extracted_source: RawImageInfo) -> Session: ), ], ) + stimulus_name = "PhotoStimulation" return Session( experimenter_full_name=self.job_settings.experimenter_full_name, session_start_time=self.job_settings.session_start_time, @@ -457,17 +459,105 @@ def _transform(self, extracted_source: RawImageInfo) -> Session: session_type=self.job_settings.session_type, iacuc_protocol=self.job_settings.iacuc_protocol, rig_id=self.job_settings.rig_id, - mouse_platform_name=self.job_settings.mouse_platform_name, - active_mouse_platform=self.job_settings.active_mouse_platform, data_streams=[data_stream], stimulus_epochs=[ StimulusEpoch( - stimulus_start_time=datetime(), - stimulus_end_time=datetime(), - stimulus_name="", - stimulus_modalities=List[StimulusModality.] + stimulus_name=stimulus_name, + stimulus_modalities=[ + StimulusModality.OPTOGENETICS, + ], + stimulus_parameters=[ + PhotoStimulation( + stimulus_name="PhotoStimulation", + number_groups=( + self.job_settings.num_of_photo_stim_groups + ), + groups=[ + PhotoStimulationGroup( + group_index=( + self.job_settings.photo_stim_groups[0][ + "group_index" + ] + ), + number_of_neurons=int( + np.array( + photostim_groups[0]["rois"][1][ + "scanfields" + ]["slmPattern"] + ).shape[0] + ), + stimulation_laser_power=int( + photostim_groups[0]["rois"][1][ + "scanfields" + ]["powers"] + ), + number_trials=( + self.job_settings.photo_stim_groups[0][ + "number_trials" + ] + ), + number_spirals=int( + photostim_groups[0]["rois"][1][ + "scanfields" + ]["repetitions"] + ), + spiral_duration=photostim_groups[0][ + "rois" + ][1]["scanfields"]["duration"], + inter_spiral_interval=photostim_groups[0][ + "rois" + ][2]["scanfields"]["duration"], + ), + PhotoStimulationGroup( + group_index=( + self.job_settings.photo_stim_groups[1][ + "group_index" + ] + ), + number_of_neurons=int( + np.array( + photostim_groups[0]["rois"][1][ + "scanfields" + ]["slmPattern"] + ).shape[0] + ), + stimulation_laser_power=int( + photostim_groups[0]["rois"][1][ + "scanfields" + ]["powers"] + ), + number_trials=( + self.job_settings.photo_stim_groups[1][ + "number_trials" + ] + ), + number_spirals=int( + photostim_groups[0]["rois"][1][ + "scanfields" + ]["repetitions"] + ), + spiral_duration=photostim_groups[0][ + "rois" + ][1]["scanfields"]["duration"], + inter_spiral_interval=photostim_groups[0][ + "rois" + ][2]["scanfields"]["duration"], + ), + ], + inter_trial_interval=( + self.job_settings. + photo_stim_inter_trial_interval + ), + ) + ], + stimulus_start_time=( + self.job_settings.stimulus_start_time + ), + stimulus_end_time=self.job_settings.stimulus_end_time, ) - ] + ], + mouse_platform_name=self.job_settings.mouse_platform_name, + active_mouse_platform=self.job_settings.active_mouse_platform, ) def run_job(self) -> JobResponse: From 5d263a9d4b4f2d2dfe4c5143540011986f662a64 Mon Sep 17 00:00:00 2001 From: jtyoung84 <104453205+jtyoung84@users.noreply.github.com> Date: Fri, 5 Apr 2024 09:31:33 -0700 Subject: [PATCH 3/8] feat: updates bergamo session test json --- tests/resources/bergamo/expected_session.json | 244 ++++++++++-------- 1 file changed, 134 insertions(+), 110 deletions(-) diff --git a/tests/resources/bergamo/expected_session.json b/tests/resources/bergamo/expected_session.json index 0102d318..fa1e78f0 100644 --- a/tests/resources/bergamo/expected_session.json +++ b/tests/resources/bergamo/expected_session.json @@ -1,13 +1,13 @@ { "describedBy": "https://raw.githubusercontent.com/AllenNeuralDynamics/aind-data-schema/main/src/aind_data_schema/core/session.py", - "schema_version": "0.1.4", + "schema_version": "0.2.1", "protocol_id": [], "experimenter_full_name": [ - "John Smith", - "Jane Smith" + "John Smith", + "Jane Smith" ], - "session_start_time": "2023-10-10T14:00:00", - "session_end_time": "2023-10-10T17:00:00", + "session_start_time": "2023-10-10T14:00:00Z", + "session_end_time": "2023-10-10T17:00:00Z", "session_type": "BCI", "iacuc_protocol": "2115", "rig_id": "Bergamo photostim.", @@ -18,116 +18,140 @@ "animal_weight_post": null, "weight_unit": "gram", "data_streams": [ - { - "stream_start_time": "2023-10-10T15:00:00", - "stream_end_time": "2023-10-10T16:00:00", - "daq_names": [], - "camera_names": [ - "Side Camera" - ], - "light_sources": [ - { - "device_type": "Laser", - "name": "Laser A", - "wavelength": 920, - "wavelength_unit": "nanometer", - "excitation_power": "15", - "excitation_power_unit": "percent" - } - ], - "ephys_modules": [], - "stick_microscopes": [], - "manipulator_modules": [], - "detectors": [ - { - "name": "PMT A", - "exposure_time": "0.1", - "exposure_time_unit": "millisecond", - "trigger_type": "Internal" - } - ], - "fiber_connections": [], - "fiber_modules": [], - "ophys_fovs": [ - { - "index": 0, - "imaging_depth": 150, - "imaging_depth_unit": "micrometer", - "targeted_structure": "M1", - "fov_coordinate_ml": "1.5", - "fov_coordinate_ap": "1.5", - "fov_coordinate_unit": "micrometer", - "fov_reference": "Bregma", - "fov_width": 512, - "fov_height": 512, - "fov_size_unit": "pixel", - "magnification": "16x", - "fov_scale_factor": "1.2", - "fov_scale_factor_unit": "um/pixel", - "frame_rate": "30.0119", - "frame_rate_unit": "hertz", - "coupled_fov_index": null - } - ], - "slap_fovs": null, - "stack_parameters": null, - "stimulus_device_names": [], - "mouse_platform_name": "some_mouse_platform_name", - "active_mouse_platform": true, - "stream_modalities": [ - { - "name": "Planar optical physiology", - "abbreviation": "ophys" - } - ], - "notes": null - } + { + "stream_start_time": "2023-10-10T15:00:00Z", + "stream_end_time": "2023-10-10T16:00:00Z", + "daq_names": [], + "camera_names": [ + "Side Camera" + ], + "light_sources": [ + { + "device_type": "Laser", + "name": "Laser A", + "wavelength": 920, + "wavelength_unit": "nanometer", + "excitation_power": "15", + "excitation_power_unit": "percent" + } + ], + "ephys_modules": [], + "stick_microscopes": [], + "manipulator_modules": [], + "detectors": [ + { + "name": "PMT A", + "exposure_time": "0.1", + "exposure_time_unit": "millisecond", + "trigger_type": "Internal" + } + ], + "fiber_connections": [], + "fiber_modules": [], + "ophys_fovs": [ + { + "index": 0, + "imaging_depth": 150, + "imaging_depth_unit": "micrometer", + "targeted_structure": "M1", + "fov_coordinate_ml": "1.5", + "fov_coordinate_ap": "1.5", + "fov_coordinate_unit": "micrometer", + "fov_reference": "Bregma", + "fov_width": 512, + "fov_height": 512, + "fov_size_unit": "pixel", + "magnification": "16x", + "fov_scale_factor": "1.2", + "fov_scale_factor_unit": "um/pixel", + "frame_rate": "30.0119", + "frame_rate_unit": "hertz", + "coupled_fov_index": null, + "power": null, + "power_unit": "percent", + "scanfield_z": null, + "scanfield_z_unit": "micrometer", + "scanimage_roi_index": null, + "notes": null + } + ], + "slap_fovs": null, + "stack_parameters": null, + "stream_modalities": [ + { + "name": "Planar optical physiology", + "abbreviation": "ophys" + } + ], + "notes": null + } ], "stimulus_epochs": [ - { - "stimulus": { - "stimulus_type": "Photo Stimulation", - "stimulus_name": "PhotoStimulation", - "number_groups": 2, - "groups": [ - { - "group_index": 0, - "number_of_neurons": 10, - "stimulation_laser_power": "20", - "stimulation_laser_power_unit": "milliwatt", - "number_trials": 5, - "number_spirals": 10, - "spiral_duration": "0.01", - "spiral_duration_unit": "second", - "inter_spiral_interval": "0.001", - "inter_spiral_interval_unit": "second", - "other_parameters": {}, - "notes": null - }, + { + "stimulus_start_time": "2023-10-10T15:15:00Z", + "stimulus_end_time": "2023-10-10T15:45:00Z", + "stimulus_name": "PhotoStimulation", + "session_number": null, + "software": [], + "script": null, + "stimulus_modalities": [ + "Optogenetics" + ], + "stimulus_parameters": [ { - "group_index": 0, - "number_of_neurons": 10, - "stimulation_laser_power": "20", - "stimulation_laser_power_unit": "milliwatt", - "number_trials": 5, - "number_spirals": 10, - "spiral_duration": "0.01", - "spiral_duration_unit": "second", - "inter_spiral_interval": "0.001", - "inter_spiral_interval_unit": "second", - "other_parameters": {}, - "notes": null + "stimulus_type": "Photo Stimulation", + "stimulus_name": "PhotoStimulation", + "number_groups": 2, + "groups": [ + { + "group_index": 0, + "number_of_neurons": 10, + "stimulation_laser_power": "20", + "stimulation_laser_power_unit": "milliwatt", + "number_trials": 5, + "number_spirals": 10, + "spiral_duration": "0.01", + "spiral_duration_unit": "second", + "inter_spiral_interval": "0.001", + "inter_spiral_interval_unit": "second", + "other_parameters": {}, + "notes": null + }, + { + "group_index": 0, + "number_of_neurons": 10, + "stimulation_laser_power": "20", + "stimulation_laser_power_unit": "milliwatt", + "number_trials": 5, + "number_spirals": 10, + "spiral_duration": "0.01", + "spiral_duration_unit": "second", + "inter_spiral_interval": "0.001", + "inter_spiral_interval_unit": "second", + "other_parameters": {}, + "notes": null + } + ], + "inter_trial_interval": "10", + "inter_trial_interval_unit": "second", + "other_parameters": {}, + "notes": null } - ], - "inter_trial_interval": "10", - "inter_trial_interval_unit": "second", - "other_parameters": {}, - "notes": null - }, - "stimulus_start_time": "2023-10-10T15:15:00", - "stimulus_end_time": "2023-10-10T15:45:00" - } + ], + "stimulus_device_names": [], + "speaker_config": null, + "light_source_config": null, + "output_parameters": {}, + "reward_consumed_during_epoch": null, + "reward_consumed_unit": "microliter", + "trials_total": null, + "trials_finished": null, + "trials_rewarded": null, + "notes": null + } ], + "mouse_platform_name": "some_mouse_platform_name", + "active_mouse_platform": true, "reward_delivery": null, "reward_consumed_total": null, "reward_consumed_unit": "microliter", From a3f5fa46d8fc7376aee289fb078af2f2e1eefb6d Mon Sep 17 00:00:00 2001 From: jtyoung84 <104453205+jtyoung84@users.noreply.github.com> Date: Fri, 5 Apr 2024 09:35:17 -0700 Subject: [PATCH 4/8] feat: updates bergamo tests --- tests/test_bergamo.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/test_bergamo.py b/tests/test_bergamo.py index c4667936..631e5399 100644 --- a/tests/test_bergamo.py +++ b/tests/test_bergamo.py @@ -4,7 +4,7 @@ import json import os import unittest -from datetime import datetime +from datetime import datetime, timezone from pathlib import Path from unittest.mock import MagicMock, call, patch @@ -46,12 +46,12 @@ def setUpClass(cls): active_mouse_platform=True, experimenter_full_name=["John Smith", "Jane Smith"], subject_id="12345", - session_start_time=datetime(2023, 10, 10, 14, 0, 0), - session_end_time=datetime(2023, 10, 10, 17, 0, 0), - stream_start_time=datetime(2023, 10, 10, 15, 0, 0), - stream_end_time=datetime(2023, 10, 10, 16, 0, 0), - stimulus_start_time=datetime(2023, 10, 10, 15, 15, 0), - stimulus_end_time=datetime(2023, 10, 10, 15, 45, 0), + session_start_time=datetime(2023, 10, 10, 14, 0, 0, tzinfo=timezone.utc), + session_end_time=datetime(2023, 10, 10, 17, 0, 0, tzinfo=timezone.utc), + stream_start_time=datetime(2023, 10, 10, 15, 0, 0, tzinfo=timezone.utc), + stream_end_time=datetime(2023, 10, 10, 16, 0, 0, tzinfo=timezone.utc), + stimulus_start_time=datetime(2023, 10, 10, 15, 15, 0, tzinfo=timezone.utc), + stimulus_end_time=datetime(2023, 10, 10, 15, 45, 0, tzinfo=timezone.utc), ) cls.expected_session = expected_session_contents From 88ec972addb4aca4280cfaf81402f0165867e130 Mon Sep 17 00:00:00 2001 From: jtyoung84 <104453205+jtyoung84@users.noreply.github.com> Date: Fri, 5 Apr 2024 09:38:23 -0700 Subject: [PATCH 5/8] feat: updates ephys session --- src/aind_metadata_mapper/ephys/session.py | 11 +- tests/resources/ephys/ephys_session.json | 535 +++++++++++----------- tests/test_ephys.py | 23 +- 3 files changed, 296 insertions(+), 273 deletions(-) diff --git a/src/aind_metadata_mapper/ephys/session.py b/src/aind_metadata_mapper/ephys/session.py index aab32d2e..8bdc38c1 100644 --- a/src/aind_metadata_mapper/ephys/session.py +++ b/src/aind_metadata_mapper/ephys/session.py @@ -89,8 +89,6 @@ def _transform(self, extracted_source: ParsedInformation) -> Session: ] ephys_session["maintenance"] = experiment_data["maintenance"] ephys_session["calibrations"] = experiment_data["calibrations"] - ephys_session["mouse_platform_name"] = "Running Wheel" - ephys_session["active_mouse_platform"] = False # Constant throughout data streams stick_microscopes = experiment_data["stick_microscopes"] @@ -103,7 +101,7 @@ def _transform(self, extracted_source: ParsedInformation) -> Session: ): session_stream = {} session_stream["stream_start_time"] = datetime.strptime( - stage[0][0], "%Y/%m/%d %H:%M:%S.%f" + stage[0][0], "%Y/%m/%d %H:%M:%S.%f" ) session_stream["stream_end_time"] = datetime.strptime( stage[-1][0], "%Y/%m/%d %H:%M:%S.%f" @@ -130,6 +128,13 @@ def _transform(self, extracted_source: ParsedInformation) -> Session: ephys_session["data_streams"].append(session_stream) + ephys_session["mouse_platform_name"] = data_stream[ + "mouse_platform_name" + ] + ephys_session["active_mouse_platform"] = data_stream[ + "active_mouse_platform" + ] + end_times = [ datetime.strptime(x[-1][0], "%Y/%m/%d %H:%M:%S.%f") for x in stage_logs diff --git a/tests/resources/ephys/ephys_session.json b/tests/resources/ephys/ephys_session.json index 58ccba3f..b30c9a3c 100644 --- a/tests/resources/ephys/ephys_session.json +++ b/tests/resources/ephys/ephys_session.json @@ -1,11 +1,12 @@ { "describedBy": "https://raw.githubusercontent.com/AllenNeuralDynamics/aind-data-schema/main/src/aind_data_schema/core/session.py", "schema_version": "0.2.1", + "protocol_id": [], "experimenter_full_name": [ - "Al Dente" + "Al Dente" ], - "session_start_time": "2023-04-04T10:37:28", - "session_end_time": "2023-04-06T13:47:57.558000", + "session_start_time": "2023-04-04T10:37:28Z", + "session_end_time": "2023-04-06T13:47:57.558000Z", "session_type": "Receptive field mapping", "iacuc_protocol": "2109", "rig_id": "323_EPHYS2-RF_2024-01-18_01", @@ -16,276 +17,276 @@ "animal_weight_post": null, "weight_unit": "gram", "data_streams": [ - { - "stream_start_time": "2023-04-04T10:37:28.815000", - "stream_end_time": "2023-04-06T11:47:57.558000", - "daq_names": [ - "Basestation" - ], - "camera_names": [], - "light_sources": [], - "ephys_modules": [ - { - "assembly_name": "46121", - "arc_angle": "5.3", - "module_angle": "-27.1", - "angle_unit": "degrees", - "rotation_angle": null, - "coordinate_transform": "behavior/calibration_info_np2_2024_01_17T15_04_00.npy", - "calibration_date": "2024-01-17T15:04:00Z", - "notes": "Easy insertion. Recorded 8 minutes, serially, so separate from prior insertion.", - "primary_targeted_structure": "AntComMid", - "targeted_ccf_coordinates": [ - { - "ml": "5700.0", - "ap": "5160.0", - "dv": "5260.0", - "unit": "micrometer", - "ccf_version": "CCFv3" - } - ], - "manipulator_coordinates": { - "x": "7520.0", - "y": "7505.0", - "z": "7493.0", - "unit": "micrometer" + { + "stream_start_time": "2023-04-04T10:37:28.815000Z", + "stream_end_time": "2023-04-06T11:47:57.558000Z", + "daq_names": [ + "Basestation" + ], + "camera_names": [], + "light_sources": [], + "ephys_modules": [ + { + "assembly_name": "46121", + "arc_angle": "5.3", + "module_angle": "-27.1", + "angle_unit": "degrees", + "rotation_angle": null, + "coordinate_transform": "behavior/calibration_info_np2_2024_01_17T15_04_00.npy", + "calibration_date": "2024-01-17T15:04:00Z", + "notes": "Easy insertion. Recorded 8 minutes, serially, so separate from prior insertion.", + "primary_targeted_structure": "AntComMid", + "targeted_ccf_coordinates": [ + { + "ml": "5700.0", + "ap": "5160.0", + "dv": "5260.0", + "unit": "micrometer", + "ccf_version": "CCFv3" + } + ], + "manipulator_coordinates": { + "x": "7520.0", + "y": "7505.0", + "z": "7493.0", + "unit": "micrometer" + }, + "implant_hole_number": null, + "ephys_probes": [ + { + "name": "46121", + "other_targeted_structures": [] + } + ] }, - "ephys_probes": [ - { - "name": "46121", - "other_targeted_structures": [] - } - ] - }, - { - "assembly_name": "46118", - "arc_angle": "14", - "module_angle": "20", - "angle_unit": "degrees", - "rotation_angle": null, - "coordinate_transform": "behavior/calibration_info_np2_2024_01_17T15_04_00.npy", - "calibration_date": "2024-01-17T15:04:00Z", - "notes": "Easy insertion. Recorded 8 minutes, serially, so separate from prior insertion.", - "primary_targeted_structure": "VISp", - "targeted_ccf_coordinates": [ - { - "ml": "5700.0", - "ap": "5160.0", - "dv": "5260.0", - "unit": "micrometer", - "ccf_version": "CCFv3" - } - ], - "manipulator_coordinates": { - "x": "7500.0", - "y": "7499.5", - "z": "7600.0", - "unit": "micrometer" + { + "assembly_name": "46118", + "arc_angle": "14", + "module_angle": "20", + "angle_unit": "degrees", + "rotation_angle": null, + "coordinate_transform": "behavior/calibration_info_np2_2024_01_17T15_04_00.npy", + "calibration_date": "2024-01-17T15:04:00Z", + "notes": "Easy insertion. Recorded 8 minutes, serially, so separate from prior insertion.", + "primary_targeted_structure": "VISp", + "targeted_ccf_coordinates": [ + { + "ml": "5700.0", + "ap": "5160.0", + "dv": "5260.0", + "unit": "micrometer", + "ccf_version": "CCFv3" + } + ], + "manipulator_coordinates": { + "x": "7500.0", + "y": "7499.5", + "z": "7600.0", + "unit": "micrometer" + }, + "implant_hole_number": null, + "ephys_probes": [ + { + "name": "46118", + "other_targeted_structures": [] + } + ] + } + ], + "stick_microscopes": [ + { + "assembly_name": "20516338", + "arc_angle": "-180.0", + "module_angle": "-180.0", + "angle_unit": "degrees", + "rotation_angle": null, + "coordinate_transform": null, + "calibration_date": null, + "notes": "Did not record arc or module angles, did not calibrate" }, - "ephys_probes": [ - { - "name": "46118", - "other_targeted_structures": [] - } - ] - } - ], - "stick_microscopes": [ - { - "assembly_name": "20516338", - "arc_angle": "-180.0", - "module_angle": "-180.0", - "angle_unit": "degrees", - "rotation_angle": null, - "coordinate_transform": null, - "calibration_date": null, - "notes": "Did not record arc or module angles, did not calibrate" - }, - { - "assembly_name": "22437106", - "arc_angle": "-180.0", - "module_angle": "-180.0", - "angle_unit": "degrees", - "rotation_angle": null, - "coordinate_transform": null, - "calibration_date": null, - "notes": "Did not record arc or module angles, did not calibrate" - }, - { - "assembly_name": "22437107", - "arc_angle": "-180.0", - "module_angle": "-180.0", - "angle_unit": "degrees", - "rotation_angle": null, - "coordinate_transform": null, - "calibration_date": null, - "notes": "Did not record arc or module angles, did not calibrate" - }, - { - "assembly_name": "22438379", - "arc_angle": "-180.0", - "module_angle": "-180.0", - "angle_unit": "degrees", - "rotation_angle": null, - "coordinate_transform": null, - "calibration_date": null, - "notes": "Did not record arc or module angles, did not calibrate" - } - ], - "manipulator_modules": [], - "detectors": [], - "fiber_connections": [], - "fiber_modules": [], - "ophys_fovs": [], - "slap_fovs": null, - "stack_parameters": null, - "stimulus_device_names": [], - "mouse_platform_name": "Running Wheel", - "active_mouse_platform": false, - "stream_modalities": [ - { - "name": "Extracellular electrophysiology", - "abbreviation": "ecephys" - } - ], - "notes": null - }, - { - "stream_start_time": "2023-04-04T12:37:28.815000", - "stream_end_time": "2023-04-06T13:47:57.558000", - "daq_names": [ - "Basestation" - ], - "camera_names": [], - "light_sources": [], - "ephys_modules": [ - { - "assembly_name": "46121", - "arc_angle": "5.3", - "module_angle": "-27.1", - "angle_unit": "degrees", - "rotation_angle": null, - "coordinate_transform": "behavior/calibration_info_np2_2024_01_17T15_04_00.npy", - "calibration_date": "2024-01-17T15:04:00Z", - "notes": "Easy insertion. Recorded 8 minutes, serially, so separate from prior insertion.", - "primary_targeted_structure": "AntComMid", - "targeted_ccf_coordinates": [ - { - "ml": "5700.0", - "ap": "5160.0", - "dv": "5260.0", - "unit": "micrometer", - "ccf_version": "CCFv3" - } - ], - "manipulator_coordinates": { - "x": "7520.0", - "y": "7505.0", - "z": "7493.0", - "unit": "micrometer" + { + "assembly_name": "22437106", + "arc_angle": "-180.0", + "module_angle": "-180.0", + "angle_unit": "degrees", + "rotation_angle": null, + "coordinate_transform": null, + "calibration_date": null, + "notes": "Did not record arc or module angles, did not calibrate" }, - "ephys_probes": [ - { - "name": "46121", - "other_targeted_structures": [] - } - ] - }, - { - "assembly_name": "46118", - "arc_angle": "14", - "module_angle": "20", - "angle_unit": "degrees", - "rotation_angle": null, - "coordinate_transform": "behavior/calibration_info_np2_2024_01_17T15_04_00.npy", - "calibration_date": "2024-01-17T15:04:00Z", - "notes": "Easy insertion. Recorded 8 minutes, serially, so separate from prior insertion.", - "primary_targeted_structure": "VISp", - "targeted_ccf_coordinates": [ - { - "ml": "5700.0", - "ap": "5160.0", - "dv": "5260.0", - "unit": "micrometer", - "ccf_version": "CCFv3" - } - ], - "manipulator_coordinates": { - "x": "7500.0", - "y": "7499.5", - "z": "7600.0", - "unit": "micrometer" + { + "assembly_name": "22437107", + "arc_angle": "-180.0", + "module_angle": "-180.0", + "angle_unit": "degrees", + "rotation_angle": null, + "coordinate_transform": null, + "calibration_date": null, + "notes": "Did not record arc or module angles, did not calibrate" }, - "ephys_probes": [ - { - "name": "46118", - "other_targeted_structures": [] - } - ] - } - ], - "stick_microscopes": [ - { - "assembly_name": "20516338", - "arc_angle": "-180.0", - "module_angle": "-180.0", - "angle_unit": "degrees", - "rotation_angle": null, - "coordinate_transform": null, - "calibration_date": null, - "notes": "Did not record arc or module angles, did not calibrate" - }, - { - "assembly_name": "22437106", - "arc_angle": "-180.0", - "module_angle": "-180.0", - "angle_unit": "degrees", - "rotation_angle": null, - "coordinate_transform": null, - "calibration_date": null, - "notes": "Did not record arc or module angles, did not calibrate" - }, - { - "assembly_name": "22437107", - "arc_angle": "-180.0", - "module_angle": "-180.0", - "angle_unit": "degrees", - "rotation_angle": null, - "coordinate_transform": null, - "calibration_date": null, - "notes": "Did not record arc or module angles, did not calibrate" - }, - { - "assembly_name": "22438379", - "arc_angle": "-180.0", - "module_angle": "-180.0", - "angle_unit": "degrees", - "rotation_angle": null, - "coordinate_transform": null, - "calibration_date": null, - "notes": "Did not record arc or module angles, did not calibrate" - } - ], - "manipulator_modules": [], - "detectors": [], - "fiber_connections": [], - "fiber_modules": [], - "ophys_fovs": [], - "slap_fovs": null, - "stack_parameters": null, - "stimulus_device_names": [], - "mouse_platform_name": "Running Wheel", - "active_mouse_platform": false, - "stream_modalities": [ - { - "name": "Extracellular electrophysiology", - "abbreviation": "ecephys" - } - ], - "notes": null - } + { + "assembly_name": "22438379", + "arc_angle": "-180.0", + "module_angle": "-180.0", + "angle_unit": "degrees", + "rotation_angle": null, + "coordinate_transform": null, + "calibration_date": null, + "notes": "Did not record arc or module angles, did not calibrate" + } + ], + "manipulator_modules": [], + "detectors": [], + "fiber_connections": [], + "fiber_modules": [], + "ophys_fovs": [], + "slap_fovs": null, + "stack_parameters": null, + "stream_modalities": [ + { + "name": "Extracellular electrophysiology", + "abbreviation": "ecephys" + } + ], + "notes": null + }, + { + "stream_start_time": "2023-04-04T12:37:28.815000Z", + "stream_end_time": "2023-04-06T13:47:57.558000Z", + "daq_names": [ + "Basestation" + ], + "camera_names": [], + "light_sources": [], + "ephys_modules": [ + { + "assembly_name": "46121", + "arc_angle": "5.3", + "module_angle": "-27.1", + "angle_unit": "degrees", + "rotation_angle": null, + "coordinate_transform": "behavior/calibration_info_np2_2024_01_17T15_04_00.npy", + "calibration_date": "2024-01-17T15:04:00Z", + "notes": "Easy insertion. Recorded 8 minutes, serially, so separate from prior insertion.", + "primary_targeted_structure": "AntComMid", + "targeted_ccf_coordinates": [ + { + "ml": "5700.0", + "ap": "5160.0", + "dv": "5260.0", + "unit": "micrometer", + "ccf_version": "CCFv3" + } + ], + "manipulator_coordinates": { + "x": "7520.0", + "y": "7505.0", + "z": "7493.0", + "unit": "micrometer" + }, + "implant_hole_number": null, + "ephys_probes": [ + { + "name": "46121", + "other_targeted_structures": [] + } + ] + }, + { + "assembly_name": "46118", + "arc_angle": "14", + "module_angle": "20", + "angle_unit": "degrees", + "rotation_angle": null, + "coordinate_transform": "behavior/calibration_info_np2_2024_01_17T15_04_00.npy", + "calibration_date": "2024-01-17T15:04:00Z", + "notes": "Easy insertion. Recorded 8 minutes, serially, so separate from prior insertion.", + "primary_targeted_structure": "VISp", + "targeted_ccf_coordinates": [ + { + "ml": "5700.0", + "ap": "5160.0", + "dv": "5260.0", + "unit": "micrometer", + "ccf_version": "CCFv3" + } + ], + "manipulator_coordinates": { + "x": "7500.0", + "y": "7499.5", + "z": "7600.0", + "unit": "micrometer" + }, + "implant_hole_number": null, + "ephys_probes": [ + { + "name": "46118", + "other_targeted_structures": [] + } + ] + } + ], + "stick_microscopes": [ + { + "assembly_name": "20516338", + "arc_angle": "-180.0", + "module_angle": "-180.0", + "angle_unit": "degrees", + "rotation_angle": null, + "coordinate_transform": null, + "calibration_date": null, + "notes": "Did not record arc or module angles, did not calibrate" + }, + { + "assembly_name": "22437106", + "arc_angle": "-180.0", + "module_angle": "-180.0", + "angle_unit": "degrees", + "rotation_angle": null, + "coordinate_transform": null, + "calibration_date": null, + "notes": "Did not record arc or module angles, did not calibrate" + }, + { + "assembly_name": "22437107", + "arc_angle": "-180.0", + "module_angle": "-180.0", + "angle_unit": "degrees", + "rotation_angle": null, + "coordinate_transform": null, + "calibration_date": null, + "notes": "Did not record arc or module angles, did not calibrate" + }, + { + "assembly_name": "22438379", + "arc_angle": "-180.0", + "module_angle": "-180.0", + "angle_unit": "degrees", + "rotation_angle": null, + "coordinate_transform": null, + "calibration_date": null, + "notes": "Did not record arc or module angles, did not calibrate" + } + ], + "manipulator_modules": [], + "detectors": [], + "fiber_connections": [], + "fiber_modules": [], + "ophys_fovs": [], + "slap_fovs": null, + "stack_parameters": null, + "stream_modalities": [ + { + "name": "Extracellular electrophysiology", + "abbreviation": "ecephys" + } + ], + "notes": null + } ], "stimulus_epochs": [], + "mouse_platform_name": "Running Wheel", + "active_mouse_platform": false, "reward_delivery": null, "reward_consumed_total": null, "reward_consumed_unit": "microliter", diff --git a/tests/test_ephys.py b/tests/test_ephys.py index 4f150e13..24b1656b 100644 --- a/tests/test_ephys.py +++ b/tests/test_ephys.py @@ -6,6 +6,7 @@ import unittest from pathlib import Path from xml.dom import minidom +import zoneinfo from aind_data_schema.core.session import Session @@ -29,6 +30,8 @@ class TestSchemaWriter(unittest.TestCase): """Test methods in SchemaWriter class.""" + maxDiff = None # show full diff without truncation + @classmethod def setUpClass(cls): """Load record object and user settings before running tests.""" @@ -41,8 +44,6 @@ def setUpClass(cls): "rig_id": "323_EPHYS2-RF_2024-01-18_01", "animal_weight_prior": None, "animal_weight_post": None, - "mouse_platform_name": "Running Wheel", - "active_mouse_platform": False, "calibrations": [], "maintenance": [], "camera_names": [], @@ -125,6 +126,8 @@ def setUpClass(cls): } ], }, + "mouse_platform_name": "Running Wheel", + "active_mouse_platform": False, "notes": "699889_2024-01-18_12-12-04", }, { @@ -168,6 +171,8 @@ def setUpClass(cls): } ], }, + "mouse_platform_name": "Running Wheel", + "active_mouse_platform": False, "notes": "699889_2024-01-18_12-24-55; Surface Finding", }, ], @@ -216,7 +221,19 @@ def test_transform(self): ) parsed_info = etl_job1._extract() actual_session = etl_job1._transform(parsed_info) - self.assertEqual(self.expected_session, actual_session) + actual_session.session_start_time = actual_session.session_start_time \ + .replace(tzinfo=zoneinfo.ZoneInfo("UTC")) + actual_session.session_end_time = actual_session.session_end_time \ + .replace(tzinfo=zoneinfo.ZoneInfo("UTC")) + for stream in actual_session.data_streams: + stream.stream_start_time = stream.stream_start_time.replace( + tzinfo=zoneinfo.ZoneInfo("UTC")) + stream.stream_end_time = stream.stream_end_time.replace( + tzinfo=zoneinfo.ZoneInfo("UTC")) + self.assertEqual( + self.expected_session.model_dump(), + actual_session.model_dump(), + ) if __name__ == "__main__": From d4ced3c2621247114a3e254f1624b3ec5540b49d Mon Sep 17 00:00:00 2001 From: jtyoung84 <104453205+jtyoung84@users.noreply.github.com> Date: Fri, 5 Apr 2024 09:41:27 -0700 Subject: [PATCH 6/8] feat: updates fip session --- src/aind_metadata_mapper/fib/session.py | 34 +-- tests/resources/fib/000000_ophys_session.json | 196 ++++++++++-------- tests/test_fib.py | 18 +- 3 files changed, 143 insertions(+), 105 deletions(-) diff --git a/src/aind_metadata_mapper/fib/session.py b/src/aind_metadata_mapper/fib/session.py index 44f1a3e2..c413af1d 100644 --- a/src/aind_metadata_mapper/fib/session.py +++ b/src/aind_metadata_mapper/fib/session.py @@ -12,13 +12,11 @@ LightEmittingDiodeConfig, Session, StimulusEpoch, - Stream, StimulusModality, + StimulusModality, + Stream, ) from aind_data_schema.models.modalities import Modality -from aind_data_schema.models.stimulus import ( - OptoStimulation, - PulseShape, -) +from aind_data_schema.models.stimulus import OptoStimulation, PulseShape from pydantic import Field from pydantic_settings import BaseSettings @@ -148,10 +146,18 @@ def _transform(self, extracted_source: ParsedMetadata) -> Session: opto_stim = OptoStimulation( stimulus_name=stimulus_name, pulse_shape=PulseShape.SQUARE, - pulse_frequency=[frequency], - number_pulse_trains=[trial_num], - pulse_width=[pulse_width], - pulse_train_duration=[opto_duration], + pulse_frequency=[ + frequency, + ], + number_pulse_trains=[ + trial_num, + ], + pulse_width=[ + pulse_width, + ], + pulse_train_duration=[ + opto_duration, + ], pulse_train_interval=opto_interval, baseline_duration=opto_base, fixed_pulse_train_interval=True, # TODO: Check this is right @@ -166,10 +172,12 @@ def _transform(self, extracted_source: ParsedMetadata) -> Session: ) stimulus_epochs = StimulusEpoch( stimulus_name=stimulus_name, + stimulus_modalities=[StimulusModality.OPTOGENETICS], + stimulus_parameters=[ + opto_stim, + ], stimulus_start_time=session_start_time, stimulus_end_time=end_datetime, - stimulus_modalities=[StimulusModality.NONE], - stimulus_parameters=[opto_stim] ) # create light source instance @@ -195,8 +203,6 @@ def _transform(self, extracted_source: ParsedMetadata) -> Session: stream_end_time=end_datetime, light_sources=light_source, stream_modalities=[Modality.FIB], - mouse_platform_name=mouse_platform_name, - active_mouse_platform=active_mouse_platform, detectors=detectors, fiber_connections=fiber_connections, ) @@ -213,6 +219,8 @@ def _transform(self, extracted_source: ParsedMetadata) -> Session: session_type=session_type, notes=notes, data_streams=data_stream, + mouse_platform_name=mouse_platform_name, + active_mouse_platform=active_mouse_platform, ) return ophys_session diff --git a/tests/resources/fib/000000_ophys_session.json b/tests/resources/fib/000000_ophys_session.json index c6375853..d635667c 100644 --- a/tests/resources/fib/000000_ophys_session.json +++ b/tests/resources/fib/000000_ophys_session.json @@ -3,10 +3,10 @@ "schema_version": "0.2.1", "protocol_id": [], "experimenter_full_name": [ - "Don Key" + "Don Key" ], - "session_start_time": "1999-10-04T00:00:00", - "session_end_time": "1999-10-04T00:00:48", + "session_start_time": "1999-10-04T00:00:00Z", + "session_end_time": "1999-10-04T00:00:48Z", "session_type": "Foraging_Photometry", "iacuc_protocol": "2115", "rig_id": "ophys_rig", @@ -17,91 +17,117 @@ "animal_weight_post": null, "weight_unit": "gram", "data_streams": [ - { - "stream_start_time": "1999-10-04T00:00:00", - "stream_end_time": "1999-10-04T00:00:48", - "daq_names": [], - "camera_names": [], - "light_sources": [ - { - "device_type": "LightEmittingDiode", - "name": "470nm LED", - "excitation_power": "0.02", - "excitation_power_unit": "milliwatt" - }, - { - "device_type": "LightEmittingDiode", - "name": "415nm LED", - "excitation_power": "0.02", - "excitation_power_unit": "milliwatt" - }, - { - "device_type": "LightEmittingDiode", - "name": "565nm LED", - "excitation_power": "0.02", - "excitation_power_unit": "milliwatt" - } - ], - "ephys_modules": [], - "stick_microscopes": [], - "manipulator_modules": [], - "detectors": [ - { - "name": "Hamamatsu Camera", - "exposure_time": "10", - "exposure_time_unit": "millisecond", - "trigger_type": "Internal" - } - ], - "fiber_connections": [ - { - "patch_cord_name": "Patch Cord A", - "patch_cord_output_power": "40", - "output_power_unit": "microwatt", - "fiber_name": "Fiber A" - } - ], - "fiber_modules": [], - "ophys_fovs": [], - "slap_fovs": null, - "stack_parameters": null, - "stimulus_device_names": [], - "mouse_platform_name": "Disc", - "active_mouse_platform": false, - "stream_modalities": [ - { - "name": "Fiber photometry", - "abbreviation": "fib" - } - ], - "notes": null - } + { + "stream_start_time": "1999-10-04T00:00:00Z", + "stream_end_time": "1999-10-04T00:00:48Z", + "daq_names": [], + "camera_names": [], + "light_sources": [ + { + "device_type": "LightEmittingDiode", + "name": "470nm LED", + "excitation_power": "0.02", + "excitation_power_unit": "milliwatt" + }, + { + "device_type": "LightEmittingDiode", + "name": "415nm LED", + "excitation_power": "0.02", + "excitation_power_unit": "milliwatt" + }, + { + "device_type": "LightEmittingDiode", + "name": "565nm LED", + "excitation_power": "0.02", + "excitation_power_unit": "milliwatt" + } + ], + "ephys_modules": [], + "stick_microscopes": [], + "manipulator_modules": [], + "detectors": [ + { + "name": "Hamamatsu Camera", + "exposure_time": "10", + "exposure_time_unit": "millisecond", + "trigger_type": "Internal" + } + ], + "fiber_connections": [ + { + "patch_cord_name": "Patch Cord A", + "patch_cord_output_power": "40", + "output_power_unit": "microwatt", + "fiber_name": "Fiber A" + } + ], + "fiber_modules": [], + "ophys_fovs": [], + "slap_fovs": null, + "stack_parameters": null, + "stream_modalities": [ + { + "name": "Fiber photometry", + "abbreviation": "fib" + } + ], + "notes": null + } ], "stimulus_epochs": [ - { - "stimulus": { - "stimulus_type": "Opto Stimulation", - "stimulus_name": "OptoStim10Hz", - "pulse_shape": "Square", - "pulse_frequency": 10, - "pulse_frequency_unit": "hertz", - "number_pulse_trains": 2, - "pulse_width": 5000, - "pulse_width_unit": "millisecond", - "pulse_train_duration": "2.0", - "pulse_train_duration_unit": "second", - "fixed_pulse_train_interval": true, - "pulse_train_interval": "18.0", - "pulse_train_interval_unit": "second", - "baseline_duration": "10.0", - "baseline_duration_unit": "second", - "other_parameters": {}, - "notes": null - }, - "stimulus_start_time": "1999-10-04T00:00:00", - "stimulus_end_time": "1999-10-04T00:00:48" - } + { + "stimulus_start_time": "1999-10-04T00:00:00Z", + "stimulus_end_time": "1999-10-04T00:00:48Z", + "stimulus_name": "OptoStim10Hz", + "session_number": null, + "software": [], + "script": null, + "stimulus_modalities": [ + "Optogenetics" + ], + "stimulus_parameters": [ + { + "stimulus_type": "Opto Stimulation", + "stimulus_name": "OptoStim10Hz", + "pulse_shape": "Square", + "pulse_frequency": [ + "10" + ], + "pulse_frequency_unit": "hertz", + "number_pulse_trains": [ + 2 + ], + "pulse_width": [ + 5000 + ], + "pulse_width_unit": "millisecond", + "pulse_train_duration": [ + "2.0" + ], + "pulse_train_duration_unit": "second", + "fixed_pulse_train_interval": true, + "pulse_train_interval": "18.0", + "pulse_train_interval_unit": "second", + "baseline_duration": "10.0", + "baseline_duration_unit": "second", + "other_parameters": {}, + "notes": null + } + ], + "stimulus_device_names": [], + "speaker_config": null, + "light_source_config": null, + "output_parameters": {}, + "reward_consumed_during_epoch": null, + "reward_consumed_unit": "microliter", + "trials_total": null, + "trials_finished": null, + "trials_rewarded": null, + "notes": null + } ], + "mouse_platform_name": "Disc", + "active_mouse_platform": false, "reward_delivery": null, "reward_consumed_total": null, "reward_consumed_unit": "microliter", diff --git a/tests/test_fib.py b/tests/test_fib.py index 06638af9..f335a483 100644 --- a/tests/test_fib.py +++ b/tests/test_fib.py @@ -1,5 +1,6 @@ """Tests parsing of session information from fib rig.""" +import zoneinfo import json import os import unittest @@ -26,13 +27,14 @@ def setUpClass(cls): with open(EXAMPLE_MD_PATH, "r") as f: raw_md_contents = f.read() - # with open(EXPECTED_SESSION, "r") as f: - # expected_session_contents = Session(**json.load(f)) + with open(EXPECTED_SESSION, "r") as f: + expected_session_contents = Session(**json.load(f)) cls.example_job_settings = JobSettings( string_to_parse=raw_md_contents, experimenter_full_name=["Don Key"], - session_start_time=datetime(1999, 10, 4), + session_start_time=datetime( + 1999, 10, 4, tzinfo=zoneinfo.ZoneInfo("UTC")), notes="brabrabrabra....", labtracks_id="000000", iacuc_protocol="2115", @@ -74,7 +76,7 @@ def setUpClass(cls): active_mouse_platform=False, ) - # cls.expected_session = expected_session_contents + cls.expected_session = expected_session_contents def test_constructor_from_string(self) -> None: """Tests that the settings can be constructed from a json string""" @@ -97,7 +99,9 @@ def test_extract(self): self.example_job_settings.string_to_parse, parsed_info.teensy_str ) self.assertEqual( - datetime(1999, 10, 4), self.example_job_settings.session_start_time + datetime( + 1999, 10, 4, tzinfo=zoneinfo.ZoneInfo("UTC")), + self.example_job_settings.session_start_time ) def test_transform(self): @@ -106,8 +110,7 @@ def test_transform(self): etl_job1 = FIBEtl(job_settings=self.example_job_settings) parsed_info = etl_job1._extract() actual_session = etl_job1._transform(parsed_info) - print(actual_session) - # self.assertEqual(self.expected_session, actual_session) + self.assertEqual(self.expected_session, actual_session) def test_run_job(self): """Tests that the teensy response maps correctly to ophys session.""" @@ -120,4 +123,5 @@ def test_run_job(self): if __name__ == "__main__": + unittest.main() From 32de7f80b3bf7bc8d04596f1a4f5e29078c46b4d Mon Sep 17 00:00:00 2001 From: jtyoung84 <104453205+jtyoung84@users.noreply.github.com> Date: Fri, 5 Apr 2024 09:43:16 -0700 Subject: [PATCH 7/8] feat: updates mesoscope session --- src/aind_metadata_mapper/mesoscope/session.py | 14 +-- .../resources/mesoscope/expected_session.json | 107 ++++++++++++------ tests/test_mesoscope.py | 36 ++++-- 3 files changed, 100 insertions(+), 57 deletions(-) diff --git a/src/aind_metadata_mapper/mesoscope/session.py b/src/aind_metadata_mapper/mesoscope/session.py index 6cb396af..90b4d236 100644 --- a/src/aind_metadata_mapper/mesoscope/session.py +++ b/src/aind_metadata_mapper/mesoscope/session.py @@ -1,4 +1,5 @@ """Mesoscope ETL""" + import argparse import json import sys @@ -99,7 +100,8 @@ def _extract(self) -> dict: behavior_source = self.job_settings.behavior_source session_metadata = {} if behavior_source.is_dir(): - for ftype in behavior_source.glob("*json"): + # deterministic order + for ftype in sorted(list(behavior_source.glob("*json"))): if ( "Behavior" in ftype.stem or "Eye" in ftype.stem @@ -167,8 +169,6 @@ def _transform(self, extracted_source: dict) -> Session: stream_start_time=self.job_settings.session_start_time, stream_end_time=self.job_settings.session_end_time, ophys_fovs=fovs, - mouse_platform_name=self.job_settings.mouse_platform_name, - active_mouse_platform=True, stream_modalities=[Modality.POPHYS], ) ) @@ -188,10 +188,6 @@ def _transform(self, extracted_source: dict) -> Session: camera_names=[camera_name], stream_start_time=start_time, stream_end_time=end_time, - mouse_platform_name=( - self.job_settings.mouse_platform_name - ), - active_mouse_platform=True, stream_modalities=[Modality.BEHAVIOR_VIDEOS], ) ) @@ -216,8 +212,6 @@ def _transform(self, extracted_source: dict) -> Session: camera_names=["Vasculature"], stream_start_time=vasculature_dt, stream_end_time=vasculature_dt, - mouse_platform_name=self.job_settings.mouse_platform_name, - active_mouse_platform=True, stream_modalities=[ Modality.CONFOCAL ], # TODO: ask Saskia about this @@ -232,6 +226,8 @@ def _transform(self, extracted_source: dict) -> Session: session_end_time=self.job_settings.session_end_time, rig_id=extracted_source["platform"]["rig_id"], data_streams=data_streams, + mouse_platform_name=self.job_settings.mouse_platform_name, + active_mouse_platform=True, ) def run_job(self) -> None: diff --git a/tests/resources/mesoscope/expected_session.json b/tests/resources/mesoscope/expected_session.json index cc2a6d8d..be4ae30f 100644 --- a/tests/resources/mesoscope/expected_session.json +++ b/tests/resources/mesoscope/expected_session.json @@ -1,12 +1,12 @@ { "describedBy": "https://raw.githubusercontent.com/AllenNeuralDynamics/aind-data-schema/main/src/aind_data_schema/core/session.py", - "schema_version": "0.1.4", + "schema_version": "0.2.1", "protocol_id": [], "experimenter_full_name": [ "John Doe" ], - "session_start_time": "2024-02-22T15:30:00", - "session_end_time": "2024-02-22T17:30:00", + "session_start_time": "2024-02-22T15:30:00Z", + "session_end_time": "2024-02-22T17:30:00Z", "session_type": "Mesoscope", "iacuc_protocol": "12345", "rig_id": "MESO.1", @@ -18,8 +18,8 @@ "weight_unit": "gram", "data_streams": [ { - "stream_start_time": "2024-02-22T15:30:00", - "stream_end_time": "2024-02-22T17:30:00", + "stream_start_time": "2024-02-22T15:30:00Z", + "stream_end_time": "2024-02-22T17:30:00Z", "daq_names": [], "camera_names": [ "Mesoscope" @@ -49,7 +49,13 @@ "fov_scale_factor_unit": "um/pixel", "frame_rate": "9.48", "frame_rate_unit": "hertz", - "coupled_fov_index": null + "coupled_fov_index": null, + "power": null, + "power_unit": "percent", + "scanfield_z": null, + "scanfield_z_unit": "micrometer", + "scanimage_roi_index": null, + "notes": null }, { "index": 0, @@ -68,7 +74,13 @@ "fov_scale_factor_unit": "um/pixel", "frame_rate": "9.48", "frame_rate_unit": "hertz", - "coupled_fov_index": null + "coupled_fov_index": null, + "power": null, + "power_unit": "percent", + "scanfield_z": null, + "scanfield_z_unit": "micrometer", + "scanimage_roi_index": null, + "notes": null }, { "index": 1, @@ -87,7 +99,13 @@ "fov_scale_factor_unit": "um/pixel", "frame_rate": "9.48", "frame_rate_unit": "hertz", - "coupled_fov_index": null + "coupled_fov_index": null, + "power": null, + "power_unit": "percent", + "scanfield_z": null, + "scanfield_z_unit": "micrometer", + "scanimage_roi_index": null, + "notes": null }, { "index": 1, @@ -106,7 +124,13 @@ "fov_scale_factor_unit": "um/pixel", "frame_rate": "9.48", "frame_rate_unit": "hertz", - "coupled_fov_index": null + "coupled_fov_index": null, + "power": null, + "power_unit": "percent", + "scanfield_z": null, + "scanfield_z_unit": "micrometer", + "scanimage_roi_index": null, + "notes": null }, { "index": 2, @@ -125,7 +149,13 @@ "fov_scale_factor_unit": "um/pixel", "frame_rate": "9.48", "frame_rate_unit": "hertz", - "coupled_fov_index": null + "coupled_fov_index": null, + "power": null, + "power_unit": "percent", + "scanfield_z": null, + "scanfield_z_unit": "micrometer", + "scanimage_roi_index": null, + "notes": null }, { "index": 2, @@ -144,7 +174,13 @@ "fov_scale_factor_unit": "um/pixel", "frame_rate": "9.48", "frame_rate_unit": "hertz", - "coupled_fov_index": null + "coupled_fov_index": null, + "power": null, + "power_unit": "percent", + "scanfield_z": null, + "scanfield_z_unit": "micrometer", + "scanimage_roi_index": null, + "notes": null }, { "index": 3, @@ -163,7 +199,13 @@ "fov_scale_factor_unit": "um/pixel", "frame_rate": "9.48", "frame_rate_unit": "hertz", - "coupled_fov_index": null + "coupled_fov_index": null, + "power": null, + "power_unit": "percent", + "scanfield_z": null, + "scanfield_z_unit": "micrometer", + "scanimage_roi_index": null, + "notes": null }, { "index": 3, @@ -182,14 +224,17 @@ "fov_scale_factor_unit": "um/pixel", "frame_rate": "9.48", "frame_rate_unit": "hertz", - "coupled_fov_index": null + "coupled_fov_index": null, + "power": null, + "power_unit": "percent", + "scanfield_z": null, + "scanfield_z_unit": "micrometer", + "scanimage_roi_index": null, + "notes": null } ], "slap_fovs": null, "stack_parameters": null, - "stimulus_device_names": [], - "mouse_platform_name": "disc", - "active_mouse_platform": true, "stream_modalities": [ { "name": "Planar optical physiology", @@ -199,8 +244,8 @@ "notes": null }, { - "stream_start_time": "2024-02-12T09:14:43", - "stream_end_time": "2024-02-12T10:30:54", + "stream_start_time": "2024-02-12T09:14:43Z", + "stream_end_time": "2024-02-12T10:30:54Z", "daq_names": [], "camera_names": [ "Behavior" @@ -215,9 +260,6 @@ "ophys_fovs": [], "slap_fovs": null, "stack_parameters": null, - "stimulus_device_names": [], - "mouse_platform_name": "disc", - "active_mouse_platform": true, "stream_modalities": [ { "name": "Behavior videos", @@ -227,8 +269,8 @@ "notes": null }, { - "stream_start_time": "2024-02-12T09:14:44", - "stream_end_time": "2024-02-12T10:30:55", + "stream_start_time": "2024-02-12T09:14:44Z", + "stream_end_time": "2024-02-12T10:30:55Z", "daq_names": [], "camera_names": [ "Eye" @@ -243,9 +285,6 @@ "ophys_fovs": [], "slap_fovs": null, "stack_parameters": null, - "stimulus_device_names": [], - "mouse_platform_name": "disc", - "active_mouse_platform": true, "stream_modalities": [ { "name": "Behavior videos", @@ -255,8 +294,8 @@ "notes": null }, { - "stream_start_time": "2024-02-12T09:14:44", - "stream_end_time": "2024-02-12T10:30:55", + "stream_start_time": "2024-02-12T09:14:44Z", + "stream_end_time": "2024-02-12T10:30:55Z", "daq_names": [], "camera_names": [ "Face" @@ -271,9 +310,6 @@ "ophys_fovs": [], "slap_fovs": null, "stack_parameters": null, - "stimulus_device_names": [], - "mouse_platform_name": "disc", - "active_mouse_platform": true, "stream_modalities": [ { "name": "Behavior videos", @@ -283,8 +319,8 @@ "notes": null }, { - "stream_start_time": "2024-02-12T11:02:22", - "stream_end_time": "2024-02-12T11:02:22", + "stream_start_time": "2024-02-12T11:02:22Z", + "stream_end_time": "2024-02-12T11:02:22Z", "daq_names": [], "camera_names": [ "Vasculature" @@ -299,9 +335,6 @@ "ophys_fovs": [], "slap_fovs": null, "stack_parameters": null, - "stimulus_device_names": [], - "mouse_platform_name": "disc", - "active_mouse_platform": true, "stream_modalities": [ { "name": "Confocal microscopy", @@ -312,6 +345,8 @@ } ], "stimulus_epochs": [], + "mouse_platform_name": "disc", + "active_mouse_platform": true, "reward_delivery": null, "reward_consumed_total": null, "reward_consumed_unit": "microliter", diff --git a/tests/test_mesoscope.py b/tests/test_mesoscope.py index 268abdc3..c908a00f 100644 --- a/tests/test_mesoscope.py +++ b/tests/test_mesoscope.py @@ -6,6 +6,7 @@ from datetime import datetime from pathlib import Path from unittest.mock import MagicMock, patch +import zoneinfo from aind_data_schema.core.session import Session from PIL import Image @@ -27,6 +28,8 @@ class TestMesoscope(unittest.TestCase): """Tests methods in MesoscopeEtl class""" + maxDiff = None # show full diff without truncation + @classmethod def setUpClass(cls) -> None: """Set up the test suite""" @@ -44,8 +47,12 @@ def setUpClass(cls) -> None: behavior_source=RESOURCES_DIR, output_directory=RESOURCES_DIR, subject_id="12345", - session_start_time=datetime(2024, 2, 22, 15, 30, 0), - session_end_time=datetime(2024, 2, 22, 17, 30, 0), + session_start_time=datetime( + 2024, 2, 22, 15, 30, 0, + tzinfo=zoneinfo.ZoneInfo("UTC")), + session_end_time=datetime( + 2024, 2, 22, 17, 30, 0, + tzinfo=zoneinfo.ZoneInfo("UTC")), project="some_project", experimenter_full_name=["John Doe"], magnification="16x", @@ -158,6 +165,7 @@ def test_extract_no_input_source( def test_transform(self, mock_open, mock_scanimage) -> None: """Tests that the platform json is extracted and transfromed into a session object correctly""" + etl = MesoscopeEtl( job_settings=self.example_job_settings, ) @@ -168,20 +176,24 @@ def test_transform(self, mock_open, mock_scanimage) -> None: # mock scanimage metadata mock_meta = [{}] - mock_meta[0][ - "SI.hRoiManager.linesPerFrame" - ] = self.example_scanimage_meta["lines_per_frame"] - mock_meta[0][ - "SI.hRoiManager.pixelsPerLine" - ] = self.example_scanimage_meta["pixels_per_line"] - mock_meta[0][ - "SI.hRoiManager.scanZoomFactor" - ] = self.example_scanimage_meta["fov_scale_factor"] + mock_meta[0]["SI.hRoiManager.linesPerFrame"] = ( + self.example_scanimage_meta["lines_per_frame"] + ) + mock_meta[0]["SI.hRoiManager.pixelsPerLine"] = ( + self.example_scanimage_meta["pixels_per_line"] + ) + mock_meta[0]["SI.hRoiManager.scanZoomFactor"] = ( + self.example_scanimage_meta["fov_scale_factor"] + ) mock_scanimage.return_value = mock_meta extract = etl._extract() transformed_session = etl._transform(extract) - + for stream in transformed_session.data_streams: + stream.stream_start_time = stream.stream_start_time.replace( + tzinfo=zoneinfo.ZoneInfo("UTC")) + stream.stream_end_time = stream.stream_end_time.replace( + tzinfo=zoneinfo.ZoneInfo("UTC")) self.assertEqual( self.example_session, json.loads(transformed_session.model_dump_json()), From f735151ff46d39981aea1da023a38063389b396c Mon Sep 17 00:00:00 2001 From: jtyoung84 <104453205+jtyoung84@users.noreply.github.com> Date: Fri, 5 Apr 2024 09:52:19 -0700 Subject: [PATCH 8/8] feat: runs linters --- src/aind_metadata_mapper/bergamo/session.py | 6 ++-- src/aind_metadata_mapper/ephys/session.py | 2 +- tests/test_bergamo.py | 24 +++++++++++---- tests/test_ephys.py | 22 ++++++++----- tests/test_fib.py | 11 +++---- tests/test_mesoscope.py | 34 +++++++++++---------- 6 files changed, 59 insertions(+), 40 deletions(-) diff --git a/src/aind_metadata_mapper/bergamo/session.py b/src/aind_metadata_mapper/bergamo/session.py index 1914875a..960d66e3 100644 --- a/src/aind_metadata_mapper/bergamo/session.py +++ b/src/aind_metadata_mapper/bergamo/session.py @@ -451,6 +451,7 @@ def _transform(self, extracted_source: RawImageInfo) -> Session: ], ) stimulus_name = "PhotoStimulation" + photostim_interval = self.job_settings.photo_stim_inter_trial_interval return Session( experimenter_full_name=self.job_settings.experimenter_full_name, session_start_time=self.job_settings.session_start_time, @@ -544,10 +545,7 @@ def _transform(self, extracted_source: RawImageInfo) -> Session: ][2]["scanfields"]["duration"], ), ], - inter_trial_interval=( - self.job_settings. - photo_stim_inter_trial_interval - ), + inter_trial_interval=(photostim_interval), ) ], stimulus_start_time=( diff --git a/src/aind_metadata_mapper/ephys/session.py b/src/aind_metadata_mapper/ephys/session.py index 8bdc38c1..6cf68a68 100644 --- a/src/aind_metadata_mapper/ephys/session.py +++ b/src/aind_metadata_mapper/ephys/session.py @@ -101,7 +101,7 @@ def _transform(self, extracted_source: ParsedInformation) -> Session: ): session_stream = {} session_stream["stream_start_time"] = datetime.strptime( - stage[0][0], "%Y/%m/%d %H:%M:%S.%f" + stage[0][0], "%Y/%m/%d %H:%M:%S.%f" ) session_stream["stream_end_time"] = datetime.strptime( stage[-1][0], "%Y/%m/%d %H:%M:%S.%f" diff --git a/tests/test_bergamo.py b/tests/test_bergamo.py index 631e5399..77f0ac59 100644 --- a/tests/test_bergamo.py +++ b/tests/test_bergamo.py @@ -46,12 +46,24 @@ def setUpClass(cls): active_mouse_platform=True, experimenter_full_name=["John Smith", "Jane Smith"], subject_id="12345", - session_start_time=datetime(2023, 10, 10, 14, 0, 0, tzinfo=timezone.utc), - session_end_time=datetime(2023, 10, 10, 17, 0, 0, tzinfo=timezone.utc), - stream_start_time=datetime(2023, 10, 10, 15, 0, 0, tzinfo=timezone.utc), - stream_end_time=datetime(2023, 10, 10, 16, 0, 0, tzinfo=timezone.utc), - stimulus_start_time=datetime(2023, 10, 10, 15, 15, 0, tzinfo=timezone.utc), - stimulus_end_time=datetime(2023, 10, 10, 15, 45, 0, tzinfo=timezone.utc), + session_start_time=datetime( + 2023, 10, 10, 14, 0, 0, tzinfo=timezone.utc + ), + session_end_time=datetime( + 2023, 10, 10, 17, 0, 0, tzinfo=timezone.utc + ), + stream_start_time=datetime( + 2023, 10, 10, 15, 0, 0, tzinfo=timezone.utc + ), + stream_end_time=datetime( + 2023, 10, 10, 16, 0, 0, tzinfo=timezone.utc + ), + stimulus_start_time=datetime( + 2023, 10, 10, 15, 15, 0, tzinfo=timezone.utc + ), + stimulus_end_time=datetime( + 2023, 10, 10, 15, 45, 0, tzinfo=timezone.utc + ), ) cls.expected_session = expected_session_contents diff --git a/tests/test_ephys.py b/tests/test_ephys.py index 24b1656b..4d87043e 100644 --- a/tests/test_ephys.py +++ b/tests/test_ephys.py @@ -4,9 +4,9 @@ import json import os import unittest +import zoneinfo from pathlib import Path from xml.dom import minidom -import zoneinfo from aind_data_schema.core.session import Session @@ -221,15 +221,23 @@ def test_transform(self): ) parsed_info = etl_job1._extract() actual_session = etl_job1._transform(parsed_info) - actual_session.session_start_time = actual_session.session_start_time \ - .replace(tzinfo=zoneinfo.ZoneInfo("UTC")) - actual_session.session_end_time = actual_session.session_end_time \ - .replace(tzinfo=zoneinfo.ZoneInfo("UTC")) + actual_session.session_start_time = ( + actual_session.session_start_time.replace( + tzinfo=zoneinfo.ZoneInfo("UTC") + ) + ) + actual_session.session_end_time = ( + actual_session.session_end_time.replace( + tzinfo=zoneinfo.ZoneInfo("UTC") + ) + ) for stream in actual_session.data_streams: stream.stream_start_time = stream.stream_start_time.replace( - tzinfo=zoneinfo.ZoneInfo("UTC")) + tzinfo=zoneinfo.ZoneInfo("UTC") + ) stream.stream_end_time = stream.stream_end_time.replace( - tzinfo=zoneinfo.ZoneInfo("UTC")) + tzinfo=zoneinfo.ZoneInfo("UTC") + ) self.assertEqual( self.expected_session.model_dump(), actual_session.model_dump(), diff --git a/tests/test_fib.py b/tests/test_fib.py index f335a483..626c3cda 100644 --- a/tests/test_fib.py +++ b/tests/test_fib.py @@ -1,9 +1,9 @@ """Tests parsing of session information from fib rig.""" -import zoneinfo import json import os import unittest +import zoneinfo from datetime import datetime from pathlib import Path @@ -34,7 +34,8 @@ def setUpClass(cls): string_to_parse=raw_md_contents, experimenter_full_name=["Don Key"], session_start_time=datetime( - 1999, 10, 4, tzinfo=zoneinfo.ZoneInfo("UTC")), + 1999, 10, 4, tzinfo=zoneinfo.ZoneInfo("UTC") + ), notes="brabrabrabra....", labtracks_id="000000", iacuc_protocol="2115", @@ -99,9 +100,8 @@ def test_extract(self): self.example_job_settings.string_to_parse, parsed_info.teensy_str ) self.assertEqual( - datetime( - 1999, 10, 4, tzinfo=zoneinfo.ZoneInfo("UTC")), - self.example_job_settings.session_start_time + datetime(1999, 10, 4, tzinfo=zoneinfo.ZoneInfo("UTC")), + self.example_job_settings.session_start_time, ) def test_transform(self): @@ -123,5 +123,4 @@ def test_run_job(self): if __name__ == "__main__": - unittest.main() diff --git a/tests/test_mesoscope.py b/tests/test_mesoscope.py index c908a00f..ed23e9db 100644 --- a/tests/test_mesoscope.py +++ b/tests/test_mesoscope.py @@ -3,10 +3,10 @@ import json import os import unittest +import zoneinfo from datetime import datetime from pathlib import Path from unittest.mock import MagicMock, patch -import zoneinfo from aind_data_schema.core.session import Session from PIL import Image @@ -48,11 +48,11 @@ def setUpClass(cls) -> None: output_directory=RESOURCES_DIR, subject_id="12345", session_start_time=datetime( - 2024, 2, 22, 15, 30, 0, - tzinfo=zoneinfo.ZoneInfo("UTC")), + 2024, 2, 22, 15, 30, 0, tzinfo=zoneinfo.ZoneInfo("UTC") + ), session_end_time=datetime( - 2024, 2, 22, 17, 30, 0, - tzinfo=zoneinfo.ZoneInfo("UTC")), + 2024, 2, 22, 17, 30, 0, tzinfo=zoneinfo.ZoneInfo("UTC") + ), project="some_project", experimenter_full_name=["John Doe"], magnification="16x", @@ -176,24 +176,26 @@ def test_transform(self, mock_open, mock_scanimage) -> None: # mock scanimage metadata mock_meta = [{}] - mock_meta[0]["SI.hRoiManager.linesPerFrame"] = ( - self.example_scanimage_meta["lines_per_frame"] - ) - mock_meta[0]["SI.hRoiManager.pixelsPerLine"] = ( - self.example_scanimage_meta["pixels_per_line"] - ) - mock_meta[0]["SI.hRoiManager.scanZoomFactor"] = ( - self.example_scanimage_meta["fov_scale_factor"] - ) + mock_meta[0][ + "SI.hRoiManager.linesPerFrame" + ] = self.example_scanimage_meta["lines_per_frame"] + mock_meta[0][ + "SI.hRoiManager.pixelsPerLine" + ] = self.example_scanimage_meta["pixels_per_line"] + mock_meta[0][ + "SI.hRoiManager.scanZoomFactor" + ] = self.example_scanimage_meta["fov_scale_factor"] mock_scanimage.return_value = mock_meta extract = etl._extract() transformed_session = etl._transform(extract) for stream in transformed_session.data_streams: stream.stream_start_time = stream.stream_start_time.replace( - tzinfo=zoneinfo.ZoneInfo("UTC")) + tzinfo=zoneinfo.ZoneInfo("UTC") + ) stream.stream_end_time = stream.stream_end_time.replace( - tzinfo=zoneinfo.ZoneInfo("UTC")) + tzinfo=zoneinfo.ZoneInfo("UTC") + ) self.assertEqual( self.example_session, json.loads(transformed_session.model_dump_json()),