From 3503ebd8d59c8518c492651402c60497e645cdad Mon Sep 17 00:00:00 2001 From: Carter Peene Date: Wed, 2 Oct 2024 13:53:49 -0700 Subject: [PATCH 1/4] Simplifiy np deps (#174) * except other errors for getting ecephys paths * fix logic for overwriting opto table, remove npc_sessions dependency * working on simplifying * get mtrain without np_session * factor out np_session * linting fixes * sort imports * remove deps from toml * refactor json_settings into job_settings and inherit from genericetl * have camstimephys inherit from genericetl * add model for camstimephys job settings * lint * get job_settings models to work in camstimephys class when passed as a dict * linting and cleanup * use genericetl.load in run_job * fix mekhlas reviews #174 * lint * finish mekhla feedback * fixes npc_mvr pin * removes hardcoded path --------- Co-authored-by: Mekhla Kapoor <54870020+mekhlakapoor@users.noreply.github.com> --- pyproject.toml | 2 +- .../open_ephys/camstim_ephys_session.py | 146 ++++++++---------- src/aind_metadata_mapper/open_ephys/models.py | 82 ++++++++++ .../open_ephys/utils/behavior_utils.py | 12 +- .../open_ephys/utils/constants.py | 58 ------- src/aind_metadata_mapper/stimulus/camstim.py | 77 ++++++--- 6 files changed, 209 insertions(+), 168 deletions(-) create mode 100644 src/aind_metadata_mapper/open_ephys/models.py diff --git a/pyproject.toml b/pyproject.toml index 0eb8fa9d..6df7dff9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -66,11 +66,11 @@ mesoscope = [ openephys = [ "aind-metadata-mapper[schema]", "h5py >= 3.11.0", - "np_session >= 0.1.39", "npc_ephys >= 0.1.18 ; python_version >= '3.9'", "scipy >= 1.11.0", "pandas >= 2.2.2", "numpy >= 1.26.4", + "npc_mvr >= 0.1.6" ] dynamicrouting = [ diff --git a/src/aind_metadata_mapper/open_ephys/camstim_ephys_session.py b/src/aind_metadata_mapper/open_ephys/camstim_ephys_session.py index 0c9187f1..9cc1890e 100644 --- a/src/aind_metadata_mapper/open_ephys/camstim_ephys_session.py +++ b/src/aind_metadata_mapper/open_ephys/camstim_ephys_session.py @@ -1,42 +1,45 @@ """ -File containing CamstimEphysSession class +File containing CamstimEphysSessionEtl class """ -import argparse import datetime import json import logging import re from pathlib import Path +from typing import Union -import np_session import npc_ephys import npc_mvr -import npc_sessions import numpy as np import pandas as pd from aind_data_schema.components.coordinates import Coordinates3d from aind_data_schema.core.session import ManipulatorModule, Session, Stream from aind_data_schema_models.modalities import Modality -import aind_metadata_mapper.open_ephys.utils.constants as constants import aind_metadata_mapper.open_ephys.utils.sync_utils as sync import aind_metadata_mapper.stimulus.camstim +from aind_metadata_mapper.core import GenericEtl +from aind_metadata_mapper.open_ephys.models import JobSettings logger = logging.getLogger(__name__) -class CamstimEphysSession(aind_metadata_mapper.stimulus.camstim.Camstim): +class CamstimEphysSessionEtl( + aind_metadata_mapper.stimulus.camstim.Camstim, GenericEtl +): """ An Ephys session, designed for OpenScope, employing neuropixel probes with visual and optogenetic stimulus from Camstim. """ json_settings: dict = None - npexp_path: Path + session_path: Path recording_dir: Path - def __init__(self, session_id: str, json_settings: dict) -> None: + def __init__( + self, session_id: str, job_settings: Union[JobSettings, str, dict] + ) -> None: """ Determine needed input filepaths from np-exp and lims, get session start and end times from sync file, write stim tables and extract @@ -47,42 +50,42 @@ def __init__(self, session_id: str, json_settings: dict) -> None: different laser states for this experiment. Otherwise, the default is used from naming_utils. """ - if json_settings.get("opto_conditions_map", None) is None: - self.opto_conditions_map = constants.DEFAULT_OPTO_CONDITIONS + if isinstance(job_settings, str): + job_settings_model = JobSettings.model_validate_json(job_settings) + elif isinstance(job_settings, dict): + job_settings_model = JobSettings(**job_settings) else: - self.opto_conditions_map = json_settings["opto_conditions_map"] - overwrite_tables = json_settings.get("overwrite_tables", False) - - self.json_settings = json_settings - session_inst = np_session.Session(session_id) - self.mtrain = session_inst.mtrain - self.npexp_path = session_inst.npexp_path - self.folder = session_inst.folder + job_settings_model = job_settings + GenericEtl.__init__(self, job_settings=job_settings_model) + + sessions_root = Path(self.job_settings.sessions_root) + self.folder = self.get_folder(session_id, sessions_root) + self.session_path = self.get_session_path(session_id, sessions_root) # sometimes data files are deleted on npexp so try files on lims - try: - self.recording_dir = npc_ephys.get_single_oebin_path( - session_inst.lims_path - ).parent - except FileNotFoundError: - self.recording_dir = npc_ephys.get_single_oebin_path( - session_inst.npexp_path - ).parent + # try: + # self.recording_dir = npc_ephys.get_single_oebin_path( + # session_inst.lims_path + # ).parent + # except: + self.recording_dir = npc_ephys.get_single_oebin_path( + self.session_path + ).parent self.motor_locs_path = ( - self.npexp_path / f"{self.folder}.motor-locs.csv" + self.session_path / f"{self.folder}.motor-locs.csv" ) - self.pkl_path = self.npexp_path / f"{self.folder}.stim.pkl" - self.opto_pkl_path = self.npexp_path / f"{self.folder}.opto.pkl" + self.pkl_path = self.session_path / f"{self.folder}.stim.pkl" + self.opto_pkl_path = self.session_path / f"{self.folder}.opto.pkl" self.opto_table_path = ( - self.npexp_path / f"{self.folder}_opto_epochs.csv" + self.session_path / f"{self.folder}_opto_epochs.csv" ) self.stim_table_path = ( - self.npexp_path / f"{self.folder}_stim_epochs.csv" + self.session_path / f"{self.folder}_stim_epochs.csv" ) - self.sync_path = self.npexp_path / f"{self.folder}.sync" + self.sync_path = self.session_path / f"{self.folder}.sync" platform_path = next( - self.npexp_path.glob(f"{self.folder}_platform*.json") + self.session_path.glob(f"{self.folder}_platform*.json") ) self.platform_json = json.loads(platform_path.read_text()) self.project_name = self.platform_json["project"] @@ -95,13 +98,17 @@ def __init__(self, session_id: str, json_settings: dict) -> None: f" session end: {self.session_end}" ) - if not self.stim_table_path.exists() or overwrite_tables: + self.session_uuid = self.get_session_uuid() + self.mtrain_regimen = self.get_mtrain() + + if not self.stim_table_path.exists() or ( + self.job_settings.overwrite_tables + ): logger.debug("building stim table") self.build_stimulus_table() - if ( - self.opto_pkl_path.exists() - and not self.opto_table_path.exists() - or overwrite_tables + if self.opto_pkl_path.exists() and ( + not self.opto_table_path.exists() or + self.job_settings.overwrite_tables ): logger.debug("building opto table") self.build_optogenetics_table() @@ -113,7 +120,17 @@ def __init__(self, session_id: str, json_settings: dict) -> None: self.available_probes = self.get_available_probes() - def generate_session_json(self) -> Session: + def run_job(self): + """Transforms all metadata for the session into relevant files""" + self._extract() + self._transform() + return self._load(self.session_json, self.session_path) + + def _extract(self): + """TODO: refactor a lot of the __init__ code here""" + pass + + def _transform(self) -> Session: """ Creates the session schema json """ @@ -123,30 +140,19 @@ def generate_session_json(self) -> Session: ], session_start_time=self.session_start, session_end_time=self.session_end, - session_type=self.json_settings.get("session_type", ""), - iacuc_protocol=self.json_settings.get("iacuc_protocol", ""), + session_type=self.job_settings.session_type, + iacuc_protocol=self.job_settings.iacuc_protocol, rig_id=self.platform_json["rig_id"], subject_id=self.folder.split("_")[1], data_streams=self.data_streams(), stimulus_epochs=self.stim_epochs, - mouse_platform_name=self.json_settings.get( - "mouse_platform", "Mouse Platform" - ), - active_mouse_platform=self.json_settings.get( - "active_mouse_platform", False - ), + mouse_platform_name=self.job_settings.mouse_platform_name, + active_mouse_platform=self.job_settings.active_mouse_platform, reward_consumed_unit="milliliter", notes="", ) return self.session_json - def write_session_json(self) -> None: - """ - Writes the session json to a session.json file - """ - self.session_json.write_standard_file(self.npexp_path) - logger.debug(f"File created at {str(self.npexp_path)}/session.json") - @staticmethod def extract_probe_letter(probe_exp, s): """ @@ -214,7 +220,7 @@ def ephys_modules(self) -> list: """ Return list of schema ephys modules for each available probe. """ - newscale_coords = npc_sessions.get_newscale_coordinates( + newscale_coords = npc_ephys.get_newscale_coordinates( self.motor_locs_path ) @@ -289,7 +295,7 @@ def video_stream(self) -> Stream: Returns schema behavior videos stream for video timing """ video_frame_times = npc_mvr.mvr.get_video_frame_times( - self.sync_path, self.npexp_path + self.sync_path, self.session_path ) stream_first_time = min( @@ -320,36 +326,12 @@ def data_streams(self) -> tuple[Stream, ...]: return tuple(data_streams) -def parse_args() -> argparse.Namespace: - """ - Parse Arguments - """ - parser = argparse.ArgumentParser( - description="Generate a session.json file for an ephys session" - ) - parser.add_argument( - "session_id", - help=( - "session ID (lims or np-exp foldername) or path to session" - "folder" - ), - ) - parser.add_argument( - "json-settings", - help=( - 'json containing at minimum the fields "session_type" and' - '"iacuc protocol"' - ), - ) - return parser.parse_args() - - def main() -> None: """ Run Main """ - sessionETL = CamstimEphysSession(**vars(parse_args())) - sessionETL.generate_session_json() + sessionETL = CamstimEphysSessionEtl(**vars) + sessionETL.run_job() if __name__ == "__main__": diff --git a/src/aind_metadata_mapper/open_ephys/models.py b/src/aind_metadata_mapper/open_ephys/models.py new file mode 100644 index 00000000..d85965f8 --- /dev/null +++ b/src/aind_metadata_mapper/open_ephys/models.py @@ -0,0 +1,82 @@ +"""Module defining JobSettings for Mesoscope ETL""" + +from typing import Literal, Union +from pathlib import Path +from aind_metadata_mapper.core_models import BaseJobSettings + +DEFAULT_OPTO_CONDITIONS = { + "0": { + "duration": 0.01, + "name": "1Hz_10ms", + "condition": "10 ms pulse at 1 Hz", + }, + "1": { + "duration": 0.002, + "name": "1Hz_2ms", + "condition": "2 ms pulse at 1 Hz", + }, + "2": { + "duration": 1.0, + "name": "5Hz_2ms", + "condition": "2 ms pulses at 5 Hz", + }, + "3": { + "duration": 1.0, + "name": "10Hz_2ms", + "condition": "2 ms pulses at 10 Hz", + }, + "4": { + "duration": 1.0, + "name": "20Hz_2ms", + "condition": "2 ms pulses at 20 Hz", + }, + "5": { + "duration": 1.0, + "name": "30Hz_2ms", + "condition": "2 ms pulses at 30 Hz", + }, + "6": { + "duration": 1.0, + "name": "40Hz_2ms", + "condition": "2 ms pulses at 40 Hz", + }, + "7": { + "duration": 1.0, + "name": "50Hz_2ms", + "condition": "2 ms pulses at 50 Hz", + }, + "8": { + "duration": 1.0, + "name": "60Hz_2ms", + "condition": "2 ms pulses at 60 Hz", + }, + "9": { + "duration": 1.0, + "name": "80Hz_2ms", + "condition": "2 ms pulses at 80 Hz", + }, + "10": { + "duration": 1.0, + "name": "square_1s", + "condition": "1 second square pulse: continuously on for 1s", + }, + "11": {"duration": 1.0, "name": "cosine_1s", "condition": "cosine pulse"}, +} + + +class JobSettings(BaseJobSettings): + """Data to be entered by the user.""" + + job_settings_name: Literal["OpenEphys"] = "OpenEphys" + session_type: str + project_name: str + iacuc_protocol: str + description: str + sessions_root: Union[Path, str] + opto_conditions_map: dict = DEFAULT_OPTO_CONDITIONS + overwrite_tables: bool = False + mtrain_server: str + # TODO: use input_source and replace sessions_root, camstimephys.getfolder + input_source: str = "blah" + active_mouse_platform: bool = False + mouse_platform_name: str = "Mouse Platform" diff --git a/src/aind_metadata_mapper/open_ephys/utils/behavior_utils.py b/src/aind_metadata_mapper/open_ephys/utils/behavior_utils.py index a58e366e..49a08295 100644 --- a/src/aind_metadata_mapper/open_ephys/utils/behavior_utils.py +++ b/src/aind_metadata_mapper/open_ephys/utils/behavior_utils.py @@ -730,9 +730,9 @@ def fix_omitted_end_frame(stim_pres_table: pd.DataFrame) -> pd.DataFrame: stim_pres_table[stim_pres_table["omitted"]]["start_frame"] + median_stim_frame_duration ) - stim_pres_table.loc[ - stim_pres_table["omitted"], "end_frame" - ] = omitted_end_frames + stim_pres_table.loc[stim_pres_table["omitted"], "end_frame"] = ( + omitted_end_frames + ) stim_dtypes = stim_pres_table.dtypes.to_dict() stim_dtypes["start_frame"] = int @@ -796,9 +796,9 @@ def compute_is_sham_change( if np.array_equal( active_images, stim_image_names[passive_block_mask].values ): - stim_df.loc[ - passive_block_mask, "is_sham_change" - ] = stim_df[active_block_mask]["is_sham_change"].values + stim_df.loc[passive_block_mask, "is_sham_change"] = ( + stim_df[active_block_mask]["is_sham_change"].values + ) return stim_df.sort_index() diff --git a/src/aind_metadata_mapper/open_ephys/utils/constants.py b/src/aind_metadata_mapper/open_ephys/utils/constants.py index aafdab8b..e74f4c70 100644 --- a/src/aind_metadata_mapper/open_ephys/utils/constants.py +++ b/src/aind_metadata_mapper/open_ephys/utils/constants.py @@ -3,64 +3,6 @@ import re INT_NULL = -99 -DEFAULT_OPTO_CONDITIONS = { - "0": { - "duration": 0.01, - "name": "1Hz_10ms", - "condition": "10 ms pulse at 1 Hz", - }, - "1": { - "duration": 0.002, - "name": "1Hz_2ms", - "condition": "2 ms pulse at 1 Hz", - }, - "2": { - "duration": 1.0, - "name": "5Hz_2ms", - "condition": "2 ms pulses at 5 Hz", - }, - "3": { - "duration": 1.0, - "name": "10Hz_2ms", - "condition": "2 ms pulses at 10 Hz", - }, - "4": { - "duration": 1.0, - "name": "20Hz_2ms", - "condition": "2 ms pulses at 20 Hz", - }, - "5": { - "duration": 1.0, - "name": "30Hz_2ms", - "condition": "2 ms pulses at 30 Hz", - }, - "6": { - "duration": 1.0, - "name": "40Hz_2ms", - "condition": "2 ms pulses at 40 Hz", - }, - "7": { - "duration": 1.0, - "name": "50Hz_2ms", - "condition": "2 ms pulses at 50 Hz", - }, - "8": { - "duration": 1.0, - "name": "60Hz_2ms", - "condition": "2 ms pulses at 60 Hz", - }, - "9": { - "duration": 1.0, - "name": "80Hz_2ms", - "condition": "2 ms pulses at 80 Hz", - }, - "10": { - "duration": 1.0, - "name": "square_1s", - "condition": "1 second square pulse: continuously on for 1s", - }, - "11": {"duration": 1.0, "name": "cosine_1s", "condition": "cosine pulse"}, -} default_stimulus_renames = { "": "spontaneous", diff --git a/src/aind_metadata_mapper/stimulus/camstim.py b/src/aind_metadata_mapper/stimulus/camstim.py index 814a577c..64176804 100644 --- a/src/aind_metadata_mapper/stimulus/camstim.py +++ b/src/aind_metadata_mapper/stimulus/camstim.py @@ -4,17 +4,20 @@ import datetime import functools +from pathlib import Path +from typing import Union import aind_data_schema import aind_data_schema.core.session as session_schema -import np_session import pandas as pd +import requests import aind_metadata_mapper.open_ephys.utils.constants as constants import aind_metadata_mapper.open_ephys.utils.naming_utils as names import aind_metadata_mapper.open_ephys.utils.pkl_utils as pkl import aind_metadata_mapper.open_ephys.utils.stim_utils as stim import aind_metadata_mapper.open_ephys.utils.sync_utils as sync +from aind_metadata_mapper.open_ephys.models import JobSettings class Camstim: @@ -25,7 +28,7 @@ class Camstim: def __init__( self, session_id: str, - json_settings: dict, + job_settings: Union[JobSettings, str], ) -> None: """ Determine needed input filepaths from np-exp and lims, get session @@ -36,27 +39,29 @@ def __init__( settings to specify the different laser states for this experiment. Otherwise, the default is used from naming_utils. """ - if json_settings.get("opto_conditions_map", None) is None: + if isinstance(job_settings, str): + self.job_settings = JobSettings.model_validate_json(job_settings) + else: + self.job_settings = job_settings + + if self.job_settings.get("opto_conditions_map", None) is None: self.opto_conditions_map = names.DEFAULT_OPTO_CONDITIONS else: - self.opto_conditions_map = json_settings["opto_conditions_map"] - overwrite_tables = json_settings.get("overwrite_tables", False) + self.opto_conditions_map = self.job_settings["opto_conditions_map"] - self.json_settings = json_settings - session_inst = np_session.Session(session_id) - self.mtrain = session_inst.mtrain - self.npexp_path = session_inst.npexp_path - self.folder = session_inst.folder + sessions_root = Path(self.job_settings.get('sessions_root')) + self.session_path = self.get_session_path(session_id, sessions_root) + self.folder = self.get_folder(session_id, sessions_root) - self.pkl_path = self.npexp_path / f"{self.folder}.stim.pkl" - self.opto_pkl_path = self.npexp_path / f"{self.folder}.opto.pkl" + self.pkl_path = self.session_path / f"{self.folder}.stim.pkl" + self.opto_pkl_path = self.session_path / f"{self.folder}.opto.pkl" self.opto_table_path = ( - self.npexp_path / f"{self.folder}_opto_epochs.csv" + self.session_path / f"{self.folder}_opto_epochs.csv" ) self.stim_table_path = ( - self.npexp_path / f"{self.folder}_stim_epochs.csv" + self.session_path / f"{self.folder}_stim_epochs.csv" ) - self.sync_path = self.npexp_path / f"{self.folder}.sync" + self.sync_path = self.session_path / f"{self.folder}.sync" sync_data = sync.load_sync(self.sync_path) self.session_start = sync.get_start_time(sync_data) @@ -68,13 +73,20 @@ def __init__( self.session_end, ) - if not self.stim_table_path.exists() or overwrite_tables: + self.mouse_id = self.folder.split("_")[1] + self.session_uuid = self.get_session_uuid() + self.mtrain_regimen = self.get_mtrain() + + if ( + not self.stim_table_path.exists() + or self.job_settings['overwrite_tables'] + ): print("building stim table") self.build_stimulus_table() if ( self.opto_pkl_path.exists() and not self.opto_table_path.exists() - or overwrite_tables + or self.job_settings['overwrite_tables'] ): print("building opto table") self.build_optogenetics_table() @@ -84,6 +96,29 @@ def __init__( if self.opto_table_path.exists(): self.stim_epochs.append(self.epoch_from_opto_table()) + def get_folder(self, session_id, npexp_root) -> str: + """returns the directory name of the session on the np-exp directory""" + for subfolder in npexp_root.iterdir(): + if subfolder.name.split("_")[0] == session_id: + return subfolder.name + else: + raise Exception("Session folder not found in np-exp") + + def get_session_path(self, session_id, npexp_root) -> Path: + """returns the path to the session on allen's np-exp directory""" + return npexp_root / self.get_folder(session_id, npexp_root) + + def get_session_uuid(self) -> str: + """returns session uuid from pickle file""" + return pkl.load_pkl(self.pkl_path)["session_uuid"] + + def get_mtrain(self) -> dict: + """Returns dictionary containing 'id', 'name', 'stages', 'states'""" + server = self.job_settings.mtrain_server + req = f"{server}/behavior_session/{self.session_uuid}/details" + mtrain_response = requests.get(req).json() + return mtrain_response["result"]["regimen"] + def build_stimulus_table( self, minimum_spontaneous_activity_duration=0.0, @@ -232,9 +267,9 @@ def epoch_from_opto_table(self) -> session_schema.StimulusEpoch: stim = aind_data_schema.core.session.StimulusModality script_obj = aind_data_schema.components.devices.Software( - name=self.mtrain["regimen"]["name"], + name=self.mtrain_regimen["name"], version="1.0", - url=self.mtrain["regimen"]["script"], + url=self.mtrain_regimen, ) opto_table = pd.read_csv(self.opto_table_path) @@ -346,9 +381,9 @@ def epochs_from_stim_table(self) -> list[session_schema.StimulusEpoch]: ) script_obj = aind_data_schema.components.devices.Software( - name=self.mtrain["regimen"]["name"], + name=self.mtrain_regimen["name"], version="1.0", - url=self.mtrain["regimen"]["script"], + url=self.mtrain_regimen["script"], ) schema_epochs = [] From 3ebb3f4ac5705419c8e94d4219e7016901417dd3 Mon Sep 17 00:00:00 2001 From: Mekhla Kapoor <54870020+mekhlakapoor@users.noreply.github.com> Date: Tue, 22 Oct 2024 18:11:21 -0700 Subject: [PATCH 2/4] overwrite example versions w/ default (#184) --- pyproject.toml | 2 +- .../base-missing-probe_rig.json | 2 +- tests/resources/dynamic_routing/base_rig.json | 2 +- .../open_ephys/open-ephys-inferred_rig.json | 2 +- .../resources/open_ephys/open-ephys_rig.json | 2 +- tests/test_bergamo/test_session.py | 4 ++++ tests/test_bruker/test_session.py | 5 +++- tests/test_dynamic_routing/test_mvr_rig.py | 19 +++++++++------ tests/test_dynamic_routing/test_sync_rig.py | 19 ++++++++------- tests/test_fip/test_session.py | 4 +++- tests/test_gather_metadata.py | 8 +++---- tests/test_mesoscope/test_session.py | 24 +++++++++++-------- tests/test_open_ephys/test_rig.py | 10 +++++--- 13 files changed, 64 insertions(+), 39 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 6df7dff9..924f3ded 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,7 +42,7 @@ all = [ ] schema = [ - "aind-data-schema>=1.0.0,<2.0", + "aind-data-schema>=1.1.1,<2.0", "pydantic<2.9" ] diff --git a/tests/resources/dynamic_routing/base-missing-probe_rig.json b/tests/resources/dynamic_routing/base-missing-probe_rig.json index 91186522..a3c4fb5b 100644 --- a/tests/resources/dynamic_routing/base-missing-probe_rig.json +++ b/tests/resources/dynamic_routing/base-missing-probe_rig.json @@ -1,6 +1,6 @@ { "describedBy": "https://raw.githubusercontent.com/AllenNeuralDynamics/aind-data-schema/main/src/aind_data_schema/core/rig.py", - "schema_version": "1.0.0", + "schema_version": "1.0.1", "rig_id": "327_NP2_20240401", "modification_date": "2024-04-01", "mouse_platform": { diff --git a/tests/resources/dynamic_routing/base_rig.json b/tests/resources/dynamic_routing/base_rig.json index 2e51b291..e89ee467 100644 --- a/tests/resources/dynamic_routing/base_rig.json +++ b/tests/resources/dynamic_routing/base_rig.json @@ -1,6 +1,6 @@ { "describedBy": "https://raw.githubusercontent.com/AllenNeuralDynamics/aind-data-schema/main/src/aind_data_schema/core/rig.py", - "schema_version": "1.0.0", + "schema_version": "1.0.1", "rig_id": "327_NP2_20240401", "modification_date": "2024-04-01", "mouse_platform": { diff --git a/tests/resources/open_ephys/open-ephys-inferred_rig.json b/tests/resources/open_ephys/open-ephys-inferred_rig.json index 51e52fad..9133c67d 100644 --- a/tests/resources/open_ephys/open-ephys-inferred_rig.json +++ b/tests/resources/open_ephys/open-ephys-inferred_rig.json @@ -1,6 +1,6 @@ { "describedBy": "https://raw.githubusercontent.com/AllenNeuralDynamics/aind-data-schema/main/src/aind_data_schema/core/rig.py", - "schema_version": "1.0.0", + "schema_version": "1.0.1", "rig_id": "327_NP2_20240401", "modification_date": "2024-04-01", "mouse_platform": { diff --git a/tests/resources/open_ephys/open-ephys_rig.json b/tests/resources/open_ephys/open-ephys_rig.json index ef845368..8d51b537 100644 --- a/tests/resources/open_ephys/open-ephys_rig.json +++ b/tests/resources/open_ephys/open-ephys_rig.json @@ -1,6 +1,6 @@ { "describedBy": "https://raw.githubusercontent.com/AllenNeuralDynamics/aind-data-schema/main/src/aind_data_schema/core/rig.py", - "schema_version": "1.0.0", + "schema_version": "1.0.1", "rig_id": "327_NP2_20240401", "modification_date": "2024-04-01", "mouse_platform": { diff --git a/tests/test_bergamo/test_session.py b/tests/test_bergamo/test_session.py index da0c2fdd..f3ad2d2b 100644 --- a/tests/test_bergamo/test_session.py +++ b/tests/test_bergamo/test_session.py @@ -8,6 +8,7 @@ from pathlib import Path from unittest.mock import MagicMock, patch +from aind_data_schema.core.session import Session from aind_metadata_mapper.bergamo.session import BergamoEtl, JobSettings RESOURCES_DIR = ( @@ -41,6 +42,9 @@ def setUpClass(cls): fov_targeted_structure="Primary Motor Cortex", notes="test upload", ) + expected_session_contents["schema_version"] = Session.model_fields[ + "schema_version" + ].default cls.expected_session = expected_session_contents def test_class_constructor(self): diff --git a/tests/test_bruker/test_session.py b/tests/test_bruker/test_session.py index 603ab2ba..c83eb7cb 100644 --- a/tests/test_bruker/test_session.py +++ b/tests/test_bruker/test_session.py @@ -16,7 +16,7 @@ MagneticStrength, ScannerLocation, ) - +from aind_data_schema.core.session import Session from aind_metadata_mapper.bruker.session import JobSettings, MRIEtl RESOURCES_DIR = ( @@ -73,6 +73,9 @@ def setUpClass(cls): with open(EXPECTED_SESSION, "r") as f: contents = json.load(f) + contents["schema_version"] = Session.model_fields[ + "schema_version" + ].default cls.expected_session = contents example_job_settings = JobSettings( diff --git a/tests/test_dynamic_routing/test_mvr_rig.py b/tests/test_dynamic_routing/test_mvr_rig.py index 878e8fec..385b47dd 100644 --- a/tests/test_dynamic_routing/test_mvr_rig.py +++ b/tests/test_dynamic_routing/test_mvr_rig.py @@ -1,10 +1,12 @@ """Tests for the MVR rig ETL.""" import os +import json import unittest from pathlib import Path from unittest.mock import MagicMock, patch +from aind_data_schema.core.rig import Rig from aind_metadata_mapper.dynamic_routing.mvr_rig import ( # type: ignore MvrRigEtl, ) @@ -17,6 +19,8 @@ / "dynamic_routing" ) +MVR_PATH = Path(RESOURCES_DIR / "mvr_rig.json") + class TestMvrRigEtl(unittest.TestCase): """Tests dxdiag utilities in for the dynamic_routing project.""" @@ -76,13 +80,14 @@ def test_run_job_bad_mapping(self, mock_write_standard_file: MagicMock): def setUp(self): """Sets up test resources.""" - ( - self.input_source, - self.output_dir, - self.expected, - ) = test_utils.setup_neuropixels_etl_resources( - RESOURCES_DIR / "mvr_rig.json", - ) + self.input_source = RESOURCES_DIR / "base_rig.json" + self.output_dir = Path("abc") + with open(MVR_PATH, "r") as f: + mvr_contents = json.load(f) + mvr_contents["schema_version"] = Rig.model_fields[ + "schema_version" + ].default + self.expected = Rig(**mvr_contents) if __name__ == "__main__": diff --git a/tests/test_dynamic_routing/test_sync_rig.py b/tests/test_dynamic_routing/test_sync_rig.py index e1bba043..62c57c4a 100644 --- a/tests/test_dynamic_routing/test_sync_rig.py +++ b/tests/test_dynamic_routing/test_sync_rig.py @@ -1,6 +1,7 @@ """Tests for Sync rig ETL.""" import os +import json import unittest from pathlib import Path from unittest.mock import MagicMock, patch @@ -8,7 +9,7 @@ from aind_metadata_mapper.dynamic_routing.sync_rig import ( # type: ignore SyncRigEtl, ) -from tests.test_dynamic_routing import test_utils as test_utils +from aind_data_schema.core.rig import Rig RESOURCES_DIR = ( Path(os.path.dirname(os.path.realpath(__file__))) @@ -16,6 +17,7 @@ / "resources" / "dynamic_routing" ) +SYNC_PATH = Path(RESOURCES_DIR / "sync_rig.json") class TestSyncRigEtl(unittest.TestCase): @@ -51,13 +53,14 @@ def test_etl( def setUp(self): """Sets up test resources.""" - ( - self.input_source, - self.output_dir, - self.expected, - ) = test_utils.setup_neuropixels_etl_resources( - RESOURCES_DIR / "sync_rig.json" - ) + self.input_source = RESOURCES_DIR / "base_rig.json" + self.output_dir = Path("abc") + with open(SYNC_PATH, "r") as f: + mvr_contents = json.load(f) + mvr_contents["schema_version"] = Rig.model_fields[ + "schema_version" + ].default + self.expected = Rig(**mvr_contents) if __name__ == "__main__": diff --git a/tests/test_fip/test_session.py b/tests/test_fip/test_session.py index fcfb05c1..79203821 100644 --- a/tests/test_fip/test_session.py +++ b/tests/test_fip/test_session.py @@ -80,7 +80,9 @@ def setUpClass(cls): mouse_platform_name="Disc", active_mouse_platform=False, ) - + expected_session_contents["schema_version"] = Session.model_fields[ + "schema_version" + ].default cls.expected_session = Session.model_validate_json( json.dumps(expected_session_contents) ) diff --git a/tests/test_gather_metadata.py b/tests/test_gather_metadata.py index 8b4ab166..2bd4c773 100644 --- a/tests/test_gather_metadata.py +++ b/tests/test_gather_metadata.py @@ -1031,13 +1031,13 @@ def test_from_job_settings_file(self): """Tests that users can set a session config file when requesting GatherMetadataJob""" + bergamo_settings = BergamoSessionJobSettings( + user_settings_config_file=EXAMPLE_BERGAMO_CONFIGS + ) test_configs = { "directory_to_write_to": RESOURCES_DIR, "session_settings": { - "job_settings": { - "job_settings_name": "Bergamo", - "user_settings_config_file": EXAMPLE_BERGAMO_CONFIGS, - } + "job_settings": bergamo_settings.model_dump(), }, } job_settings = JobSettings.model_validate_json( diff --git a/tests/test_mesoscope/test_session.py b/tests/test_mesoscope/test_session.py index 3be844b9..87c0b069 100644 --- a/tests/test_mesoscope/test_session.py +++ b/tests/test_mesoscope/test_session.py @@ -37,7 +37,11 @@ def setUpClass(cls) -> None: with open(EXAMPLE_EXTRACT, "r") as f: cls.example_extract = json.load(f) with open(EXAMPLE_SESSION, "r") as f: - cls.example_session = json.load(f) + expected_session = json.load(f) + expected_session["schema_version"] = Session.model_fields[ + "schema_version" + ].default + cls.example_session = expected_session cls.example_scanimage_meta = { "lines_per_frame": 512, "pixels_per_line": 512, @@ -177,15 +181,15 @@ def test_transform(self, mock_open, mock_scanimage) -> None: # mock scanimage metadata mock_meta = [{}] - mock_meta[0][ - "SI.hRoiManager.linesPerFrame" - ] = self.example_scanimage_meta["lines_per_frame"] - mock_meta[0][ - "SI.hRoiManager.pixelsPerLine" - ] = self.example_scanimage_meta["pixels_per_line"] - mock_meta[0][ - "SI.hRoiManager.scanZoomFactor" - ] = self.example_scanimage_meta["fov_scale_factor"] + mock_meta[0]["SI.hRoiManager.linesPerFrame"] = ( + self.example_scanimage_meta["lines_per_frame"] + ) + mock_meta[0]["SI.hRoiManager.pixelsPerLine"] = ( + self.example_scanimage_meta["pixels_per_line"] + ) + mock_meta[0]["SI.hRoiManager.scanZoomFactor"] = ( + self.example_scanimage_meta["fov_scale_factor"] + ) mock_scanimage.return_value = mock_meta extract = etl._extract() diff --git a/tests/test_open_ephys/test_rig.py b/tests/test_open_ephys/test_rig.py index 99209f05..8120e475 100644 --- a/tests/test_open_ephys/test_rig.py +++ b/tests/test_open_ephys/test_rig.py @@ -1,6 +1,7 @@ """Tests for the dynamic_routing open open_ephys rig ETL.""" import os +import json import unittest from pathlib import Path from unittest.mock import MagicMock, patch @@ -27,9 +28,12 @@ class TestOpenEphysRigEtl(unittest.TestCase): def load_rig(self, model_path: Path): """Convenience function to load a rig model.""" - return Rig.model_validate_json( - (model_path).read_text(), - ) + with open(model_path, "r") as f: + expected_rig = json.load(f) + expected_rig["schema_version"] = Rig.model_fields[ + "schema_version" + ].default + return Rig(**expected_rig) def test_transform(self): """Tests etl transform.""" From c9bdbcb4aec455412b896140940ef49c1af0a00f Mon Sep 17 00:00:00 2001 From: Mekhla Kapoor <54870020+mekhlakapoor@users.noreply.github.com> Date: Wed, 23 Oct 2024 10:38:52 -0700 Subject: [PATCH 3/4] adds schema version hack to integration test (#186) --- tests/integration/bergamo/session.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/integration/bergamo/session.py b/tests/integration/bergamo/session.py index 5ee1c2b4..2c2bba8f 100644 --- a/tests/integration/bergamo/session.py +++ b/tests/integration/bergamo/session.py @@ -7,6 +7,7 @@ import unittest from pathlib import Path +from aind_data_schema.core.session import Session from aind_metadata_mapper.bergamo.models import JobSettings from aind_metadata_mapper.bergamo.session import BergamoEtl @@ -28,6 +29,9 @@ def setUpClass(cls) -> None: """Set up the class.""" with open(EXPECTED_OUTPUT_FILE_PATH, "r") as f: expected_output_json = json.load(f) + expected_output_json["schema_version"] = Session.model_fields[ + "schema_version" + ].default cls.expected_output = expected_output_json def test_run_job(self): From 65ab8a78a489cdb392d717cc6bb2f955a13821a7 Mon Sep 17 00:00:00 2001 From: Mekhla Kapoor <54870020+mekhlakapoor@users.noreply.github.com> Date: Wed, 23 Oct 2024 11:07:11 -0700 Subject: [PATCH 4/4] release v0.19.0 --- src/aind_metadata_mapper/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aind_metadata_mapper/__init__.py b/src/aind_metadata_mapper/__init__.py index 366b5f5b..099e394c 100644 --- a/src/aind_metadata_mapper/__init__.py +++ b/src/aind_metadata_mapper/__init__.py @@ -1,3 +1,3 @@ """Init package""" -__version__ = "0.18.4" +__version__ = "0.19.0"