diff --git a/.github/workflows/run_dev_tests.yml b/.github/workflows/run_dev_tests.yml index a5a1a187..b91f1529 100644 --- a/.github/workflows/run_dev_tests.yml +++ b/.github/workflows/run_dev_tests.yml @@ -10,7 +10,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [ '3.10', '3.11' ] + python-version: [ '3.9', '3.10', '3.11' ] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} diff --git a/.github/workflows/run_integration_test_dev.yml b/.github/workflows/run_integration_test_dev.yml new file mode 100644 index 00000000..82c96ae1 --- /dev/null +++ b/.github/workflows/run_integration_test_dev.yml @@ -0,0 +1,51 @@ +name: Run integration tests + +on: + push: + branches: + - dev + +jobs: + integration_tests: + runs-on: ubuntu-latest + permissions: + id-token: write + contents: read + env: + AWS_METADATA_MAPPER_ROLE: ${{ secrets.AWS_METADATA_MAPPER_ROLE_PROD }} + AWS_METADATA_MAPPER_BUCKET: ${{ vars.AWS_METADATA_MAPPER_BUCKET_PROD }} + AWS_REGION: ${{ vars.AWS_REGION_PROD }} + MOUNT_S3_URL: ${{ vars.MOUNT_S3_URL }} + steps: + - uses: actions/checkout@v3 + - name: Set up Python 3.10 + uses: actions/setup-python@v3 + with: + python-version: '3.10' + - name: Install dependencies + run: + python -m pip install -e .[all] + - name: Configure aws credentials + uses: aws-actions/configure-aws-credentials@v2 + with: + role-to-assume: ${{ env.AWS_METADATA_MAPPER_ROLE }} + role-session-name: github-integration-test-session + aws-region: ${{ env.AWS_REGION }} + - name: install mountpoint-s3 + run: | + wget $MOUNT_S3_URL + sudo apt-get update + sudo apt-get install ./mount-s3.deb + - name: mount s3 bucket + run: | + mkdir bucket_mt + mount-s3 $AWS_METADATA_MAPPER_BUCKET bucket_mt + - name: run integration tests + run: | + python tests/integration/bergamo/session.py --input_source "bucket_mt/metadata-mapper-integration-testing/bergamo" IntegrationTestBergamo + umount bucket_mt + + + + + diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 931f9ed8..b256b399 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -21,7 +21,7 @@ Then in bash, run pip install -e .[dev] ``` to set up your environment. Once these steps are complete, you can get developing🚀. Data is organized by acquisition machine, so if you're adding a new machine, create a new directory. Otherwise, put your code in its corresponding directory. -### Testing +### Unit Testing Testing is required to open a PR in this repository to ensure robustness and reliability of our codebase. - **1:1 Correspondence:** Structure unit tests in a manner that mirrors the module structure. @@ -39,5 +39,12 @@ Testing is required to open a PR in this repository to ensure robustness and rel coverage html ``` and find the report in the htmlcov/index.html. - + There are several libraries used to run linters and check documentation. We've included these in the development package. You can run them as described [here](https://github.com/AllenNeuralDynamics/aind-metadata-mapper/blob/main/README.md#linters-and-testing). + +### Integration Testing +To ensure that an ETL runs as expected against data on the VAST, you can run an integration test locally by pointing to the input directory on VAST. For example, to test the 'bergamo' package: +```bash + python tests/integration/bergamo/session.py --input_source "/allen/aind/scratch/svc_aind_upload/test_data_sets/bergamo" IntegrationTestBergamo + ``` + diff --git a/pyproject.toml b/pyproject.toml index f6a80f43..ac04ed75 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta" name = "aind-metadata-mapper" description = "Generated from aind-library-template" license = {text = "MIT"} -requires-python = ">=3.10" +requires-python = ">=3.8" authors = [ {name = "Allen Institute for Neural Dynamics"} ] @@ -20,14 +20,11 @@ dependencies = [ "aind-data-schema==0.36.0", "aind-data-schema-models==0.1.7", "pydantic-settings>=2.0", - "pandas", - "numpy", - "pytz" ] [project.optional-dependencies] dev = [ - "aind-metadata-mapper[all]", + "aind-metadata-mapper[all] ; python_version >= '3.9'", "black", "coverage", "flake8", @@ -35,18 +32,19 @@ dev = [ "isort", "Sphinx", "furo", - "pyyaml>=6.0.0", ] all = [ "aind-metadata-mapper[bergamo]", "aind-metadata-mapper[bruker]", "aind-metadata-mapper[mesoscope]", - "aind-metadata-mapper[openephys]" + "aind-metadata-mapper[openephys]", + "aind-metadata-mapper[dynamicrouting]", ] bergamo = [ - "scanimage-tiff-reader==1.4.1.4" + "scanimage-tiff-reader==1.4.1.4", + "numpy", ] bruker = [ @@ -64,6 +62,12 @@ openephys = [ "np_session", "npc_ephys", "scipy", + "pandas", + "numpy", +] + +dynamicrouting = [ + "pyyaml>=6.0.0", ] [tool.setuptools.packages.find] diff --git a/src/aind_metadata_mapper/__init__.py b/src/aind_metadata_mapper/__init__.py index 77cd89c4..b335a61e 100644 --- a/src/aind_metadata_mapper/__init__.py +++ b/src/aind_metadata_mapper/__init__.py @@ -1,3 +1,3 @@ """Init package""" -__version__ = "0.14.0" +__version__ = "0.15.0" diff --git a/src/aind_metadata_mapper/bergamo/models.py b/src/aind_metadata_mapper/bergamo/models.py new file mode 100644 index 00000000..c2827328 --- /dev/null +++ b/src/aind_metadata_mapper/bergamo/models.py @@ -0,0 +1,79 @@ +"""Module defining JobSettings for Bergamo ETL""" +from decimal import Decimal +from pathlib import Path +from typing import List, Literal, Optional + +from pydantic import Field +from pydantic_settings import BaseSettings + + +class JobSettings(BaseSettings): + """Data that needs to be input by user. Can be pulled from env vars with + BERGAMO prefix or set explicitly.""" + + job_settings_name: Literal["Bergamo"] = "Bergamo" + input_source: Path = Field( + ..., description="Directory of files that need to be parsed." + ) + output_directory: Optional[Path] = Field( + default=None, + description=( + "Directory where to save the json file to. If None, then json" + " contents will be returned in the Response message." + ), + ) + # mandatory fields: + experimenter_full_name: List[str] + subject_id: str + imaging_laser_wavelength: int # user defined + fov_imaging_depth: int + fov_targeted_structure: str + notes: Optional[str] + + # fields with default values + mouse_platform_name: str = "Standard Mouse Tube" # FROM RIG JSON + active_mouse_platform: bool = False + session_type: str = "BCI" + iacuc_protocol: str = "2109" + # should match rig json: + rig_id: str = "442 Bergamo 2p photostim" + behavior_camera_names: List[str] = [ + "Side Face Camera", + "Bottom Face Camera", + ] + ch1_filter_names: List[str] = [ + "Green emission filter", + "Emission dichroic", + ] + ch1_detector_name: str = "Green PMT" + ch1_daq_name: str = "PXI" + ch2_filter_names: List[str] = ["Red emission filter", "Emission dichroic"] + ch2_detector_name: str = "Red PMT" + ch2_daq_name: str = "PXI" + imaging_laser_name: str = "Chameleon Laser" + + photostim_laser_name: str = "Monaco Laser" + stimulus_device_names: List[str] = ["speaker", "lickport"] # FROM RIG JSON + photostim_laser_wavelength: int = 1040 + fov_coordinate_ml: Decimal = Decimal("1.5") + fov_coordinate_ap: float = Decimal("1.5") + fov_reference: str = "Bregma" + + starting_lickport_position: List[float] = [ + 0, + -6, + 0, + ] # in mm from face of the mouse + behavior_task_name: str = "single neuron BCI conditioning" + hit_rate_trials_0_10: Optional[float] = None + hit_rate_trials_20_40: Optional[float] = None + total_hits: Optional[float] = None + average_hit_rate: Optional[float] = None + trial_num: Optional[float] = None + # ZoneInfo object doesn't serialize well, so we can define it as a str + timezone: str = "US/Pacific" + + class Config: + """Config to set env var prefix to BERGAMO""" + + env_prefix = "BERGAMO_" diff --git a/src/aind_metadata_mapper/bergamo/session.py b/src/aind_metadata_mapper/bergamo/session.py index 4aea2313..1a23bb69 100644 --- a/src/aind_metadata_mapper/bergamo/session.py +++ b/src/aind_metadata_mapper/bergamo/session.py @@ -11,7 +11,7 @@ from decimal import Decimal from enum import Enum from pathlib import Path -from typing import Dict, List, Optional, Tuple, Union +from typing import Dict, List, Tuple, Union from zoneinfo import ZoneInfo import numpy as np @@ -45,87 +45,13 @@ TriggerType, ) from aind_data_schema_models.units import PowerUnit -from pydantic import Field -from pydantic_settings import BaseSettings from ScanImageTiffReader import ScanImageTiffReader +from aind_metadata_mapper.bergamo.models import JobSettings from aind_metadata_mapper.core import GenericEtl, JobResponse -class JobSettings(BaseSettings): - """Data that needs to be input by user. Can be pulled from env vars with - BERGAMO prefix or set explicitly.""" - - input_source: Path = Field( - ..., description="Directory of files that need to be parsed." - ) - output_directory: Optional[Path] = Field( - default=None, - description=( - "Directory where to save the json file to. If None, then json" - " contents will be returned in the Response message." - ), - ) - # mandatory fields: - experimenter_full_name: List[str] - subject_id: str - imaging_laser_wavelength: int # user defined - fov_imaging_depth: int - fov_targeted_structure: str - notes: Optional[str] - - # fields with default values - mouse_platform_name: str = "Standard Mouse Tube" # FROM RIG JSON - active_mouse_platform: bool = False - session_type: str = "BCI" - iacuc_protocol: str = "2109" - # should match rig json: - rig_id: str = "442 Bergamo 2p photostim" - behavior_camera_names: List[str] = [ - "Side Face Camera", - "Bottom Face Camera", - ] - ch1_filter_names: List[str] = [ - "Green emission filter", - "Emission dichroic", - ] - ch1_detector_name: str = "Green PMT" - ch1_daq_name: str = "PXI" - ch2_filter_names: List[str] = ["Red emission filter", "Emission dichroic"] - ch2_detector_name: str = "Red PMT" - ch2_daq_name: str = "PXI" - imaging_laser_name: str = "Chameleon Laser" - - photostim_laser_name: str = "Monaco Laser" - stimulus_device_names: List[str] = ["speaker", "lickport"] # FROM RIG JSON - photostim_laser_wavelength: int = 1040 - fov_coordinate_ml: Decimal = Decimal("1.5") - fov_coordinate_ap: float = Decimal("1.5") - fov_reference: str = "Bregma" - - starting_lickport_position: list[float] = [ - 0, - -6, - 0, - ] # in mm from face of the mouse - behavior_task_name: str = "single neuron BCI conditioning" - hit_rate_trials_0_10: Optional[float] = None - hit_rate_trials_20_40: Optional[float] = None - total_hits: Optional[float] = None - average_hit_rate: Optional[float] = None - trial_num: Optional[float] = None - # ZoneInfo object doesn't serialize well, so we can define it as a str - timezone: str = "US/Pacific" - - class Config: - """Config to set env var prefix to BERGAMO""" - - env_prefix = "BERGAMO_" - - # This class makes it easier to flag which tif files are which expected type - - class TifFileGroup(str, Enum): """Type of stimulation a group of files belongs to""" @@ -137,8 +63,6 @@ class TifFileGroup(str, Enum): # This class will hold the metadata information pulled from the tif files # with minimal parsing. - - @dataclass(frozen=True) class RawImageInfo: """Raw metadata from a tif file""" diff --git a/src/aind_metadata_mapper/bruker/models.py b/src/aind_metadata_mapper/bruker/models.py new file mode 100644 index 00000000..2232c2b1 --- /dev/null +++ b/src/aind_metadata_mapper/bruker/models.py @@ -0,0 +1,40 @@ +"""Module defining JobSettings for Bruker ETL""" + +from pathlib import Path +from typing import List, Literal, Optional + +from aind_data_schema.components.devices import ( + MagneticStrength, + ScannerLocation, +) +from pydantic import Field +from pydantic_settings import BaseSettings + + +class JobSettings(BaseSettings): + """Data that needs to be input by user.""" + + job_settings_name: Literal["Bruker"] = "Bruker" + data_path: Path + output_directory: Optional[Path] = Field( + default=None, + description=( + "Directory where to save the json file to. If None, then json" + " contents will be returned in the Response message." + ), + ) + experimenter_full_name: List[str] + protocol_id: str = Field(default="", description="Protocol ID") + collection_tz: str = Field( + default="America/Los_Angeles", + description="Timezone string of the collection site", + ) + session_type: str + primary_scan_number: int + setup_scan_number: int + scanner_name: str + scan_location: ScannerLocation + magnetic_strength: MagneticStrength + subject_id: str + iacuc_protocol: str + session_notes: str diff --git a/src/aind_metadata_mapper/bruker/session.py b/src/aind_metadata_mapper/bruker/session.py index 2e231213..23fbc339 100644 --- a/src/aind_metadata_mapper/bruker/session.py +++ b/src/aind_metadata_mapper/bruker/session.py @@ -7,8 +7,7 @@ import traceback from datetime import datetime, timedelta from decimal import Decimal -from pathlib import Path -from typing import List, Optional, Union +from typing import List, Union from zoneinfo import ZoneInfo from aind_data_schema.components.coordinates import ( @@ -16,11 +15,7 @@ Scale3dTransform, Translation3dTransform, ) -from aind_data_schema.components.devices import ( - MagneticStrength, - Scanner, - ScannerLocation, -) +from aind_data_schema.components.devices import Scanner from aind_data_schema.core.session import ( MRIScan, MriScanSequence, @@ -32,40 +27,10 @@ from aind_data_schema_models.modalities import Modality from aind_data_schema_models.units import TimeUnit from bruker2nifti._metadata import BrukerMetadata -from pydantic import Field -from pydantic_settings import BaseSettings +from aind_metadata_mapper.bruker.models import JobSettings from aind_metadata_mapper.core import GenericEtl, JobResponse - -class JobSettings(BaseSettings): - """Data that needs to be input by user.""" - - data_path: Path - output_directory: Optional[Path] = Field( - default=None, - description=( - "Directory where to save the json file to. If None, then json" - " contents will be returned in the Response message." - ), - ) - experimenter_full_name: List[str] - protocol_id: str = Field(default="", description="Protocol ID") - collection_tz: str = Field( - default="America/Los_Angeles", - description="Timezone string of the collection site", - ) - session_type: str - primary_scan_number: int - setup_scan_number: int - scanner_name: str - scan_location: ScannerLocation - magnetic_strength: MagneticStrength - subject_id: str - iacuc_protocol: str - session_notes: str - - DATETIME_FORMAT = "%H:%M:%S %d %b %Y" LENGTH_FORMAT = "%Hh%Mm%Ss%fms" diff --git a/src/aind_metadata_mapper/core.py b/src/aind_metadata_mapper/core.py index ee2f57a8..46c192df 100644 --- a/src/aind_metadata_mapper/core.py +++ b/src/aind_metadata_mapper/core.py @@ -8,21 +8,13 @@ from typing import Any, Generic, Optional, TypeVar, Union from aind_data_schema.base import AindCoreModel -from pydantic import BaseModel, ConfigDict, Field, ValidationError +from pydantic import ValidationError from pydantic_settings import BaseSettings +from aind_metadata_mapper.models import JobResponse _T = TypeVar("_T", bound=BaseSettings) -class JobResponse(BaseModel): - """Standard model of a JobResponse.""" - - model_config = ConfigDict(extra="forbid") - status_code: int - message: Optional[str] = Field(None) - data: Optional[str] = Field(None) - - class GenericEtl(ABC, Generic[_T]): """A generic etl class. Child classes will need to create a JobSettings object that is json serializable. Child class will also need to implement diff --git a/src/aind_metadata_mapper/fip/models.py b/src/aind_metadata_mapper/fip/models.py new file mode 100644 index 00000000..608a521c --- /dev/null +++ b/src/aind_metadata_mapper/fip/models.py @@ -0,0 +1,36 @@ +"""Module defining JobSettings for FIP ETL""" + +from datetime import datetime +from pathlib import Path +from typing import List, Literal, Optional + +from pydantic import Field +from pydantic_settings import BaseSettings + + +class JobSettings(BaseSettings): + """Data that needs to be input by user.""" + + job_settings_name: Literal["FIP"] = "FIP" + output_directory: Optional[Path] = Field( + default=None, + description=( + "Directory where to save the json file to. If None, then json" + " contents will be returned in the Response message." + ), + ) + + string_to_parse: str + experimenter_full_name: List[str] + session_start_time: datetime + notes: str + labtracks_id: str + iacuc_protocol: str + light_source_list: List[dict] + detector_list: List[dict] + fiber_connections_list: List[dict] + + rig_id: str = "ophys_rig" + session_type: str = "Foraging_Photometry" + mouse_platform_name: str = "Disc" + active_mouse_platform: bool = False diff --git a/src/aind_metadata_mapper/fip/session.py b/src/aind_metadata_mapper/fip/session.py index 35233f6b..49deef09 100644 --- a/src/aind_metadata_mapper/fip/session.py +++ b/src/aind_metadata_mapper/fip/session.py @@ -2,9 +2,8 @@ import re from dataclasses import dataclass -from datetime import datetime, timedelta -from pathlib import Path -from typing import List, Optional, Union +from datetime import timedelta +from typing import Union from aind_data_schema.components.stimulus import OptoStimulation, PulseShape from aind_data_schema.core.session import ( @@ -17,37 +16,9 @@ Stream, ) from aind_data_schema_models.modalities import Modality -from pydantic import Field -from pydantic_settings import BaseSettings from aind_metadata_mapper.core import GenericEtl, JobResponse - - -class JobSettings(BaseSettings): - """Data that needs to be input by user.""" - - output_directory: Optional[Path] = Field( - default=None, - description=( - "Directory where to save the json file to. If None, then json" - " contents will be returned in the Response message." - ), - ) - - string_to_parse: str - experimenter_full_name: List[str] - session_start_time: datetime - notes: str - labtracks_id: str - iacuc_protocol: str - light_source_list: List[dict] - detector_list: List[dict] - fiber_connections_list: List[dict] - - rig_id: str = "ophys_rig" - session_type: str = "Foraging_Photometry" - mouse_platform_name: str = "Disc" - active_mouse_platform: bool = False +from aind_metadata_mapper.fip.models import JobSettings @dataclass(frozen=True) diff --git a/src/aind_metadata_mapper/gather_metadata.py b/src/aind_metadata_mapper/gather_metadata.py index 58bbce51..97070453 100644 --- a/src/aind_metadata_mapper/gather_metadata.py +++ b/src/aind_metadata_mapper/gather_metadata.py @@ -5,7 +5,7 @@ import logging import sys from pathlib import Path -from typing import List, Optional, Type +from typing import Optional, Type import requests from aind_data_schema.base import AindCoreModel @@ -17,97 +17,28 @@ from aind_data_schema.core.instrument import Instrument from aind_data_schema.core.metadata import Metadata from aind_data_schema.core.procedures import Procedures -from aind_data_schema.core.processing import PipelineProcess, Processing +from aind_data_schema.core.processing import Processing from aind_data_schema.core.rig import Rig from aind_data_schema.core.session import Session from aind_data_schema.core.subject import Subject -from aind_data_schema_models.modalities import Modality -from aind_data_schema_models.organizations import Organization from aind_data_schema_models.pid_names import PIDName -from pydantic import Field, ValidationError -from pydantic_settings import BaseSettings +from pydantic import ValidationError -from aind_metadata_mapper.smartspim.acquisition import ( - JobSettings as SmartSpimAcquisitionJobSettings, +from aind_metadata_mapper.bergamo.models import ( + JobSettings as BergamoSessionJobSettings, ) +from aind_metadata_mapper.bergamo.session import BergamoEtl +from aind_metadata_mapper.bruker.models import ( + JobSettings as BrukerSessionJobSettings, +) +from aind_metadata_mapper.bruker.session import MRIEtl +from aind_metadata_mapper.fip.models import ( + JobSettings as FipSessionJobSettings, +) +from aind_metadata_mapper.fip.session import FIBEtl +from aind_metadata_mapper.mesoscope.session import MesoscopeEtl from aind_metadata_mapper.smartspim.acquisition import SmartspimETL - - -class AcquisitionSettings(BaseSettings): - """Fields needed to retrieve acquisition metadata""" - - # TODO: we can change this to a tagged union once more acquisition settings - # are added - job_settings: SmartSpimAcquisitionJobSettings - - -class SubjectSettings(BaseSettings): - """Fields needed to retrieve subject metadata""" - - subject_id: str - metadata_service_path: str = "subject" - - -class ProceduresSettings(BaseSettings): - """Fields needed to retrieve procedures metadata""" - - subject_id: str - metadata_service_path: str = "procedures" - - -class RawDataDescriptionSettings(BaseSettings): - """Fields needed to retrieve data description metadata""" - - name: str - project_name: str - modality: List[Modality.ONE_OF] - institution: Optional[Organization.ONE_OF] = Organization.AIND - metadata_service_path: str = "funding" - - -class ProcessingSettings(BaseSettings): - """Fields needed to retrieve processing metadata""" - - pipeline_process: PipelineProcess - - -class MetadataSettings(BaseSettings): - """Fields needed to retrieve main Metadata""" - - name: str - location: str - subject_filepath: Optional[Path] = None - data_description_filepath: Optional[Path] = None - procedures_filepath: Optional[Path] = None - session_filepath: Optional[Path] = None - rig_filepath: Optional[Path] = None - processing_filepath: Optional[Path] = None - acquisition_filepath: Optional[Path] = None - instrument_filepath: Optional[Path] = None - - -class JobSettings(BaseSettings): - """Fields needed to gather all metadata""" - - metadata_service_domain: Optional[str] = None - subject_settings: Optional[SubjectSettings] = None - acquisition_settings: Optional[AcquisitionSettings] = None - raw_data_description_settings: Optional[RawDataDescriptionSettings] = None - procedures_settings: Optional[ProceduresSettings] = None - processing_settings: Optional[ProcessingSettings] = None - metadata_settings: Optional[MetadataSettings] = None - directory_to_write_to: Path - metadata_dir: Optional[Path] = Field( - default=None, - description="Optional path where user defined metadata files might be", - ) - metadata_dir_force: bool = Field( - default=False, - description=( - "Whether to override the user defined files in metadata_dir with " - "those pulled from metadata service" - ), - ) +from aind_metadata_mapper.models import JobSettings class GatherMetadataJob: @@ -338,6 +269,21 @@ def get_session_metadata(self) -> Optional[dict]: file_name=file_name ) return contents + elif self.settings.session_settings is not None: + session_settings = self.settings.session_settings.job_settings + if isinstance(session_settings, BergamoSessionJobSettings): + session_job = BergamoEtl(job_settings=session_settings) + elif isinstance(session_settings, BrukerSessionJobSettings): + session_job = MRIEtl(job_settings=session_settings) + elif isinstance(session_settings, FipSessionJobSettings): + session_job = FIBEtl(job_settings=session_settings) + else: + session_job = MesoscopeEtl(job_settings=session_settings) + job_response = session_job.run_job() + if job_response.status_code != 500: + return json.loads(job_response.data) + else: + return None else: return None diff --git a/src/aind_metadata_mapper/mesoscope/models.py b/src/aind_metadata_mapper/mesoscope/models.py new file mode 100644 index 00000000..c7e37402 --- /dev/null +++ b/src/aind_metadata_mapper/mesoscope/models.py @@ -0,0 +1,30 @@ +"""Module defining JobSettings for Mesoscope ETL""" + +from datetime import datetime +from pathlib import Path +from typing import List, Literal + +from pydantic import Field +from pydantic_settings import BaseSettings + + +class JobSettings(BaseSettings): + """Data to be entered by the user.""" + + job_settings_name: Literal["Mesoscope"] = "Mesoscope" + input_source: Path + behavior_source: Path + output_directory: Path + session_start_time: datetime + session_end_time: datetime + subject_id: str + project: str + iacuc_protocol: str = "2115" + magnification: str = "16x" + fov_coordinate_ml: float = 1.5 + fov_coordinate_ap: float = 1.5 + fov_reference: str = "Bregma" + experimenter_full_name: List[str] = Field( + ..., title="Full name of the experimenter" + ) + mouse_platform_name: str = "disc" diff --git a/src/aind_metadata_mapper/mesoscope/session.py b/src/aind_metadata_mapper/mesoscope/session.py index eb48e7cc..06c85343 100644 --- a/src/aind_metadata_mapper/mesoscope/session.py +++ b/src/aind_metadata_mapper/mesoscope/session.py @@ -5,41 +5,16 @@ import sys from datetime import datetime from pathlib import Path -from typing import List, Union +from typing import Union import tifffile from aind_data_schema.core.session import FieldOfView, Session, Stream from aind_data_schema_models.modalities import Modality from PIL import Image from PIL.TiffTags import TAGS -from pydantic import Field -from pydantic_settings import BaseSettings from aind_metadata_mapper.core import GenericEtl - - -class JobSettings(BaseSettings): - """Data to be entered by the user.""" - - # TODO: for now this will need to be directly input by the user. - # In the future, the workflow sequencing engine should be able to put - # this in a json or we can extract it from SLIMS - input_source: Path - behavior_source: Path - output_directory: Path - session_start_time: datetime - session_end_time: datetime - subject_id: str - project: str - iacuc_protocol: str = "2115" - magnification: str = "16x" - fov_coordinate_ml: float = 1.5 - fov_coordinate_ap: float = 1.5 - fov_reference: str = "Bregma" - experimenter_full_name: List[str] = Field( - ..., title="Full name of the experimenter" - ) - mouse_platform_name: str = "disc" +from aind_metadata_mapper.mesoscope.models import JobSettings class MesoscopeEtl(GenericEtl[JobSettings]): diff --git a/src/aind_metadata_mapper/models.py b/src/aind_metadata_mapper/models.py new file mode 100644 index 00000000..d45afbc8 --- /dev/null +++ b/src/aind_metadata_mapper/models.py @@ -0,0 +1,129 @@ +"""Module to define models for Gather Metadata Job""" + +from pathlib import Path +from typing import List, Optional, Union +from typing_extensions import Annotated + +from aind_data_schema.core.processing import PipelineProcess +from aind_data_schema_models.modalities import Modality +from aind_data_schema_models.organizations import Organization + +from pydantic import Field, BaseModel, ConfigDict +from pydantic_settings import BaseSettings + +from aind_metadata_mapper.bergamo.models import ( + JobSettings as BergamoSessionJobSettings, +) +from aind_metadata_mapper.bruker.models import ( + JobSettings as BrukerSessionJobSettings, +) +from aind_metadata_mapper.fip.models import ( + JobSettings as FipSessionJobSettings, +) +from aind_metadata_mapper.mesoscope.models import ( + JobSettings as MesoscopeSessionJobSettings, +) +from aind_metadata_mapper.smartspim.models import ( + JobSettings as SmartSpimAcquisitionJobSettings, +) + + +class JobResponse(BaseModel): + """Standard model of a JobResponse.""" + + model_config = ConfigDict(extra="forbid") + status_code: int + message: Optional[str] = Field(None) + data: Optional[str] = Field(None) + + +class SessionSettings(BaseSettings): + """Settings needed to retrieve session metadata""" + + job_settings: Annotated[ + Union[ + BergamoSessionJobSettings, + BrukerSessionJobSettings, + FipSessionJobSettings, + MesoscopeSessionJobSettings, + ], + Field(discriminator="job_settings_name"), + ] + + +class AcquisitionSettings(BaseSettings): + """Fields needed to retrieve acquisition metadata""" + + # TODO: we can change this to a tagged union once more acquisition settings + # are added + job_settings: SmartSpimAcquisitionJobSettings + + +class SubjectSettings(BaseSettings): + """Fields needed to retrieve subject metadata""" + + subject_id: str + metadata_service_path: str = "subject" + + +class ProceduresSettings(BaseSettings): + """Fields needed to retrieve procedures metadata""" + + subject_id: str + metadata_service_path: str = "procedures" + + +class RawDataDescriptionSettings(BaseSettings): + """Fields needed to retrieve data description metadata""" + + name: str + project_name: str + modality: List[Modality.ONE_OF] + institution: Optional[Organization.ONE_OF] = Organization.AIND + metadata_service_path: str = "funding" + + +class ProcessingSettings(BaseSettings): + """Fields needed to retrieve processing metadata""" + + pipeline_process: PipelineProcess + + +class MetadataSettings(BaseSettings): + """Fields needed to retrieve main Metadata""" + + name: str + location: str + subject_filepath: Optional[Path] = None + data_description_filepath: Optional[Path] = None + procedures_filepath: Optional[Path] = None + session_filepath: Optional[Path] = None + rig_filepath: Optional[Path] = None + processing_filepath: Optional[Path] = None + acquisition_filepath: Optional[Path] = None + instrument_filepath: Optional[Path] = None + + +class JobSettings(BaseSettings): + """Fields needed to gather all metadata""" + + metadata_service_domain: Optional[str] = None + subject_settings: Optional[SubjectSettings] = None + session_settings: Optional[SessionSettings] = None + acquisition_settings: Optional[AcquisitionSettings] = None + raw_data_description_settings: Optional[RawDataDescriptionSettings] = None + procedures_settings: Optional[ProceduresSettings] = None + processing_settings: Optional[ProcessingSettings] = None + metadata_settings: Optional[MetadataSettings] = None + directory_to_write_to: Path + metadata_dir: Optional[Path] = Field( + default=None, + description="Optional path where user defined metadata files might be", + ) + metadata_dir_force: bool = Field( + default=False, + description=( + "Whether to override the user defined files in metadata_dir with " + "those pulled from metadata service" + ), + ) diff --git a/src/aind_metadata_mapper/smartspim/acquisition.py b/src/aind_metadata_mapper/smartspim/acquisition.py index e269a931..38d27caf 100644 --- a/src/aind_metadata_mapper/smartspim/acquisition.py +++ b/src/aind_metadata_mapper/smartspim/acquisition.py @@ -3,14 +3,14 @@ import os import re from datetime import datetime -from pathlib import Path -from typing import Dict, Literal, Optional, Union +from typing import Dict, Union from aind_data_schema.components.coordinates import ImageAxis from aind_data_schema.core import acquisition from pydantic_settings import BaseSettings from aind_metadata_mapper.core import GenericEtl, JobResponse +from aind_metadata_mapper.smartspim.models import JobSettings from aind_metadata_mapper.smartspim.utils import ( get_anatomical_direction, get_excitation_emission_waves, @@ -20,24 +20,6 @@ ) -class JobSettings(BaseSettings): - """Data to be entered by the user.""" - - # Field can be used to switch between different acquisition etl jobs - job_settings_name: Literal["SmartSPIM"] = "SmartSPIM" - - subject_id: str - raw_dataset_path: Path - output_directory: Optional[Path] = None - - # Metadata names - asi_filename: str = "derivatives/ASI_logging.txt" - mdata_filename_json: str = "derivatives/metadata.json" - - # Metadata provided by microscope operators - processing_manifest_path: str = "derivatives/processing_manifest.json" - - class SmartspimETL(GenericEtl): """ This class contains the methods to write the metadata diff --git a/src/aind_metadata_mapper/smartspim/models.py b/src/aind_metadata_mapper/smartspim/models.py new file mode 100644 index 00000000..87ef4db3 --- /dev/null +++ b/src/aind_metadata_mapper/smartspim/models.py @@ -0,0 +1,24 @@ +"""Module defining JobSettings for SmartSPIM ETL""" + +from pathlib import Path +from typing import Literal, Optional + +from pydantic_settings import BaseSettings + + +class JobSettings(BaseSettings): + """Data to be entered by the user.""" + + # Field can be used to switch between different acquisition etl jobs + job_settings_name: Literal["SmartSPIM"] = "SmartSPIM" + + subject_id: str + raw_dataset_path: Path + output_directory: Optional[Path] = None + + # Metadata names + asi_filename: str = "derivatives/ASI_logging.txt" + mdata_filename_json: str = "derivatives/metadata.json" + + # Metadata provided by microscope operators + processing_manifest_path: str = "derivatives/processing_manifest.json" diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py new file mode 100644 index 00000000..f4b3db4e --- /dev/null +++ b/tests/integration/__init__.py @@ -0,0 +1 @@ +"""Integration Testing package""" diff --git a/tests/integration/bergamo/__init__.py b/tests/integration/bergamo/__init__.py new file mode 100644 index 00000000..7ca4fad1 --- /dev/null +++ b/tests/integration/bergamo/__init__.py @@ -0,0 +1 @@ +"""Package for bergamo integration tests.""" diff --git a/tests/integration/bergamo/session.py b/tests/integration/bergamo/session.py new file mode 100644 index 00000000..5ee1c2b4 --- /dev/null +++ b/tests/integration/bergamo/session.py @@ -0,0 +1,64 @@ +"""Integration test for bergamo session""" + +import argparse +import json +import os +import sys +import unittest +from pathlib import Path + +from aind_metadata_mapper.bergamo.models import JobSettings +from aind_metadata_mapper.bergamo.session import BergamoEtl + +EXPECTED_OUTPUT_FILE_PATH = ( + Path(os.path.dirname(os.path.realpath(__file__))) + / ".." + / ".." + / "resources" + / "bergamo" + / "session.json" +) + + +class IntegrationTestBergamo(unittest.TestCase): + """Integration test for Bergamo""" + + @classmethod + def setUpClass(cls) -> None: + """Set up the class.""" + with open(EXPECTED_OUTPUT_FILE_PATH, "r") as f: + expected_output_json = json.load(f) + cls.expected_output = expected_output_json + + def test_run_job(self): + """Tests run_job on actual raw data source.""" + input_source: str = getattr(IntegrationTestBergamo, "input_source") + job_settings = JobSettings( + input_source=Path(input_source), + experimenter_full_name=["Jane Doe"], + subject_id="706957", + imaging_laser_wavelength=405, + fov_imaging_depth=150, + fov_targeted_structure="M1", + notes=None, + ) + bergamo_job = BergamoEtl(job_settings=job_settings) + response = bergamo_job.run_job() + actual_session = json.loads(response.data) + self.assertEqual(self.expected_output, actual_session) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "--input_source", + type=str, + required=True, + help="The input source for the ETL job.", + ) + parser.add_argument("unittest_args", nargs="*") + + args = parser.parse_args() + setattr(IntegrationTestBergamo, "input_source", args.input_source) + sys.argv[1:] = args.unittest_args + unittest.main() diff --git a/tests/resources/bergamo/session.json b/tests/resources/bergamo/session.json new file mode 100755 index 00000000..ba628d2d --- /dev/null +++ b/tests/resources/bergamo/session.json @@ -0,0 +1,507 @@ +{ + "describedBy": "https://raw.githubusercontent.com/AllenNeuralDynamics/aind-data-schema/main/src/aind_data_schema/core/session.py", + "schema_version": "0.2.6", + "protocol_id": [], + "experimenter_full_name": [ + "Jane Doe" + ], + "session_start_time": "2023-03-22T15:10:35.604999-07:00", + "session_end_time": "2023-03-22T15:21:33.039442-07:00", + "session_type": "BCI", + "iacuc_protocol": "2109", + "rig_id": "442 Bergamo 2p photostim", + "calibrations": [], + "maintenance": [], + "subject_id": "706957", + "animal_weight_prior": null, + "animal_weight_post": null, + "weight_unit": "gram", + "anaesthesia": null, + "data_streams": [ + { + "stream_start_time": "2023-03-22T15:12:48.350999-07:00", + "stream_end_time": "2023-03-22T15:13:20.870256-07:00", + "daq_names": [ + "PXI" + ], + "camera_names": [], + "light_sources": [ + { + "device_type": "Laser", + "name": "Chameleon Laser", + "wavelength": 405, + "wavelength_unit": "nanometer", + "excitation_power": "10.0", + "excitation_power_unit": "percent" + } + ], + "ephys_modules": [], + "stick_microscopes": [], + "manipulator_modules": [], + "detectors": [ + { + "name": "Green PMT", + "exposure_time": null, + "exposure_time_unit": "millisecond", + "trigger_type": "Internal" + } + ], + "fiber_connections": [], + "fiber_modules": [], + "ophys_fovs": [ + { + "index": 0, + "imaging_depth": 150, + "imaging_depth_unit": "micrometer", + "targeted_structure": "M1", + "fov_coordinate_ml": "1.5", + "fov_coordinate_ap": "1.5", + "fov_coordinate_unit": "micrometer", + "fov_reference": "there is no reference", + "fov_width": 800, + "fov_height": 800, + "fov_size_unit": "pixel", + "magnification": "1.2", + "fov_scale_factor": "1.0416666666666667", + "fov_scale_factor_unit": "um/pixel", + "frame_rate": "19.4188", + "frame_rate_unit": "hertz", + "coupled_fov_index": null, + "power": null, + "power_unit": "percent", + "power_ratio": null, + "scanfield_z": null, + "scanfield_z_unit": "micrometer", + "scanimage_roi_index": null, + "notes": null + } + ], + "slap_fovs": null, + "stack_parameters": null, + "mri_scans": [], + "stream_modalities": [ + { + "name": "Planar optical physiology", + "abbreviation": "ophys" + } + ], + "software": [], + "notes": "tiff_stem:spont2" + }, + { + "stream_start_time": "2023-03-22T15:15:08.795999-07:00", + "stream_end_time": "2023-03-22T15:15:52.396783-07:00", + "daq_names": [ + "PXI" + ], + "camera_names": [], + "light_sources": [ + { + "device_type": "Laser", + "name": "Chameleon Laser", + "wavelength": 405, + "wavelength_unit": "nanometer", + "excitation_power": "10.0", + "excitation_power_unit": "percent" + } + ], + "ephys_modules": [], + "stick_microscopes": [], + "manipulator_modules": [], + "detectors": [ + { + "name": "Green PMT", + "exposure_time": null, + "exposure_time_unit": "millisecond", + "trigger_type": "Internal" + } + ], + "fiber_connections": [], + "fiber_modules": [], + "ophys_fovs": [ + { + "index": 0, + "imaging_depth": 150, + "imaging_depth_unit": "micrometer", + "targeted_structure": "M1", + "fov_coordinate_ml": "1.5", + "fov_coordinate_ap": "1.5", + "fov_coordinate_unit": "micrometer", + "fov_reference": "there is no reference", + "fov_width": 800, + "fov_height": 800, + "fov_size_unit": "pixel", + "magnification": "1.2", + "fov_scale_factor": "1.0416666666666667", + "fov_scale_factor_unit": "um/pixel", + "frame_rate": "19.4188", + "frame_rate_unit": "hertz", + "coupled_fov_index": null, + "power": null, + "power_unit": "percent", + "power_ratio": null, + "scanfield_z": null, + "scanfield_z_unit": "micrometer", + "scanimage_roi_index": null, + "notes": null + } + ], + "slap_fovs": null, + "stack_parameters": null, + "mri_scans": [], + "stream_modalities": [ + { + "name": "Planar optical physiology", + "abbreviation": "ophys" + } + ], + "software": [], + "notes": "tiff_stem:spont3" + }, + { + "stream_start_time": "2023-03-22T15:10:35.604999-07:00", + "stream_end_time": "2023-03-22T15:11:01.328279-07:00", + "daq_names": [ + "PXI" + ], + "camera_names": [], + "light_sources": [ + { + "device_type": "Laser", + "name": "Chameleon Laser", + "wavelength": 405, + "wavelength_unit": "nanometer", + "excitation_power": "10.0", + "excitation_power_unit": "percent" + } + ], + "ephys_modules": [], + "stick_microscopes": [], + "manipulator_modules": [], + "detectors": [ + { + "name": "Green PMT", + "exposure_time": null, + "exposure_time_unit": "millisecond", + "trigger_type": "Internal" + } + ], + "fiber_connections": [], + "fiber_modules": [], + "ophys_fovs": [ + { + "index": 0, + "imaging_depth": 150, + "imaging_depth_unit": "micrometer", + "targeted_structure": "M1", + "fov_coordinate_ml": "1.5", + "fov_coordinate_ap": "1.5", + "fov_coordinate_unit": "micrometer", + "fov_reference": "there is no reference", + "fov_width": 800, + "fov_height": 800, + "fov_size_unit": "pixel", + "magnification": "1.2", + "fov_scale_factor": "1.0416666666666667", + "fov_scale_factor_unit": "um/pixel", + "frame_rate": "19.4188", + "frame_rate_unit": "hertz", + "coupled_fov_index": null, + "power": null, + "power_unit": "percent", + "power_ratio": null, + "scanfield_z": null, + "scanfield_z_unit": "micrometer", + "scanimage_roi_index": null, + "notes": null + } + ], + "slap_fovs": null, + "stack_parameters": null, + "mri_scans": [], + "stream_modalities": [ + { + "name": "Planar optical physiology", + "abbreviation": "ophys" + } + ], + "software": [], + "notes": "tiff_stem:spont" + }, + { + "stream_start_time": "2023-03-22T15:20:45.973999-07:00", + "stream_end_time": "2023-03-22T15:21:33.039442-07:00", + "daq_names": [ + "PXI" + ], + "camera_names": [ + "Side Face Camera", + "Bottom Face Camera" + ], + "light_sources": [ + { + "device_type": "Laser", + "name": "Chameleon Laser", + "wavelength": 405, + "wavelength_unit": "nanometer", + "excitation_power": "10.0", + "excitation_power_unit": "percent" + } + ], + "ephys_modules": [], + "stick_microscopes": [], + "manipulator_modules": [], + "detectors": [ + { + "name": "Green PMT", + "exposure_time": null, + "exposure_time_unit": "millisecond", + "trigger_type": "Internal" + } + ], + "fiber_connections": [], + "fiber_modules": [], + "ophys_fovs": [ + { + "index": 0, + "imaging_depth": 150, + "imaging_depth_unit": "micrometer", + "targeted_structure": "M1", + "fov_coordinate_ml": "1.5", + "fov_coordinate_ap": "1.5", + "fov_coordinate_unit": "micrometer", + "fov_reference": "there is no reference", + "fov_width": 800, + "fov_height": 800, + "fov_size_unit": "pixel", + "magnification": "1.2", + "fov_scale_factor": "1.0416666666666667", + "fov_scale_factor_unit": "um/pixel", + "frame_rate": "19.4188", + "frame_rate_unit": "hertz", + "coupled_fov_index": null, + "power": null, + "power_unit": "percent", + "power_ratio": null, + "scanfield_z": null, + "scanfield_z_unit": "micrometer", + "scanimage_roi_index": null, + "notes": null + } + ], + "slap_fovs": null, + "stack_parameters": null, + "mri_scans": [], + "stream_modalities": [ + { + "name": "Planar optical physiology", + "abbreviation": "ophys" + }, + { + "name": "Behavior", + "abbreviation": "behavior" + }, + { + "name": "Behavior videos", + "abbreviation": "behavior-videos" + } + ], + "software": [], + "notes": "tiff_stem:pair3_7vs29" + } + ], + "stimulus_epochs": [ + { + "stimulus_start_time": "2023-03-22T15:12:48.350999-07:00", + "stimulus_end_time": "2023-03-22T15:13:20.870256-07:00", + "stimulus_name": "spontaneous activity", + "session_number": null, + "software": [], + "script": null, + "stimulus_modalities": [ + "None" + ], + "stimulus_parameters": null, + "stimulus_device_names": [], + "speaker_config": null, + "light_source_config": null, + "output_parameters": { + "tiff_files": [ + "spont2_00001.tif", + "spont2_00002.tif", + "spont2_00003.tif" + ], + "tiff_stem": "spont2" + }, + "reward_consumed_during_epoch": null, + "reward_consumed_unit": "microliter", + "trials_total": null, + "trials_finished": null, + "trials_rewarded": null, + "notes": "absence of any kind of stimulus" + }, + { + "stimulus_start_time": "2023-03-22T15:15:08.795999-07:00", + "stimulus_end_time": "2023-03-22T15:15:52.396783-07:00", + "stimulus_name": "spontaneous activity", + "session_number": null, + "software": [], + "script": null, + "stimulus_modalities": [ + "None" + ], + "stimulus_parameters": null, + "stimulus_device_names": [], + "speaker_config": null, + "light_source_config": null, + "output_parameters": { + "tiff_files": [ + "spont3_00001.tif", + "spont3_00002.tif", + "spont3_00003.tif", + "spont3_00004.tif" + ], + "tiff_stem": "spont3" + }, + "reward_consumed_during_epoch": null, + "reward_consumed_unit": "microliter", + "trials_total": null, + "trials_finished": null, + "trials_rewarded": null, + "notes": "absence of any kind of stimulus" + }, + { + "stimulus_start_time": "2023-03-22T15:10:35.604999-07:00", + "stimulus_end_time": "2023-03-22T15:11:01.328279-07:00", + "stimulus_name": "spontaneous activity", + "session_number": null, + "software": [], + "script": null, + "stimulus_modalities": [ + "None" + ], + "stimulus_parameters": null, + "stimulus_device_names": [], + "speaker_config": null, + "light_source_config": null, + "output_parameters": { + "tiff_files": [ + "spont_00001.tif" + ], + "tiff_stem": "spont" + }, + "reward_consumed_during_epoch": null, + "reward_consumed_unit": "microliter", + "trials_total": null, + "trials_finished": null, + "trials_rewarded": null, + "notes": "absence of any kind of stimulus" + }, + { + "stimulus_start_time": "2023-03-22T15:20:45.973999-07:00", + "stimulus_end_time": "2023-03-22T15:21:33.039442-07:00", + "stimulus_name": "single neuron BCI conditioning", + "session_number": null, + "software": [ + { + "name": "pyBpod", + "version": "1.8.2", + "url": "https://github.com/pybpod/pybpod", + "parameters": {} + } + ], + "script": { + "name": "pybpod_basic.py", + "version": "2d77d15", + "url": "https://github.com/rozmar/BCI-motor-control/blob/main/BCI-pybpod-protocols/bci_basic.py", + "parameters": {} + }, + "stimulus_modalities": [ + "Auditory" + ], + "stimulus_parameters": [], + "stimulus_device_names": [ + "speaker", + "lickport" + ], + "speaker_config": null, + "light_source_config": null, + "output_parameters": { + "tiff_files": [ + "pair3_7vs29_00001.tif", + "pair3_7vs29_00002.tif", + "pair3_7vs29_00003.tif", + "pair3_7vs29_00004.tif" + ], + "tiff_stem": "pair3_7vs29", + "hit_rate_trials_0_10": null, + "hit_rate_trials_20_40": null, + "total_hits": null, + "average_hit_rate": null + }, + "reward_consumed_during_epoch": null, + "reward_consumed_unit": "microliter", + "trials_total": null, + "trials_finished": null, + "trials_rewarded": null, + "notes": null + } + ], + "mouse_platform_name": "Standard Mouse Tube", + "active_mouse_platform": false, + "reward_delivery": { + "reward_solution": "Water", + "reward_spouts": [ + { + "side": "Center", + "starting_position": { + "device_position_transformations": [ + { + "type": "translation", + "translation": [ + "0.0", + "-6.0", + "0.0" + ] + }, + { + "type": "rotation", + "rotation": [ + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0" + ] + } + ], + "device_origin": "tip of the lickspout", + "device_axes": [ + { + "name": "X", + "direction": "lateral motion" + }, + { + "name": "Y", + "direction": "rostro-caudal motion positive is towards mouse, negative is away" + }, + { + "name": "Z", + "direction": "up/down" + } + ], + "notes": null + }, + "variable_position": true + } + ], + "notes": null + }, + "reward_consumed_total": null, + "reward_consumed_unit": "microliter", + "notes": null +} \ No newline at end of file diff --git a/tests/test_dynamic_routing/test_neuropixels_rig.py b/tests/test_dynamic_routing/test_neuropixels_rig.py index 221c0214..6f1d7ff4 100644 --- a/tests/test_dynamic_routing/test_neuropixels_rig.py +++ b/tests/test_dynamic_routing/test_neuropixels_rig.py @@ -24,7 +24,7 @@ def test_update_modification_date(self): """Test ETL workflow with inferred probe mapping.""" etl = NeuropixelsRigEtl( RESOURCES_DIR / "base_rig.json", - Path("./"), + Path("abc"), ) extracted = etl._extract() transformed = etl._transform(extracted) diff --git a/tests/test_dynamic_routing/test_utils.py b/tests/test_dynamic_routing/test_utils.py index 83be53d8..8340ca60 100644 --- a/tests/test_dynamic_routing/test_utils.py +++ b/tests/test_dynamic_routing/test_utils.py @@ -40,6 +40,6 @@ def setup_neuropixels_etl_resources( """ return ( RESOURCES_DIR / "base_rig.json", - Path("./"), # hopefully file writes are mocked + Path("abc"), # hopefully file writes are mocked Rig.model_validate_json(expected_json.read_text()), ) diff --git a/tests/test_gather_metadata.py b/tests/test_gather_metadata.py index baafea61..2b0ad995 100644 --- a/tests/test_gather_metadata.py +++ b/tests/test_gather_metadata.py @@ -11,19 +11,37 @@ from aind_data_schema.core.processing import DataProcess, PipelineProcess from aind_data_schema_models.modalities import Modality from aind_data_schema_models.process_names import ProcessName +from pydantic import ValidationError from requests import Response +from aind_metadata_mapper.bergamo.models import ( + JobSettings as BergamoSessionJobSettings, +) +from aind_metadata_mapper.bergamo.session import BergamoEtl +from aind_metadata_mapper.bruker.models import ( + JobSettings as BrukerSessionJobSettings, +) +from aind_metadata_mapper.bruker.session import MRIEtl from aind_metadata_mapper.core import JobResponse -from aind_metadata_mapper.gather_metadata import ( +from aind_metadata_mapper.fip.models import ( + JobSettings as FipSessionJobSettings, +) +from aind_metadata_mapper.fip.session import FIBEtl +from aind_metadata_mapper.models import ( AcquisitionSettings, - GatherMetadataJob, JobSettings, MetadataSettings, ProceduresSettings, ProcessingSettings, RawDataDescriptionSettings, + SessionSettings, SubjectSettings, ) +from aind_metadata_mapper.gather_metadata import GatherMetadataJob +from aind_metadata_mapper.mesoscope.models import ( + JobSettings as MesoscopeSessionJobSettings, +) +from aind_metadata_mapper.mesoscope.session import MesoscopeEtl from aind_metadata_mapper.smartspim.acquisition import ( JobSettings as SmartSpimAcquisitionJobSettings, ) @@ -477,6 +495,118 @@ def test_get_session_metadata(self): contents = metadata_job.get_session_metadata() self.assertIsNotNone(contents) + @patch("aind_metadata_mapper.bergamo.session.BergamoEtl.run_job") + def test_get_session_metadata_bergamo_success( + self, mock_run_job: MagicMock + ): + """Tests get_session_metadata bergamo""" + mock_run_job.return_value = JobResponse( + status_code=200, data=json.dumps({"some_key": "some_value"}) + ) + bergamo_session_settings = BergamoSessionJobSettings.model_construct() + job_settings = JobSettings( + directory_to_write_to=RESOURCES_DIR, + session_settings=SessionSettings( + job_settings=bergamo_session_settings + ), + ) + metadata_job = GatherMetadataJob(settings=job_settings) + contents = metadata_job.get_session_metadata() + self.assertEqual({"some_key": "some_value"}, contents) + BergamoEtl( + job_settings=bergamo_session_settings + ).run_job.assert_called_once() + + @patch("aind_metadata_mapper.bruker.session.MRIEtl.run_job") + def test_get_session_metadata_bruker_success( + self, mock_run_job: MagicMock + ): + """Tests get_session_metadata bruker creates MRIEtl""" + mock_run_job.return_value = JobResponse( + status_code=200, data=json.dumps({"some_key": "some_value"}) + ) + bruker_session_settings = BrukerSessionJobSettings.model_construct() + job_settings = JobSettings( + directory_to_write_to=RESOURCES_DIR, + session_settings=SessionSettings( + job_settings=bruker_session_settings, + ), + ) + metadata_job = GatherMetadataJob(settings=job_settings) + contents = metadata_job.get_session_metadata() + self.assertEqual({"some_key": "some_value"}, contents) + MRIEtl( + job_settings=bruker_session_settings + ).run_job.assert_called_once() + + @patch("aind_metadata_mapper.fip.session.FIBEtl.run_job") + def test_get_session_metadata_fip_success(self, mock_run_job: MagicMock): + """Tests get_session_metadata bruker creates FibEtl""" + mock_run_job.return_value = JobResponse( + status_code=200, data=json.dumps({"some_key": "some_value"}) + ) + fip_session_settings = FipSessionJobSettings.model_construct() + job_settings = JobSettings( + directory_to_write_to=RESOURCES_DIR, + session_settings=SessionSettings( + job_settings=fip_session_settings, + ), + ) + metadata_job = GatherMetadataJob(settings=job_settings) + contents = metadata_job.get_session_metadata() + self.assertEqual({"some_key": "some_value"}, contents) + FIBEtl(job_settings=fip_session_settings).run_job.assert_called_once() + + @patch("aind_metadata_mapper.mesoscope.session.MesoscopeEtl.run_job") + def test_get_session_metadata_mesoscope_success( + self, mock_run_job: MagicMock + ): + """Tests get_session_metadata bruker creates MRIEtl""" + mock_run_job.return_value = JobResponse( + status_code=200, data=json.dumps({"some_key": "some_value"}) + ) + mesoscope_session_settings = ( + MesoscopeSessionJobSettings.model_construct() + ) + job_settings = JobSettings( + directory_to_write_to=RESOURCES_DIR, + session_settings=SessionSettings( + job_settings=mesoscope_session_settings, + ), + ) + metadata_job = GatherMetadataJob(settings=job_settings) + contents = metadata_job.get_session_metadata() + self.assertEqual({"some_key": "some_value"}, contents) + MesoscopeEtl( + job_settings=mesoscope_session_settings + ).run_job.assert_called_once() + + def test_session_settings_error(self): + """Tests SessionSettings raises error if JobSettings is not expected""" + session_settings = SmartSpimAcquisitionJobSettings.model_construct() + with self.assertRaises(ValidationError): + JobSettings( + directory_to_write_to=RESOURCES_DIR, + session_settings=SessionSettings( + job_settings=session_settings, + ), + ) + + @patch("aind_metadata_mapper.bergamo.session.BergamoEtl.run_job") + def test_get_session_metadata_error(self, mock_run_job: MagicMock): + """Tests get_session_metadata returns None when requesting + Bergamo metadata and a 500 response is returned.""" + mock_run_job.return_value = JobResponse(status_code=500, data=None) + job_settings = JobSettings( + directory_to_write_to=RESOURCES_DIR, + session_settings=SessionSettings( + job_settings=BergamoSessionJobSettings.model_construct() + ), + ) + metadata_job = GatherMetadataJob(settings=job_settings) + contents = metadata_job.get_session_metadata() + self.assertIsNone(contents) + def test_get_session_metadata_none(self): """Tests get_session_metadata returns none""" diff --git a/tests/test_mesoscope/test_session.py b/tests/test_mesoscope/test_session.py index bfe801b3..3be844b9 100644 --- a/tests/test_mesoscope/test_session.py +++ b/tests/test_mesoscope/test_session.py @@ -148,7 +148,7 @@ def test_extract_no_input_source( """Tests that _extract raises a ValueError""" mock_is_dir.return_value = True mock_path_exists.return_value = False - mock_path_glob.return_value = iter([Path("/somedir/a")]) + mock_path_glob.return_value = iter([Path("somedir/a")]) etl1 = MesoscopeEtl( job_settings=self.example_job_settings, ) diff --git a/tests/test_open_ephys/test_rig.py b/tests/test_open_ephys/test_rig.py index 8024badc..99209f05 100644 --- a/tests/test_open_ephys/test_rig.py +++ b/tests/test_open_ephys/test_rig.py @@ -19,7 +19,7 @@ BASE_RIG_MISSING_PROBE_PATH = ( RESOURCES_DIR / "dynamic_routing" / "base-missing-probe_rig.json" ) -OUTPUT_DIR = Path(".") # File writes will be mocked +OUTPUT_DIR = Path("abc") # File writes will be mocked class TestOpenEphysRigEtl(unittest.TestCase): diff --git a/tests/test_open_ephys/test_utils/__init__.py b/tests/test_open_ephys/test_utils/__init__.py new file mode 100644 index 00000000..45410b01 --- /dev/null +++ b/tests/test_open_ephys/test_utils/__init__.py @@ -0,0 +1 @@ +"""Test modules in utils package""" diff --git a/tests/test_open_ephys/test_utils/test_stim_utils.py b/tests/test_open_ephys/test_utils/test_stim_utils.py index 3f01f32a..2858f4fc 100644 --- a/tests/test_open_ephys/test_utils/test_stim_utils.py +++ b/tests/test_open_ephys/test_utils/test_stim_utils.py @@ -116,7 +116,7 @@ def test_enforce_df_column_order(self): } ) result_df = stim.enforce_df_column_order(df, column_order) - pd.testing.assert_frame_equal(result_df, expected_df) + pd.testing.assert_frame_equal(result_df, expected_df, check_like=True) # Test case: Specified column order with all columns column_order = ["C", "A", "D", "B"]