Merge pull request #116 from AllenNeuralDynamics/release-v0.15.0

Release v0.15.0

.github/workflows/run_dev_tests.yml

@@ -10,7 +10,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [ '3.10', '3.11' ]
+        python-version: [ '3.9', '3.10', '3.11' ]
     steps:
     - uses: actions/checkout@v3
     - name: Set up Python ${{ matrix.python-version }}

.github/workflows/run_integration_test_dev.yml

@@ -0,0 +1,51 @@
+name: Run integration tests
+
+on:
+  push:
+    branches:
+      - dev
+
+jobs:
+  integration_tests:
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write
+      contents: read
+    env:
+      AWS_METADATA_MAPPER_ROLE: ${{ secrets.AWS_METADATA_MAPPER_ROLE_PROD }}
+      AWS_METADATA_MAPPER_BUCKET: ${{ vars.AWS_METADATA_MAPPER_BUCKET_PROD }}
+      AWS_REGION: ${{ vars.AWS_REGION_PROD }}
+      MOUNT_S3_URL: ${{ vars.MOUNT_S3_URL }}
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v3
+        with:
+          python-version: '3.10'
+      - name: Install dependencies
+        run:
+          python -m pip install -e .[all]
+      - name: Configure aws credentials
+        uses: aws-actions/configure-aws-credentials@v2
+        with:
+          role-to-assume: ${{ env.AWS_METADATA_MAPPER_ROLE }}
+          role-session-name: github-integration-test-session
+          aws-region: ${{ env.AWS_REGION }}
+      - name: install mountpoint-s3
+        run: |
+          wget $MOUNT_S3_URL
+          sudo apt-get update 
+          sudo apt-get install ./mount-s3.deb
+      - name: mount s3 bucket
+        run: |
+          mkdir bucket_mt
+          mount-s3 $AWS_METADATA_MAPPER_BUCKET bucket_mt
+      - name: run integration tests
+        run: |
+          python tests/integration/bergamo/session.py --input_source "bucket_mt/metadata-mapper-integration-testing/bergamo" IntegrationTestBergamo
+          umount bucket_mt
+
+
+
+
+

CONTRIBUTING.md

@@ -21,7 +21,7 @@ Then in bash, run
 pip install -e .[dev]
 ```
 to set up your environment. Once these steps are complete, you can get developing🚀. Data is organized by acquisition machine, so if you're adding a new machine, create a new directory. Otherwise, put your code in its corresponding directory.
-### Testing
+### Unit Testing
 
 Testing is required to open a PR in this repository to ensure robustness and reliability of our codebase. 
 - **1:1 Correspondence:** Structure unit tests in a manner that mirrors the module structure. 
@@ -39,5 +39,12 @@ Testing is required to open a PR in this repository to ensure robustness and rel
     coverage html
     ```
     and find the report in the htmlcov/index.html.
-
+  
 There are several libraries used to run linters and check documentation. We've included these in the development package. You can run them as described [here](https://github.com/AllenNeuralDynamics/aind-metadata-mapper/blob/main/README.md#linters-and-testing). 
+
+### Integration Testing
+To ensure that an ETL runs as expected against data on the VAST, you can run an integration test locally by pointing to the input directory on VAST. For example, to test the 'bergamo' package:
+```bash
+    python tests/integration/bergamo/session.py --input_source "/allen/aind/scratch/svc_aind_upload/test_data_sets/bergamo" IntegrationTestBergamo
+ ```
+

pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
 name = "aind-metadata-mapper"
 description = "Generated from aind-library-template"
 license = {text = "MIT"}
-requires-python = ">=3.10"
+requires-python = ">=3.8"
 authors = [
     {name = "Allen Institute for Neural Dynamics"}
 ]
@@ -20,33 +20,31 @@ dependencies = [
     "aind-data-schema==0.36.0",
     "aind-data-schema-models==0.1.7",
     "pydantic-settings>=2.0",
-    "pandas",
-    "numpy",
-    "pytz"
 ]
 
 [project.optional-dependencies]
 dev = [
-    "aind-metadata-mapper[all]",
+    "aind-metadata-mapper[all] ; python_version >= '3.9'",
     "black",
     "coverage",
     "flake8",
     "interrogate",
     "isort",
     "Sphinx",
     "furo",
-    "pyyaml>=6.0.0",
 ]
 
 all = [
     "aind-metadata-mapper[bergamo]",
     "aind-metadata-mapper[bruker]",
     "aind-metadata-mapper[mesoscope]",
-    "aind-metadata-mapper[openephys]"
+    "aind-metadata-mapper[openephys]",
+    "aind-metadata-mapper[dynamicrouting]",
 ]
 
 bergamo = [
-    "scanimage-tiff-reader==1.4.1.4"
+    "scanimage-tiff-reader==1.4.1.4",
+    "numpy",
 ]
 
 bruker = [
@@ -64,6 +62,12 @@ openephys = [
     "np_session",
     "npc_ephys",
     "scipy",
+    "pandas",
+    "numpy",
+]
+
+dynamicrouting = [
+    "pyyaml>=6.0.0",
 ]
 
 [tool.setuptools.packages.find]

src/aind_metadata_mapper/__init__.py

@@ -1,3 +1,3 @@
 """Init package"""
 
-__version__ = "0.14.0"
+__version__ = "0.15.0"

src/aind_metadata_mapper/bergamo/models.py

@@ -0,0 +1,79 @@
+"""Module defining JobSettings for Bergamo ETL"""
+from decimal import Decimal
+from pathlib import Path
+from typing import List, Literal, Optional
+
+from pydantic import Field
+from pydantic_settings import BaseSettings
+
+
+class JobSettings(BaseSettings):
+    """Data that needs to be input by user. Can be pulled from env vars with
+    BERGAMO prefix or set explicitly."""
+
+    job_settings_name: Literal["Bergamo"] = "Bergamo"
+    input_source: Path = Field(
+        ..., description="Directory of files that need to be parsed."
+    )
+    output_directory: Optional[Path] = Field(
+        default=None,
+        description=(
+            "Directory where to save the json file to. If None, then json"
+            " contents will be returned in the Response message."
+        ),
+    )
+    # mandatory fields:
+    experimenter_full_name: List[str]
+    subject_id: str
+    imaging_laser_wavelength: int  # user defined
+    fov_imaging_depth: int
+    fov_targeted_structure: str
+    notes: Optional[str]
+
+    # fields with default values
+    mouse_platform_name: str = "Standard Mouse Tube"  # FROM RIG JSON
+    active_mouse_platform: bool = False
+    session_type: str = "BCI"
+    iacuc_protocol: str = "2109"
+    # should match rig json:
+    rig_id: str = "442 Bergamo 2p photostim"
+    behavior_camera_names: List[str] = [
+        "Side Face Camera",
+        "Bottom Face Camera",
+    ]
+    ch1_filter_names: List[str] = [
+        "Green emission filter",
+        "Emission dichroic",
+    ]
+    ch1_detector_name: str = "Green PMT"
+    ch1_daq_name: str = "PXI"
+    ch2_filter_names: List[str] = ["Red emission filter", "Emission dichroic"]
+    ch2_detector_name: str = "Red PMT"
+    ch2_daq_name: str = "PXI"
+    imaging_laser_name: str = "Chameleon Laser"
+
+    photostim_laser_name: str = "Monaco Laser"
+    stimulus_device_names: List[str] = ["speaker", "lickport"]  # FROM RIG JSON
+    photostim_laser_wavelength: int = 1040
+    fov_coordinate_ml: Decimal = Decimal("1.5")
+    fov_coordinate_ap: float = Decimal("1.5")
+    fov_reference: str = "Bregma"
+
+    starting_lickport_position: List[float] = [
+        0,
+        -6,
+        0,
+    ]  # in mm from face of the mouse
+    behavior_task_name: str = "single neuron BCI conditioning"
+    hit_rate_trials_0_10: Optional[float] = None
+    hit_rate_trials_20_40: Optional[float] = None
+    total_hits: Optional[float] = None
+    average_hit_rate: Optional[float] = None
+    trial_num: Optional[float] = None
+    # ZoneInfo object doesn't serialize well, so we can define it as a str
+    timezone: str = "US/Pacific"
+
+    class Config:
+        """Config to set env var prefix to BERGAMO"""
+
+        env_prefix = "BERGAMO_"

src/aind_metadata_mapper/bergamo/session.py

@@ -11,7 +11,7 @@
 from decimal import Decimal
 from enum import Enum
 from pathlib import Path
-from typing import Dict, List, Optional, Tuple, Union
+from typing import Dict, List, Tuple, Union
 from zoneinfo import ZoneInfo
 
 import numpy as np
@@ -45,87 +45,13 @@
     TriggerType,
 )
 from aind_data_schema_models.units import PowerUnit
-from pydantic import Field
-from pydantic_settings import BaseSettings
 from ScanImageTiffReader import ScanImageTiffReader
 
+from aind_metadata_mapper.bergamo.models import JobSettings
 from aind_metadata_mapper.core import GenericEtl, JobResponse
 
 
-class JobSettings(BaseSettings):
-    """Data that needs to be input by user. Can be pulled from env vars with
-    BERGAMO prefix or set explicitly."""
-
-    input_source: Path = Field(
-        ..., description="Directory of files that need to be parsed."
-    )
-    output_directory: Optional[Path] = Field(
-        default=None,
-        description=(
-            "Directory where to save the json file to. If None, then json"
-            " contents will be returned in the Response message."
-        ),
-    )
-    # mandatory fields:
-    experimenter_full_name: List[str]
-    subject_id: str
-    imaging_laser_wavelength: int  # user defined
-    fov_imaging_depth: int
-    fov_targeted_structure: str
-    notes: Optional[str]
-
-    # fields with default values
-    mouse_platform_name: str = "Standard Mouse Tube"  # FROM RIG JSON
-    active_mouse_platform: bool = False
-    session_type: str = "BCI"
-    iacuc_protocol: str = "2109"
-    # should match rig json:
-    rig_id: str = "442 Bergamo 2p photostim"
-    behavior_camera_names: List[str] = [
-        "Side Face Camera",
-        "Bottom Face Camera",
-    ]
-    ch1_filter_names: List[str] = [
-        "Green emission filter",
-        "Emission dichroic",
-    ]
-    ch1_detector_name: str = "Green PMT"
-    ch1_daq_name: str = "PXI"
-    ch2_filter_names: List[str] = ["Red emission filter", "Emission dichroic"]
-    ch2_detector_name: str = "Red PMT"
-    ch2_daq_name: str = "PXI"
-    imaging_laser_name: str = "Chameleon Laser"
-
-    photostim_laser_name: str = "Monaco Laser"
-    stimulus_device_names: List[str] = ["speaker", "lickport"]  # FROM RIG JSON
-    photostim_laser_wavelength: int = 1040
-    fov_coordinate_ml: Decimal = Decimal("1.5")
-    fov_coordinate_ap: float = Decimal("1.5")
-    fov_reference: str = "Bregma"
-
-    starting_lickport_position: list[float] = [
-        0,
-        -6,
-        0,
-    ]  # in mm from face of the mouse
-    behavior_task_name: str = "single neuron BCI conditioning"
-    hit_rate_trials_0_10: Optional[float] = None
-    hit_rate_trials_20_40: Optional[float] = None
-    total_hits: Optional[float] = None
-    average_hit_rate: Optional[float] = None
-    trial_num: Optional[float] = None
-    # ZoneInfo object doesn't serialize well, so we can define it as a str
-    timezone: str = "US/Pacific"
-
-    class Config:
-        """Config to set env var prefix to BERGAMO"""
-
-        env_prefix = "BERGAMO_"
-
-
 # This class makes it easier to flag which tif files are which expected type
-
-
 class TifFileGroup(str, Enum):
     """Type of stimulation a group of files belongs to"""
 
@@ -137,8 +63,6 @@ class TifFileGroup(str, Enum):
 
 # This class will hold the metadata information pulled from the tif files
 # with minimal parsing.
-
-
 @dataclass(frozen=True)
 class RawImageInfo:
     """Raw metadata from a tif file"""

src/aind_metadata_mapper/bruker/models.py

@@ -0,0 +1,40 @@
+"""Module defining JobSettings for Bruker ETL"""
+
+from pathlib import Path
+from typing import List, Literal, Optional
+
+from aind_data_schema.components.devices import (
+    MagneticStrength,
+    ScannerLocation,
+)
+from pydantic import Field
+from pydantic_settings import BaseSettings
+
+
+class JobSettings(BaseSettings):
+    """Data that needs to be input by user."""
+
+    job_settings_name: Literal["Bruker"] = "Bruker"
+    data_path: Path
+    output_directory: Optional[Path] = Field(
+        default=None,
+        description=(
+            "Directory where to save the json file to. If None, then json"
+            " contents will be returned in the Response message."
+        ),
+    )
+    experimenter_full_name: List[str]
+    protocol_id: str = Field(default="", description="Protocol ID")
+    collection_tz: str = Field(
+        default="America/Los_Angeles",
+        description="Timezone string of the collection site",
+    )
+    session_type: str
+    primary_scan_number: int
+    setup_scan_number: int
+    scanner_name: str
+    scan_location: ScannerLocation
+    magnetic_strength: MagneticStrength
+    subject_id: str
+    iacuc_protocol: str
+    session_notes: str