Enh/neuropixels rig

pyproject.toml

@@ -17,9 +17,10 @@ readme = "README.md"
 dynamic = ["version"]
 
 dependencies = [
-    "aind-data-schema==0.15.9",
+    "aind-data-schema==0.17.3",
     "aind-metadata-service[client]",
-    "scanimage-tiff-reader==1.4.1.4"
+    "scanimage-tiff-reader==1.4.1.4",
+    "pyyaml==6.0.1"
 ]
 
 [project.optional-dependencies]

src/aind_metadata_mapper/neuropixels/__init__.py

@@ -0,0 +1 @@
+"""Maps neuropixels related metadata."""

src/aind_metadata_mapper/neuropixels/rig.py

@@ -0,0 +1,197 @@
+"""ETL class for neuropixels rigs."""
+
+import json
+import yaml
+import pathlib
+import datetime
+import configparser
+from aind_data_schema import rig
+
+from ..core import BaseEtl
+
+
+class NeuropixelsRigException(Exception):
+    """General error for MVR."""
+
+
+# assembly name, serial number, height, width, frame rate
+MVRCamera = tuple[str, str, int, int, float]
+SyncChannel = tuple[int, str]  # di line index, name
+# device name, sample rate, channels
+SyncContext = tuple[str, float, list[SyncChannel]]
+RigContext = tuple[dict, list[MVRCamera], SyncContext]
+
+
+class NeuropixelsRigEtl(BaseEtl):
+    """Neuropixels rig ETL class. Extracts information from rig-related files
+    and transforms them into an aind-data-schema rig.Rig instance.
+    """
+
+    def __init__(
+        self,
+        input_source: pathlib.Path,
+        output_directory: pathlib.Path,
+    ):
+        """Class constructor for Neuropixels rig etl class.
+
+        Parameters
+        ----------
+        input_source : Path
+          Can be a string or a Path
+        output_directory : Path
+          The directory where to save the json files.
+        """
+        super().__init__(input_source, output_directory)
+
+    def _load_resource(self, path: pathlib.Path) -> str:
+        """Loads a rig-related resource from input source.
+        """
+        if not path.exists():
+            raise NeuropixelsRigException("%s not found." % path)
+
+        return path.read_text()
+
+    def _extract(self) -> RigContext:
+        """Extracts rig-related information from config files.
+        """
+        if not self.input_source.is_dir():
+            raise NeuropixelsRigException(
+                "Input source is not a directory. %s" % self.input_source
+            )
+
+        return (
+            json.loads(
+                self._load_resource(
+                    self.input_source / "rig.partial.json",
+                )
+            ),
+            self._extract_mvr(
+                self._load_resource(
+                    self.input_source / "mvr.ini",
+                ),
+                json.loads(
+                    self._load_resource(
+                        self.input_source / "mvr.mapping.json",
+                    )
+                ),
+            ),
+            self._extract_sync(
+                self._load_resource(
+                    self.input_source / "sync.yml",
+                )
+            ),
+        )
+
+    def _extract_mvr(self, content: str, mapping: dict) -> list[MVRCamera]:
+        """Extracts camera-related information from MPE mvr config.
+        """
+        config = configparser.ConfigParser()
+        config.read_string(content)
+        frame_rate = float(
+            config["CAMERA_DEFAULT_CONFIG"]["frames_per_second"]
+        )
+        height = int(config["CAMERA_DEFAULT_CONFIG"]["height"])
+        width = int(config["CAMERA_DEFAULT_CONFIG"]["width"])
+
+        extracted = []
+        for mvr_name, assembly_name in mapping.items():
+            try:
+                extracted.append(
+                    (
+                        assembly_name,
+                        "".join(config[mvr_name]["sn"]),
+                        height,
+                        width,
+                        frame_rate,
+                    )
+                )
+            except KeyError:
+                raise NeuropixelsRigException(
+                    "No camera found for: %s in mvr.ini" %
+                    mvr_name
+                )
+        return extracted
+
+    def _extract_sync(self, content: str) -> SyncContext:
+        """Extracts DAQ-related information from MPE sync config.
+        """
+        config = yaml.safe_load(content)
+        return (
+            config["device"],
+            config["freq"],
+            list(config["line_labels"].items()),
+        )
+
+    def _transform(self, extracted_source: RigContext) -> rig.Rig:
+        """Transforms extracted rig context into aind-data-schema rig.Rig
+        instance.
+        """
+        partial, mvr_cameras, sync_context = extracted_source
+
+        # search for partial sync daq
+        sync_device_name, sync_sample_rate, sync_channels = sync_context
+        sync_daq_name = "Sync"
+        for idx, partial_daq in enumerate(partial["daqs"]):
+            if partial_daq["name"] == sync_daq_name:
+                # remove from daqs for later spread operation
+                partial_sync_daq = partial["daqs"].pop(idx)
+                break
+        else:
+            raise NeuropixelsRigException(
+                "Sync daq not found in partial rig. expected=%s" %
+                sync_daq_name
+            )
+
+        sync_daq = {
+            **partial_sync_daq,
+            "channels": [
+                {
+                    "channel_name": name,
+                    "channel_type": "Digital Input",
+                    "device_name": sync_device_name,
+                    "event_based_sampling": False,
+                    "channel_index": line,
+                    "sample_rate": sync_sample_rate,
+                    "sample_rate_unit": "hertz"
+                }
+                for line, name in sync_channels
+            ],
+        }
+
+        camera_assemblies = []
+        for partial_camera_assembly in partial["cameras"]:
+            name = partial_camera_assembly["camera_assembly_name"]
+
+            found = list(filter(
+                lambda camera: camera[0] == name,
+                mvr_cameras,
+            ))
+            if len(found) < 1:
+                raise NeuropixelsRigException(
+                    "Camera assembly not found in partial rig. expected=%s"
+                    % name
+                )
+
+            assembly_name, serial_number, height, width, frame_rate = found[0]
+            camera_assemblies.append({
+                **partial_camera_assembly,
+                "camera": {
+                    **partial_camera_assembly["camera"],
+                    "name": assembly_name,
+                    "serial_number": serial_number,
+                    "pixel_height": height,
+                    "pixel_width": width,
+                    "size_unit": "pixel",
+                    "max_frame_rate": frame_rate,
+                }
+            })
+
+        return rig.Rig.parse_obj({
+            **partial,
+            "modification_date": datetime.date.today(),
+            "daqs": [
+                *partial["daqs"],
+                sync_daq,
+            ],
+            "cameras": camera_assemblies,
+        })

tests/resources/neuropixels/bad/mvr.ini

@@ -0,0 +1,71 @@
+[MVR_BACKEND]
+CAMERA_INPUT_SIGNAL_DWELL_MS=400
+DEFAULT_RECORDING_TIME_MINUTES=180
+CAMERA_START_STOP_RECORDING_SPACING_TIME_MS=50
+SNAPSHOT_FILE_FORMAT=image/jpeg
+CAMERAS_TO_IGNORE=<none>
+CODEC=h264_nvenc
+FFMPEG_INPUT_PIXEL_FORMAT=gray
+FFMPEG_OUTPUT_PIXEL_FORMAT=yuv420p
+H264_CRF=17
+H264_PRESET=medium
+H264_HW_ACCELERATION_MODE=cuvid
+HIDE_FFMPEG_BANNER=true
+
+[CAMERA_DEFAULT_CONFIG]
+LABEL=none
+TAG=-1
+PERCENTAGE_OF_FRAMES_TO_SHOW_IN_UI=50
+CAMERA_FRAME_BUFFER_SIZE=10
+CAMERA_GAIN=0.0000000000
+CAMERA_EXPOSURE_MS=10.0000000000
+STREAM_BYTES_PER_SECOND=45000000
+FRAMES_PER_SECOND=60.0000000000
+BINNING_FACTOR=1
+WIDTH=658
+HEIGHT=492
+PAINT_FRAME_ID=true
+FRAME_ID_OFFSET_X=0
+FRAME_ID_OFFSET_Y=0
+FLIP_IMAGE_VERTICALLY=false
+FLIP_IMAGE_HORIZONTALLY=false
+ROTATE_IMAGE=0
+UI_FUNCTION=panel
+SN=none
+INITIAL_CUSTOM_EXPOSURE_TIME_MS=1.0000000000
+INITIAL_CUSTOM_RECORDED_VIDEO_FRAMES=10
+
+[UI_PROPERTIES]
+LOG_LEVEL=INFO
+GRID_MODE=1x3
+STARTUP_UI_IN_BASIC_MODE=false
+SOCKET_SERVER_PORT=50000
+UI_LAYOUT_MODE=landscape
+MAX_FRAME_RATE_ON_GRID_PANEL=10.0000000000
+MAX_FRAME_RATE_ON_FOCUS_PANEL=30.0000000000
+LOG_ISSUE_URL=https://alleninstitute.sharepoint.com/sites/Instrumentation/MPE_Software/MPE%20Software/MultiVideoRecorder.aspx
+LOG_SERVER_HOST_NAME=eng-logtools
+LOG_SERVER_HOST_PORT=9000
+ADMIN_PASSWORD=5f4dcc3b5aa765d61d8327deb882cf99
+KILL_FILE_NAME=mvr.kill
+KILL_FILE_FOLDER=c:\ProgramData\AIBS_MPE\kfs
+IN_PRODUCTION=true
+USE_LOG_TIME_FOR_LOG_FILE_NAME=false
+
+[Camera 2]
+ID=DEV_000F315C1468
+SN=50-0536905703
+LABEL=Eye
+TAG=2
+CAMERA_EXPOSURE_MS=16.0000000000
+CAMERA_GAIN=22.0000000000
+INITIAL_CUSTOM_EXPOSURE_TIME_MS=2.0000000000
+
+[Camera 3]
+ID=DEV_000F315C1467
+SN=50-0536905703
+LABEL=Face
+TAG=3
+CAMERA_EXPOSURE_MS=3.0000000000
+CAMERA_GAIN=13.0000000000
+INITIAL_CUSTOM_EXPOSURE_TIME_MS=3.0000000000

tests/resources/neuropixels/bad/mvr.mapping.json

@@ -0,0 +1,5 @@
+{
+    "Camera 5": "Behavior",
+    "Camera 2": "Eye",
+    "Camera 3": "Face forward"
+}