add converter; some debugs

src/leifer_lab_to_nwb/randi_nature_2023/_randi_nature_2023_converter.py

@@ -6,8 +6,28 @@
 import pynwb
 from pydantic import FilePath
 
+from leifer_lab_to_nwb.randi_nature_2023.interfaces import (
+    ExtraOphysMetadataInterface,
+    NeuroPALImagingInterface,
+    NeuroPALSegmentationInterface,
+    OptogeneticStimulationInterface,
+    PumpProbeImagingInterface,
+    PumpProbeSegmentationInterface,
+)
+
+
+class RandiNature2023Converter(neuroconv.NWBConverter):
+    data_interface_classes = {
+        "PumpProbeImagingInterfaceGreen": PumpProbeImagingInterface,
+        "PumpProbeImagingInterfaceRed": PumpProbeImagingInterface,
+        "PumpProbeSegmentationInterfaceGreed": PumpProbeSegmentationInterface,
+        "PumpProbeSegmentationInterfaceRed": PumpProbeSegmentationInterface,
+        "NeuroPALImagingInterface": NeuroPALImagingInterface,
+        "NeuroPALSegmentationInterface": NeuroPALSegmentationInterface,
+        "OptogeneticStimulationInterface": OptogeneticStimulationInterface,
+        "ExtraOphysMetadataInterface": ExtraOphysMetadataInterface,
+    }
 
-class RandiNature2023Converter(neuroconv.ConverterPipe):
     def get_metadata_schema(self) -> dict:
         base_metadata_schema = super().get_metadata_schema()
 

src/leifer_lab_to_nwb/randi_nature_2023/convert_session.py

@@ -4,67 +4,82 @@
 import pathlib
 
 import pandas
+import pynwb
 from dateutil import tz
 
 from leifer_lab_to_nwb.randi_nature_2023 import RandiNature2023Converter
-from leifer_lab_to_nwb.randi_nature_2023.interfaces import (
-    ExtraOphysMetadataInterface,
-    OptogeneticStimulationInterface,
-    PumpProbeImagingInterface,
-    SubjectInterface,
-)
+
+# STUB_TEST=True creates 'preview' files that truncate all major data blocks; useful for ensuring process runs smoothly
+# STUB_TEST=False performs a full file conversion
+STUB_TEST = True
+
 
 # Define base folder of source data
-base_folder_path = pathlib.Path("E:/Leifer")
-session_folder_path = base_folder_path / "Hermaphrodite"
+# Change these as needed on new systems
+BASE_FOLDER_PATH = pathlib.Path("D:/Leifer")
+SESSION_FOLDER_PATH = BASE_FOLDER_PATH / "20211104"
+# LOGBOOK_FILE_PATH = SESSION_FOLDER_PATH / "logbook.txt"
+
+PUMPPROBE_FOLDER_PATH = SESSION_FOLDER_PATH / "pumpprobe_20211104_163944"
+MULTICOLOR_FOLDER_PATH = SESSION_FOLDER_PATH / "multicolorworm_20211104_162630"
+
+NWB_OUTPUT_FOLDER_PATH = BASE_FOLDER_PATH / "nwbfiles"
+
+# *************************************************************************
+# Everything below this line is automated and should not need to be changed
+# *************************************************************************
 
-pumpprobe_folder_path = session_folder_path / "pumpprobe_20210830_111646"
+NWB_OUTPUT_FOLDER_PATH.mkdir(exist_ok=True)
 
-# Parse session start time from top path
-session_string = pumpprobe_folder_path.stem.removeprefix("pumpprobe_")
+# Parse session start time from the pumpprobe path
+session_string = PUMPPROBE_FOLDER_PATH.stem.removeprefix("pumpprobe_")
 session_start_time = datetime.datetime.strptime(session_string, "%Y%m%d_%H%M%S")
 session_start_time = session_start_time.replace(tzinfo=tz.gettz("US/Eastern"))
 
-# Define specific paths for interfaces and output
-raw_pumpprobe_folder_path = pumpprobe_folder_path / "raw"
-raw_data_file_path = raw_pumpprobe_folder_path / "sCMOS_Frames_U16_1024x512.dat"
-logbook_file_path = raw_pumpprobe_folder_path.parent / "logbook.txt"
-
-nwbfile_folder_path = base_folder_path / "nwbfiles"
-nwbfile_folder_path.mkdir(exist_ok=True)
-nwbfile_path = nwbfile_folder_path / f"{session_string}.nwb"
+# TODO: might be able to remove these when NeuroConv supports better schema validation
+PUMPPROBE_FOLDER_PATH = str(PUMPPROBE_FOLDER_PATH)
+MULTICOLOR_FOLDER_PATH = str(MULTICOLOR_FOLDER_PATH)
 
 # Initialize interfaces
 data_interfaces = list()
 
-# TODO: pending logbook consistency across sessions (still uploading)
-# subject_interface = SubjectInterface(file_path=logbook_file_path, session_id=session_string)
-# data_interfaces.append(subject_interface)
-
-# TODO: pending extension
-# one_photon_series_interface = PumpProbeImagingInterface(folder_path=raw_pumpprobe_folder_path)
-# data_interfaces.append(one_photon_series_interface)
-
-extra_ophys_metadata_interface = ExtraOphysMetadataInterface(folder_path=raw_pumpprobe_folder_path)
-data_interfaces.append(extra_ophys_metadata_interface)
-
-optogenetic_stimulation_interface = OptogeneticStimulationInterface(folder_path=raw_pumpprobe_folder_path)
-data_interfaces.append(optogenetic_stimulation_interface)
+source_data = {
+    "PumpProbeImagingInterfaceGreen": {"pumpprobe_folder_path": PUMPPROBE_FOLDER_PATH, "channel_name": "Green"},
+    "PumpProbeImagingInterfaceRed": {"pumpprobe_folder_path": PUMPPROBE_FOLDER_PATH, "channel_name": "Red"},
+    "PumpProbeSegmentationInterfaceGreed": {"pumpprobe_folder_path": PUMPPROBE_FOLDER_PATH, "channel_name": "Green"},
+    "PumpProbeSegmentationInterfaceRed": {"pumpprobe_folder_path": PUMPPROBE_FOLDER_PATH, "channel_name": "Red"},
+    "NeuroPALImagingInterface": {"multicolor_folder_path": MULTICOLOR_FOLDER_PATH},
+    "NeuroPALSegmentationInterface": {"multicolor_folder_path": MULTICOLOR_FOLDER_PATH},
+    "OptogeneticStimulationInterface": {"pumpprobe_folder_path": PUMPPROBE_FOLDER_PATH},
+    "ExtraOphysMetadataInterface": {"pumpprobe_folder_path": PUMPPROBE_FOLDER_PATH},
+}
 
 # Initialize converter
-converter = RandiNature2023Converter(data_interfaces=data_interfaces)
+converter = RandiNature2023Converter(source_data=source_data)
 
 metadata = converter.get_metadata()
 
 metadata["NWBFile"]["session_start_time"] = session_start_time
 
-# TODO: shouldn't need most of this once logbook parsing is done
-metadata["Subject"]["subject_id"] = session_start_time.strftime("%y%m%d")
+# TODO: these are placeholders that would be read in from a logbook read+lookup
+subject_id = session_start_time.strftime("%y%m%d")
+metadata["Subject"]["subject_id"] = subject_id
 metadata["Subject"]["species"] = "C. elegans"
 metadata["Subject"]["sex"] = "XX"
 metadata["Subject"]["age"] = "P1D"
 # metadata["Subject"]["growth_stage_time"] = pandas.Timedelta(hours=2, minutes=30).isoformat()  # TODO: request
 metadata["Subject"]["growth_stage"] = "YA"
 metadata["Subject"]["cultivation_temp"] = 20.0
 
-converter.run_conversion(nwbfile_path=nwbfile_path, metadata=metadata, overwrite=True)
+conversion_options = {
+    "PumpProbeImagingInterfaceGreen": {"stub_test": True},
+    "PumpProbeImagingInterfaceRed": {"stub_test": True},
+    "PumpProbeSegmentationInterfaceGreed": {"stub_test": True},
+    "PumpProbeSegmentationInterfaceRed": {"stub_test": True},
+    "NeuroPALImagingInterface": {"stub_test": True},
+}
+
+nwbfile_path = NWB_OUTPUT_FOLDER_PATH / f"sub-{subject_id}_ses-{session_string}.nwb"
+converter.run_conversion(
+    nwbfile_path=nwbfile_path, metadata=metadata, overwrite=True, conversion_options=conversion_options
+)

src/leifer_lab_to_nwb/randi_nature_2023/interfaces/_extra_ophys_metadata.py

@@ -4,12 +4,11 @@
 import neuroconv
 import pandas
 import pynwb
-from pydantic import DirectoryPath
 
 
 class ExtraOphysMetadataInterface(neuroconv.BaseDataInterface):
 
-    def __init__(self, *, pumpprobe_folder_path: DirectoryPath) -> None:
+    def __init__(self, *, pumpprobe_folder_path: str | pathlib.Path) -> None:
         """
         A custom interface for adding extra table metadata for the ophys rig.
 

src/leifer_lab_to_nwb/randi_nature_2023/interfaces/_neuropal_imaging_interface.py

@@ -8,13 +8,12 @@
 import numpy
 import pandas
 import pynwb
-from pydantic import DirectoryPath
 
 
 class NeuroPALImagingInterface(neuroconv.basedatainterface.BaseDataInterface):
     """Custom interface for automatically setting metadata and conversion options for this experiment."""
 
-    def __init__(self, *, multicolor_folder_path: DirectoryPath) -> None:
+    def __init__(self, *, multicolor_folder_path: str | pathlib.Path) -> None:
         """
         A custom interface for the raw volumetric PumpProbe data.
 

src/leifer_lab_to_nwb/randi_nature_2023/interfaces/_neuropal_segmentation_interface.py

@@ -5,12 +5,11 @@
 import neuroconv
 import pynwb
 from neuroconv.basedatainterface import BaseDataInterface
-from pydantic import DirectoryPath
 
 
 class NeuroPALSegmentationInterface(BaseDataInterface):
 
-    def __init__(self, *, multicolor_folder_path: DirectoryPath):
+    def __init__(self, *, multicolor_folder_path: str | pathlib.Path):
         """
         A custom interface for the raw volumetric NeuroPAL data.
 

src/leifer_lab_to_nwb/randi_nature_2023/interfaces/_optogenetic_stimulation.py

@@ -6,12 +6,11 @@
 import numpy
 import pandas
 import pynwb
-from pydantic import DirectoryPath
 
 
 class OptogeneticStimulationInterface(neuroconv.BaseDataInterface):
 
-    def __init__(self, *, pumpprobe_folder_path: DirectoryPath):
+    def __init__(self, *, pumpprobe_folder_path: str | pathlib.Path):
         """
         A custom interface for the two photon optogenetic stimulation data.
 
@@ -93,8 +92,10 @@ def add_to_nwbfile(
                 "its lateral position. The pulsed beam was then combined with the imaging light path by a dichroic "
                 "mirror immediately before entering the back of the objective."
             ),
-            lateral_point_spread_function_in_um="9 ± 0.7",  # TODO
-            axial_point_spread_function_in_um="32 ± 1.6",  # TODO
+            # Calculated manually from the 'source data' of Supplementary Figure 2a
+            # https://www.nature.com/articles/s41586-023-06683-4#MOESM10
+            lateral_point_spread_function_in_um="(-0.245, 0.059) ± (0.396, 0.264)",
+            axial_point_spread_function_in_um="0.444 ± 0.536",
         )
         nwbfile.add_lab_meta_data(temporal_focusing)
 
@@ -105,12 +106,15 @@ def add_to_nwbfile(
             description="Table for storing the target centroids, defined by a one-voxel mask.",
             imaging_plane=imaging_plane,
         )
-        for target_x, target_y, target_z in zip(
+        targeted_plane_segmentation.add_column(name="depth_in_um", description="Targeted depth in micrometers.")
+        for target_x_index, target_y_index, depth_in_mm in zip(
             self.optogenetic_stimulus_table["optogTargetX"],
             self.optogenetic_stimulus_table["optogTargetY"],
             self.optogenetic_stimulus_table["optogTargetZ"],
         ):
-            targeted_plane_segmentation.add_roi(voxel_mask=[(target_x, target_y, target_z, 1.0)])
+            targeted_plane_segmentation.add_roi(
+                pixel_mask=[(int(target_x_index), int(target_y_index), 1.0)], depth_in_um=depth_in_mm * 1e3
+            )
         image_segmentation.add_plane_segmentation(targeted_plane_segmentation)
 
         # Hardcoded duration from the methods section of paper

src/leifer_lab_to_nwb/randi_nature_2023/interfaces/_pump_probe_imaging_interface.py

@@ -7,7 +7,6 @@
 import numpy
 import pandas
 import pynwb
-from pydantic import DirectoryPath
 
 _DEFAULT_CHANNEL_NAMES = ["Green", "Red"]
 _DEFAULT_CHANNEL_FRAME_SLICING = {
@@ -21,7 +20,7 @@ class PumpProbeImagingInterface(neuroconv.basedatainterface.BaseDataInterface):
     def __init__(
         self,
         *,
-        pumpprobe_folder_path: DirectoryPath,
+        pumpprobe_folder_path: str | pathlib.Path,
         channel_name: Literal[_DEFAULT_CHANNEL_NAMES] | str,
         channel_frame_slicing: tuple[slice, slice] | None = None,
     ) -> None:
@@ -161,7 +160,7 @@ def add_to_nwbfile(
         timestamps = self.timestamps if not stub_test else self.timestamps[:stub_frames]
 
         variable_depth_microscopy_series = ndx_microscopy.VariableDepthMicroscopySeries(
-            name="PumpProbeImaging",
+            name=f"PumpProbeImaging{self.channel_name}",
             description="",  # TODO
             microscope=microscope,
             light_source=light_source,

src/leifer_lab_to_nwb/randi_nature_2023/interfaces/_pump_probe_segmentation_extractor.py

@@ -1,8 +1,8 @@
+import pathlib
 import pickle
 from typing import Iterable, Literal, Tuple, Union
 
 import numpy
-from pydantic import FilePath
 from roiextractors import SegmentationExtractor
 
 
@@ -12,7 +12,7 @@ class PumpProbeSegmentationExtractor(SegmentationExtractor):
     def __init__(
         self,
         *,
-        file_path: FilePath,
+        file_path: str | pathlib.Path,
         timestamps: Iterable[float],
         image_shape: Tuple[int, int],
     ) -> None:

src/leifer_lab_to_nwb/randi_nature_2023/interfaces/_pump_probe_segmentation_interface.py

@@ -6,15 +6,14 @@
 from neuroconv.datainterfaces.ophys.basesegmentationextractorinterface import (
     BaseSegmentationExtractorInterface,
 )
-from pydantic import DirectoryPath
 from pynwb import NWBFile
 
 
 class PumpProbeSegmentationInterface(BaseSegmentationExtractorInterface):
     ExtractorModuleName = "leifer_lab_to_nwb.randi_nature_2023.interfaces._pump_probe_segmentation_extractor"
     ExtractorName = "PumpProbeSegmentationExtractor"
 
-    def __init__(self, *, pumpprobe_folder_path: DirectoryPath, channel_name: Literal["Green", "Red"] | str):
+    def __init__(self, *, pumpprobe_folder_path: str | pathlib.Path, channel_name: Literal["Green", "Red"] | str):
         """
         A custom interface for the raw volumetric pumpprobe data.
 

src/leifer_lab_to_nwb/randi_nature_2023/test_all_interfaces.py

@@ -1,4 +1,10 @@
-"""Main conversion script for a single session of data for the Randi et al. Nature 2023 paper."""
+"""
+Test each individual interface by performing standalone file creations.
+
+An actual conversion should use the `convert_session.py` or `convert_data.py` scripts.
+
+This just makes debugging easier.
+"""
 
 import datetime
 import pathlib