diff --git a/README.md b/README.md index 7cd7078..4e8ec3c 100644 --- a/README.md +++ b/README.md @@ -7,8 +7,14 @@ Includes the publication Neural signal propagation atlas of Caenorhabditis elega ## Installation -``` +```bash git clone https://github.com/catalystneuro/leifer-lab-to-nwb cd leifer-lab-to-nwb pip install -e . ``` + +Then to install the specific set of dependencies for a particular conversion, such as `randi_nature_2023`: + +```bash +pip install --requirement src/leifer_lab_to_nwb/randi_nature_2023/requirements.txt +``` diff --git a/src/leifer_lab_to_nwb/randi_nature_2023/_randi_nature_2023_converter.py b/src/leifer_lab_to_nwb/randi_nature_2023/_randi_nature_2023_converter.py index 5008720..243b049 100644 --- a/src/leifer_lab_to_nwb/randi_nature_2023/_randi_nature_2023_converter.py +++ b/src/leifer_lab_to_nwb/randi_nature_2023/_randi_nature_2023_converter.py @@ -1,18 +1,29 @@ +import copy from typing import Union import ndx_multichannel_volume import neuroconv import pynwb +from pydantic import FilePath class RandiNature2023Converter(neuroconv.ConverterPipe): + def get_metadata_schema(self) -> dict: + base_metadata_schema = super().get_metadata_schema() + + # Suppress special Subject field validations + metadata_schema = copy.deepcopy(base_metadata_schema) + metadata_schema["properties"].pop("Subject") + + return metadata_schema + def run_conversion( self, - nwbfile_path: Union[str, None] = None, - nwbfile: Union[pynwb.NWBFile, None] = None, - metadata: Union[dict, None] = None, + nwbfile_path: FilePath | None = None, + nwbfile: pynwb.NWBFile | None = None, + metadata: dict | None = None, overwrite: bool = False, - conversion_options: Union[dict, None] = None, + conversion_options: dict | None = None, ) -> pynwb.NWBFile: if metadata is None: metadata = self.get_metadata() @@ -20,7 +31,7 @@ def run_conversion( metadata_copy = dict(metadata) subject_metadata = metadata_copy.pop("Subject") # Must remove from base metadata - ibl_subject = ndx_multichannel_volume.CElegansSubject(**subject_metadata) + subject = ndx_multichannel_volume.CElegansSubject(**subject_metadata) conversion_options = conversion_options or dict() self.validate_conversion_options(conversion_options=conversion_options) @@ -32,7 +43,7 @@ def run_conversion( overwrite=overwrite, verbose=self.verbose, ) as nwbfile_out: - nwbfile_out.subject = ibl_subject + nwbfile_out.subject = subject for interface_name, data_interface in self.data_interface_objects.items(): data_interface.add_to_nwbfile( nwbfile=nwbfile_out, metadata=metadata_copy, **conversion_options.get(interface_name, dict()) diff --git a/src/leifer_lab_to_nwb/randi_nature_2023/convert_session.py b/src/leifer_lab_to_nwb/randi_nature_2023/convert_session.py index 2081300..0ee6bbd 100644 --- a/src/leifer_lab_to_nwb/randi_nature_2023/convert_session.py +++ b/src/leifer_lab_to_nwb/randi_nature_2023/convert_session.py @@ -9,8 +9,8 @@ from leifer_lab_to_nwb.randi_nature_2023 import RandiNature2023Converter from leifer_lab_to_nwb.randi_nature_2023.interfaces import ( ExtraOphysMetadataInterface, - OnePhotonSeriesInterface, OptogeneticStimulationInterface, + PumpProbeImagingInterface, SubjectInterface, ) @@ -30,38 +30,41 @@ raw_data_file_path = raw_pumpprobe_folder_path / "sCMOS_Frames_U16_1024x512.dat" logbook_file_path = raw_pumpprobe_folder_path.parent / "logbook.txt" -nwbfile_path = base_folder_path / "nwbfiles" / f"{session_string}.nwb" +nwbfile_folder_path = base_folder_path / "nwbfiles" +nwbfile_folder_path.mkdir(exist_ok=True) +nwbfile_path = nwbfile_folder_path / f"{session_string}.nwb" # Initialize interfaces data_interfaces = list() +# TODO: pending logbook consistency across sessions (still uploading) # subject_interface = SubjectInterface(file_path=logbook_file_path, session_id=session_string) +# data_interfaces.append(subject_interface) -# one_photon_series_interface = OnePhotonSeriesInterface(folder_path=raw_pumpprobe_folder_path) +# TODO: pending extension +# one_photon_series_interface = PumpProbeImagingInterface(folder_path=raw_pumpprobe_folder_path) +# data_interfaces.append(one_photon_series_interface) extra_ophys_metadata_interface = ExtraOphysMetadataInterface(folder_path=raw_pumpprobe_folder_path) +data_interfaces.append(extra_ophys_metadata_interface) optogenetic_stimulation_interface = OptogeneticStimulationInterface(folder_path=raw_pumpprobe_folder_path) +data_interfaces.append(optogenetic_stimulation_interface) # Initialize converter -data_interfaces = [ - # subject_interface, # TODO: pending logbook consistency across sessions (still uploading) - # one_photon_series_interface, # TODO: pending extension - extra_ophys_metadata_interface, - optogenetic_stimulation_interface, -] converter = RandiNature2023Converter(data_interfaces=data_interfaces) metadata = converter.get_metadata() metadata["NWBFile"]["session_start_time"] = session_start_time -# metadata["Subject"]["subject_id"] = session_start_time.strftime("%y%m%d") # TODO: hopefully come up with better ID -# metadata["Subject"]["species"] = "C. elegans" -# metadata["Subject"]["sex"] = "XX" # TODO: pull from global listing by subject -# metadata["Subject"]["age"] = "P1D" # TODO: request +# TODO: shouldn't need most of this once logbook parsing is done +metadata["Subject"]["subject_id"] = session_start_time.strftime("%y%m%d") +metadata["Subject"]["species"] = "C. elegans" +metadata["Subject"]["sex"] = "XX" +metadata["Subject"]["age"] = "P1D" # metadata["Subject"]["growth_stage_time"] = pandas.Timedelta(hours=2, minutes=30).isoformat() # TODO: request -# metadata["Subject"]["growth_stage"] = "YA" # TODO: request -# metadata["Subject"]["cultivation_temp"] = "20." # TODO: request, schema says in units Celsius +metadata["Subject"]["growth_stage"] = "YA" +metadata["Subject"]["cultivation_temp"] = 20.0 converter.run_conversion(nwbfile_path=nwbfile_path, metadata=metadata, overwrite=True) diff --git a/src/leifer_lab_to_nwb/randi_nature_2023/interfaces/__init__.py b/src/leifer_lab_to_nwb/randi_nature_2023/interfaces/__init__.py index ce007c6..39d83e8 100644 --- a/src/leifer_lab_to_nwb/randi_nature_2023/interfaces/__init__.py +++ b/src/leifer_lab_to_nwb/randi_nature_2023/interfaces/__init__.py @@ -1,13 +1,17 @@ """Collection of interfaces for the conversion of data related to the Randi (Nature 2023) paper from the Leifer lab.""" from ._extra_ophys_metadata import ExtraOphysMetadataInterface -from ._logbook_metadata import SubjectInterface -from ._onephotonseries import OnePhotonSeriesInterface +from ._neuropal_imaging_interface import NeuroPALImagingInterface +from ._neuropal_segmentation_interface import NeuroPALSegmentationInterface from ._optogenetic_stimulation import OptogeneticStimulationInterface +from ._pump_probe_imaging_interface import PumpProbeImagingInterface +from ._pump_probe_segmentation_interface import PumpProbeSegmentationInterface __all__ = [ "ExtraOphysMetadataInterface", - "OnePhotonSeriesInterface", + "PumpProbeImagingInterface", + "PumpProbeSegmentationInterface", + "NeuroPALImagingInterface", + "NeuroPALSegmentationInterface", "OptogeneticStimulationInterface", - "SubjectInterface", ] diff --git a/src/leifer_lab_to_nwb/randi_nature_2023/interfaces/_binaryimagingextractor.py b/src/leifer_lab_to_nwb/randi_nature_2023/interfaces/_binaryimagingextractor.py deleted file mode 100644 index 82f80a3..0000000 --- a/src/leifer_lab_to_nwb/randi_nature_2023/interfaces/_binaryimagingextractor.py +++ /dev/null @@ -1,75 +0,0 @@ -from typing import Iterable, Literal, Tuple, Union - -import numpy -from pydantic import FilePath -from roiextractors import ImagingExtractor - - -# TODO: propagate to ROIExtractors -class BinaryImagingExtractor(ImagingExtractor): - """A generic class for reading binary imaging data.""" - - def __init__( - self, - *, - file_path: FilePath, - dtype: str, - shape: Tuple[int, int, int], - starting_time: float | None = None, - sampling_frequency: float | None = None, - timestamps: Iterable[float] | None = None, - offset: int = 0, - order: Literal["C", "F"] = "C", - ) -> None: - if sampling_frequency is None and timestamps is None: - raise ValueError("At least one of `sampling_frequency` or `timestamps` must be specified.") - if sampling_frequency is not None and starting_time is None: - raise ValueError("Since `sampling_frequency` was specified, please also specify the `starting_time`.") - - super().__init__( - file_path=file_path, - dtype=dtype, - shape=shape, - starting_time=starting_time, - sampling_frequency=sampling_frequency, - timestamps=timestamps, - offset=offset, - order=order, - ) - self._memmap = numpy.memmap(filename=file_path, dtype=dtype, shape=shape, offset=offset, order=order, mode="r") - - def get_image_size(self) -> Tuple[int, int]: - return self._kwargs["shape"][1:] - - def get_num_frames(self) -> int: - return self._kwargs["shape"][0] - - def get_sampling_frequency(self) -> float: - if self._kwargs["sampling_frequency"] is None: - raise ValueError( - "This ImagingExtractor does not have a constant sampling frequency! Use `get_timestamps` instead." - ) - - return self._kwargs["sampling_frequency"] - - def get_timestamps(self) -> Iterable[float]: - if self._kwargs["timestamps"] is None: - raise ValueError( - "This ImagingExtractor does not have timestamps set! Use `get_sampling_frequency` instead." - ) - - return self._kwargs["timestamps"] - - def get_channel_names(self) -> list: - raise NotImplementedError - - def get_num_channels(self) -> int: - return 1 - - def get_dtype(self) -> str: - return self._kwargs["dtype"] - - def get_video( - self, start_frame: Union[int, None] = None, end_frame: Union[int, None] = None, channel: int = 0 - ) -> numpy.ndarray: - return self._memmap[start_frame:end_frame, ...] diff --git a/src/leifer_lab_to_nwb/randi_nature_2023/interfaces/_extra_ophys_metadata.py b/src/leifer_lab_to_nwb/randi_nature_2023/interfaces/_extra_ophys_metadata.py index d0976ff..0531547 100644 --- a/src/leifer_lab_to_nwb/randi_nature_2023/interfaces/_extra_ophys_metadata.py +++ b/src/leifer_lab_to_nwb/randi_nature_2023/interfaces/_extra_ophys_metadata.py @@ -8,46 +8,29 @@ class ExtraOphysMetadataInterface(neuroconv.BaseDataInterface): - """A custom interface for adding extra table metadata for the ophys rig.""" - def __init__(self, *, folder_path: DirectoryPath) -> None: + def __init__(self, *, pumpprobe_folder_path: DirectoryPath) -> None: """ A custom interface for adding extra table metadata for the ophys rig. Parameters ---------- - folder_path : DirectoryPath + pumpprobe_folder_path : DirectoryPath Path to the raw pumpprobe folder. """ - folder_path = pathlib.Path(folder_path) + pumpprobe_folder_path = pathlib.Path(pumpprobe_folder_path) - super().__init__(folder_path=folder_path) - - self.z_scan_file_path = folder_path / "zScan.json" - with open(file=self.z_scan_file_path, mode="r") as fp: + z_scan_file_path = pumpprobe_folder_path / "zScan.json" + with open(file=z_scan_file_path, mode="r") as fp: self.z_scan = json.load(fp=fp) - self.sync_table_file_path = folder_path / "other-frameSynchronous.txt" - self.sync_table = pandas.read_table(filepath_or_buffer=self.sync_table_file_path, index_col=False) + sync_table_file_path = pumpprobe_folder_path / "other-frameSynchronous.txt" + self.sync_table = pandas.read_table(filepath_or_buffer=sync_table_file_path, index_col=False) - def add_to_nwbfile(self, nwbfile: pynwb.NWBFile): + def add_to_nwbfile(self, nwbfile: pynwb.NWBFile, metadata: dict): # Plane depths volt_per_um = 0.125 # Hardcoded value by the lab depth_in_um_per_pixel = 0.42 # Hardcoded value by the lab - frame_depth_table = pynwb.file.DynamicTable( - name="FrameDepths", - description=( - "Each frame was acquired at a different depth as tracked by the voltage supplied to an " - "Electrically Tunable Lense (ETL)." - ), - columns=[ - pynwb.file.VectorData( - name="depth_in_um", - # Referred to in file as 'piezo' but it's really the ETL - data=self.sync_table["Piezo position (V)"] / volt_per_um, - ) - ], - ) # zScan contents diff --git a/src/leifer_lab_to_nwb/randi_nature_2023/interfaces/_logbook_metadata.py b/src/leifer_lab_to_nwb/randi_nature_2023/interfaces/_logbook_metadata.py index 0609e28..f58c5ce 100644 --- a/src/leifer_lab_to_nwb/randi_nature_2023/interfaces/_logbook_metadata.py +++ b/src/leifer_lab_to_nwb/randi_nature_2023/interfaces/_logbook_metadata.py @@ -1,48 +1 @@ -import json -import pathlib - -import ndx_multichannel_volume -import neuroconv -import pandas -import pynwb -from pydantic import FilePath - - -class SubjectInterface(neuroconv.BaseDataInterface): - """A custom interface for adding extra subject metadata from the logbook.""" - - def __init__(self, *, file_path: FilePath, session_id: str) -> None: - """ - A custom interface for adding extra subject metadata from the logbook. - - Parameters - ---------- - file_path : FilePath - Path to the logbook for this session. - """ - file_path = pathlib.Path(file_path) - - super().__init__(file_path=file_path, session_id=session_id) - - with open(file=file_path, mode="r") as io: - self.logbook = io.readlines() - - def add_to_nwbfile(self, nwbfile: pynwb.NWBFile): - session_id = self.source_data["session_id"] - - logbook_growth_stage_mapping = { - "L4": "L4", - "young adult": "YA", - "L4/ya": "YA", # TODO: consult them on how to handle this case - } - - subject_start_line = self.logbook - - subject = ndx_multichannel_volume.CElegansSubject( - subject_id=session_id, # Sessions are effectively defined by the subject number on that day - description="", # TODO: find something from paper - species="Caenorhabditis elegans", - growth_stage=logbook_growth_stage_mapping[growth_stage], - strain=strain, - ) - nwbfile.subject = subject +# TODO: write a simple function to read the logbook YAML file and lookup the information for this session diff --git a/src/leifer_lab_to_nwb/randi_nature_2023/interfaces/_neuropal_imaging_interface.py b/src/leifer_lab_to_nwb/randi_nature_2023/interfaces/_neuropal_imaging_interface.py new file mode 100644 index 0000000..b31d556 --- /dev/null +++ b/src/leifer_lab_to_nwb/randi_nature_2023/interfaces/_neuropal_imaging_interface.py @@ -0,0 +1,122 @@ +import pathlib +import shutil +from typing import Literal + +import ndx_microscopy +import neuroconv +import numpy +import pandas +import pynwb +from pydantic import DirectoryPath + + +class NeuroPALImagingInterface(neuroconv.basedatainterface.BaseDataInterface): + """Custom interface for automatically setting metadata and conversion options for this experiment.""" + + def __init__(self, *, multicolor_folder_path: DirectoryPath) -> None: + """ + A custom interface for the raw volumetric PumpProbe data. + + Parameters + ---------- + multicolor_folder_path : directory + Path to the multicolor folder. + """ + super().__init__(multicolor_folder_path=multicolor_folder_path) + multicolor_folder_path = pathlib.Path(multicolor_folder_path) + + # If the device setup is ever changed, these may need to be exposed as keyword arguments + dtype = numpy.dtype("uint16") + frame_shape = (2048, 2048) + number_of_channels = 4 + number_of_depths = 26 + + # This file always has a few bytes on the end that make it not automatically reshapable as expected + # No clue where it comes from but they ignore those bytes even in their own processing code + dat_file_path = multicolor_folder_path / "frames-2048x2048.dat" + unshaped_data = numpy.memmap(filename=dat_file_path, dtype=dtype, mode="r") + clipped_data = unshaped_data[: number_of_channels * number_of_depths * frame_shape[0] * frame_shape[1]] + + # The reshape here still preserves the memory map + self.data_shape = (number_of_depths, number_of_channels, frame_shape[0], frame_shape[1]) + shaped_data = clipped_data.reshape(self.data_shape) + + self.data = shaped_data + + # def get_metadata(self) -> dict: + # one_photon_metadata = super().get_metadata(photon_series_type=self.photon_series_type) + # + # # Hardcoded value from lab + # # This is also an average in a sense - the exact depth is tracked by the Piezo and written + # # as a custom DynamicTable in the ExtraOphysMetadataInterface + # # depth_per_pixel = 0.42 + # + # # one_photon_metadata["Ophys"]["grid_spacing"] = (um_per_pixel, um_per_pixel, um_per_pixel) + # + # return one_photon_metadata + # + # def get_metadata_schema(self) -> dict: + # return super().get_metadata(photon_series_type=self.photon_series_type) + + def add_to_nwbfile( + self, + *, + nwbfile: pynwb.NWBFile, + metadata: dict | None = None, + stub_test: bool = False, + stub_depths: int = 3, + ) -> None: + # TODO: enhance all metadata + if "Microscope" not in nwbfile.devices: + microscope = ndx_microscopy.Microscope(name="Microscope") + nwbfile.add_device(devices=microscope) + else: + microscope = nwbfile.devices["Microscope"] + + if "LightSource" not in nwbfile.devices: + light_source = ndx_microscopy.MicroscopyLightSource(name="LightSource") + nwbfile.add_device(devices=light_source) + else: + light_source = nwbfile.devices["LightSource"] + + if "NeuroPALImagingSpace" not in nwbfile.lab_meta_data: + imaging_space = ndx_microscopy.VolumetricImagingSpace( + name="NeuroPALImagingSpace", description="", microscope=microscope + ) + nwbfile.add_lab_meta_data(lab_meta_data=imaging_space) + else: + imaging_space = nwbfile.lab_meta_data["PlanarImagingSpace"] + + # TODO: confirm the order of the channels in the data + neuropal_channel_names = ["mtagBFP2", "CyOFP1.5", "tagRFP-T", "mNeptune2.5"] + optical_channels = list() + light_sources = list() + for channel_name in neuropal_channel_names: + light_source = ndx_microscopy.MicroscopyLightSource(name=f"{channel_name}LightSource") + nwbfile.add_device(devices=light_source) + light_sources.append(light_source) + + optical_channel = ndx_microscopy.MicroscopyOpticalChannel( + name=f"{channel_name}Filter", description="", indicator=channel_name + ) + nwbfile.add_lab_meta_data(lab_meta_data=optical_channel) + optical_channels.append(optical_channel) + + # Not exposing chunking/buffering control here for simplicity; note that a single frame is about 8 MB + chunk_shape = (1, 1, self.data_shape[-2], self.data_shape[-1]) + + # Best we can do is limit the number of depths that are written by stub + imaging_data = self.data if not stub_test else self.data[:stub_depths, :, :, :] + data_iterator = neuroconv.tools.hdmf.SliceableDataChunkIterator(data=imaging_data, chunk_shape=chunk_shape) + + multi_channel_microscopy_volume = ndx_microscopy.MultiChannelMicroscopyVolume( + name="NeuroPALImaging", + description="", + microscope=microscope, + light_sources=light_sources[0], # TODO + imaging_space=imaging_space, + optical_channels=optical_channels[0], # TODO + data=data_iterator, + unit="n.a.", + ) + nwbfile.add_acquisition(multi_channel_microscopy_volume) diff --git a/src/leifer_lab_to_nwb/randi_nature_2023/interfaces/_neuropal_segmentation_interface.py b/src/leifer_lab_to_nwb/randi_nature_2023/interfaces/_neuropal_segmentation_interface.py new file mode 100644 index 0000000..bba3425 --- /dev/null +++ b/src/leifer_lab_to_nwb/randi_nature_2023/interfaces/_neuropal_segmentation_interface.py @@ -0,0 +1,103 @@ +import json +import pathlib + +import ndx_microscopy +import neuroconv +import pynwb +from neuroconv.basedatainterface import BaseDataInterface +from pydantic import DirectoryPath + + +class NeuroPALSegmentationInterface(BaseDataInterface): + + def __init__(self, *, multicolor_folder_path: DirectoryPath): + """ + A custom interface for the raw volumetric NeuroPAL data. + + Parameters + ---------- + multicolor_folder_path : DirectoryPath + Path to the multicolor folder. + """ + super().__init__(multicolor_folder_path=multicolor_folder_path) + multicolor_folder_path = pathlib.Path(multicolor_folder_path) + + brains_file_path = multicolor_folder_path / "brains.json" + with open(brains_file_path, "r") as io: + self.brains_info = json.load(fp=io) + + # Some basic homogeneity checks + assert len(self.brains_info["nInVolume"]) == 1, "Only one labeling is supported." + assert ( + len(self.brains_info["nInVolume"]) == len(self.brains_info["zOfFrame"]) + and len(self.brains_info["zOfFrame"]) == len(self.brains_info["labels"]) + and len(self.brains_info["labels"]) == len(self.brains_info["labels_confidences"]) + and len(self.brains_info["labels_confidences"]) == len(self.brains_info["labels_comments"]) + ), "Mismatch in JSON substructure lengths." + assert ( + self.brains_info["nInVolume"][0] == len(self.brains_info["labels"][0]) + and self.brains_info["nInVolume"][0] == len(self.brains_info["labels_confidences"][0]) + and self.brains_info["nInVolume"][0] == len(self.brains_info["labels_comments"][0]) + ), "Length of contents does not match number of ROIs." + + def add_to_nwbfile( + self, + *, + nwbfile: pynwb.NWBFile, + metadata: dict | None = None, + ) -> None: + # TODO: probably centralize this in a helper function + if "Microscope" not in nwbfile.devices: + microscope = ndx_microscopy.Microscope(name="Microscope") + nwbfile.add_device(devices=microscope) + else: + microscope = nwbfile.devices["Microscope"] + + if "NeuroPALImagingSpace" not in nwbfile.lab_meta_data: + imaging_space = ndx_microscopy.VolumetricImagingSpace( + name="NeuroPALImagingSpace", description="", microscope=microscope + ) + nwbfile.add_lab_meta_data(lab_meta_data=imaging_space) + else: + imaging_space = nwbfile.lab_meta_data["PlanarImagingSpace"] + + plane_segmentation = ndx_microscopy.MicroscopyPlaneSegmentation( + name="NeuroPALPlaneSegmentation", + description="The NeuroPAL segmentation of the C. elegans brain with cell labels.", + imaging_space=imaging_space, + ) + plane_segmentation.add_column( + name="labels", + description="The C. elegans cell names labeled from the NeuroPAL imaging.", + ) + plane_segmentation.add_column( + name="labels_confidences", + description="The C. elegans cell names labeled from the NeuroPAL imaging.", + ) + plane_segmentation.add_column( + name="labels_comments", + description="Various comments about the cell label classification process.", + ) + + number_of_rois = self.brains_info["nInVolume"][0] + for neuropal_roi_id in range(number_of_rois): + coordinate_info = self.brains_info["coordZYX"][neuropal_roi_id] + coordinates = (coordinate_info[1], coordinate_info[2], coordinate_info[0], 1.0) + + plane_segmentation.add_row( + id=neuropal_roi_id, + voxel_mask=[coordinates], + labels=self.brains_info["labels"][0][neuropal_roi_id], + labels_confidences=self.brains_info["labels_confidences"][0][neuropal_roi_id], + labels_comments=self.brains_info["labels_comments"][0][neuropal_roi_id], + ) + + image_segmentation = ndx_microscopy.MicroscopyImageSegmentation( + name="NeuroPALImageSegmentation", microscopy_plane_segmentations=[plane_segmentation] + ) + + ophys_module = neuroconv.tools.nwb_helpers.get_module(nwbfile=nwbfile, name="ophys") + ophys_module.add(image_segmentation) + + # TODO: include z frame depths for NeuroPAL side - maybe in the 'extra ophys interface'? + # Or just in a custom irregular grid spacing volumetric imaging space? diff --git a/src/leifer_lab_to_nwb/randi_nature_2023/interfaces/_one_photon_series.py b/src/leifer_lab_to_nwb/randi_nature_2023/interfaces/_one_photon_series.py deleted file mode 100644 index e8f31bf..0000000 --- a/src/leifer_lab_to_nwb/randi_nature_2023/interfaces/_one_photon_series.py +++ /dev/null @@ -1,89 +0,0 @@ -import pathlib - -import numpy -import pandas -from neuroconv.datainterfaces.ophys.baseimagingextractorinterface import ( - BaseImagingExtractorInterface, -) -from pydantic import DirectoryPath -from pynwb import NWBFile - - -class OnePhotonSeriesInterface(BaseImagingExtractorInterface): - """Custom interface for automatically setting metadata and conversion options for this experiment.""" - - ExtractorModuleName = "leifer_lab_to_nwb.randi_nature_2023.interfaces.binaryimagingextractor" # TODO: propagate - ExtractorName = "BinaryImagingExtractor" - - def __init__(self, *, folder_path: DirectoryPath): - """ - A custom interface for the raw volumetric pumpprobe data. - - Parameters - ---------- - folder_path : DirectoryPath - Path to the raw pumpprobe folder. - """ - folder_path = pathlib.Path(folder_path) - dat_file_path = next(folder_path.glob("*.dat")) - assert ( - "U16" in dat_file_path.stem - ), "Raw .dat file '{dat_file_path}' does not indicate uint16 dtype in filename." - dtype = numpy.dtype("uint16") - frame_shape = tuple(int(value) for value in dat_file_path.stem.split("_")[-1].split("x")) - - timestamps_file_path = folder_path / "framesDetails.txt" - timestamps_table = pandas.read_table(filepath_or_buffer=timestamps_file_path, index_col=False) - number_of_frames = timestamps_table.shape[0] - timestamps = numpy.array(timestamps_table["Timestamp"]) - - self.sync_table_file_path = folder_path / "other-frameSynchronous.txt" - self.sync_table = pandas.read_table(filepath_or_buffer=self.sync_table_file_path, index_col=False) - - scanning_direction = self.sync_table["Piezo direction (+-1)"] - scanning_direction_change_indices = numpy.where(numpy.diff(scanning_direction) != 0)[0] - number_of_depths_per_scanning_cycle = numpy.diff(scanning_direction_change_indices) - # This is only an estimate; individual - median_number_of_depths_per_scanning_cycle = int(numpy.median(number_of_depths_per_scanning_cycle)) - - volume_partitions_file_path = folder_path / "other-volumeMetadataUtilities.txt" - volume_partitions_table = pandas.read_table(filepath_or_buffer=volume_partitions_file_path, index_col=False) - number_of_frames = volume_partitions_table.shape[0] - - shape = (number_of_frames, frame_shape[0], frame_shape[1], number_of_depths) - - super.__init__(file_path=dat_file_path, dtype=dtype, shape=shape, timestamps=timestamps) - - def get_metadata(self) -> dict: - one_photon_metadata = super().get_metadata(photon_series_type="OnePhotonSeries") - - # Hardcoded value from lab - # This is also an average in a sense - the exact depth is tracked by the Piezo and written - # as a custom DynamicTable in the ExtraOphysMetadataInterface - depth_per_pixel = 0.42 - - one_photon_metadata["Ophys"]["grid_spacing"] = (um_per_pixel, um_per_pixel, um_per_pixel) - - return one_photon_metadata - - def get_metadata_schema(self) -> dict: - return super().get_metadata(photon_series_type="OnePhotonSeries") - - def add_to_nwbfile( - self, - *, - nwbfile: NWBFile, - metadata: dict | None = None, - photon_series_index: int = 0, - stub_test: bool = False, - stub_frames: int = 100, - ) -> None: - super().add_to_nwbfile( - nwbfile=nwbfile, - metadata=metadata, - photon_series_index=photon_series_index, - stub_test=stub_test, - stub_frames=stub_frames, - photon_series_type="OnePhotonSeries", - parent_container="acquisition", - ) diff --git a/src/leifer_lab_to_nwb/randi_nature_2023/interfaces/_optogenetic_stimulation.py b/src/leifer_lab_to_nwb/randi_nature_2023/interfaces/_optogenetic_stimulation.py index 3dd2060..ec1d422 100644 --- a/src/leifer_lab_to_nwb/randi_nature_2023/interfaces/_optogenetic_stimulation.py +++ b/src/leifer_lab_to_nwb/randi_nature_2023/interfaces/_optogenetic_stimulation.py @@ -10,26 +10,25 @@ class OptogeneticStimulationInterface(neuroconv.BaseDataInterface): - """Custom interface for the two photon optogenetic stimulation data.""" - def __init__(self, *, folder_path: DirectoryPath): + def __init__(self, *, pumpprobe_folder_path: DirectoryPath): """ - A custom interface for the two photon optogenetic volumetric pumpprobe data. + A custom interface for the two photon optogenetic stimulation data. Parameters ---------- - folder_path : DirectoryPath + pumpprobe_folder_path : DirectoryPath Path to the raw pumpprobe folder. """ - folder_path = pathlib.Path(folder_path) + pumpprobe_folder_path = pathlib.Path(pumpprobe_folder_path) - self.optogenetic_stimulus_file_path = folder_path / "pharosTriggers.txt" + optogenetic_stimulus_file_path = pumpprobe_folder_path / "pharosTriggers.txt" self.optogenetic_stimulus_table = pandas.read_table( - filepath_or_buffer=self.optogenetic_stimulus_file_path, index_col=False + filepath_or_buffer=optogenetic_stimulus_file_path, index_col=False ) - self.timestamps_file_path = folder_path / "framesDetails.txt" - self.timestamps_table = pandas.read_table(filepath_or_buffer=self.timestamps_file_path, index_col=False) + timestamps_file_path = pumpprobe_folder_path / "framesDetails.txt" + self.timestamps_table = pandas.read_table(filepath_or_buffer=timestamps_file_path, index_col=False) self.timestamps = numpy.array(self.timestamps_table["Timestamp"]) def add_to_nwbfile( @@ -37,24 +36,23 @@ def add_to_nwbfile( *, nwbfile: pynwb.NWBFile, metadata: Union[dict, None] = None, - photon_series_index: int = 0, - stub_test: bool = False, - stub_frames: int = 100, ) -> None: assert "Microscope" in nwbfile.devices, ( "The `Microscope` must be added before this interface! Make sure the call to " - "`.run_conversion` for this interface occurs after the `OnePhotonSeriesInterface`." + "`.run_conversion` for this interface occurs after the `PumpProbeSegmentationInterface`." ) microscope = nwbfile.devices["Microscope"] - assert "PlaneSegmentation" in nwbfile.processing["ophys"], ( + assert ( + "ImageSegmentation" in nwbfile.processing["ophys"].data_interfaces + and "PlaneSegmentation" in nwbfile.processing["ophys"]["ImageSegmentation"].plane_segmentations + ), ( "The `PlaneSegmentation` must be added before this interface! Make sure the call to " - "`.run_conversion` for this interface occurs after the `OnePhotonSeriesInterface`." + "`.run_conversion` for this interface occurs after the `PumpProbeSegmentationInterface`." ) - plane_segmentation = nwbfile.processing["ophys"]["PlaneSegmentation"] - - pharos_device = pynwb.Device(name="PHAROSLaser", description="") - nwbfile.add_device(device=pharos_device) + image_segmentation = nwbfile.processing["ophys"]["ImageSegmentation"] + plane_segmentation = image_segmentation.plane_segmentations["PlaneSegmentation"] + imaging_plane = plane_segmentation.imaging_plane light_source = ndx_patterned_ogen.LightSource( name="AmplifiedLaser", @@ -64,20 +62,19 @@ def add_to_nwbfile( ), model="ORPHEUS amplifier and PHAROS laser", manufacturer="Light Conversion", - stimulation_wavelength=850.0, # nm + stimulation_wavelength_in_nm=850.0, filter_description="Short pass at 1040 nm", - peak_power=1.2 / 1e3, # Hardcoded from the paper - # intensity=0.005, # TODO: issue raised - pulse_rate=500e3, # Hardcoded from the paper + peak_power_in_W=1.2 / 1e3, # Hardcoded from the paper + pulse_rate_in_Hz=500e3, # Hardcoded from the paper ) nwbfile.add_device(light_source) site = ndx_patterned_ogen.PatternedOptogeneticStimulusSite( name="PatternedOptogeneticStimulusSite", - description="Scanning", + description="Scanning", # TODO excitation_lambda=850.0, # nm effector="GUR-3/PRDX-2", - location="whole-brain", + location="whole brain", device=microscope, light_source=light_source, ) @@ -96,16 +93,25 @@ def add_to_nwbfile( "its lateral position. The pulsed beam was then combined with the imaging light path by a dichroic " "mirror immediately before entering the back of the objective." ), - lateral_point_spread_function="9 um ± 0.7 um", # TODO - axial_point_spread_function="32 um ± 1.6 um", # TODO + lateral_point_spread_function_in_um="9 ± 0.7", # TODO + axial_point_spread_function_in_um="32 ± 1.6", # TODO ) nwbfile.add_lab_meta_data(temporal_focusing) - # TODO: utilize objective registration - # for index, - # target = OptogeneticStimulusTarget(name=f"Target{index}", targeted_rois=targeted_rois_1) - # targets.append(target) - # nwbfile.add_lab_meta_data(target) + # Assume all targets are unique; if retargeting of the same location is ever enabled, it would be nice + # to refactor this to make proper reuse of target locations. + targeted_plane_segmentation = pynwb.ophys.PlaneSegmentation( + name="TargetPlaneSegmentation", + description="Table for storing the target centroids, defined by a one-voxel mask.", + imaging_plane=imaging_plane, + ) + for target_x, target_y, target_z in zip( + self.optogenetic_stimulus_table["optogTargetX"], + self.optogenetic_stimulus_table["optogTargetY"], + self.optogenetic_stimulus_table["optogTargetZ"], + ): + targeted_plane_segmentation.add_roi(voxel_mask=[(target_x, target_y, target_z, 1.0)]) + image_segmentation.add_plane_segmentation(targeted_plane_segmentation) # Hardcoded duration from the methods section of paper # TODO: may have to adjust this for unc-31 mutant strain subjects @@ -116,22 +122,33 @@ def add_to_nwbfile( stimulus_table = ndx_patterned_ogen.PatternedOptogeneticStimulusTable( name="OptogeneticStimulusTable", description=( - "Every 30 seconds, a random neuron was selected among the neurons found in the current volumetric " + "Every 30 seconds, a random neuron was selected among the neurons found in the current volumetric " "image, on the basis of only its tagRFP-T signal. After galvo-mirrors and the tunable lens set the " "position of the two-photon spot on that neuron, a 500-ms (300-ms for the unc-31-mutant strain) " "train of light pulses was used to optogenetically stimulate that neuron. The duration of stimulus " "illumination for the unc-31-mutant strain was selected to elicit calcium transients in stimulated " "neurons with a distribution of amplitudes such that the maximum amplitude was similar to those in " "WT-background animals. The output of the laser was controlled through the external interface to its " - "built-in pulse picker, and the power of the laser at the sample was 1.2 mW at 500 kHz. Neuron " + "built-in pulse picker, and the power of the laser at the sample was 1.2mW at 500kHz. Neuron " "identities were assigned to stimulated neurons after the completion of experiments using NeuroPAL." ), ) for index, start_time_in_s in enumerate(stimulus_start_times_in_s): + targeted_roi_reference = targeted_plane_segmentation.create_roi_table_region( + name="targeted_rois", description="The targeted ROI.", region=[index] + ) + # TODO: create a container of targets so they don't all get dumped to outer level of general + stimulus_target = ndx_patterned_ogen.OptogeneticStimulusTarget( + name=f"OptogeneticStimulusTarget{index}", targeted_rois=targeted_roi_reference + ) + nwbfile.add_lab_meta_data(stimulus_target) + stimulus_table.add_interval( start_time=start_time_in_s, stop_time=start_time_in_s + stimulus_duration_in_s, - targets=nwbfile.lab_meta_data[f"Target{index}"], + targets=stimulus_target, stimulus_pattern=temporal_focusing, stimulus_site=site, + power=1.2 / 1e3, # Hardcoded from the paper; TODO: should be 'power_in_W' ) + nwbfile.add_time_intervals(stimulus_table) diff --git a/src/leifer_lab_to_nwb/randi_nature_2023/interfaces/_pump_probe_imaging_interface.py b/src/leifer_lab_to_nwb/randi_nature_2023/interfaces/_pump_probe_imaging_interface.py new file mode 100644 index 0000000..eb05980 --- /dev/null +++ b/src/leifer_lab_to_nwb/randi_nature_2023/interfaces/_pump_probe_imaging_interface.py @@ -0,0 +1,175 @@ +import pathlib +import shutil +from typing import Literal + +import ndx_microscopy +import neuroconv +import numpy +import pandas +import pynwb +from pydantic import DirectoryPath + +_DEFAULT_CHANNEL_NAMES = ["Green", "Red"] +_DEFAULT_CHANNEL_FRAME_SLICING = { + "Green": (slice(0, 512), slice(0, 512)), + "Red": (slice(512, 1024), slice(0, 512)), +} + + +class PumpProbeImagingInterface(neuroconv.basedatainterface.BaseDataInterface): + + def __init__( + self, + *, + pumpprobe_folder_path: DirectoryPath, + channel_name: Literal[_DEFAULT_CHANNEL_NAMES] | str, + channel_frame_slicing: tuple[slice, slice] | None = None, + ) -> None: + """ + A custom interface for the raw volumetric PumpProbe data. + + Parameters + ---------- + pumpprobe_folder_path : directory + Path to the pumpprobe folder. + channel_name : either of "GreenChannel", "RedChannel" or an arbitrary string + The name given to the optical channel responsible for collecting this data. + The two allowed defaults determine other properties automatically; but when specifying an arbitrary string, + you will need to specify the other information (slicing range, wavelength metadata, etc.) manually. + channel_frame_slicing : tuple of slices, optional + If the `channel_name` is not one of the defaults, then then you must specify how to slice the frame + to extract the data for this channel. + + The default slicing is: + GreenChannel=(slice(0, 512), slice(0, 512)) + RedChannel=(slice(512, 1024), slice(0, 512)) + """ + if channel_name not in _DEFAULT_CHANNEL_NAMES and channel_frame_slicing is None: + message = ( + f"A custom `optical_channel_name` was specified ('{channel_name}') and was not one of the " + f"known defaults ('{_DEFAULT_CHANNEL_NAMES}'), but no frame slicing pattern was passed." + ) + raise ValueError(message=message) + + self.channel_name = channel_name + self.channel_frame_slicing = channel_frame_slicing or _DEFAULT_CHANNEL_FRAME_SLICING[channel_name] + + pumpprobe_folder_path = pathlib.Path(pumpprobe_folder_path) + + # If the device setup is ever changed, these may need to be exposed as keyword arguments + dtype = numpy.dtype("uint16") + frame_shape = (1024, 512) + + # From prototyping data, the frameSync seems to start first... + sync_table_file_path = pumpprobe_folder_path / "other-frameSynchronous.txt" + sync_table = pandas.read_table(filepath_or_buffer=sync_table_file_path, index_col=False) + frame_indices = sync_table["Frame index"] + + # ...then the frameDetails has timestamps for a subset of the frame indices + timestamps_file_path = pumpprobe_folder_path / "framesDetails.txt" + timestamps_table = pandas.read_table(filepath_or_buffer=timestamps_file_path, index_col=False) + number_of_frames = timestamps_table.shape[0] + + frame_count_delay = timestamps_table["frameCount"][0] - frame_indices[0] + frame_count_end = frame_count_delay + number_of_frames + + sync_subtable = sync_table.iloc[frame_count_delay:frame_count_end] + + self.timestamps = numpy.array(timestamps_table["Timestamp"]) + + # This was hardcoded via discussion in + # https://github.com/catalystneuro/leifer_lab_to_nwb/issues/2 + depth_scanning_piezo_volts_to_um = 1 / 0.125 + self.series_depth_per_frame_in_um = numpy.array( + sync_subtable["Piezo position (V)"] * depth_scanning_piezo_volts_to_um + ) + + full_shape = (number_of_frames, frame_shape[0], frame_shape[1]) + + dat_file_path = pumpprobe_folder_path / "sCMOS_Frames_U16_1024x512.dat" + self.imaging_data_memory_map = numpy.memmap(filename=dat_file_path, dtype=dtype, mode="r", shape=full_shape) + + # This slicing operation *should* be lazy since it does not usually include fancy indexing + full_slice = (slice(0, number_of_frames), self.channel_frame_slicing[0], self.channel_frame_slicing[1]) + self.imaging_data_for_channel = self.imaging_data_memory_map[full_slice] + + self.data_shape = ( + number_of_frames, + full_slice[1].stop - full_slice[1].start, + full_slice[2].stop - full_slice[2].start, + ) + + # def get_metadata(self) -> dict: + # one_photon_metadata = super().get_metadata(photon_series_type=self.photon_series_type) + # + # # Hardcoded value from lab + # # This is also an average in a sense - the exact depth is tracked by the Piezo and written + # # as a custom DynamicTable in the ExtraOphysMetadataInterface + # # depth_per_pixel = 0.42 + # + # # one_photon_metadata["Ophys"]["grid_spacing"] = (um_per_pixel, um_per_pixel, um_per_pixel) + # + # return one_photon_metadata + # + # def get_metadata_schema(self) -> dict: + # return super().get_metadata(photon_series_type=self.photon_series_type) + + def add_to_nwbfile( + self, + *, + nwbfile: pynwb.NWBFile, + metadata: dict | None = None, + stub_test: bool = False, + stub_frames: int = 70, + ) -> None: + # TODO: enhance all metadata + if "Microscope" not in nwbfile.devices: + microscope = ndx_microscopy.Microscope(name="Microscope") + nwbfile.add_device(devices=microscope) + else: + microscope = nwbfile.devices["Microscope"] + + if "MicroscopyLightSource" not in nwbfile.devices: + light_source = ndx_microscopy.MicroscopyLightSource(name="MicroscopyLightSource") # TODO + nwbfile.add_device(devices=light_source) + else: + light_source = nwbfile.devices["MicroscopyLightSource"] + + if "PlanarImagingSpace" not in nwbfile.lab_meta_data: + imaging_space = ndx_microscopy.PlanarImagingSpace( + name="PlanarImagingSpace", description="", microscope=microscope + ) + nwbfile.add_lab_meta_data(lab_meta_data=imaging_space) + else: + imaging_space = nwbfile.lab_meta_data["PlanarImagingSpace"] + + optical_channel = ndx_microscopy.MicroscopyOpticalChannel(name=self.channel_name, description="", indicator="") + nwbfile.add_lab_meta_data(lab_meta_data=optical_channel) + + # Not exposing chunking/buffering control here for simplicity; but this is where they would be passed + num_frames = self.data_shape[0] if not stub_test else stub_frames + x = self.data_shape[1] + y = self.data_shape[2] + frame_size_bytes = x * y * self.imaging_data_for_channel.dtype.itemsize + chunk_size_bytes = 10.0 * 1e6 # 10 MB default + num_frames_per_chunk = int(chunk_size_bytes / frame_size_bytes) + chunk_shape = (max(min(num_frames_per_chunk, num_frames), 1), x, y) + + imaging_data = self.imaging_data_for_channel if not stub_test else self.imaging_data_for_channel[:stub_frames] + data_iterator = neuroconv.tools.hdmf.SliceableDataChunkIterator(data=imaging_data, chunk_shape=chunk_shape) + + timestamps = self.timestamps if not stub_test else self.timestamps[:stub_frames] + + variable_depth_microscopy_series = ndx_microscopy.VariableDepthMicroscopySeries( + name="PumpProbeImaging", + description="", # TODO + microscope=microscope, + light_source=light_source, + imaging_space=imaging_space, + optical_channel=optical_channel, + data=data_iterator, + depth_per_frame_in_mm=self.series_depth_per_frame_in_um * 1e-3, + unit="n.a.", + timestamps=timestamps, + ) + nwbfile.add_acquisition(variable_depth_microscopy_series) diff --git a/src/leifer_lab_to_nwb/randi_nature_2023/interfaces/_pump_probe_segmentation_extractor.py b/src/leifer_lab_to_nwb/randi_nature_2023/interfaces/_pump_probe_segmentation_extractor.py new file mode 100644 index 0000000..2e0259d --- /dev/null +++ b/src/leifer_lab_to_nwb/randi_nature_2023/interfaces/_pump_probe_segmentation_extractor.py @@ -0,0 +1,52 @@ +import pickle +from typing import Iterable, Literal, Tuple, Union + +import numpy +from pydantic import FilePath +from roiextractors import SegmentationExtractor + + +class PumpProbeSegmentationExtractor(SegmentationExtractor): + """A very custom segmentation extractor for the .pickle files that correspond to wormdatamodel objects.""" + + def __init__( + self, + *, + file_path: FilePath, + timestamps: Iterable[float], + image_shape: Tuple[int, int], + ) -> None: + super().__init__() + self._file_path = file_path + self._times = timestamps + self._image_shape = image_shape + + self._channel_names = [self._file_path.stem.upper()] + + with open(file=file_path, mode="rb") as file: + self._all_pickled_info = pickle.load(file=file) + + self._roi_response_raw = self._all_pickled_info.data + + # TODO: this isn't quite true, I don't think it's normalized + self._roi_response_dff = self._all_pickled_info.derivative + + # TODO + number_of_rois = self._roi_response_raw.shape[1] + self._image_masks = numpy.zeros(shape=(self._image_shape[0], self._image_shape[1], number_of_rois), dtype=bool) + + def get_accepted_list(self) -> list: + return list(range(self._roi_response_raw.shape[1])) + + def get_rejected_list(self) -> list: + return list() + + def get_image_size(self) -> Tuple[int, int]: + return self._image_shape + + # TODO: might want to load these from brains file + def get_roi_ids(self) -> list: + return super().get_roi_ids() + + def get_channel_names(self) -> list: + return self._channel_names diff --git a/src/leifer_lab_to_nwb/randi_nature_2023/interfaces/_pump_probe_segmentation_interface.py b/src/leifer_lab_to_nwb/randi_nature_2023/interfaces/_pump_probe_segmentation_interface.py new file mode 100644 index 0000000..fb495ca --- /dev/null +++ b/src/leifer_lab_to_nwb/randi_nature_2023/interfaces/_pump_probe_segmentation_interface.py @@ -0,0 +1,95 @@ +import pathlib +from typing import Literal + +import numpy +import pandas +from neuroconv.datainterfaces.ophys.basesegmentationextractorinterface import ( + BaseSegmentationExtractorInterface, +) +from pydantic import DirectoryPath +from pynwb import NWBFile + + +class PumpProbeSegmentationInterface(BaseSegmentationExtractorInterface): + ExtractorModuleName = "leifer_lab_to_nwb.randi_nature_2023.interfaces._pump_probe_segmentation_extractor" + ExtractorName = "PumpProbeSegmentationExtractor" + + def __init__(self, *, pumpprobe_folder_path: DirectoryPath, channel_name: Literal["Green", "Red"] | str): + """ + A custom interface for the raw volumetric pumpprobe data. + + Parameters + ---------- + pumpprobe_folder_path : DirectoryPath + Path to the pumpprobe folder. + """ + pumpprobe_folder_path = pathlib.Path(pumpprobe_folder_path) + + # Other interfaces use CamelCase to refer to the NWB object the channel data will end up as + # The files on the other hand are all lower case + lower_channel_name = channel_name.lower() + pickle_file_path = pumpprobe_folder_path / f"{lower_channel_name}.pickle" + + # TODO: generalize this timestamp extraction to a common utility function + # From prototyping data, the frameSync seems to start first... + sync_table_file_path = pumpprobe_folder_path / "other-frameSynchronous.txt" + sync_table = pandas.read_table(filepath_or_buffer=sync_table_file_path, index_col=False) + frame_indices = sync_table["Frame index"] + + # ...then the frameDetails has timestamps for a subset of the frame indices + timestamps_file_path = pumpprobe_folder_path / "framesDetails.txt" + timestamps_table = pandas.read_table(filepath_or_buffer=timestamps_file_path, index_col=False) + number_of_frames = timestamps_table.shape[0] + + frame_count_delay = timestamps_table["frameCount"][0] - frame_indices[0] + frame_count_end = frame_count_delay + number_of_frames + + sync_subtable = sync_table.iloc[frame_count_delay:frame_count_end] + + self.timestamps = numpy.array(timestamps_table["Timestamp"]) + + # Hardcoding this for now + image_shape = (512, 512) + super().__init__(file_path=pickle_file_path, timestamps=self.timestamps, image_shape=image_shape) + + # Hardcode a special plane segmentation name for this interface + # self.plane_segmentation_name = "PumpProbeSegmentation" + + # def get_metadata(self) -> dict: + # metadata = super().get_metadata() + # + # # Hardcoded value from lab + # # This is also an average in a sense - the exact depth is tracked by the Piezo and written + # # as a custom DynamicTable in the ExtraOphysMetadataInterface + # depth_per_pixel = 0.42 + # + # # one_photon_metadata["Ophys"]["grid_spacing"] = (um_per_pixel, um_per_pixel, um_per_pixel) + # + # metadata["Ophys"]["ImageSegmentation"]["plane_segmentations"][0]["name"] = self.plane_segmentation_name + # + # metadata["Ophys"]["Fluorescence"]= {'name': 'Fluorescence', self.plane_segmentation_name: {'raw': { + # 'name': 'BaselineSignal', 'description': 'Array of raw fluorescence traces.', 'unit': 'n.a.'}}} + # metadata["Ophys"]["DfOverF"] = {'name': 'Derivative', self.plane_segmentation_name: { + # 'dff': {'name': 'DerivativeOfSignal', 'description': 'Array of filtered fluorescence traces; ' + # 'approximately the derivative (unnormalized) of the ' + # 'baseline signal' + # '.', 'unit': 'n.a.'}}} + # + # return metadata + # + # def get_metadata_schema(self) -> dict: + # return super().get_metadata(photon_series_type="OnePhotonSeries") + + # def add_to_nwbfile( + # self, + # *, + # nwbfile: NWBFile, + # metadata: dict | None = None, + # stub_test: bool = False, + # ) -> None: + # super().add_to_nwbfile( + # nwbfile=nwbfile, + # metadata=metadata, + # stub_test=stub_test, + # plane_segmentation_name="PumpProbeSegmentation", + # ) diff --git a/src/leifer_lab_to_nwb/randi_nature_2023/notes/meeting_5_15_2024.md b/src/leifer_lab_to_nwb/randi_nature_2023/notes/meeting_5_15_2024.md index 4218309..b65ed98 100644 --- a/src/leifer_lab_to_nwb/randi_nature_2023/notes/meeting_5_15_2024.md +++ b/src/leifer_lab_to_nwb/randi_nature_2023/notes/meeting_5_15_2024.md @@ -20,13 +20,13 @@ Also ignore redgreen registration files; those get copied over to pumpprobe Segmentation; masks are per volume (unique), not interpolated -Separate segmentation for multicolor and pumprobe; masks not saved +Separate segmentation for multicolor and pumprobe; masks not saved ## Mmatches in matches.txt/matches.pickle -matches.txt/.pickle; In matches file, -1 means not matches, matched tracking vs matchless tracking. +matches.txt/.pickle; In matches file, -1 means not matches, matched tracking vs matchless tracking. MMmatch might be motion tracking? @@ -42,5 +42,3 @@ Mapping from multicolor to pumpprobe is a separate file. Excitation of multicolor is excitation of pumprobe red/green Same filter on red; the green would be different for GCamp? - - diff --git a/src/leifer_lab_to_nwb/randi_nature_2023/notes/meeting_6_14_2024.md b/src/leifer_lab_to_nwb/randi_nature_2023/notes/meeting_6_14_2024.md index 705fa2f..099b9db 100644 --- a/src/leifer_lab_to_nwb/randi_nature_2023/notes/meeting_6_14_2024.md +++ b/src/leifer_lab_to_nwb/randi_nature_2023/notes/meeting_6_14_2024.md @@ -13,11 +13,11 @@ Main part I'm stuck on now is how to find the `Neuron` coordinates passed into t ## Code for generating figures from paper I have found all information for this and have no further questions - + ## Processed levels of segmeentaiton -They are hearing frome collaborators that unprocessed forms of data may also be useful +They are hearing from collaborators that unprocessed forms of data may also be useful Need to check with Emily how many different combinations of preprocessing (such as photobleaching, nan interpolation) are desired diff --git a/src/leifer_lab_to_nwb/randi_nature_2023/requirements.txt b/src/leifer_lab_to_nwb/randi_nature_2023/requirements.txt index 1f24724..ee54c08 100644 --- a/src/leifer_lab_to_nwb/randi_nature_2023/requirements.txt +++ b/src/leifer_lab_to_nwb/randi_nature_2023/requirements.txt @@ -3,3 +3,4 @@ roiextractors==0.5.6 ndx-patterned-ogen @ git+https://github.com/catalystneuro/ndx-patterned-ogen.git@1880684f33c220c502283dba88a458739df9174e ndx-multichannel-volume==0.1.12 scikit-image==0.23.2 # Required to import ndx-multichannel-volume +ndx_microscopy @ git+https://github.com/catalystneuro/ndx-microscopy.git diff --git a/src/leifer_lab_to_nwb/randi_nature_2023/test_all_interfaces.py b/src/leifer_lab_to_nwb/randi_nature_2023/test_all_interfaces.py new file mode 100644 index 0000000..bc0c00c --- /dev/null +++ b/src/leifer_lab_to_nwb/randi_nature_2023/test_all_interfaces.py @@ -0,0 +1,126 @@ +"""Main conversion script for a single session of data for the Randi et al. Nature 2023 paper.""" + +import datetime +import pathlib + +import pandas +import pynwb +from dateutil import tz + +from leifer_lab_to_nwb.randi_nature_2023 import RandiNature2023Converter +from leifer_lab_to_nwb.randi_nature_2023.interfaces import ( + ExtraOphysMetadataInterface, + NeuroPALImagingInterface, + NeuroPALSegmentationInterface, + OptogeneticStimulationInterface, + PumpProbeImagingInterface, + PumpProbeSegmentationInterface, +) + +# Define base folder of source data +# Change these as needed on new systems +BASE_FOLDER_PATH = pathlib.Path("D:/Leifer") +SESSION_FOLDER_PATH = BASE_FOLDER_PATH / "20211104" +# LOGBOOK_FILE_PATH = SESSION_FOLDER_PATH / "logbook.txt" + +PUMPPROBE_FOLDER_PATH = SESSION_FOLDER_PATH / "pumpprobe_20211104_163944" +MULTICOLOR_FOLDER_PATH = SESSION_FOLDER_PATH / "multicolorworm_20211104_162630" + +NWB_OUTPUT_FOLDER_PATH = BASE_FOLDER_PATH / "nwbfiles" + +# ************************************************************************* +# Everything below this line is automated and should not need to be changed +# ************************************************************************* + +NWB_OUTPUT_FOLDER_PATH.mkdir(exist_ok=True) + +# Parse session start time from the pumpprobe path +session_string = PUMPPROBE_FOLDER_PATH.stem.removeprefix("pumpprobe_") +session_start_time = datetime.datetime.strptime(session_string, "%Y%m%d_%H%M%S") +session_start_time = session_start_time.replace(tzinfo=tz.gettz("US/Eastern")) + +interfaces_classes_to_test = { + "PumpProbeImagingInterfaceGreen": { + "class": PumpProbeImagingInterface, + "source_data": {"pumpprobe_folder_path": PUMPPROBE_FOLDER_PATH, "channel_name": "Green"}, + "conversion_options": {"stub_test": True}, + }, + "PumpProbeImagingInterfaceRed": { + "class": PumpProbeImagingInterface, + "source_data": {"pumpprobe_folder_path": PUMPPROBE_FOLDER_PATH, "channel_name": "Red"}, + "conversion_options": {"stub_test": True}, + }, + "PumpProbeSegmentationInterfaceGreed": { + "class": PumpProbeSegmentationInterface, + "source_data": {"pumpprobe_folder_path": PUMPPROBE_FOLDER_PATH, "channel_name": "Green"}, + "conversion_options": {"stub_test": True}, + }, + "PumpProbeSegmentationInterfaceRed": { + "class": PumpProbeSegmentationInterface, + "source_data": {"pumpprobe_folder_path": PUMPPROBE_FOLDER_PATH, "channel_name": "Red"}, + "conversion_options": {"stub_test": True}, + }, + "NeuroPALImagingInterface": { + "class": NeuroPALImagingInterface, + "source_data": {"multicolor_folder_path": MULTICOLOR_FOLDER_PATH}, + "conversion_options": {"stub_test": True}, + }, + "NeuroPALSegmentationInterface": { + "class": NeuroPALSegmentationInterface, + "source_data": {"multicolor_folder_path": MULTICOLOR_FOLDER_PATH}, + }, + "OptogeneticStimulationInterface": { + "class": OptogeneticStimulationInterface, + "source_data": {"pumpprobe_folder_path": PUMPPROBE_FOLDER_PATH}, + }, + "ExtraOphysMetadataInterface": { + "class": ExtraOphysMetadataInterface, + "source_data": {"pumpprobe_folder_path": PUMPPROBE_FOLDER_PATH}, + }, +} + + +for test_case_name, interface_options in interfaces_classes_to_test.items(): + nwbfile_path = NWB_OUTPUT_FOLDER_PATH / f"test_{test_case_name}.nwb" + + data_interfaces = list() + + # Special case of the OptogeneticStimulationInterface; requires the PumpProbeSegmentationInterface to be added first + if test_case_name == "OptogeneticStimulationInterface": + pump_probe_segmentation_interface = PumpProbeSegmentationInterface( + pumpprobe_folder_path=PUMPPROBE_FOLDER_PATH, channel_name="Green" + ) + data_interfaces.append(pump_probe_segmentation_interface) + + InterfaceClassToTest = interface_options["class"] + interface = InterfaceClassToTest(**interface_options["source_data"]) + data_interfaces.append(interface) + + # Initialize converter + converter = RandiNature2023Converter(data_interfaces=data_interfaces) + + metadata = converter.get_metadata() + + metadata["NWBFile"]["session_start_time"] = session_start_time + metadata["Subject"]["subject_id"] = session_start_time.strftime("%y%m%d") + + # TODO: these are placeholders that would be read in from a logbook read+lookup + metadata["Subject"]["subject_id"] = session_start_time.strftime("%y%m%d") + metadata["Subject"]["species"] = "C. elegans" + metadata["Subject"]["sex"] = "XX" + metadata["Subject"]["age"] = "P1D" + # metadata["Subject"]["growth_stage_time"] = pandas.Timedelta(hours=2, minutes=30).isoformat() # TODO: request + metadata["Subject"]["growth_stage"] = "YA" + metadata["Subject"]["cultivation_temp"] = 20.0 + + if "conversion_options" in interface_options: + conversion_options = {InterfaceClassToTest.__name__: interface_options["conversion_options"]} + else: + conversion_options = None + converter.run_conversion( + nwbfile_path=nwbfile_path, metadata=metadata, conversion_options=conversion_options, overwrite=True + ) + + # Test roundtrip to make sure PyNWB can read the file back + with pynwb.NWBHDF5IO(path=nwbfile_path, mode="r") as io: + read_nwbfile = io.read()