Add complete session start time to FicTrac

CHANGELOG.md

@@ -1,14 +1,18 @@
 # (Upcoming)
-### Fixes
-Remove `starting_time` reset to default value (0.0) when adding the rate and updating the `photon_series_kwargs` or `roi_response_series_kwargs`, in `add_photon_series` or `add_fluorescence_traces`. [PR #595](https://github.com/catalystneuro/neuroconv/pull/595)
+
+### Features
+Addedd `session_start_time` extraction to `FicTracDataInterface`. [PR #598](https://github.com/catalystneuro/neuroconv/pull/598)
+
 
 ### Fixes
+Remove `starting_time` reset to default value (0.0) when adding the rate and updating the `photon_series_kwargs` or `roi_response_series_kwargs`, in `add_photon_series` or `add_fluorescence_traces`. [PR #595](https://github.com/catalystneuro/neuroconv/pull/595)
 
 * Changed the date parsing in `OpenEphysLegacyRecordingInterface` to `datetime.strptime` with the expected date format explicitly set to `"%d-%b-%Y %H%M%S"`. [PR #577](https://github.com/catalystneuro/neuroconv/pull/577)
 * Pin lower bound HDMF version to `3.10.0`. [PR #586](https://github.com/catalystneuro/neuroconv/pull/586)
 
 
 
+
 # v0.4.4
 
 ### Features

src/neuroconv/datainterfaces/behavior/fictrac/fictracdatainterface.py

@@ -1,7 +1,13 @@
+import warnings
+from datetime import datetime
 from pathlib import Path
 from typing import Optional
 
-import numpy as np
+try:
+    from zoneinfo import ZoneInfo
+except ImportError:
+    from backports.zoneinfo import ZoneInfo
+
 from pynwb.behavior import CompassDirection, SpatialSeries
 from pynwb.file import NWBFile
 
@@ -22,6 +28,9 @@ class FicTracDataInterface(BaseDataInterface):
         "visual fixation",
     ]
 
+    # The full header can be found in:
+    # https://github.com/rjdmoore/fictrac/blob/master/doc/data_header.txt
+
     data_columns = [
         "frame_counter",
         "rotation_delta_x_cam",
@@ -56,7 +65,7 @@ def __init__(
         verbose: bool = True,
     ):
         """
-        Interface for writing fictract files to nwb.
+        Interface for writing FicTrac files to nwb.
 
         Parameters
         ----------
@@ -72,17 +81,12 @@ def __init__(
 
     def get_metadata(self):
         metadata = super().get_metadata()
-        from datetime import datetime
 
-        config_file = self.file_path.parent / "fictrac_config.txt"
-        if config_file.exists():
-            self._config_file = parse_fictrac_config(config_file)
-            date_string = self._config_file["build_date"]
-            date_object = datetime.strptime(date_string, "%b %d %Y")
-
-            metadata["NWBFile"].update(
-                session_start_time=date_object,
-            )
+        try:
+            session_start_time = extract_session_start_time(self.file_path)
+            metadata["NWBFile"].update(session_start_time=session_start_time)
+        except:
+            warnings.warn("Unable to extract the session start time from the FicTrac data file. ")
 
         return metadata
 
@@ -102,17 +106,25 @@ def add_to_nwbfile(
 
         import pandas as pd
 
-        fictrac_data_df = pd.read_csv(self.file_path, sep=",", header=None, names=self.data_columns)
+        # The first row only contains the session start time and invalid data
+        fictrac_data_df = pd.read_csv(self.file_path, sep=",", skiprows=1, header=None, names=self.data_columns)
 
         # Get the timestamps
         timestamps_milliseconds = fictrac_data_df["timestamp"].values
         timestamps = timestamps_milliseconds / 1000.0
-        rate = calculate_regular_series_rate(series=timestamps)  # Returns None if it is not regular
-        write_timestamps = True
-        if rate:
-            write_timestamps = False
 
-        processing_module = get_module(nwbfile=nwbfile, name="Behavior")
+        # Shift to the session start time
+        metadata = metadata or self.get_metadata()
+        interface_session_start_time = extract_session_start_time(self.file_path)
+        conversion_session_start_time = metadata["NWBFile"]["session_start_time"]
+        starting_time = interface_session_start_time.timestamp() - conversion_session_start_time.timestamp()
+        timestamps_shifted = timestamps + starting_time
+
+        # Note: The last values of the timestamps look very irregular for the sample file in catalyst neuro gin repo
+        # The reason, most likely, is that FicTrac is relying on OpenCV to get the timestamps from the video
+        # And we know that OpenCV is not very accurate with the timestamps, specially the last ones.
+        rate = calculate_regular_series_rate(series=timestamps_shifted)
+        write_timestamps = False if rate else True  # calculate_regular_series_rate returns None if it is not regular
 
         # All the units in FicTrac are in radians, the radius of the ball required to transform to
         # Distances is not specified in the format
@@ -148,6 +160,7 @@ def add_to_nwbfile(
             spatial_seriess_kwargs["timestamps"] = timestamps
         else:
             spatial_seriess_kwargs["rate"] = rate
+            spatial_seriess_kwargs["starting_time"] = starting_time
 
         spatial_series = SpatialSeries(**spatial_seriess_kwargs)
         compass_direction_container.add_spatial_series(spatial_series)
@@ -182,6 +195,7 @@ def add_to_nwbfile(
             spatial_seriess_kwargs["timestamps"] = timestamps
         else:
             spatial_seriess_kwargs["rate"] = rate
+            spatial_seriess_kwargs["starting_time"] = starting_time
 
         spatial_series = SpatialSeries(**spatial_seriess_kwargs)
         compass_direction_container.add_spatial_series(spatial_series)
@@ -216,6 +230,7 @@ def add_to_nwbfile(
             spatial_seriess_kwargs["timestamps"] = timestamps
         else:
             spatial_seriess_kwargs["rate"] = rate
+            spatial_seriess_kwargs["starting_time"] = starting_time
 
         spatial_series = SpatialSeries(**spatial_seriess_kwargs)
         compass_direction_container.add_spatial_series(spatial_series)
@@ -250,6 +265,7 @@ def add_to_nwbfile(
             spatial_seriess_kwargs["timestamps"] = timestamps
         else:
             spatial_seriess_kwargs["rate"] = rate
+            spatial_seriess_kwargs["starting_time"] = starting_time
 
         spatial_series = SpatialSeries(**spatial_seriess_kwargs)
         compass_direction_container.add_spatial_series(spatial_series)
@@ -274,6 +290,7 @@ def add_to_nwbfile(
             spatial_seriess_kwargs["timestamps"] = timestamps
         else:
             spatial_seriess_kwargs["rate"] = rate
+            spatial_seriess_kwargs["starting_time"] = starting_time
 
         spatial_series = SpatialSeries(**spatial_seriess_kwargs)
         compass_direction_container.add_spatial_series(spatial_series)
@@ -300,6 +317,7 @@ def add_to_nwbfile(
             spatial_seriess_kwargs["timestamps"] = timestamps
         else:
             spatial_seriess_kwargs["rate"] = rate
+            spatial_seriess_kwargs["starting_time"] = starting_time
 
         spatial_series = SpatialSeries(**spatial_seriess_kwargs)
         compass_direction_container.add_spatial_series(spatial_series)
@@ -321,6 +339,7 @@ def add_to_nwbfile(
             spatial_seriess_kwargs["timestamps"] = timestamps
         else:
             spatial_seriess_kwargs["rate"] = rate
+            spatial_seriess_kwargs["starting_time"] = starting_time
 
         spatial_series = SpatialSeries(**spatial_seriess_kwargs)
         compass_direction_container.add_spatial_series(spatial_series)
@@ -351,6 +370,7 @@ def add_to_nwbfile(
             spatial_seriess_kwargs["timestamps"] = timestamps
         else:
             spatial_seriess_kwargs["rate"] = rate
+            spatial_seriess_kwargs["starting_time"] = starting_time
 
         spatial_series = SpatialSeries(**spatial_seriess_kwargs)
         compass_direction_container.add_spatial_series(spatial_series)
@@ -379,6 +399,7 @@ def add_to_nwbfile(
             spatial_seriess_kwargs["timestamps"] = timestamps
         else:
             spatial_seriess_kwargs["rate"] = rate
+            spatial_seriess_kwargs["starting_time"] = starting_time
 
         spatial_series = SpatialSeries(**spatial_seriess_kwargs)
         compass_direction_container.add_spatial_series(spatial_series)
@@ -401,14 +422,36 @@ def add_to_nwbfile(
 
         else:
             spatial_seriess_kwargs["rate"] = rate
+            spatial_seriess_kwargs["starting_time"] = starting_time
 
         spatial_series = SpatialSeries(**spatial_seriess_kwargs)
         compass_direction_container.add_spatial_series(spatial_series)
 
         # Add the compass direction container to the processing module
+        processing_module = get_module(nwbfile=nwbfile, name="Behavior")
         processing_module.add_data_interface(compass_direction_container)
 
 
+def extract_session_start_time(file_path: FilePathType) -> datetime:
+    """
+    Lazily extract the session start datetime from a FicTrac data file.
+
+    In FicTrac the column 22 in the data has the timestamps which are given in milliseconds since the epoch.
+
+    The epoch in Linux is 1970-01-01 00:00:00 UTC.
+    """
+    with open(file_path, "r") as file:
+        # Read the first data line
+        first_line = file.readline()
+
+        # Split by comma and extract the timestamp (the 22nd column)
+        utc_timestamp = float(first_line.split(",")[21]) / 1000.0  # Transform to seconds
+
+    utc_datetime = datetime.utcfromtimestamp(utc_timestamp).replace(tzinfo=ZoneInfo("UTC"))
+
+    return utc_datetime
+
+
 def parse_fictrac_config(filename) -> dict:
     """
     Parse a FicTrac configuration file and return a dictionary of its parameters. See the

src/neuroconv/datainterfaces/behavior/fictrac/requirements.txt

@@ -0,0 +1 @@
+backports.zoneinfo;python_version<"3.9"

tests/test_on_data/test_behavior_interfaces.py

@@ -1,6 +1,7 @@
 import unittest
 from datetime import datetime
 from pathlib import Path
+from zoneinfo import ZoneInfo
 
 import numpy as np
 from natsort import natsorted
@@ -37,6 +38,10 @@ class TestFicTracDataInterface(DataInterfaceTestMixin, unittest.TestCase):
 
     save_directory = OUTPUT_PATH
 
+    def check_extracted_metadata(self, metadata: dict):
+        expected_session_start_time = datetime(2023, 7, 24, 9, 30, 55, 440600, tzinfo=ZoneInfo("UTC"))
+        assert metadata["NWBFile"]["session_start_time"] == expected_session_start_time
+
 
 class TestVideoInterface(VideoInterfaceMixin, unittest.TestCase):
     data_interface_cls = VideoInterface