Add complete session start time to FicTrac

CHANGELOG.md

@@ -1,4 +1,9 @@
 # (Upcoming)
+
+### Features
+Added `session_start_time` extraction to `FicTracDataInterface`. [PR #598](https://github.com/catalystneuro/neuroconv/pull/598)
+
+
 ### Fixes
 Remove `starting_time` reset to default value (0.0) when adding the rate and updating the `photon_series_kwargs` or `roi_response_series_kwargs`, in `add_photon_series` or `add_fluorescence_traces`. [PR #595](https://github.com/catalystneuro/neuroconv/pull/595)
 
@@ -17,6 +22,7 @@ Remove `starting_time` reset to default value (0.0) when adding the rate and upd
 
 
 
+
 # v0.4.4
 
 ### Features

src/neuroconv/datainterfaces/behavior/fictrac/fictracdatainterface.py

@@ -1,13 +1,14 @@
+import importlib.util
+from datetime import datetime, timezone
 from pathlib import Path
 from typing import Optional
 
-import numpy as np
 from pynwb.behavior import CompassDirection, SpatialSeries
 from pynwb.file import NWBFile
 
 # from ....basetemporalalignmentinterface import BaseTemporalAlignmentInterface TODO: Add timing methods
 from ....basedatainterface import BaseDataInterface
-from ....tools import get_module, get_package
+from ....tools import get_module
 from ....utils import FilePathType, calculate_regular_series_rate
 
 
@@ -22,6 +23,9 @@ class FicTracDataInterface(BaseDataInterface):
         "visual fixation",
     ]
 
+    # The full header can be found in:
+    # https://github.com/rjdmoore/fictrac/blob/master/doc/data_header.txt
+
     data_columns = [
         "frame_counter",
         "rotation_delta_x_cam",
@@ -56,7 +60,7 @@ def __init__(
         verbose: bool = True,
     ):
         """
-        Interface for writing fictract files to nwb.
+        Interface for writing FicTrac files to nwb.
 
         Parameters
         ----------
@@ -72,17 +76,9 @@ def __init__(
 
     def get_metadata(self):
         metadata = super().get_metadata()
-        from datetime import datetime
-
-        config_file = self.file_path.parent / "fictrac_config.txt"
-        if config_file.exists():
-            self._config_file = parse_fictrac_config(config_file)
-            date_string = self._config_file["build_date"]
-            date_object = datetime.strptime(date_string, "%b %d %Y")
 
-            metadata["NWBFile"].update(
-                session_start_time=date_object,
-            )
+        session_start_time = extract_session_start_time(self.file_path)
+        metadata["NWBFile"].update(session_start_time=session_start_time)
 
         return metadata
 
@@ -102,21 +98,29 @@ def add_to_nwbfile(
 
         import pandas as pd
 
-        fictrac_data_df = pd.read_csv(self.file_path, sep=",", header=None, names=self.data_columns)
+        # The first row only contains the session start time and invalid data
+        fictrac_data_df = pd.read_csv(self.file_path, sep=",", skiprows=1, header=None, names=self.data_columns)
 
         # Get the timestamps
         timestamps_milliseconds = fictrac_data_df["timestamp"].values
         timestamps = timestamps_milliseconds / 1000.0
-        rate = calculate_regular_series_rate(series=timestamps)  # Returns None if it is not regular
-        write_timestamps = True
+
+        # Note: The last values of the timestamps look very irregular for the sample file in catalyst neuro gin repo
+        # The reason, most likely, is that FicTrac is relying on OpenCV to get the timestamps from the video
+        # In my experience, OpenCV is not very accurate with the timestamps at the end of the video.
+        rate = calculate_regular_series_rate(series=timestamps)  # Returns None if the series is not regular
         if rate:
             write_timestamps = False
-
-        processing_module = get_module(nwbfile=nwbfile, name="Behavior")
+            starting_time = timestamps[0]
+        else:
+            write_timestamps = True
 
         # All the units in FicTrac are in radians, the radius of the ball required to transform to
         # Distances is not specified in the format
-        compass_direction_container = CompassDirection(name="FicTrac")
+        compass_direction_container = CompassDirection(
+            name="FicTrac"
+        )  # TODO: this is just for the heading angle. I will change this to Position in a subsequent PR.
+        # Position and SpatialSeries nested into it
 
         # Add rotation delta from camera
         rotation_delta_cam_columns = [
@@ -148,6 +152,7 @@ def add_to_nwbfile(
             spatial_seriess_kwargs["timestamps"] = timestamps
         else:
             spatial_seriess_kwargs["rate"] = rate
+            spatial_seriess_kwargs["starting_time"] = starting_time
 
         spatial_series = SpatialSeries(**spatial_seriess_kwargs)
         compass_direction_container.add_spatial_series(spatial_series)
@@ -182,6 +187,7 @@ def add_to_nwbfile(
             spatial_seriess_kwargs["timestamps"] = timestamps
         else:
             spatial_seriess_kwargs["rate"] = rate
+            spatial_seriess_kwargs["starting_time"] = starting_time
 
         spatial_series = SpatialSeries(**spatial_seriess_kwargs)
         compass_direction_container.add_spatial_series(spatial_series)
@@ -216,6 +222,7 @@ def add_to_nwbfile(
             spatial_seriess_kwargs["timestamps"] = timestamps
         else:
             spatial_seriess_kwargs["rate"] = rate
+            spatial_seriess_kwargs["starting_time"] = starting_time
 
         spatial_series = SpatialSeries(**spatial_seriess_kwargs)
         compass_direction_container.add_spatial_series(spatial_series)
@@ -250,6 +257,7 @@ def add_to_nwbfile(
             spatial_seriess_kwargs["timestamps"] = timestamps
         else:
             spatial_seriess_kwargs["rate"] = rate
+            spatial_seriess_kwargs["starting_time"] = starting_time
 
         spatial_series = SpatialSeries(**spatial_seriess_kwargs)
         compass_direction_container.add_spatial_series(spatial_series)
@@ -274,6 +282,7 @@ def add_to_nwbfile(
             spatial_seriess_kwargs["timestamps"] = timestamps
         else:
             spatial_seriess_kwargs["rate"] = rate
+            spatial_seriess_kwargs["starting_time"] = starting_time
 
         spatial_series = SpatialSeries(**spatial_seriess_kwargs)
         compass_direction_container.add_spatial_series(spatial_series)
@@ -300,6 +309,7 @@ def add_to_nwbfile(
             spatial_seriess_kwargs["timestamps"] = timestamps
         else:
             spatial_seriess_kwargs["rate"] = rate
+            spatial_seriess_kwargs["starting_time"] = starting_time
 
         spatial_series = SpatialSeries(**spatial_seriess_kwargs)
         compass_direction_container.add_spatial_series(spatial_series)
@@ -321,6 +331,7 @@ def add_to_nwbfile(
             spatial_seriess_kwargs["timestamps"] = timestamps
         else:
             spatial_seriess_kwargs["rate"] = rate
+            spatial_seriess_kwargs["starting_time"] = starting_time
 
         spatial_series = SpatialSeries(**spatial_seriess_kwargs)
         compass_direction_container.add_spatial_series(spatial_series)
@@ -351,6 +362,7 @@ def add_to_nwbfile(
             spatial_seriess_kwargs["timestamps"] = timestamps
         else:
             spatial_seriess_kwargs["rate"] = rate
+            spatial_seriess_kwargs["starting_time"] = starting_time
 
         spatial_series = SpatialSeries(**spatial_seriess_kwargs)
         compass_direction_container.add_spatial_series(spatial_series)
@@ -379,6 +391,7 @@ def add_to_nwbfile(
             spatial_seriess_kwargs["timestamps"] = timestamps
         else:
             spatial_seriess_kwargs["rate"] = rate
+            spatial_seriess_kwargs["starting_time"] = starting_time
 
         spatial_series = SpatialSeries(**spatial_seriess_kwargs)
         compass_direction_container.add_spatial_series(spatial_series)
@@ -401,14 +414,37 @@ def add_to_nwbfile(
 
         else:
             spatial_seriess_kwargs["rate"] = rate
+            spatial_seriess_kwargs["starting_time"] = starting_time
 
         spatial_series = SpatialSeries(**spatial_seriess_kwargs)
         compass_direction_container.add_spatial_series(spatial_series)
 
         # Add the compass direction container to the processing module
+        processing_module = get_module(nwbfile=nwbfile, name="Behavior")
         processing_module.add_data_interface(compass_direction_container)
 
 
+def extract_session_start_time(file_path: FilePathType) -> datetime:
+    """
+    Lazily extract the session start datetime from a FicTrac data file.
+
+    In FicTrac the column 22 in the data has the timestamps which are given in milliseconds since the epoch.
+
+    The epoch in Linux is 1970-01-01 00:00:00 UTC.
+    """
+    with open(file_path, "r") as file:
+        # Read the first data line
+        first_line = file.readline()
+
+        # Split by comma and extract the timestamp (the 22nd column)
+        utc_timestamp = float(first_line.split(",")[21]) / 1000.0  # Transform to seconds
+
+    utc_datetime = datetime.utcfromtimestamp(utc_timestamp).replace(tzinfo=timezone.utc)
+
+    return utc_datetime
+
+
+# TODO: Parse probably will do this in a simpler way.
 def parse_fictrac_config(filename) -> dict:
     """
     Parse a FicTrac configuration file and return a dictionary of its parameters. See the

tests/test_on_data/test_behavior_interfaces.py

@@ -1,5 +1,5 @@
 import unittest
-from datetime import datetime
+from datetime import datetime, timezone
 from pathlib import Path
 
 import numpy as np
@@ -37,6 +37,10 @@ class TestFicTracDataInterface(DataInterfaceTestMixin, unittest.TestCase):
 
     save_directory = OUTPUT_PATH
 
+    def check_extracted_metadata(self, metadata: dict):
+        expected_session_start_time = datetime(2023, 7, 24, 9, 30, 55, 440600, tzinfo=timezone.utc)
+        assert metadata["NWBFile"]["session_start_time"] == expected_session_start_time
+
 
 class TestVideoInterface(VideoInterfaceMixin, unittest.TestCase):
     data_interface_cls = VideoInterface