🐛 Dynamically capture 1D header (#2153)

CHANGELOG.md

@@ -29,6 +29,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 - A bug in which AWS S3 encryption was looked for in Nipype config instead of pipeline config (only affected uploading logs).
 - Restored `bids-validator` functionality.
+- A bug in which bandpass filters always assumed 1D regressor files have exactly 5 header rows.
 
 ### Removed
 

CPAC/nuisance/bandpass.py

@@ -1,6 +1,8 @@
 import os
+from pathlib import Path
 
 import numpy as np
+from numpy.typing import NDArray
 import nibabel as nib
 from scipy.fftpack import fft, ifft
 
@@ -44,6 +46,22 @@ def ideal_bandpass(data, sample_period, bandpass_freqs):
     return np.real_if_close(ifft(f_data)[:sample_length])
 
 
+def read_1D(one_D: Path | str) -> tuple[list[str], NDArray]:
+    """Parse a header from a 1D file, returing that header and a Numpy Array."""
+    header = []
+    with open(one_D, "r") as _f:
+        # Each leading line that doesn't start with a number goes into the header
+        for line in _f.readlines():
+            try:
+                float(line.split()[0])
+                break
+            except ValueError:
+                header.append(line)
+
+    regressor = np.loadtxt(one_D, skiprows=len(header))
+    return header, regressor
+
+
 def bandpass_voxels(realigned_file, regressor_file, bandpass_freqs, sample_period=None):
     """Performs ideal bandpass filtering on each voxel time-series.
 
@@ -106,18 +124,9 @@ def bandpass_voxels(realigned_file, regressor_file, bandpass_freqs, sample_perio
             img.to_filename(regressor_bandpassed_file)
 
         else:
-            with open(regressor_file, "r") as f:
-                header = []
-
-                # header wouldn't be longer than 5, right? I don't want to
-                # loop over the whole file
-                for i in range(5):
-                    line = f.readline()
-                    if line.startswith("#") or isinstance(line[0], str):
-                        header.append(line)
-
-            # usecols=[list]
-            regressor = np.loadtxt(regressor_file, skiprows=len(header))
+            header: list[str]
+            regressor: NDArray
+            header, regressor = read_1D(regressor_file)
             Yc = regressor - np.tile(regressor.mean(0), (regressor.shape[0], 1))
             Y_bp = np.zeros_like(Yc)
 

CPAC/nuisance/tests/regressors.1D

@@ -0,0 +1,15 @@
+# Extra header
+# extra header
+# C-PAC 1.8.7.dev1
+# Nuisance regressors:
+# RotY	RotYDelay	RotYSq	RotYDelaySq	RotX	RotXDelay	RotXSq	RotXDelaySq	RotZ	RotZDelay	RotZSq	RotZDelaySq	Y	YDelay	YSq	YDelaySq	X	XDelay	XSq	XDelaySq	Z	ZDelay	ZSq	ZDelaySq	aCompCorDetrendPC0	aCompCorDetrendPC1	aCompCorDetrendPC2	aCompCorDetrendPC3	aCompCorDetrendPC4
+0.064503015618032941	0.000000000000000000	0.004160639023820202	0.000000000000000000	0.071612848897811346	0.000000000000000000	0.005128400127260760	0.000000000000000000	-0.045875642036314265	0.000000000000000000	0.002104574532244045	0.000000000000000000	0.132890000000000008	0.000000000000000000	0.017659752100000002	0.000000000000000000	0.014942199999999999	0.000000000000000000	0.000223269340840000	0.000000000000000000	0.000408556000000000	0.000000000000000000	0.000000166918005136	0.000000000000000000	-0.022348500000000000	0.024816700000000001	-0.096326200000000001	0.157762999999999987	-0.097873799999999997
+0.031640849390966043	0.064503015618032941	0.001001143350181796	0.004160639023820202	0.128928108975928074	0.071612848897811346	0.016622457284108785	0.005128400127260760	-0.067560891370646151	-0.045875642036314265	0.004564474042796250	0.002104574532244045	0.031627599999999999	0.132890000000000008	0.001000305081760000	0.017659752100000002	0.038095700000000003	0.014942199999999999	0.001451282358490000	0.000223269340840000	-0.005307810000000000	0.000408556000000000	0.000028172846996100	0.000000166918005136	-0.064876000000000003	-0.013603499999999999	0.009020350000000000	-0.160142000000000007	-0.177807999999999994
+0.014566182605406878	0.011659654350684051	0.001782025477654622	0.001708282485349496	0.087538262826358210	0.084814056613328720	0.003050410763897181	0.001983217145137512	-0.041453889502682612	-0.041248724781196566	0.000887045295189055	0.001102853114798172	0.024593061637466357	0.019123515563283400	-0.001171834437865083	-0.001702326740091272	0.013267008686230538	0.014908354440170480	-0.000023048542269668	0.000030800663864303	-0.003147503026503373	-0.002156489951271478	-0.000212523379574746	-0.000134571632225604	-0.005279020489008680	0.003309414394962159	0.006218425399968431	0.006926438427946187	0.031874911370701621
+0.023012917432044880	0.023641462459337223	0.001353826869739763	0.001428748088263631	0.128401517423642225	0.127907328597750475	0.002936077845255422	0.001732591121410621	-0.064041009402203836	-0.065349619535801984	-0.001376339705694537	-0.000867347717315630	0.055371528230890282	0.047838664356472604	-0.003939704578469714	-0.004413819725322955	0.008626921921677059	0.013521224060128565	-0.000524131399781458	-0.000509996162422567	0.001399646015426790	0.002426771079716165	-0.000697817034458711	-0.000644064148730770	0.003453684797811343	0.004439728633043883	0.005528130051255496	-0.000681564743845684	0.027088427450170843
+0.025438893846313822	0.030058212923250879	0.000838561693597976	0.001085005134557843	0.158696217127646116	0.160188595362451003	0.002834979654468744	0.001871305030454243	-0.079495085073931035	-0.083080090516398086	-0.003568788021910289	-0.002826331376429190	0.082500133838064399	0.073831252771084988	-0.006214900864815498	-0.006543763203955914	0.000519334243296480	0.008630341137520037	-0.000923363158038725	-0.000927750776503564	0.005165821347348335	0.005851034226762506	-0.001054704872395450	-0.001043041584010332	0.012752740283469200	0.004786640061712925	0.012289830660907162	-0.008745532683606035	0.014261415118720363
+0.021743016120035281	0.029688950895877426	0.000290547599874028	0.000682055571198300	0.175549364989970313	0.178338230890874111	0.002486643991830800	0.002149192970833630	-0.085377454115175486	-0.091489126463240492	-0.005383312549059558	-0.004535185285883645	0.102288251365551003	0.094066918293276736	-0.007766221033112258	-0.007876677356441979	-0.010112433374632405	0.000319385240548675	-0.001198648271548705	-0.001193505340585474	0.008037366757553616	0.007980258888708817	-0.001242736103775270	-0.001273598198058523	0.020974706057590668	0.005751802778007228	0.025351389814577394	-0.017180756363741379	-0.003956879522184370
+0.013525050094767123	0.023039913400015079	-0.000213791695822321	0.000249432472712464	0.178794499964418374	0.182090614749512603	0.001668344371008412	0.002226367140418777	-0.081444170893389012	-0.089634493861210238	-0.006575553895215308	-0.005785817468059847	0.112188805335497160	0.106323654207989879	-0.008527087208204130	-0.008379970470761666	-0.021551792557092900	-0.010410526495855658	-0.001350988613004632	-0.001312369367927021	0.010150399365352503	0.009047754995696919	-0.001267065761949068	-0.001322015050183638	0.027086406796860162	0.008769045224622980	0.041260717228531141	-0.025783341088905919	-0.023130294003556602
+0.003710293144471088	0.012303012925884141	-0.000591683949645386	-0.000159272972606234	0.170628799324984620	0.173931286958495634	0.000283113801796188	0.001792439708046661	-0.069705778794223461	-0.078851018906234180	-0.007027047515815758	-0.006451875040969878	0.111350260801486828	0.109587738981351920	-0.008599721245876775	-0.008202102875302755	-0.031732073497397532	-0.021834007346710128	-0.001401093147591972	-0.001318145135788918	0.011803916636990694	0.009558331079300939	-0.001174308952196117	-0.001222014617445004	0.030766413414606002	0.014584179038094797	0.055050504861566943	-0.034070573320800129	-0.038211308729750156
+-0.004234132549489960	0.000832649040004215	-0.000775011343076728	-0.000476609472781693	0.154706450373287646	0.158080166174354941	-0.001594467727120446	0.000677513943272737	-0.053855672812893871	-0.062403402297765788	-0.006775880525707196	-0.006520249171448194	0.100795197589240160	0.104212055737311265	-0.008213823778438519	-0.007620791647640451	-0.038786879957853938	-0.031921725342639970	-0.001382244593213621	-0.001261079549196342	0.013326020461721926	0.010106105453167591	-0.001035584634549134	-0.001043215412680393	0.032184969062050033	0.022685626981519318	0.061584251842384724	-0.041209431181336478	-0.044960991839340970
+-0.007277482359979987	-0.007732524875134966	-0.000730772864804432	-0.000639221128232586	0.134982773383533428	0.139112331565989317	-0.003731084894624379	-0.001083447029356628	-0.038226479638264539	-0.044608858813448345	-0.006004861339980094	-0.006093216205261694	0.083174466753920082	0.091804401122952045	-0.007653794404152313	-0.006957769820022970	-0.041563433510437883	-0.038870841259792399	-0.001331418827716966	-0.001192680917598046	0.014941405278128142	0.011168243606804554	-0.000922586836031674	-0.000868492699062700	0.031582380224112708	0.031133381053542616	0.057080842480777161	-0.046093261482679442	-0.041188612349529960

CPAC/nuisance/tests/test_bandpass.py

@@ -0,0 +1,48 @@
+# Copyright (C) 2022 - 2024  C-PAC Developers
+
+# This file is part of C-PAC.
+
+# C-PAC is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Lesser General Public License as published by the
+# Free Software Foundation, either version 3 of the License, or (at your
+# option) any later version.
+
+# C-PAC is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
+# License for more details.
+
+# You should have received a copy of the GNU Lesser General Public
+# License along with C-PAC. If not, see <https://www.gnu.org/licenses/>.
+"""Tests for bandpass filters."""
+
+from importlib.abc import Traversable
+from importlib.resources import files
+from pathlib import Path
+
+from numpy.typing import NDArray
+import pytest
+
+from CPAC.nuisance.bandpass import read_1D
+
+RAW_ONE_D: Traversable = files("CPAC").joinpath("nuisance/tests/regressors.1D")
+
+
+@pytest.mark.parametrize("start_line", list(range(6)))
+def test_read_1D(start_line: int, tmp_path: Path) -> None:
+    """Test the correct number of rows are read when reading a 1D file."""
+    regressor: Path = tmp_path / f"regressor_startAtL{start_line}.1D"
+    # create a regressor.1D file with (5 - ``start_line``) lines of header
+    with (
+        RAW_ONE_D.open("r", encoding="utf-8") as _raw,
+        regressor.open("w", encoding="utf-8") as _test_file,
+    ):
+        for line in _raw.readlines()[start_line:]:
+            _test_file.write(line)
+    header: list[str]
+    data: NDArray
+    header, data = read_1D(regressor)
+    # should get the same array no matter how many lines of header
+    assert data.shape == (10, 29)
+    # all header lines should be captured
+    assert len(header) == 5 - start_line