1-preprocess_acoustics-zebra_finch.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Preprocess acoustic data and sync to neural data\n",
    "\n",
    "Notebook within the chronic ephys processing pipeline\n",
    "- **1-preprocess_acoustics**\n",
    "- 2-curate_acoustics\n",
    "- 3-sort_spikes\n",
    "- 4-curate_spikes\n",
    "\n",
    "Use the environment **songproc** to run this notebook"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/mnt/cube/lo/envs/songproc/lib/python3.8/site-packages/spikeextractors/__init__.py:21: DeprecationWarning: distutils Version classes are deprecated. Use packaging.version instead.\n",
      "  if StrictVersion(h5py.__version__) > '2.10.0':\n",
      "2024-11-14 10:00:00,473 root         INFO     Running on pakhi.ucsd.edu\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "h5py version > 2.10.0. Some extractors might not work properly. It is recommended to downgrade to version 2.10.0: \n",
      ">>> pip install h5py==2.10.0\n"
     ]
    }
   ],
   "source": [
    "%matplotlib widget\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import os\n",
    "import pickle\n",
    "from scipy.io import wavfile\n",
    "import traceback\n",
    "\n",
    "import sys\n",
    "sys.path.append('/mnt/cube/lo/envs/ceciestunepipe')\n",
    "from ceciestunepipe.file import bcistructure as et\n",
    "from ceciestunepipe.util.sound import boutsearch as bs\n",
    "from ceciestunepipe.pipeline import searchbout as sb\n",
    "from ceciestunepipe.util import stimutil as su\n",
    "from ceciestunepipe.util import sglxutil as sglu\n",
    "from ceciestunepipe.util import sglxsync as sy\n",
    "from ceciestunepipe.util.spikeextractors.extractors.spikeglxrecordingextractor import spikeglxrecordingextractor as sglex\n",
    "from ceciestunepipe.util import oeutil as oeu\n",
    "from ceciestunepipe.mods import preproc_sglx, preproc_oe\n",
    "\n",
    "import logging\n",
    "logger = logging.getLogger()\n",
    "handler = logging.StreamHandler()\n",
    "formatter = logging.Formatter(\n",
    "        '%(asctime)s %(name)-12s %(levelname)-8s %(message)s')\n",
    "handler.setFormatter(formatter)\n",
    "logger.addHandler(handler)\n",
    "logger.setLevel(logging.WARNING) # set to logging.INFO if you'd like to see the full readout"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Set parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "## default bout detection parameters that work well for zebra finches\n",
    "hparams = {\n",
    "    # spectrogram\n",
    "    'num_freq':1024, # how many channels to use in a spectrogram\n",
    "    'preemphasis':0.97,\n",
    "    'frame_shift_ms':5, # step size for fft\n",
    "    'frame_length_ms':10, # frame length for fft FRAME SAMPLES < NUM_FREQ!!!\n",
    "    'min_level_db':-55, # minimum threshold db for computing spectrogram\n",
    "    'ref_level_db':110, # reference db for computing spectrogram\n",
    "    'sample_rate':None, # sample rate of your data\n",
    "    \n",
    "    # mel filter\n",
    "    'mel_filter':False, # should a mel filter be used?\n",
    "    'num_mels':1024, # how many channels to use in the mel-spectrogram\n",
    "    'fmin':300, # low frequency cutoff for mel filter\n",
    "    'fmax':12000, # high frequency cutoff for mel filter\n",
    "    \n",
    "    # spectrogram inversion\n",
    "    'max_iters':200,\n",
    "    'griffin_lim_iters':20,\n",
    "    'power':1.5,\n",
    "    \n",
    "    # bout searching\n",
    "    'bout_auto_file':'bout_auto.pickle', # extension for saving the auto found files\n",
    "    'bout_sync_file':'bout_sync.pickle', # extension for saving the synchronized auto bouts\n",
    "    'stim_sync_file':'stim_sync.pickle', # extension for saving the synchronized stim if stim session\n",
    "    'bout_curated_file':'bout_curated.pickle', # extension for manually curated files\n",
    "    \n",
    "    # if using deep_bout_search = False, the following parameters will apply for automatic bout detection:\n",
    "    'read_wav_fun':bs.read_npy_chan, # function for loading the wav_like_stream (returns fs, ndarray)\n",
    "    'file_order_fun':bs.sess_file_id, # function for extracting the file ID within the session\n",
    "    'min_segment':20, # minimum length of supra_threshold to consider a 'syllable' (ms)\n",
    "    'min_silence':3000, # minmum distance between groups of syllables to consider separate bouts (ms)\n",
    "    'min_bout':500, # min bout duration (ms)\n",
    "    'peak_thresh_rms':0.55, # threshold (rms) for peak acceptance,\n",
    "    'thresh_rms':0.25, # threshold for detection of syllables\n",
    "    'mean_syl_rms_thresh':0.3, # threshold for acceptance of mean rms across the syllable (relative to rms of the file)\n",
    "    'max_bout':180000, # exclude bouts too long (ms)\n",
    "    'l_p_r_thresh':100, # threshold for n of len_ms/peaks (typycally about 2-3 syllable spans)\n",
    "    'waveform_edges':1000, # get number of ms before and after the edges of the bout for the waveform sample\n",
    "}\n",
    "\n",
    "## other processing parameters\n",
    "n_jobs = 1 # n_jobs for deriving bout info (errors when increased)\n",
    "mic_file_ext = 'npy' # npy method more efficient than wav\n",
    "force_preprocess = False # skip preprocessing for previously failed epochs\n",
    "deep_bout_search = True # detect bouts using deep search -- see ceciestunepipe.mods.bout_detection_mf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# dictionaries for birds / recordings\n",
    "bird_rec_dict = {\n",
    "###### Zeke/Pablo had 3 successful finch recordings in the past: z_w12m7_20, z_r12r13_21, z_y19o20_21\n",
    "#     'z_w12m7_20':[\n",
    "#         {'sess_par_list':['2020-11-04'], # sessions with this configuration\n",
    "#          'stim_sess_list':[], # sessions where stimuli were presented\n",
    "#          'mic_list':['microphone_0'], # list of mics of interest, by signal name in rig.json\n",
    "#          'adc_list':[], # list of adc channels of interest\n",
    "#          'stim_list':[], # list of adc chans with the stimulus\n",
    "#          'nidq_ttl_list':[], # list of TTL signals form the nidq digital inputs to extract (besides the 'sync')\n",
    "#          'ref_stream':'ap_0', # what to syncrhonize everything to (sglx only, oe already synced)\n",
    "#          'trial_tag_chan':[], # sglx, what was the tag channel in the stimulus wave (this should come from meta et. al)\n",
    "#          'on_signal':[], # sglx, whether signal on is hi or lo\n",
    "#          'sort':'sort_0', # sort index\n",
    "#          'ephys_software':'sglx' # sglx or oe\n",
    "#         }],\n",
    "#    'z_r12r13_21':[\n",
    "#        {'sess_par_list':['2021-06-27','2021-06-28'], # sessions with this configuration\n",
    "#         'stim_sess_list':[], # sessions where stimuli were presented\n",
    "#         'mic_list':['microphone_0'], # list of mics of interest, by signal name in rig.json\n",
    "#         'adc_list':[], # list of adc channels of interest\n",
    "#         'stim_list':[], # list of adc chans with the stimulus\n",
    "#         'nidq_ttl_list': [], # list of TTL signals form the nidq digital inputs to extract (besides the 'sync')\n",
    "#         'ref_stream':'ap_0', # what to syncrhonize everything to (sglx only, oe already synced)\n",
    "#         'trial_tag_chan':[], # sglx, what was the tag channel in the stimulus wave (this should come from meta et. al)\n",
    "#         'on_signal':[], # sglx, whether signal on is hi or lo\n",
    "#         'sort':'sort_0', # sort index\n",
    "#         'ephys_software':'sglx' # sglx or oe\n",
    "#        }], \n",
    "#     'z_y19o20_21':[   # two implants, 10/26-27 and 12/21-23, with same rig.json info (just different probe serial #)\n",
    "#         {'sess_par_list':['2021-10-27'],# ['2021-10-26','2021-10-27','2021-12-21','2021-12-22','2021-12-23'], # sessions with this configuration\n",
    "#          'stim_sess_list':[], # sessions where stimuli were presented\n",
    "#          'probes':['probe_0'],\n",
    "#          'mic_list':['microphone_1'], # list of mics of interest, by signal name in rig.json\n",
    "#          'adc_list':[], # list of adc channels of interest\n",
    "#          'stim_list':[], # list of adc chans with the stimulus\n",
    "#          'nidq_ttl_list':[], # list of TTL signals form the nidq digital inputs to extract (besides the 'sync')\n",
    "#          'ref_stream':'ap_0', # what to syncrhonize everything to (sglx only, oe already synced)\n",
    "#          'trial_tag_chan':[], # sglx, what was the tag channel in the stimulus wave (this should come from meta et. al)\n",
    "#          'on_signal':[], # sglx, whether signal on is hi or lo\n",
    "#          'sort':'sort_0', # sort index\n",
    "#          'ephys_software':'sglx' # sglx or oe\n",
    "#         }]\n",
    "###### Pablo and I also collected another finch's data: z_c5o30_23\n",
    "    ## Session 2023-06-12 was directly after the surgery, no song collected\n",
    "    ##   -- Epoch 1752: neuropixel came unplugged, data length mismatch\n",
    "    ## Session 2023-06-15 is the best session (according to Pablo)\n",
    "#     'z_c5o30_23':[\n",
    "#         {'sess_par_list':['2023-06-15'], # sessions with this configuration '2023-06-12','2023-06-13','2023-06-14','2023-06-15','2023-06-15'\n",
    "#          'stim_sess_list':[], # sessions where stimuli were presented\n",
    "#          'mic_list':['microphone_0'], # list of mics of interest, by signal name in rig.json\n",
    "#          'adc_list':[], # list of adc channels of interest\n",
    "#          'stim_list':[], # list of adc chans with the stimulus\n",
    "#          'nidq_ttl_list':[], # list of TTL signals form the nidq digital inputs to extract (besides the 'sync')\n",
    "#          'ref_stream':'ap_0', # what to synchronize everything to (sglx only, oe already synced)\n",
    "#          'trial_tag_chan':0, # sglx, what was the tag channel in the stimulus wave (this should come from meta et. al)\n",
    "#          'on_signal':1, # sglx, whether signal on is hi or lo\n",
    "#          'sort':'sort_0', # sort index\n",
    "#          'ephys_software':'sglx' # sglx or oe\n",
    "#         }]\n",
    "###### My data:\n",
    "#     'z_c4y1_23':[\n",
    "#         {'sess_par_list':['2024-02-08','2024-02-09','2024-02-10','2024-02-11','2024-02-12'], # sessions with this configuration\n",
    "#          'stim_sess_list':[], # sessions where stimuli were presented\n",
    "#          'mic_list':['microphone_M','microphone_F'], # list of mics of interest, by signal name in rig.json\n",
    "#          'adc_list':[], # list of adc channels of interest\n",
    "#          'stim_list':['wav_stim','wav_syn'], # list of adc chans with the stimulus\n",
    "#          'nidq_ttl_list':[], # list of TTL signals form the nidq digital inputs to extract (besides the 'sync')\n",
    "#          'ref_stream':'ap_0', # what to synchronize everything to (sglx only, oe already synced)\n",
    "#          'trial_tag_chan':2, # sglx, what was the tag channel in the stimulus wave (this should come from meta et. al)\n",
    "#          'on_signal':1, # sglx, whether signal on is hi or lo\n",
    "#          'sort':'sort_0', # sort info\n",
    "#          'ephys_software':'sglx' # sglx or oe\n",
    "#         }],\n",
    "#     'z_g9y18_23':[\n",
    "#         {'sess_par_list':['2024-04-17','2024-04-18','2024-04-19'], # sessions with this configuration\n",
    "#          'stim_sess_list':[], # sessions where stimuli were presented\n",
    "#          'mic_list':['microphone_M','microphone_F'], # list of mics of interest, by signal name in rig.json\n",
    "#          'adc_list':[], # list of adc channels of interest\n",
    "#          'stim_list':['wav_stim','wav_syn'], # list of adc chans with the stimulus\n",
    "#          'nidq_ttl_list':[], # list of TTL signals form the nidq digital inputs to extract (besides the 'sync')\n",
    "#          'ref_stream':'ap_0', # what to synchronize everything to (sglx only, oe already synced)\n",
    "#          'trial_tag_chan':2, # sglx, what was the tag channel in the stimulus wave (this should come from meta et. al)\n",
    "#          'on_signal':1, # sglx, whether signal on is hi or lo\n",
    "#          'sort':'sort_0', # sort info\n",
    "#          'ephys_software':'sglx' # sglx or oe\n",
    "#         }],\n",
    "#     'z_p5y10_23':[\n",
    "#         {'sess_par_list':['2024-05-16','2024-05-17'], # sessions with this configuration\n",
    "#          'stim_sess_list':[], # sessions where stimuli were presented\n",
    "#          'mic_list':['microphone_M','microphone_F'], # list of mics of interest, by signal name in rig.json\n",
    "#          'adc_list':[], # list of adc channels of interest\n",
    "#          'stim_list':['wav_stim','wav_syn'], # list of adc chans with the stimulus\n",
    "#          'nidq_ttl_list':[], # list of TTL signals form the nidq digital inputs to extract (besides the 'sync')\n",
    "#          'ref_stream':'ap_0', # what to synchronize everything to (sglx only, oe already synced)\n",
    "#          'trial_tag_chan':2, # sglx, what was the tag channel in the stimulus wave (this should come from meta et. al)\n",
    "#          'on_signal':1, # sglx, whether signal on is hi or lo\n",
    "#          'sort':'sort_0', # sort info\n",
    "#          'ephys_software':'sglx' # sglx or oe\n",
    "#         }]\n",
    "#     'z_r5r13_24':[\n",
    "#         {'sess_par_list':['2024-08-05','2024-08-05_reduced_chans','2024-08-06','2024-08-07','2024-08-08'], # sessions with this configuration\n",
    "#          'stim_sess_list':[], # sessions where stimuli were presented\n",
    "#          'mic_list':['microphone_M','microphone_F'], # list of mics of interest, by signal name in rig.json\n",
    "#          'adc_list':[], # list of adc channels of interest\n",
    "#          'stim_list':['wav_stim','wav_syn'], # list of adc chans with the stimulus\n",
    "#          'nidq_ttl_list':[], # list of TTL signals form the nidq digital inputs to extract (besides the 'sync')\n",
    "#          'ref_stream':'ap_0', # what to synchronize everything to (sglx only, oe already synced)\n",
    "#          'trial_tag_chan':2, # sglx, what was the tag channel in the stimulus wave (this should come from meta et. al)\n",
    "#          'on_signal':1, # sglx, whether signal on is hi or lo\n",
    "#          'sort':'sort_0', # sort info\n",
    "#          'ephys_software':'sglx' # sglx or oe\n",
    "#         }]\n",
    "#     'z_g10r16_24':[\n",
    "#         {'sess_par_list':['2024-09-24','2024-09-25','2024-09-26','2024-09-26_overnight','2024-09-27'], # sessions with this configuration\n",
    "#          'stim_sess_list':[], # sessions where stimuli were presented\n",
    "#          'mic_list':['microphone_M','microphone_F'], # list of mics of interest, by signal name in rig.json\n",
    "#          'adc_list':[], # list of adc channels of interest\n",
    "#          'stim_list':['wav_stim','wav_syn'], # list of adc chans with the stimulus\n",
    "#          'nidq_ttl_list':[], # list of TTL signals form the nidq digital inputs to extract (besides the 'sync')\n",
    "#          'ref_stream':'ap_0', # what to synchronize everything to (sglx only, oe already synced)\n",
    "#          'trial_tag_chan':2, # sglx, what was the tag channel in the stimulus wave (this should come from meta et. al)\n",
    "#          'on_signal':1, # sglx, whether signal on is hi or lo\n",
    "#          'sort':'sort_0', # sort info\n",
    "#          'ephys_software':'sglx' # sglx or oe\n",
    "#         }]\n",
    "#     'z_p6p10_24':[\n",
    "#         {'sess_par_list':['2024-10-01','2024-10-02','2024-10-03'], # sessions with this configuration\n",
    "#          'stim_sess_list':[], # sessions where stimuli were presented\n",
    "#          'mic_list':['microphone_M','microphone_F'], # list of mics of interest, by signal name in rig.json\n",
    "#          'adc_list':['opto_stim'], # list of adc channels of interest\n",
    "#          'stim_list':['wav_stim','wav_syn'], # list of adc chans with the stimulus\n",
    "#          'nidq_ttl_list':[], # list of TTL signals form the nidq digital inputs to extract (besides the 'sync')\n",
    "#          'ref_stream':'ap_0', # what to synchronize everything to (sglx only, oe already synced)\n",
    "#          'trial_tag_chan':2, # sglx, what was the tag channel in the stimulus wave (this should come from meta et. al)\n",
    "#          'on_signal':1, # sglx, whether signal on is hi or lo\n",
    "#          'sort':'sort_0', # sort info\n",
    "#          'ephys_software':'sglx' # sglx or oe\n",
    "#         }]\n",
    "#     'z_r7c3_23':[\n",
    "#         {'sess_par_list':['2024-06-03','2024-06-04'], # sessions with this configuration\n",
    "#          'stim_sess_list':[], # sessions where stimuli were presented\n",
    "#          'mic_list':['microphone_M','microphone_F'], # list of mics of interest, by signal name in rig.json\n",
    "#          'adc_list':[], # list of adc channels of interest\n",
    "#          'stim_list':['wav_stim','wav_syn'], # list of adc chans with the stimulus\n",
    "#          'nidq_ttl_list':[], # list of TTL signals form the nidq digital inputs to extract (besides the 'sync')\n",
    "#          'ref_stream':'ap_0', # what to synchronize everything to (sglx only, oe already synced)\n",
    "#          'trial_tag_chan':2, # sglx, what was the tag channel in the stimulus wave (this should come from meta et. al)\n",
    "#          'on_signal':1, # sglx, whether signal on is hi or lo\n",
    "#          'sort':'sort_0', # sort info\n",
    "#          'ephys_software':'sglx' # sglx or oe\n",
    "#         }]\n",
    "#     'z_r1p1_24':[\n",
    "#         {'sess_par_list':['2024-10-18','2024-10-19'], # sessions with this configuration\n",
    "#          'stim_sess_list':[], # sessions where stimuli were presented\n",
    "#          'mic_list':['microphone_M','microphone_F'], # list of mics of interest, by signal name in rig.json\n",
    "#          'adc_list':[], # list of adc channels of interest\n",
    "#          'stim_list':[], # list of adc chans with the stimulus\n",
    "#          'nidq_ttl_list':[], # list of TTL signals form the nidq digital inputs to extract (besides the 'sync')\n",
    "#          'ref_stream':'ap_0', # what to synchronize everything to (sglx only, oe already synced)\n",
    "#          'trial_tag_chan':2, # sglx, what was the tag channel in the stimulus wave (this should come from meta et. al)\n",
    "#          'on_signal':1, # sglx, whether signal on is hi or lo\n",
    "#          'sort':'sort_0', # sort info\n",
    "#          'ephys_software':'sglx' # sglx or oe\n",
    "#         }]\n",
    "#     'z_c7r3_24':[\n",
    "#         {'sess_par_list':['2024-10-26','2024-10-27'], # sessions with this configuration\n",
    "#          'stim_sess_list':[], # sessions where stimuli were presented\n",
    "#          'mic_list':['microphone_M','microphone_F'], # list of mics of interest, by signal name in rig.json\n",
    "#          'adc_list':['opto_stim'], # list of adc channels of interest\n",
    "#          'stim_list':[], # list of adc chans with the stimulus\n",
    "#          'nidq_ttl_list':[], # list of TTL signals form the nidq digital inputs to extract (besides the 'sync')\n",
    "#          'ref_stream':'ap_0', # what to synchronize everything to (sglx only, oe already synced)\n",
    "#          'trial_tag_chan':2, # sglx, what was the tag channel in the stimulus wave (this should come from meta et. al)\n",
    "#          'on_signal':1, # sglx, whether signal on is hi or lo\n",
    "#          'sort':'sort_0', # sort info\n",
    "#          'ephys_software':'sglx' # sglx or oe\n",
    "#         }]\n",
    "    'z_b1w4_24':[\n",
    "        {'sess_par_list':['2024-11-11','2024-11-12','2024-11-13'], # sessions with this configuration\n",
    "         'stim_sess_list':[], # sessions where stimuli were presented\n",
    "         'mic_list':['microphone_M','microphone_F'], # list of mics of interest, by signal name in rig.json\n",
    "         'adc_list':[], # list of adc channels of interest\n",
    "         'stim_list':['wav_stim','wav_syn'], # list of adc chans with the stimulus\n",
    "         'nidq_ttl_list':[], # list of TTL signals form the nidq digital inputs to extract (besides the 'sync')\n",
    "         'ref_stream':'ap_0', # what to synchronize everything to (sglx only, oe already synced)\n",
    "         'trial_tag_chan':2, # sglx, what was the tag channel in the stimulus wave (this should come from meta et. al)\n",
    "         'on_signal':1, # sglx, whether signal on is hi or lo\n",
    "         'sort':'sort_0', # sort info\n",
    "         'ephys_software':'sglx' # sglx or oe\n",
    "        }]\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Preprocess and synchronize recordings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "z_b1w4_24 2024-11-11 already exists -- skipping preprocessing\n",
      "z_b1w4_24 2024-11-12 already exists -- skipping preprocessing\n",
      "z_b1w4_24 2024-11-13 no log file found -- running preprocessing\n",
      "z_b1w4_24 2024-11-13 sglx preprocessing session..\n",
      "preprocessing..\n",
      "z_b1w4_24 2024-11-13 deriving bout information..\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ea09c157d066423cad14c5e41ee42237",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/9 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "90d8012cd1a54d8e9acaf858a9a300dc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/9 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b88f3fa54277458d82cd95ef1fbe52c6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/9 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "z_b1w4_24 2024-11-13 0700_g0 syncing..\n",
      "z_b1w4_24 2024-11-13 1000_g0 syncing..\n",
      "z_b1w4_24 2024-11-13 1301_g0 syncing..\n",
      "z_b1w4_24 2024-11-13 preprocessing complete without error\n",
      "CPU times: user 1h 22min 52s, sys: 41min 2s, total: 2h 3min 54s\n",
      "Wall time: 3h 1min 36s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "# loop through all birds / recordings\n",
    "for this_bird in bird_rec_dict.keys():\n",
    "    # get session parameters\n",
    "    sess_par_all = bird_rec_dict[this_bird]\n",
    "    # get all sessions\n",
    "    all_bird_sess = et.list_sessions(this_bird)\n",
    "    \n",
    "    # loop through sessions\n",
    "    for this_sess in all_bird_sess:\n",
    "        \n",
    "        # loop though possible session parameter sets\n",
    "        sess_par_match = []\n",
    "        for this_sess_par in sess_par_all:\n",
    "            # save if there is a match\n",
    "            if this_sess in this_sess_par['sess_par_list']:\n",
    "                sess_par_match.append(this_sess_par)      \n",
    "        \n",
    "        # ensure only one match\n",
    "        if len(sess_par_match) == 1:\n",
    "            sess_par = sess_par_match[0]\n",
    "            # add bird and sess variables\n",
    "            sess_par['bird'] = this_bird        \n",
    "            sess_par['sess'] = this_sess\n",
    "            # stim session\n",
    "            if this_sess in this_sess_par['stim_sess_list']:\n",
    "                sess_par['stim_sess'] = this_sess\n",
    "            else:\n",
    "                sess_par['stim_sess'] = []\n",
    "            \n",
    "            # get experiment structure\n",
    "            exp_struct = et.get_exp_struct(sess_par['bird'],sess_par['sess'],sort=sess_par['sort'],ephys_software=sess_par['ephys_software'])\n",
    "            \n",
    "            # get preprocessing log\n",
    "            log_dir = os.path.join('/mnt/cube/chronic_ephys/log', this_bird, this_sess)\n",
    "            log_file = os.path.join(log_dir,'preprocessing.log')\n",
    "            try:\n",
    "                with open(log_file, 'r') as f:\n",
    "                    log_message=f.readline() # read the first line of the log file\n",
    "                if log_message[:-1] == sess_par['bird']+' '+sess_par['sess']+' preprocessing complete without error':\n",
    "                    print(sess_par['bird'],sess_par['sess'],'already exists -- skipping preprocessing')\n",
    "                    run_preprocess = False\n",
    "                elif log_message[:-1] == sess_par['bird']+' '+sess_par['sess']+' preprocessing failed':\n",
    "                    if force_preprocess:\n",
    "                        print(sess_par['bird'],sess_par['sess'],'failed previously -- running preprocessing')\n",
    "                        run_preprocess = True\n",
    "                    else:\n",
    "                        print(sess_par['bird'],sess_par['sess'],'failed previously -- skipping preprocessing')\n",
    "                        run_preprocess = False\n",
    "                else: # uninterpretable log file\n",
    "                    print(sess_par['bird'],sess_par['sess'],'couldn\\'t interpret log file -- running preprocessing')\n",
    "                    run_preprocess = True\n",
    "            except: # no existing log file\n",
    "                print(sess_par['bird'],sess_par['sess'],'no log file found -- running preprocessing')\n",
    "                run_preprocess = True\n",
    "            \n",
    "            # run preprocessing \n",
    "            if run_preprocess:\n",
    "                \n",
    "                ##### preprocess acoustics #####\n",
    "                print(this_bird,this_sess,sess_par['ephys_software'],'preprocessing session..')\n",
    "                if sess_par['ephys_software'] == 'sglx':\n",
    "                    preproc_sglx.preprocess_session(sess_par,force_redo=True)\n",
    "                elif sess_par['ephys_software'] == 'oe':\n",
    "                    preproc_oe.preprocess_session(sess_par,force_redo=True)\n",
    "                \n",
    "                ###### derive bout information #####\n",
    "                print(this_bird,this_sess,'deriving bout information..')\n",
    "                sess_bout_pd = sb.get_all_day_bouts(sess_par,hparams,n_jobs=n_jobs,ephys_software=sess_par['ephys_software'],\n",
    "                                                    file_ext=mic_file_ext, deep_search=deep_bout_search)\n",
    "                \n",
    "                # get epochs\n",
    "                sess_epochs = et.list_ephys_epochs(sess_par)\n",
    "                \n",
    "                # store bout info for each epoch\n",
    "                bout_syn_pd_all = []\n",
    "                # stim sess collect all stim as well\n",
    "                if len(sess_par['stim_sess']) > 0:\n",
    "                    trial_syn_pd_all = []\n",
    "                \n",
    "                # loop through epochs\n",
    "                epoch_fail = np.zeros_like(sess_epochs, dtype=bool) # track failed preprocessing per epoch\n",
    "                error_tracker = {}\n",
    "                for epoch_n,this_epoch in enumerate(sess_epochs):\n",
    "                    try:\n",
    "                        print(this_bird,this_sess,this_epoch,'syncing..')\n",
    "                        # add to session parameter dictionary\n",
    "                        sess_par['epoch'] = this_epoch\n",
    "                        # get epoch info\n",
    "                        epoch_struct = et.sgl_struct(sess_par,this_epoch,ephys_software=sess_par['ephys_software'])\n",
    "\n",
    "                        ##### synchronization - sglx #####\n",
    "                        if sess_par['ephys_software'] == 'sglx':\n",
    "                            # get epoch files\n",
    "                            sgl_folders, sgl_files = sglu.sgl_file_struct(epoch_struct['folders']['sglx'])\n",
    "                            run_meta_files = {k:v[0] for k,v in sgl_files.items()}\n",
    "                            run_recordings = {k:sglex.SpikeGLXRecordingExtractor(sglu.get_data_meta_path(v)[0]) for k,v in run_meta_files.items()}\n",
    "\n",
    "                            # get streams, from raw recording extractors and preprocessed data\n",
    "                            all_streams = list(run_recordings.keys()) + ['wav'] ### might want to just remove this\n",
    "                            # get sync pattern\n",
    "                            all_syn_dict = {k:sy.get_syn_pattern(run_recordings,epoch_struct,k,force=False) for k in all_streams}\n",
    "                            # run sync\n",
    "                            sy.sync_all(all_syn_dict,sess_par['ref_stream'],force=False)\n",
    "\n",
    "                            # load bouts\n",
    "                            hparams, bout_pd = sb.load_bouts(sess_par['bird'],sess_par['sess'],'', derived_folder='bouts_sglx',bout_file_key='bout_auto_file')\n",
    "                            # keep only epoch bouts\n",
    "                            logger.info('bouts from this epoch {}'.format(sess_par['epoch']))\n",
    "                            drop_condition = ~bout_pd['file'].str.contains(sess_par['epoch'])\n",
    "                            bout_pd.drop(bout_pd[drop_condition].index, inplace=True)\n",
    "                            bout_pd.reset_index(drop=True, inplace=True)\n",
    "                            # sync bouts to spike time base\n",
    "                            bout_dict, bout_syn_pd = sy.bout_dict_from_pd(bout_pd,all_syn_dict,s_f_key='wav')\n",
    "                            # store epoch synced bout info\n",
    "                            bout_syn_pd['bird'] = this_bird\n",
    "                            bout_syn_pd['sess'] = this_sess\n",
    "                            bout_syn_pd['epoch'] = this_epoch\n",
    "                            bout_syn_pd_all.append(bout_syn_pd)\n",
    "                            # save synced bouts\n",
    "                            bout_dict_path = os.path.join(epoch_struct['folders']['derived'],'bout_dict_ap0.pkl')\n",
    "                            with open(bout_dict_path, 'wb') as handle:\n",
    "                                pickle.dump(bout_dict, handle)\n",
    "                            bout_pd_path = os.path.join(epoch_struct['folders']['derived'],'bout_pd_ap0.pkl')\n",
    "                            bout_pd.to_pickle(bout_pd_path)\n",
    "                            logger.info('saved syncronized bout dict and pandas dataframe to {}, {}'.format(bout_dict_path, bout_pd_path))\n",
    "\n",
    "                            if len(sess_par['stim_sess']) > 0:\n",
    "                                # syn_ttl comes from the digital pin, syn_sine_ttl from the sine\n",
    "                                event_name = 'wav_stim'\n",
    "                                ttl_ev_name = event_name + '_sync_sine_ttl' \n",
    "                                # get the events npy file\n",
    "                                npy_stim_path = os.path.join(epoch_struct['folders']['derived'],ttl_ev_name + '_evt.npy')\n",
    "                                stream_stim_path = os.path.join(epoch_struct['folders']['derived'],event_name + '.npy')\n",
    "                                trial_ttl = np.load(npy_stim_path)\n",
    "                                # epoch may not have trials - if so ttl file will be empty\n",
    "                                if len(trial_ttl) > 0:\n",
    "                                    trial_stream = np.load(stream_stim_path,mmap_mode='r')\n",
    "                                    # get sampling frequency\n",
    "                                    stim_s_f = int(all_syn_dict['nidq']['s_f'])\n",
    "                                    # load the stimulus name - frequency tag dictionary\n",
    "                                    stim_tags_dict = preproc_sglx.load_stim_tags_dict(sess_par['stim_sess'],sess_par['bird'])\n",
    "                                    # get trial tagged dataframe\n",
    "                                    trial_tagged_pd = su.get_trials_pd(trial_ttl, trial_stream, stim_s_f,on_signal=sess_par['on_signal'],\n",
    "                                                                       tag_chan=sess_par['trial_tag_chan'],stim_tags_dict=stim_tags_dict,\n",
    "                                                                       trial_is_onof=True)\n",
    "                                    # sync stim\n",
    "                                    trial_dict, trial_syn_pd = sy.trial_syn_from_pd(trial_tagged_pd,all_syn_dict,s_f_key='nidq')\n",
    "                                    # store epoch synced stim info\n",
    "                                    trial_syn_pd['bird'] = this_bird\n",
    "                                    trial_syn_pd['sess'] = this_sess\n",
    "                                    trial_syn_pd['epoch'] = this_epoch\n",
    "                                    trial_syn_pd_all.append(trial_syn_pd)\n",
    "                                    # save synced stim\n",
    "                                    stim_dict_path = os.path.join(epoch_struct['folders']['derived'],'stim_dict_ap0.pkl')\n",
    "                                    stim_pd_path = os.path.join(epoch_struct['folders']['derived'],'stim_pd_ap0.pkl')\n",
    "                                    with open(stim_dict_path,'wb') as handle:\n",
    "                                        pickle.dump(trial_dict,handle)\n",
    "                                    trial_syn_pd.to_pickle(stim_pd_path)\n",
    "                                    logger.info('saved syncronized stim dict and pandas dataframe to {}, {}'.format(stim_dict_path, stim_pd_path))\n",
    "\n",
    "                        ###### synchronization - oe #####\n",
    "                        elif sess_par['ephys_software'] == 'oe':\n",
    "                            # get epoch files\n",
    "                            run_recordings = {'oeb':preproc_oe.get_oe_cont_recording(exp_struct,this_epoch)}\n",
    "\n",
    "                            # make an all_syn_dict\n",
    "                            mic_file_name = os.path.join(exp_struct['folders']['derived'],this_epoch,'wav_mic-npy_meta.pickle')\n",
    "                            with open(mic_file_name, 'rb') as handle:\n",
    "                                wav_mic_meta = pickle.load(handle)\n",
    "                            all_syn_dict = {'wav': {'s_f':wav_mic_meta['s_f']}, \n",
    "                                           'ap_0': {'s_f':run_recordings['oeb'].get_sampling_frequency()},\n",
    "                                           'nidq': {'s_f':run_recordings['oeb'].get_sampling_frequency()}}\n",
    "                            # make bouts pandas file for this session - match sglx format, streams already synced\n",
    "                            bout_oe_struct = et.get_exp_struct(sess_par['bird'],sess_par['sess'],sort=sess_par['sort'],ephys_software='bouts_oe')\n",
    "                            bout_pd_path = os.path.join(bout_oe_struct['folders']['derived'], 'bout_auto.pickle')\n",
    "                            bout_syn_pd = pd.read_pickle(bout_pd_path)\n",
    "                            bout_dict = preproc_oe.bout_dict_from_pd(bout_syn_pd,all_syn_dict)\n",
    "                            # store epoch synced bout info\n",
    "                            bout_syn_pd['bird'] = this_bird\n",
    "                            bout_syn_pd['sess'] = this_sess\n",
    "                            bout_syn_pd['epoch'] = this_epoch\n",
    "                            bout_syn_pd_all.append(bout_syn_pd)\n",
    "                            # save synced bouts\n",
    "                            bout_dict_path = os.path.join(epoch_struct['folders']['derived'],'bout_dict_oe.pkl')\n",
    "                            bout_pd_path = os.path.join(epoch_struct['folders']['derived'],'bout_pd_oe.pkl')\n",
    "                            with open(bout_dict_path,'wb') as handle:\n",
    "                                pickle.dump(bout_dict,handle)\n",
    "                            bout_syn_pd.to_pickle(bout_pd_path)\n",
    "\n",
    "                            if len(sess_par['stim_sess']) > 0:\n",
    "                                # this epoch name - get recording events path\n",
    "                                raw_folder = exp_struct['folders']['oe']\n",
    "                                epoch_path = os.path.join(raw_folder,this_epoch)\n",
    "                                node_path = preproc_oe.get_default_node(exp_struct,this_epoch)\n",
    "                                rec_path = preproc_oe.get_default_recording(node_path)\n",
    "                                events_path = os.path.join(rec_path,'events/Network_Events-102.0/TEXT_group_1/')\n",
    "                                # load stim lables / onsets\n",
    "                                stim_labels = np.load(os.path.join(events_path,'text.npy'))\n",
    "                                stim_onsets = np.load(os.path.join(events_path,'timestamps.npy'))\n",
    "\n",
    "                                # get stim onsets and offsets\n",
    "                                stim_on_all = []; stim_off_all = []; \n",
    "                                stim_proc_path_all = []; stim_exp_path_all = [];\n",
    "                                stim_map_dir_all = []; stim_id_all = [];\n",
    "                                # loop through stim\n",
    "                                for stim_i in range(len(stim_labels)):\n",
    "                                    this_stim_label = stim_labels[stim_i].astype('str')\n",
    "                                    this_stim_onset = stim_onsets[stim_i]\n",
    "                                    if this_stim_label[:4] == 'stim':\n",
    "                                        stim_exp_file = this_stim_label[5:]\n",
    "                                        # get stim preprocessing directory\n",
    "                                        stim_file_split = stim_exp_file.split('/')\n",
    "                                        stim_map_i = np.where([stim_file_split[i] in list(stim_map_dict.keys()) for i in range(len(stim_file_split))])[0][0]\n",
    "                                        stim_map_dir = stim_map_dict[stim_file_split[stim_map_i]]\n",
    "                                        # get remaining stim file path - identical for experiment and preprocessing\n",
    "                                        remaining_stim_file = '/'.join(stim_file_split[stim_map_i+1:])\n",
    "                                        # processing file location\n",
    "                                        stim_file = os.path.join(stim_map_dir,remaining_stim_file)\n",
    "                                        # load stim and get length\n",
    "                                        sf,this_wav = wavfile.read(stim_file,mmap=True)\n",
    "                                        stim_len = this_wav.shape[0]/sf\n",
    "                                        # get length of stim in samples - round up\n",
    "                                        stim_samp_len = int(np.ceil(stim_len * bout_dict['s_f']))\n",
    "                                        # get stim on / off\n",
    "                                        stim_on_all.append(this_stim_onset)\n",
    "                                        stim_off_all.append(this_stim_onset+stim_samp_len)  \n",
    "                                        stim_proc_path_all.append(stim_file)\n",
    "                                        stim_exp_path_all.append(stim_exp_file)\n",
    "                                        stim_map_dir_all.append(stim_map_dir)\n",
    "                                        stim_id_all.append(remaining_stim_file)\n",
    "\n",
    "                                # make into a pd - oe already synced\n",
    "                                stim_on_all_np = np.array(stim_on_all).astype('int')\n",
    "                                stim_off_all_np = np.array(stim_off_all).astype('int')\n",
    "                                stim_on_all_np_ms = 1000*(stim_on_all_np/bout_dict['s_f'])\n",
    "                                stim_off_all_np_ms = 1000*(stim_off_all_np/bout_dict['s_f'])\n",
    "                                trial_syn_pd = pd.DataFrame(np.vstack([stim_on_all_np,\n",
    "                                                                    stim_off_all_np,\n",
    "                                                                    stim_on_all_np_ms,\n",
    "                                                                    stim_off_all_np_ms,\n",
    "                                                                    stim_off_all_np_ms-stim_on_all_np_ms,\n",
    "                                                                    stim_proc_path_all,\n",
    "                                                                    stim_exp_path_all,\n",
    "                                                                    stim_map_dir_all,\n",
    "                                                                    stim_id_all]).T,\n",
    "                                columns=['start_sample','end_sample','start_ms','end_ms','len_ms',\n",
    "                                         'proc_file','exp_file','map_dir','stim_id'])\n",
    "                                trial_syn_pd['start_sample'] = trial_syn_pd['start_sample'].astype('int')\n",
    "                                trial_syn_pd['end_sample'] = trial_syn_pd['end_sample'].astype('int')\n",
    "                                trial_syn_pd['start_ms'] = trial_syn_pd['start_ms'].astype('float')\n",
    "                                trial_syn_pd['len_ms'] = trial_syn_pd['len_ms'].astype('float')\n",
    "                                # store epoch synced stim info\n",
    "                                trial_syn_pd['bird'] = this_bird\n",
    "                                trial_syn_pd['sess'] = this_sess\n",
    "                                trial_syn_pd['epoch'] = this_epoch\n",
    "                                trial_syn_pd_all.append(trial_syn_pd)\n",
    "                                trial_dict = {\n",
    "                                    's_f': all_syn_dict['wav']['s_f'],\n",
    "                                    'ap_0':all_syn_dict['ap_0']['s_f'],\n",
    "                                    'nidq':all_syn_dict['nidq']['s_f'],\n",
    "                                    'start_ms':trial_syn_pd['start_ms'],\n",
    "                                    'len_ms':trial_syn_pd['len_ms'],\n",
    "                                    'start_sample':trial_syn_pd['start_sample'],\n",
    "                                    'end_sample':trial_syn_pd['end_sample'],\n",
    "                                    'proc_file':trial_syn_pd['proc_file'],\n",
    "                                    'exp_file':trial_syn_pd['exp_file'],\n",
    "                                    'map_dir':trial_syn_pd['map_dir'],\n",
    "                                    'stim_id':trial_syn_pd['stim_id']}\n",
    "                                # save synced stim\n",
    "                                stim_dict_path = os.path.join(epoch_struct['folders']['derived'],'stim_dict_ap0.pkl')\n",
    "                                stim_pd_path = os.path.join(epoch_struct['folders']['derived'],'stim_pd_ap0.pkl')\n",
    "                                with open(stim_dict_path,'wb') as handle:\n",
    "                                    pickle.dump(trial_dict,handle)\n",
    "                                trial_syn_pd.to_pickle(stim_pd_path)\n",
    "                                logger.info('saved syncronized stim dict and pandas dataframe to {}, {}'.format(stim_dict_path, stim_pd_path))\n",
    "                                    \n",
    "                    # record failed epoch \n",
    "                    except Exception as e:\n",
    "                        epoch_fail[epoch_n] = True\n",
    "                        error_tracker[this_epoch] = f\"Error: {str(e)}\"\n",
    "                        \n",
    "                if not np.any(epoch_fail): # if preprocessing complete without error..\n",
    "                    print(sess_par['bird']+' '+sess_par['sess']+' preprocessing complete without error')\n",
    "                    \n",
    "                    # concatenate list of synced bout data frames from each epoch and save\n",
    "                    bout_syn_pd_all_cat = pd.concat(bout_syn_pd_all)\n",
    "                    sb.save_auto_bouts(bout_syn_pd_all_cat,sess_par,hparams,software=sess_par['ephys_software'],bout_file_key='bout_sync_file')\n",
    "\n",
    "                    # stim sess save the all sync epoch stim data frame as well\n",
    "                    if len(sess_par['stim_sess']) > 0:\n",
    "                        trial_syn_pd_all_cat = pd.concat(trial_syn_pd_all)\n",
    "                        sb.save_auto_bouts(trial_syn_pd_all_cat,sess_par,hparams,software=sess_par['ephys_software'],bout_file_key='stim_sync_file')\n",
    "                    \n",
    "                    # log preprocessing complete without error\n",
    "                    if not os.path.exists(log_dir): os.makedirs(log_dir)\n",
    "                    with open(log_file, 'w') as f:\n",
    "                        f.write(sess_par['bird']+' '+sess_par['sess']+' preprocessing complete without error\\nEpochs '+', '.join(sess_epochs)+' processed\\n')\n",
    "                else:\n",
    "                    # log failed epochs and exit without saving\n",
    "                    failed_epochs = np.array(sess_epochs)[epoch_fail]\n",
    "                    if not os.path.exists(log_dir): os.makedirs(log_dir)\n",
    "                    with open(log_file, 'w') as f:\n",
    "                        f.write(sess_par['bird']+' '+sess_par['sess']+' preprocessing failed\\nProblematic epoch(s): '+', '.join(failed_epochs)+'\\n')\n",
    "                        for err_epoch, err_msg in error_tracker.items():\n",
    "                            f.write(err_epoch+': '+err_msg+'\\n')\n",
    "                    #### common reason for failure: data length mismatch (ex. mic or neuropixel came unplugged)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "songproc",
   "language": "python",
   "name": "songproc"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}