5.2-syllable_feature_analysis.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "dd6a2c11-c0f6-4f8b-8a33-0d131ab660f5",
   "metadata": {},
   "source": [
    "# Calculate syllable features\n",
    "\n",
    "Optional notebook within the chronic ephys processing pipeline\n",
    "- 1-preprocess_acoustics\n",
    "- 2-curate_acoustics\n",
    "- 3-sort_spikes\n",
    "- 4-curate_spikes\n",
    "- 5-cluster_acoustics\n",
    "    - **5.1-syllable_feature_analysis**\n",
    "\n",
    "Use the environment **birdsong** to run this notebook"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "73143ca1-2c55-4318-863d-b8fba4d57382",
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "import os\n",
    "import sys\n",
    "import pickle\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from matplotlib import pyplot as plt\n",
    "import seaborn as sns\n",
    "from scipy import stats\n",
    "from scipy import signal\n",
    "from scipy.io import wavfile\n",
    "import IPython.display as ipd\n",
    "from scipy.stats import pearsonr\n",
    "from joblib import Parallel, delayed\n",
    "import noisereduce as nr\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "sys.path.append('/mnt/cube/lo/envs/ceciestunepipe')\n",
    "from ceciestunepipe.file import bcistructure as et\n",
    "\n",
    "sys.path.append('/mnt/cube/lo/envs/SAT')\n",
    "from calculate_SAT_features import calculate_features"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9d53bd3e-b699-46b5-a036-c0d0d857d8ed",
   "metadata": {},
   "source": [
    "## Set parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "18549a3b-ab0c-4d0a-80f2-2c34a1854f9e",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# session parameters\n",
    "sess_par = {\n",
    "    'bird':'z_c5o30_23', # bird ID\n",
    "    'sess':'2023-06-15', # session date\n",
    "    'ephys_software':'sglx', # recording software, sglx or oe\n",
    "    'stim_sess':False, # if song stimulus was played during the session, ignore detected bouts\n",
    "    'trim_bouts':True, # manually trim bouts after curation\n",
    "    'sort':'sort_0', # sort index\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4597492c-069d-469b-87da-cbf9bdb53432",
   "metadata": {},
   "source": [
    "## Load segmented syllables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "32baef4e-479d-4647-9dca-2d9e0d86bf5b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 1 epoch(s): ['0913_g0']\n"
     ]
    }
   ],
   "source": [
    "sess_epochs = et.list_ephys_epochs(sess_par)\n",
    "print(f\"Found {len(sess_epochs)} epoch(s):\", sess_epochs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "c0fdd1c2-2aa4-4f43-ba22-08d23b0aa9d9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading syllables from epoch 0913_g0\n"
     ]
    }
   ],
   "source": [
    "this_epoch = sess_epochs[0] # set epoch index\n",
    "epoch_struct = et.sgl_struct(sess_par,this_epoch,ephys_software=sess_par['ephys_software'])\n",
    "print('Loading syllables from epoch', this_epoch)\n",
    "\n",
    "# load segmented syllable dataframe\n",
    "syl_seg_df_path = os.path.join(epoch_struct['folders']['derived'],'syl_seg_df_ap0.pickle')\n",
    "with open(syl_seg_df_path, 'rb') as handle:\n",
    "    syl_seg_df = pickle.load(handle)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "089cad78-6ce2-41c9-a0a9-0288a29feaf3",
   "metadata": {},
   "source": [
    "## Calculate syllable features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "85ffbfe3-df2b-41af-93d2-9436861c4787",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Added the following syllable features: {'FM', 'goodness', 'AM', 'f0', 'duration_ms', 'mean_freq', 'amplitude', 'entropy'}\n"
     ]
    }
   ],
   "source": [
    "features_in = syl_seg_df.columns.tolist()\n",
    "syl_seg_df = calculate_features(syl_seg_df)\n",
    "features_out = syl_seg_df.columns.tolist()\n",
    "print('Added the following syllable features:',set(features_out) - set(features_in))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "181a8e42-7d92-467a-9324-8647d0e52a9f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>file</th>\n",
       "      <th>sess</th>\n",
       "      <th>epoch</th>\n",
       "      <th>sample_rate</th>\n",
       "      <th>bout_index</th>\n",
       "      <th>waveform_raw</th>\n",
       "      <th>waveform_filt</th>\n",
       "      <th>start_ms_ap_0</th>\n",
       "      <th>start_sample_ap_0</th>\n",
       "      <th>syllable_index</th>\n",
       "      <th>...</th>\n",
       "      <th>label_orig</th>\n",
       "      <th>label</th>\n",
       "      <th>duration_ms</th>\n",
       "      <th>f0</th>\n",
       "      <th>mean_freq</th>\n",
       "      <th>amplitude</th>\n",
       "      <th>goodness</th>\n",
       "      <th>FM</th>\n",
       "      <th>AM</th>\n",
       "      <th>entropy</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>/mnt/cube/chronic_ephys/der/z_c5o30_23/2023-06...</td>\n",
       "      <td>2023-06-15</td>\n",
       "      <td>0913_g0</td>\n",
       "      <td>40000</td>\n",
       "      <td>0</td>\n",
       "      <td>[14, 23, 19, 16, -2, 37, 51, 49, 42, 20, 37, 3...</td>\n",
       "      <td>[3.3851349619418847, 13.522975407927271, 20.71...</td>\n",
       "      <td>919480</td>\n",
       "      <td>27584486</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>56</td>\n",
       "      <td>[525.3561668435311, 540.7502775664981, 543.882...</td>\n",
       "      <td>[2479.6318876837963, 2479.5869596466646, 2470....</td>\n",
       "      <td>[143.27679854240318, 144.39858516668593, 145.3...</td>\n",
       "      <td>[0.10259739092556909, 0.14397099678767095, 0.1...</td>\n",
       "      <td>[0.7879531835604661, 0.7879531835604661, 0.787...</td>\n",
       "      <td>[2584544.6849159594, 3156681.601859254, 361846...</td>\n",
       "      <td>[-1.891032298654375, -1.991935816617417, -2.06...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>/mnt/cube/chronic_ephys/der/z_c5o30_23/2023-06...</td>\n",
       "      <td>2023-06-15</td>\n",
       "      <td>0913_g0</td>\n",
       "      <td>40000</td>\n",
       "      <td>0</td>\n",
       "      <td>[14, 23, 19, 16, -2, 37, 51, 49, 42, 20, 37, 3...</td>\n",
       "      <td>[3.3851349619418847, 13.522975407927271, 20.71...</td>\n",
       "      <td>919480</td>\n",
       "      <td>27584486</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>68</td>\n",
       "      <td>[493.0141887514742, 537.6357851471714, 553.389...</td>\n",
       "      <td>[2832.560229112974, 2899.5090834358793, 2938.4...</td>\n",
       "      <td>[144.49206284367574, 146.26663331306088, 147.8...</td>\n",
       "      <td>[0.20561146103683567, 0.20486051174835548, 0.1...</td>\n",
       "      <td>[0.7879531835604661, 0.7879531835604661, 0.787...</td>\n",
       "      <td>[4496961.870308619, 5987643.710685434, 8064415...</td>\n",
       "      <td>[-1.7842421751760806, -1.868247029859285, -1.7...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>/mnt/cube/chronic_ephys/der/z_c5o30_23/2023-06...</td>\n",
       "      <td>2023-06-15</td>\n",
       "      <td>0913_g0</td>\n",
       "      <td>40000</td>\n",
       "      <td>0</td>\n",
       "      <td>[14, 23, 19, 16, -2, 37, 51, 49, 42, 20, 37, 3...</td>\n",
       "      <td>[3.3851349619418847, 13.522975407927271, 20.71...</td>\n",
       "      <td>919480</td>\n",
       "      <td>27584486</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>76</td>\n",
       "      <td>[nan, 550.2024259228027, 556.5954701647858, 55...</td>\n",
       "      <td>[2755.020615371183, 2846.292909025462, 2877.64...</td>\n",
       "      <td>[144.2056433642707, 146.12090270642864, 147.92...</td>\n",
       "      <td>[0.111279747049674, 0.14580015887779169, 0.148...</td>\n",
       "      <td>[0.7879531835604661, 0.7879531835604661, 0.787...</td>\n",
       "      <td>[4914220.5674833665, 7137293.607432798, 975089...</td>\n",
       "      <td>[-1.8719154057561873, -2.0816475598860418, -2....</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3 rows × 28 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                file        sess    epoch  \\\n",
       "0  /mnt/cube/chronic_ephys/der/z_c5o30_23/2023-06...  2023-06-15  0913_g0   \n",
       "1  /mnt/cube/chronic_ephys/der/z_c5o30_23/2023-06...  2023-06-15  0913_g0   \n",
       "2  /mnt/cube/chronic_ephys/der/z_c5o30_23/2023-06...  2023-06-15  0913_g0   \n",
       "\n",
       "   sample_rate  bout_index                                       waveform_raw  \\\n",
       "0        40000           0  [14, 23, 19, 16, -2, 37, 51, 49, 42, 20, 37, 3...   \n",
       "1        40000           0  [14, 23, 19, 16, -2, 37, 51, 49, 42, 20, 37, 3...   \n",
       "2        40000           0  [14, 23, 19, 16, -2, 37, 51, 49, 42, 20, 37, 3...   \n",
       "\n",
       "                                       waveform_filt  start_ms_ap_0  \\\n",
       "0  [3.3851349619418847, 13.522975407927271, 20.71...         919480   \n",
       "1  [3.3851349619418847, 13.522975407927271, 20.71...         919480   \n",
       "2  [3.3851349619418847, 13.522975407927271, 20.71...         919480   \n",
       "\n",
       "   start_sample_ap_0  syllable_index  ...  label_orig  label  duration_ms  \\\n",
       "0           27584486               0  ...           4      0           56   \n",
       "1           27584486               1  ...           4      0           68   \n",
       "2           27584486               2  ...           0      0           76   \n",
       "\n",
       "                                                  f0  \\\n",
       "0  [525.3561668435311, 540.7502775664981, 543.882...   \n",
       "1  [493.0141887514742, 537.6357851471714, 553.389...   \n",
       "2  [nan, 550.2024259228027, 556.5954701647858, 55...   \n",
       "\n",
       "                                           mean_freq  \\\n",
       "0  [2479.6318876837963, 2479.5869596466646, 2470....   \n",
       "1  [2832.560229112974, 2899.5090834358793, 2938.4...   \n",
       "2  [2755.020615371183, 2846.292909025462, 2877.64...   \n",
       "\n",
       "                                           amplitude  \\\n",
       "0  [143.27679854240318, 144.39858516668593, 145.3...   \n",
       "1  [144.49206284367574, 146.26663331306088, 147.8...   \n",
       "2  [144.2056433642707, 146.12090270642864, 147.92...   \n",
       "\n",
       "                                            goodness  \\\n",
       "0  [0.10259739092556909, 0.14397099678767095, 0.1...   \n",
       "1  [0.20561146103683567, 0.20486051174835548, 0.1...   \n",
       "2  [0.111279747049674, 0.14580015887779169, 0.148...   \n",
       "\n",
       "                                                  FM  \\\n",
       "0  [0.7879531835604661, 0.7879531835604661, 0.787...   \n",
       "1  [0.7879531835604661, 0.7879531835604661, 0.787...   \n",
       "2  [0.7879531835604661, 0.7879531835604661, 0.787...   \n",
       "\n",
       "                                                  AM  \\\n",
       "0  [2584544.6849159594, 3156681.601859254, 361846...   \n",
       "1  [4496961.870308619, 5987643.710685434, 8064415...   \n",
       "2  [4914220.5674833665, 7137293.607432798, 975089...   \n",
       "\n",
       "                                             entropy  \n",
       "0  [-1.891032298654375, -1.991935816617417, -2.06...  \n",
       "1  [-1.7842421751760806, -1.868247029859285, -1.7...  \n",
       "2  [-1.8719154057561873, -2.0816475598860418, -2....  \n",
       "\n",
       "[3 rows x 28 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "syl_seg_df.head(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5d02441b-9607-41c3-83c8-8d007a5e28fc",
   "metadata": {},
   "source": [
    "## Save syllable features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "a891dcbb-7482-4058-bb0a-b7a5927c7fbf",
   "metadata": {},
   "outputs": [],
   "source": [
    "syl_seg_df.to_pickle(os.path.join(epoch_struct['folders']['derived'],'syl_seg_df_ap0.pickle'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "91046fec-bfa5-4fc1-9f24-ffbdd353114b",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "birdsong",
   "language": "python",
   "name": "birdsong"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}