Predict Data.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Predict Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from scipy import stats\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.model_selection import train_test_split\n",
    "from keras.layers.recurrent import GRU\n",
    "from keras.models import Sequential, load_model\n",
    "from keras.regularizers import l2\n",
    "from keras.layers import Dense, BatchNormalization"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load the CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Columns: Index(['speaker', 'word', 'token', 'pre_segment', 'fol_segment', 'vowelR',\n",
      "       'vowel_type', 'note', 'time', 'norm_time', 'c1', 'c2', 'c3', 'c4', 'c5',\n",
      "       'c6', 'c7', 'c8', 'c9', 'c10', 'c11', 'c12'],\n",
      "      dtype='object')\n",
      "Initially 2083105 rows\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>speaker</th>\n",
       "      <th>word</th>\n",
       "      <th>token</th>\n",
       "      <th>pre_segment</th>\n",
       "      <th>fol_segment</th>\n",
       "      <th>vowelR</th>\n",
       "      <th>vowel_type</th>\n",
       "      <th>note</th>\n",
       "      <th>time</th>\n",
       "      <th>norm_time</th>\n",
       "      <th>...</th>\n",
       "      <th>c3</th>\n",
       "      <th>c4</th>\n",
       "      <th>c5</th>\n",
       "      <th>c6</th>\n",
       "      <th>c7</th>\n",
       "      <th>c8</th>\n",
       "      <th>c9</th>\n",
       "      <th>c10</th>\n",
       "      <th>c11</th>\n",
       "      <th>c12</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>100_2016_9_5_10_ngTMg</td>\n",
       "      <td>weather</td>\n",
       "      <td>100_2016_9_5_10_ngTMg_recording10_1</td>\n",
       "      <td>DH</td>\n",
       "      <td>#</td>\n",
       "      <td>ER0</td>\n",
       "      <td>none</td>\n",
       "      <td>linking</td>\n",
       "      <td>2.1715</td>\n",
       "      <td>0.0101</td>\n",
       "      <td>...</td>\n",
       "      <td>-1022.492295</td>\n",
       "      <td>-674.645850</td>\n",
       "      <td>-374.559604</td>\n",
       "      <td>-33.608998</td>\n",
       "      <td>43.539283</td>\n",
       "      <td>-100.527786</td>\n",
       "      <td>-93.679401</td>\n",
       "      <td>-444.218747</td>\n",
       "      <td>-525.123463</td>\n",
       "      <td>-660.706331</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>100_2016_9_5_10_ngTMg</td>\n",
       "      <td>weather</td>\n",
       "      <td>100_2016_9_5_10_ngTMg_recording10_1</td>\n",
       "      <td>DH</td>\n",
       "      <td>#</td>\n",
       "      <td>ER0</td>\n",
       "      <td>none</td>\n",
       "      <td>linking</td>\n",
       "      <td>2.1730</td>\n",
       "      <td>0.0202</td>\n",
       "      <td>...</td>\n",
       "      <td>-1058.318886</td>\n",
       "      <td>-618.081041</td>\n",
       "      <td>-392.756742</td>\n",
       "      <td>-73.851940</td>\n",
       "      <td>9.402799</td>\n",
       "      <td>-70.048259</td>\n",
       "      <td>-76.092667</td>\n",
       "      <td>-485.080975</td>\n",
       "      <td>-514.300174</td>\n",
       "      <td>-690.813720</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>100_2016_9_5_10_ngTMg</td>\n",
       "      <td>weather</td>\n",
       "      <td>100_2016_9_5_10_ngTMg_recording10_1</td>\n",
       "      <td>DH</td>\n",
       "      <td>#</td>\n",
       "      <td>ER0</td>\n",
       "      <td>none</td>\n",
       "      <td>linking</td>\n",
       "      <td>2.1745</td>\n",
       "      <td>0.0303</td>\n",
       "      <td>...</td>\n",
       "      <td>-1111.359326</td>\n",
       "      <td>-607.571292</td>\n",
       "      <td>-404.454693</td>\n",
       "      <td>-32.069785</td>\n",
       "      <td>63.086838</td>\n",
       "      <td>-111.803538</td>\n",
       "      <td>-75.985507</td>\n",
       "      <td>-442.519672</td>\n",
       "      <td>-538.917883</td>\n",
       "      <td>-665.585262</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>100_2016_9_5_10_ngTMg</td>\n",
       "      <td>weather</td>\n",
       "      <td>100_2016_9_5_10_ngTMg_recording10_1</td>\n",
       "      <td>DH</td>\n",
       "      <td>#</td>\n",
       "      <td>ER0</td>\n",
       "      <td>none</td>\n",
       "      <td>linking</td>\n",
       "      <td>2.1761</td>\n",
       "      <td>0.0404</td>\n",
       "      <td>...</td>\n",
       "      <td>-986.829893</td>\n",
       "      <td>-567.632984</td>\n",
       "      <td>-426.516657</td>\n",
       "      <td>-14.907034</td>\n",
       "      <td>51.907415</td>\n",
       "      <td>-89.423728</td>\n",
       "      <td>-64.152095</td>\n",
       "      <td>-500.297254</td>\n",
       "      <td>-479.800648</td>\n",
       "      <td>-628.009562</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>100_2016_9_5_10_ngTMg</td>\n",
       "      <td>weather</td>\n",
       "      <td>100_2016_9_5_10_ngTMg_recording10_1</td>\n",
       "      <td>DH</td>\n",
       "      <td>#</td>\n",
       "      <td>ER0</td>\n",
       "      <td>none</td>\n",
       "      <td>linking</td>\n",
       "      <td>2.1776</td>\n",
       "      <td>0.0505</td>\n",
       "      <td>...</td>\n",
       "      <td>-1063.674469</td>\n",
       "      <td>-641.366517</td>\n",
       "      <td>-365.899597</td>\n",
       "      <td>-17.289025</td>\n",
       "      <td>26.717821</td>\n",
       "      <td>-107.038116</td>\n",
       "      <td>-52.297277</td>\n",
       "      <td>-479.613893</td>\n",
       "      <td>-519.625957</td>\n",
       "      <td>-662.952884</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 22 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                 speaker     word                                token  \\\n",
       "0  100_2016_9_5_10_ngTMg  weather  100_2016_9_5_10_ngTMg_recording10_1   \n",
       "1  100_2016_9_5_10_ngTMg  weather  100_2016_9_5_10_ngTMg_recording10_1   \n",
       "2  100_2016_9_5_10_ngTMg  weather  100_2016_9_5_10_ngTMg_recording10_1   \n",
       "3  100_2016_9_5_10_ngTMg  weather  100_2016_9_5_10_ngTMg_recording10_1   \n",
       "4  100_2016_9_5_10_ngTMg  weather  100_2016_9_5_10_ngTMg_recording10_1   \n",
       "\n",
       "  pre_segment fol_segment vowelR vowel_type     note    time  norm_time  ...  \\\n",
       "0          DH           #    ER0       none  linking  2.1715     0.0101  ...   \n",
       "1          DH           #    ER0       none  linking  2.1730     0.0202  ...   \n",
       "2          DH           #    ER0       none  linking  2.1745     0.0303  ...   \n",
       "3          DH           #    ER0       none  linking  2.1761     0.0404  ...   \n",
       "4          DH           #    ER0       none  linking  2.1776     0.0505  ...   \n",
       "\n",
       "            c3          c4          c5         c6         c7          c8  \\\n",
       "0 -1022.492295 -674.645850 -374.559604 -33.608998  43.539283 -100.527786   \n",
       "1 -1058.318886 -618.081041 -392.756742 -73.851940   9.402799  -70.048259   \n",
       "2 -1111.359326 -607.571292 -404.454693 -32.069785  63.086838 -111.803538   \n",
       "3  -986.829893 -567.632984 -426.516657 -14.907034  51.907415  -89.423728   \n",
       "4 -1063.674469 -641.366517 -365.899597 -17.289025  26.717821 -107.038116   \n",
       "\n",
       "          c9         c10         c11         c12  \n",
       "0 -93.679401 -444.218747 -525.123463 -660.706331  \n",
       "1 -76.092667 -485.080975 -514.300174 -690.813720  \n",
       "2 -75.985507 -442.519672 -538.917883 -665.585262  \n",
       "3 -64.152095 -500.297254 -479.800648 -628.009562  \n",
       "4 -52.297277 -479.613893 -519.625957 -662.952884  \n",
       "\n",
       "[5 rows x 22 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = pd.read_csv('datasets/mt.csv', sep=',')\n",
    "print(\"Columns:\", data.columns)\n",
    "print(\"Initially\", len(data), \"rows\")\n",
    "\n",
    "data.loc[data['vowel_type'].isna(), 'vowel_type'] = 'none'\n",
    "data.head()\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Encode Categorical Variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['AW1' 'AY1' 'B' 'CH' 'D' 'DH' 'ER0' 'F' 'G' 'H' 'HH' 'IY0' 'IY1' 'K' 'L'\n",
      " 'M' 'N' 'NG' 'OW1' 'P' 'R' 'S' 'SH' 'T' 'UW1' 'V' 'W' 'Y' 'Z' 'nan' 'sp']\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>speaker</th>\n",
       "      <th>word</th>\n",
       "      <th>token</th>\n",
       "      <th>pre_segment</th>\n",
       "      <th>fol_segment</th>\n",
       "      <th>vowelR</th>\n",
       "      <th>vowel_type</th>\n",
       "      <th>note</th>\n",
       "      <th>time</th>\n",
       "      <th>norm_time</th>\n",
       "      <th>...</th>\n",
       "      <th>c3</th>\n",
       "      <th>c4</th>\n",
       "      <th>c5</th>\n",
       "      <th>c6</th>\n",
       "      <th>c7</th>\n",
       "      <th>c8</th>\n",
       "      <th>c9</th>\n",
       "      <th>c10</th>\n",
       "      <th>c11</th>\n",
       "      <th>c12</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>100_2016_9_5_10_ngTMg</td>\n",
       "      <td>weather</td>\n",
       "      <td>100_2016_9_5_10_ngTMg_recording10_1</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "      <td>linking</td>\n",
       "      <td>2.1715</td>\n",
       "      <td>0.0101</td>\n",
       "      <td>...</td>\n",
       "      <td>-1022.492295</td>\n",
       "      <td>-674.645850</td>\n",
       "      <td>-374.559604</td>\n",
       "      <td>-33.608998</td>\n",
       "      <td>43.539283</td>\n",
       "      <td>-100.527786</td>\n",
       "      <td>-93.679401</td>\n",
       "      <td>-444.218747</td>\n",
       "      <td>-525.123463</td>\n",
       "      <td>-660.706331</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>100_2016_9_5_10_ngTMg</td>\n",
       "      <td>weather</td>\n",
       "      <td>100_2016_9_5_10_ngTMg_recording10_1</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "      <td>linking</td>\n",
       "      <td>2.1730</td>\n",
       "      <td>0.0202</td>\n",
       "      <td>...</td>\n",
       "      <td>-1058.318886</td>\n",
       "      <td>-618.081041</td>\n",
       "      <td>-392.756742</td>\n",
       "      <td>-73.851940</td>\n",
       "      <td>9.402799</td>\n",
       "      <td>-70.048259</td>\n",
       "      <td>-76.092667</td>\n",
       "      <td>-485.080975</td>\n",
       "      <td>-514.300174</td>\n",
       "      <td>-690.813720</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>100_2016_9_5_10_ngTMg</td>\n",
       "      <td>weather</td>\n",
       "      <td>100_2016_9_5_10_ngTMg_recording10_1</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "      <td>linking</td>\n",
       "      <td>2.1745</td>\n",
       "      <td>0.0303</td>\n",
       "      <td>...</td>\n",
       "      <td>-1111.359326</td>\n",
       "      <td>-607.571292</td>\n",
       "      <td>-404.454693</td>\n",
       "      <td>-32.069785</td>\n",
       "      <td>63.086838</td>\n",
       "      <td>-111.803538</td>\n",
       "      <td>-75.985507</td>\n",
       "      <td>-442.519672</td>\n",
       "      <td>-538.917883</td>\n",
       "      <td>-665.585262</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>100_2016_9_5_10_ngTMg</td>\n",
       "      <td>weather</td>\n",
       "      <td>100_2016_9_5_10_ngTMg_recording10_1</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "      <td>linking</td>\n",
       "      <td>2.1761</td>\n",
       "      <td>0.0404</td>\n",
       "      <td>...</td>\n",
       "      <td>-986.829893</td>\n",
       "      <td>-567.632984</td>\n",
       "      <td>-426.516657</td>\n",
       "      <td>-14.907034</td>\n",
       "      <td>51.907415</td>\n",
       "      <td>-89.423728</td>\n",
       "      <td>-64.152095</td>\n",
       "      <td>-500.297254</td>\n",
       "      <td>-479.800648</td>\n",
       "      <td>-628.009562</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>100_2016_9_5_10_ngTMg</td>\n",
       "      <td>weather</td>\n",
       "      <td>100_2016_9_5_10_ngTMg_recording10_1</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "      <td>linking</td>\n",
       "      <td>2.1776</td>\n",
       "      <td>0.0505</td>\n",
       "      <td>...</td>\n",
       "      <td>-1063.674469</td>\n",
       "      <td>-641.366517</td>\n",
       "      <td>-365.899597</td>\n",
       "      <td>-17.289025</td>\n",
       "      <td>26.717821</td>\n",
       "      <td>-107.038116</td>\n",
       "      <td>-52.297277</td>\n",
       "      <td>-479.613893</td>\n",
       "      <td>-519.625957</td>\n",
       "      <td>-662.952884</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>100_2016_9_5_10_ngTMg</td>\n",
       "      <td>weather</td>\n",
       "      <td>100_2016_9_5_10_ngTMg_recording10_1</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "      <td>linking</td>\n",
       "      <td>2.1791</td>\n",
       "      <td>0.0606</td>\n",
       "      <td>...</td>\n",
       "      <td>-1065.472565</td>\n",
       "      <td>-620.111160</td>\n",
       "      <td>-423.981473</td>\n",
       "      <td>-57.378900</td>\n",
       "      <td>68.832020</td>\n",
       "      <td>-69.322706</td>\n",
       "      <td>-110.514901</td>\n",
       "      <td>-480.604908</td>\n",
       "      <td>-516.096800</td>\n",
       "      <td>-670.000800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>100_2016_9_5_10_ngTMg</td>\n",
       "      <td>weather</td>\n",
       "      <td>100_2016_9_5_10_ngTMg_recording10_1</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "      <td>linking</td>\n",
       "      <td>2.1806</td>\n",
       "      <td>0.0707</td>\n",
       "      <td>...</td>\n",
       "      <td>-1065.865377</td>\n",
       "      <td>-675.983234</td>\n",
       "      <td>-369.431880</td>\n",
       "      <td>-42.174560</td>\n",
       "      <td>32.678901</td>\n",
       "      <td>-78.833412</td>\n",
       "      <td>-83.945704</td>\n",
       "      <td>-499.744672</td>\n",
       "      <td>-488.783056</td>\n",
       "      <td>-680.203198</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>100_2016_9_5_10_ngTMg</td>\n",
       "      <td>weather</td>\n",
       "      <td>100_2016_9_5_10_ngTMg_recording10_1</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "      <td>linking</td>\n",
       "      <td>2.1821</td>\n",
       "      <td>0.0808</td>\n",
       "      <td>...</td>\n",
       "      <td>-1085.372450</td>\n",
       "      <td>-640.074493</td>\n",
       "      <td>-407.788671</td>\n",
       "      <td>-8.526462</td>\n",
       "      <td>30.385037</td>\n",
       "      <td>-86.137435</td>\n",
       "      <td>-108.084595</td>\n",
       "      <td>-449.959978</td>\n",
       "      <td>-523.291645</td>\n",
       "      <td>-684.036196</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>100_2016_9_5_10_ngTMg</td>\n",
       "      <td>weather</td>\n",
       "      <td>100_2016_9_5_10_ngTMg_recording10_1</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "      <td>linking</td>\n",
       "      <td>2.1836</td>\n",
       "      <td>0.0909</td>\n",
       "      <td>...</td>\n",
       "      <td>-959.284014</td>\n",
       "      <td>-617.434322</td>\n",
       "      <td>-421.071931</td>\n",
       "      <td>0.850639</td>\n",
       "      <td>33.118351</td>\n",
       "      <td>-141.286203</td>\n",
       "      <td>-84.906223</td>\n",
       "      <td>-459.382941</td>\n",
       "      <td>-520.984107</td>\n",
       "      <td>-621.123415</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>100_2016_9_5_10_ngTMg</td>\n",
       "      <td>weather</td>\n",
       "      <td>100_2016_9_5_10_ngTMg_recording10_1</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "      <td>linking</td>\n",
       "      <td>2.1852</td>\n",
       "      <td>0.1010</td>\n",
       "      <td>...</td>\n",
       "      <td>-994.788822</td>\n",
       "      <td>-679.964733</td>\n",
       "      <td>-417.837474</td>\n",
       "      <td>23.585294</td>\n",
       "      <td>37.165498</td>\n",
       "      <td>-129.098794</td>\n",
       "      <td>-79.556766</td>\n",
       "      <td>-449.219367</td>\n",
       "      <td>-526.555203</td>\n",
       "      <td>-675.400315</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>100_2016_9_5_10_ngTMg</td>\n",
       "      <td>weather</td>\n",
       "      <td>100_2016_9_5_10_ngTMg_recording10_1</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "      <td>linking</td>\n",
       "      <td>2.1867</td>\n",
       "      <td>0.1111</td>\n",
       "      <td>...</td>\n",
       "      <td>-1023.384287</td>\n",
       "      <td>-642.773084</td>\n",
       "      <td>-367.782185</td>\n",
       "      <td>-5.284019</td>\n",
       "      <td>-4.179570</td>\n",
       "      <td>-97.267366</td>\n",
       "      <td>-90.008919</td>\n",
       "      <td>-488.774613</td>\n",
       "      <td>-507.899234</td>\n",
       "      <td>-662.863047</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>100_2016_9_5_10_ngTMg</td>\n",
       "      <td>weather</td>\n",
       "      <td>100_2016_9_5_10_ngTMg_recording10_1</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "      <td>linking</td>\n",
       "      <td>2.1882</td>\n",
       "      <td>0.1212</td>\n",
       "      <td>...</td>\n",
       "      <td>-1062.043479</td>\n",
       "      <td>-592.324192</td>\n",
       "      <td>-411.929318</td>\n",
       "      <td>-58.980487</td>\n",
       "      <td>47.032517</td>\n",
       "      <td>-81.177417</td>\n",
       "      <td>-101.290066</td>\n",
       "      <td>-468.385271</td>\n",
       "      <td>-532.024478</td>\n",
       "      <td>-670.322280</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>100_2016_9_5_10_ngTMg</td>\n",
       "      <td>weather</td>\n",
       "      <td>100_2016_9_5_10_ngTMg_recording10_1</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "      <td>linking</td>\n",
       "      <td>2.1897</td>\n",
       "      <td>0.1313</td>\n",
       "      <td>...</td>\n",
       "      <td>-973.859908</td>\n",
       "      <td>-596.008790</td>\n",
       "      <td>-400.989062</td>\n",
       "      <td>-110.336663</td>\n",
       "      <td>41.467889</td>\n",
       "      <td>-57.469808</td>\n",
       "      <td>-66.102633</td>\n",
       "      <td>-460.263050</td>\n",
       "      <td>-517.879455</td>\n",
       "      <td>-668.504013</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>100_2016_9_5_10_ngTMg</td>\n",
       "      <td>weather</td>\n",
       "      <td>100_2016_9_5_10_ngTMg_recording10_1</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "      <td>linking</td>\n",
       "      <td>2.1912</td>\n",
       "      <td>0.1414</td>\n",
       "      <td>...</td>\n",
       "      <td>-1097.701283</td>\n",
       "      <td>-664.891726</td>\n",
       "      <td>-372.274870</td>\n",
       "      <td>-40.995007</td>\n",
       "      <td>26.694060</td>\n",
       "      <td>-67.709100</td>\n",
       "      <td>-112.023698</td>\n",
       "      <td>-496.278069</td>\n",
       "      <td>-487.751241</td>\n",
       "      <td>-693.348117</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>100_2016_9_5_10_ngTMg</td>\n",
       "      <td>weather</td>\n",
       "      <td>100_2016_9_5_10_ngTMg_recording10_1</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "      <td>linking</td>\n",
       "      <td>2.1927</td>\n",
       "      <td>0.1515</td>\n",
       "      <td>...</td>\n",
       "      <td>-1084.806882</td>\n",
       "      <td>-599.333117</td>\n",
       "      <td>-397.589833</td>\n",
       "      <td>-24.794497</td>\n",
       "      <td>73.461545</td>\n",
       "      <td>-110.530665</td>\n",
       "      <td>-122.142781</td>\n",
       "      <td>-437.685280</td>\n",
       "      <td>-506.831723</td>\n",
       "      <td>-656.654496</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>100_2016_9_5_10_ngTMg</td>\n",
       "      <td>weather</td>\n",
       "      <td>100_2016_9_5_10_ngTMg_recording10_1</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "      <td>linking</td>\n",
       "      <td>2.1942</td>\n",
       "      <td>0.1616</td>\n",
       "      <td>...</td>\n",
       "      <td>-1017.969258</td>\n",
       "      <td>-667.043051</td>\n",
       "      <td>-414.810019</td>\n",
       "      <td>-31.457236</td>\n",
       "      <td>50.477793</td>\n",
       "      <td>-78.118045</td>\n",
       "      <td>-119.686091</td>\n",
       "      <td>-485.323087</td>\n",
       "      <td>-483.331104</td>\n",
       "      <td>-676.555043</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>100_2016_9_5_10_ngTMg</td>\n",
       "      <td>weather</td>\n",
       "      <td>100_2016_9_5_10_ngTMg_recording10_1</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "      <td>linking</td>\n",
       "      <td>2.1958</td>\n",
       "      <td>0.1717</td>\n",
       "      <td>...</td>\n",
       "      <td>-1047.727327</td>\n",
       "      <td>-627.326194</td>\n",
       "      <td>-354.243706</td>\n",
       "      <td>-44.328483</td>\n",
       "      <td>-0.114849</td>\n",
       "      <td>-86.959657</td>\n",
       "      <td>-98.176778</td>\n",
       "      <td>-482.192906</td>\n",
       "      <td>-498.370002</td>\n",
       "      <td>-666.326418</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>100_2016_9_5_10_ngTMg</td>\n",
       "      <td>weather</td>\n",
       "      <td>100_2016_9_5_10_ngTMg_recording10_1</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "      <td>linking</td>\n",
       "      <td>2.1973</td>\n",
       "      <td>0.1818</td>\n",
       "      <td>...</td>\n",
       "      <td>-1056.292893</td>\n",
       "      <td>-562.678986</td>\n",
       "      <td>-421.238856</td>\n",
       "      <td>-106.645364</td>\n",
       "      <td>41.085645</td>\n",
       "      <td>-41.879521</td>\n",
       "      <td>-104.503612</td>\n",
       "      <td>-488.691699</td>\n",
       "      <td>-508.357849</td>\n",
       "      <td>-692.791547</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>100_2016_9_5_10_ngTMg</td>\n",
       "      <td>weather</td>\n",
       "      <td>100_2016_9_5_10_ngTMg_recording10_1</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "      <td>linking</td>\n",
       "      <td>2.1988</td>\n",
       "      <td>0.1919</td>\n",
       "      <td>...</td>\n",
       "      <td>-1070.866679</td>\n",
       "      <td>-600.016910</td>\n",
       "      <td>-389.366085</td>\n",
       "      <td>-68.624698</td>\n",
       "      <td>45.238264</td>\n",
       "      <td>-69.487174</td>\n",
       "      <td>-82.938884</td>\n",
       "      <td>-480.472603</td>\n",
       "      <td>-514.861968</td>\n",
       "      <td>-650.736055</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>100_2016_9_5_10_ngTMg</td>\n",
       "      <td>weather</td>\n",
       "      <td>100_2016_9_5_10_ngTMg_recording10_1</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "      <td>linking</td>\n",
       "      <td>2.2003</td>\n",
       "      <td>0.2020</td>\n",
       "      <td>...</td>\n",
       "      <td>-1070.647542</td>\n",
       "      <td>-644.234503</td>\n",
       "      <td>-393.390540</td>\n",
       "      <td>-55.644766</td>\n",
       "      <td>31.506429</td>\n",
       "      <td>-62.231565</td>\n",
       "      <td>-112.585092</td>\n",
       "      <td>-491.784054</td>\n",
       "      <td>-497.043906</td>\n",
       "      <td>-688.502231</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>100_2016_9_5_10_ngTMg</td>\n",
       "      <td>weather</td>\n",
       "      <td>100_2016_9_5_10_ngTMg_recording10_1</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "      <td>linking</td>\n",
       "      <td>2.2018</td>\n",
       "      <td>0.2121</td>\n",
       "      <td>...</td>\n",
       "      <td>-1062.155799</td>\n",
       "      <td>-621.044103</td>\n",
       "      <td>-364.775614</td>\n",
       "      <td>-14.405735</td>\n",
       "      <td>11.431850</td>\n",
       "      <td>-87.574939</td>\n",
       "      <td>-98.935280</td>\n",
       "      <td>-467.898393</td>\n",
       "      <td>-499.075398</td>\n",
       "      <td>-644.791299</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>21 rows × 22 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                  speaker     word                                token  \\\n",
       "0   100_2016_9_5_10_ngTMg  weather  100_2016_9_5_10_ngTMg_recording10_1   \n",
       "1   100_2016_9_5_10_ngTMg  weather  100_2016_9_5_10_ngTMg_recording10_1   \n",
       "2   100_2016_9_5_10_ngTMg  weather  100_2016_9_5_10_ngTMg_recording10_1   \n",
       "3   100_2016_9_5_10_ngTMg  weather  100_2016_9_5_10_ngTMg_recording10_1   \n",
       "4   100_2016_9_5_10_ngTMg  weather  100_2016_9_5_10_ngTMg_recording10_1   \n",
       "5   100_2016_9_5_10_ngTMg  weather  100_2016_9_5_10_ngTMg_recording10_1   \n",
       "6   100_2016_9_5_10_ngTMg  weather  100_2016_9_5_10_ngTMg_recording10_1   \n",
       "7   100_2016_9_5_10_ngTMg  weather  100_2016_9_5_10_ngTMg_recording10_1   \n",
       "8   100_2016_9_5_10_ngTMg  weather  100_2016_9_5_10_ngTMg_recording10_1   \n",
       "9   100_2016_9_5_10_ngTMg  weather  100_2016_9_5_10_ngTMg_recording10_1   \n",
       "10  100_2016_9_5_10_ngTMg  weather  100_2016_9_5_10_ngTMg_recording10_1   \n",
       "11  100_2016_9_5_10_ngTMg  weather  100_2016_9_5_10_ngTMg_recording10_1   \n",
       "12  100_2016_9_5_10_ngTMg  weather  100_2016_9_5_10_ngTMg_recording10_1   \n",
       "13  100_2016_9_5_10_ngTMg  weather  100_2016_9_5_10_ngTMg_recording10_1   \n",
       "14  100_2016_9_5_10_ngTMg  weather  100_2016_9_5_10_ngTMg_recording10_1   \n",
       "15  100_2016_9_5_10_ngTMg  weather  100_2016_9_5_10_ngTMg_recording10_1   \n",
       "16  100_2016_9_5_10_ngTMg  weather  100_2016_9_5_10_ngTMg_recording10_1   \n",
       "17  100_2016_9_5_10_ngTMg  weather  100_2016_9_5_10_ngTMg_recording10_1   \n",
       "18  100_2016_9_5_10_ngTMg  weather  100_2016_9_5_10_ngTMg_recording10_1   \n",
       "19  100_2016_9_5_10_ngTMg  weather  100_2016_9_5_10_ngTMg_recording10_1   \n",
       "20  100_2016_9_5_10_ngTMg  weather  100_2016_9_5_10_ngTMg_recording10_1   \n",
       "\n",
       "    pre_segment  fol_segment  vowelR  vowel_type     note    time  norm_time  \\\n",
       "0             5            0       7           2  linking  2.1715     0.0101   \n",
       "1             5            0       7           2  linking  2.1730     0.0202   \n",
       "2             5            0       7           2  linking  2.1745     0.0303   \n",
       "3             5            0       7           2  linking  2.1761     0.0404   \n",
       "4             5            0       7           2  linking  2.1776     0.0505   \n",
       "5             5            0       7           2  linking  2.1791     0.0606   \n",
       "6             5            0       7           2  linking  2.1806     0.0707   \n",
       "7             5            0       7           2  linking  2.1821     0.0808   \n",
       "8             5            0       7           2  linking  2.1836     0.0909   \n",
       "9             5            0       7           2  linking  2.1852     0.1010   \n",
       "10            5            0       7           2  linking  2.1867     0.1111   \n",
       "11            5            0       7           2  linking  2.1882     0.1212   \n",
       "12            5            0       7           2  linking  2.1897     0.1313   \n",
       "13            5            0       7           2  linking  2.1912     0.1414   \n",
       "14            5            0       7           2  linking  2.1927     0.1515   \n",
       "15            5            0       7           2  linking  2.1942     0.1616   \n",
       "16            5            0       7           2  linking  2.1958     0.1717   \n",
       "17            5            0       7           2  linking  2.1973     0.1818   \n",
       "18            5            0       7           2  linking  2.1988     0.1919   \n",
       "19            5            0       7           2  linking  2.2003     0.2020   \n",
       "20            5            0       7           2  linking  2.2018     0.2121   \n",
       "\n",
       "    ...           c3          c4          c5          c6         c7  \\\n",
       "0   ... -1022.492295 -674.645850 -374.559604  -33.608998  43.539283   \n",
       "1   ... -1058.318886 -618.081041 -392.756742  -73.851940   9.402799   \n",
       "2   ... -1111.359326 -607.571292 -404.454693  -32.069785  63.086838   \n",
       "3   ...  -986.829893 -567.632984 -426.516657  -14.907034  51.907415   \n",
       "4   ... -1063.674469 -641.366517 -365.899597  -17.289025  26.717821   \n",
       "5   ... -1065.472565 -620.111160 -423.981473  -57.378900  68.832020   \n",
       "6   ... -1065.865377 -675.983234 -369.431880  -42.174560  32.678901   \n",
       "7   ... -1085.372450 -640.074493 -407.788671   -8.526462  30.385037   \n",
       "8   ...  -959.284014 -617.434322 -421.071931    0.850639  33.118351   \n",
       "9   ...  -994.788822 -679.964733 -417.837474   23.585294  37.165498   \n",
       "10  ... -1023.384287 -642.773084 -367.782185   -5.284019  -4.179570   \n",
       "11  ... -1062.043479 -592.324192 -411.929318  -58.980487  47.032517   \n",
       "12  ...  -973.859908 -596.008790 -400.989062 -110.336663  41.467889   \n",
       "13  ... -1097.701283 -664.891726 -372.274870  -40.995007  26.694060   \n",
       "14  ... -1084.806882 -599.333117 -397.589833  -24.794497  73.461545   \n",
       "15  ... -1017.969258 -667.043051 -414.810019  -31.457236  50.477793   \n",
       "16  ... -1047.727327 -627.326194 -354.243706  -44.328483  -0.114849   \n",
       "17  ... -1056.292893 -562.678986 -421.238856 -106.645364  41.085645   \n",
       "18  ... -1070.866679 -600.016910 -389.366085  -68.624698  45.238264   \n",
       "19  ... -1070.647542 -644.234503 -393.390540  -55.644766  31.506429   \n",
       "20  ... -1062.155799 -621.044103 -364.775614  -14.405735  11.431850   \n",
       "\n",
       "            c8          c9         c10         c11         c12  \n",
       "0  -100.527786  -93.679401 -444.218747 -525.123463 -660.706331  \n",
       "1   -70.048259  -76.092667 -485.080975 -514.300174 -690.813720  \n",
       "2  -111.803538  -75.985507 -442.519672 -538.917883 -665.585262  \n",
       "3   -89.423728  -64.152095 -500.297254 -479.800648 -628.009562  \n",
       "4  -107.038116  -52.297277 -479.613893 -519.625957 -662.952884  \n",
       "5   -69.322706 -110.514901 -480.604908 -516.096800 -670.000800  \n",
       "6   -78.833412  -83.945704 -499.744672 -488.783056 -680.203198  \n",
       "7   -86.137435 -108.084595 -449.959978 -523.291645 -684.036196  \n",
       "8  -141.286203  -84.906223 -459.382941 -520.984107 -621.123415  \n",
       "9  -129.098794  -79.556766 -449.219367 -526.555203 -675.400315  \n",
       "10  -97.267366  -90.008919 -488.774613 -507.899234 -662.863047  \n",
       "11  -81.177417 -101.290066 -468.385271 -532.024478 -670.322280  \n",
       "12  -57.469808  -66.102633 -460.263050 -517.879455 -668.504013  \n",
       "13  -67.709100 -112.023698 -496.278069 -487.751241 -693.348117  \n",
       "14 -110.530665 -122.142781 -437.685280 -506.831723 -656.654496  \n",
       "15  -78.118045 -119.686091 -485.323087 -483.331104 -676.555043  \n",
       "16  -86.959657  -98.176778 -482.192906 -498.370002 -666.326418  \n",
       "17  -41.879521 -104.503612 -488.691699 -508.357849 -692.791547  \n",
       "18  -69.487174  -82.938884 -480.472603 -514.861968 -650.736055  \n",
       "19  -62.231565 -112.585092 -491.784054 -497.043906 -688.502231  \n",
       "20  -87.574939  -98.935280 -467.898393 -499.075398 -644.791299  \n",
       "\n",
       "[21 rows x 22 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# make categorical variables numeric\n",
    "encoder = LabelEncoder()\n",
    "# encoder.classes_ = numpy.load('arrays/style_classes.npy')\n",
    "# data['speech_style'] = encoder.fit_transform(data['style'].astype(str))\n",
    "\n",
    "encoder.classes_ = np.load('arrays/preseg_classes.npy')\n",
    "print(encoder.classes_)\n",
    "data['pre_segment'] = encoder.transform(data['pre_segment'].astype(str))\n",
    "\n",
    "encoder.classes_ = np.load('arrays/folseg_classes.npy')\n",
    "data['fol_segment'] = encoder.transform(data['fol_segment'].astype(str))\n",
    "\n",
    "encoder.classes_ = np.load('arrays/vowelR_classes.npy')\n",
    "data['vowelR'] = encoder.transform(data['vowelR'].astype(str))\n",
    "\n",
    "encoder.classes_ = np.load('arrays/vowel_type_classes.npy')\n",
    "data['vowel_type'] = encoder.transform(data['vowel_type'].astype(str))\n",
    "data.head(21)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Process Time Series"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "token_length = 100\n",
    "num_features = 16 # 17\n",
    "\n",
    "def windows(data):\n",
    "    start = 0\n",
    "    end = 0\n",
    "    this_token = data.iloc[end]['token']\n",
    "    while start < len(data):\n",
    "        while end < len(data) and data.iloc[end]['token'] == this_token:\n",
    "            end += 1\n",
    "        yield start, end\n",
    "        start = end\n",
    "        if end >= len(data):\n",
    "            break\n",
    "        this_token = data.iloc[end]['token']\n",
    "        \n",
    "def get_speech_tokens_without_labels(data, window_size=21, num_features=3):\n",
    "    new_data = pd.DataFrame()  # dataframe to store categorical data\n",
    "    tokens = np.empty((0, window_size, num_features))\n",
    "    for start, end in windows(data):\n",
    "        \n",
    "        c1 = data.iloc[start:end]['c1'].astype(float)\n",
    "        c2 = data.iloc[start:end]['c2'].astype(float)\n",
    "        c3 = data.iloc[start:end]['c3'].astype(float)\n",
    "        c4 = data.iloc[start:end]['c4'].astype(float)\n",
    "        c5 = data.iloc[start:end]['c5'].astype(float)\n",
    "        c6 = data.iloc[start:end]['c6'].astype(float)\n",
    "        c7 = data.iloc[start:end]['c7'].astype(float)\n",
    "        c8 = data.iloc[start:end]['c8'].astype(float)\n",
    "        c9 = data.iloc[start:end]['c9'].astype(float)\n",
    "        c10 = data.iloc[start:end]['c10'].astype(float)\n",
    "        c11 = data.iloc[start:end]['c11'].astype(float)\n",
    "        c12 = data.iloc[start:end]['c12'].astype(float)\n",
    "        \n",
    "#         style = np.array(data.iloc[start:end]['speech_style']).astype(int)\n",
    "        pre_seg = np.array(data.iloc[start:end]['pre_segment']).astype(int)\n",
    "        fol_seg = np.array(data.iloc[start:end]['fol_segment']).astype(int)\n",
    "        vowelR = np.array(data.iloc[start:end]['vowelR']).astype(int)\n",
    "        vowel_type = np.array(data.iloc[start:end]['vowel_type']).astype(int)\n",
    "        \n",
    "        if 0 < len(c1) < window_size:\n",
    "            difference = window_size - len(c1)\n",
    "            padding = np.array([0] * difference)\n",
    "            c1 = np.hstack([padding, c1])\n",
    "            c2 = np.hstack([padding, c2])\n",
    "            c3 = np.hstack([padding, c3])\n",
    "            c4 = np.hstack([padding, c4])\n",
    "            c5 = np.hstack([padding, c5])\n",
    "            c6 = np.hstack([padding, c6])\n",
    "            c7 = np.hstack([padding, c7])\n",
    "            c8 = np.hstack([padding, c8])\n",
    "            c9 = np.hstack([padding, c9])\n",
    "            c10 = np.hstack([padding, c10])\n",
    "            c11 = np.hstack([padding, c11])\n",
    "            c12 = np.hstack([padding, c12])\n",
    "            \n",
    "#             style = np.hstack([padding, style])\n",
    "            pre_seg = np.hstack([padding, pre_seg])\n",
    "            fol_seg = np.hstack([padding, fol_seg])\n",
    "            vowelR = np.hstack([padding, vowelR])\n",
    "            vowel_type = np.hstack([padding, vowel_type])\n",
    "            \n",
    "        \n",
    "        if len(c1) == window_size:\n",
    "            # get continuous data\n",
    "            feature_stack = np.dstack([c1, c2, c3, c4, c5, c6, c7, c8, c9, c10, c11, c12,\n",
    "                                      pre_seg, fol_seg, vowelR, vowel_type]) # , style])\n",
    "            tokens = np.vstack([tokens, feature_stack])\n",
    "\n",
    "    return tokens"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Perform Preprocesssing and Reshape the Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Segment shape: (21141, 100, 16)\n",
      "X shape: (21141, 100, 16)\n"
     ]
    }
   ],
   "source": [
    "segments = get_speech_tokens_without_labels(data, token_length, num_features)\n",
    "print(\"Segment shape:\", segments.shape)\n",
    "X = segments.reshape((len(segments), token_length, num_features))\n",
    "print(\"X shape:\", X.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load the Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_classes = 2\n",
    "token_length = 100\n",
    "num_features = 16\n",
    "\n",
    "def regularization_constructor(alpha):\n",
    "    if alpha:\n",
    "        return l2(alpha)\n",
    "    else:\n",
    "        return None\n",
    "\n",
    "def create_model(dropout=0, recurrent_dropout=0, recurrent_num_hidden=50, num_hidden=200,\n",
    "                 gru_activation='tanh', recurrent_activation='tanh', activation='relu',\n",
    "                 rec_l2=0.01, ker_l2=None, bias_l2=None, act_l2=0.01, \n",
    "                 dense_ker_l2=0.01, dense_bias_l2=None, dense_act_l2=None, \n",
    "                 verbose=False):\n",
    "    rec_l2 = regularization_constructor(rec_l2)\n",
    "    ker_l2 = regularization_constructor(ker_l2)\n",
    "    bias_l2 = regularization_constructor(bias_l2)\n",
    "    act_l2 = regularization_constructor(act_l2)\n",
    "    dense_ker_l2 = regularization_constructor(dense_ker_l2)\n",
    "    dense_bias_l2 = regularization_constructor(dense_bias_l2)\n",
    "    dense_act_l2 = regularization_constructor(dense_act_l2)\n",
    "    \n",
    "    model = Sequential()\n",
    "    model.add(BatchNormalization(input_shape=(token_length, num_features,)))\n",
    "    model.add(GRU(units=recurrent_num_hidden, return_sequences=False, recurrent_dropout=recurrent_dropout,\n",
    "                        dropout=dropout, activation=gru_activation, recurrent_activation=recurrent_activation,\n",
    "                        recurrent_regularizer=rec_l2,\n",
    "                        kernel_regularizer=ker_l2,\n",
    "                        bias_regularizer=bias_l2,\n",
    "                        activity_regularizer=act_l2,\n",
    "                        input_shape=(token_length, num_features,)))\n",
    "    model.add(Dense(num_hidden, activation=activation,\n",
    "                   kernel_regularizer=dense_ker_l2,\n",
    "                   bias_regularizer=dense_bias_l2,\n",
    "                   activity_regularizer=dense_act_l2))\n",
    "    model.add(Dense(1, activation='sigmoid'))\n",
    "    if verbose:\n",
    "        print(model.summary())\n",
    "    model.compile(optimizer='rmsprop', loss='binary_crossentropy', metrics=['accuracy'])\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = create_model()\n",
    "model.load_weights(\"results/weights.best.mfcc.hdf5\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_prob = model.predict(X)\n",
    "y_pred = np.round(y_prob)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Write Prediction to CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_to_write = pd.DataFrame()\n",
    "data_to_write['token'] = pd.read_csv('datasets/mt.csv', sep=',')['token'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loaded rows: 21141\n",
      "Predicted rows: 21141\n",
      "Probability rows: 21141\n"
     ]
    }
   ],
   "source": [
    "print(\"Loaded rows:\", len(data_to_write))\n",
    "print(\"Predicted rows:\", len(y_pred))\n",
    "print(\"Probability rows:\", len(y_prob))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_to_write['rhoticity_prediction'] = y_pred\n",
    "data_to_write['rhoticity_probability'] = y_prob"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_to_write.to_csv('results/mt_prediction.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}