models_and_features.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import os\n",
    "from pitch_data import load_data # This is the custom function I built to load the data from cwd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Data Loaded!\n"
     ]
    }
   ],
   "source": [
    "pitches, metadata = load_data() # Function at work"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Replacing the column names from the sandbox notebook \n",
    "pitches_to_replace = pitches['pitch_type'].dropna().value_counts(normalize=True) # Getting pitch % for each pitch_type\n",
    "pitches_to_replace = pitches_to_replace.where(pitches_to_replace <= .05).dropna().index.to_list() # creating a list of pitches that were thrown <5% in our data\n",
    "pitches['pitch_type'] = pitches['pitch_type'].replace(pitches_to_replace, \"other\").dropna() # Replacing those pitch types with \"other\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>uid</th>\n",
       "      <th>game_pk</th>\n",
       "      <th>year</th>\n",
       "      <th>date</th>\n",
       "      <th>team_id_b</th>\n",
       "      <th>team_id_p</th>\n",
       "      <th>inning</th>\n",
       "      <th>top</th>\n",
       "      <th>at_bat_num</th>\n",
       "      <th>pcount_at_bat</th>\n",
       "      <th>...</th>\n",
       "      <th>RISP</th>\n",
       "      <th>previous_pitch</th>\n",
       "      <th>previous_outcome</th>\n",
       "      <th>handedness</th>\n",
       "      <th>run_diff</th>\n",
       "      <th>leading</th>\n",
       "      <th>trailing</th>\n",
       "      <th>mean_career_velocity</th>\n",
       "      <th>l5_velocity</th>\n",
       "      <th>%_diff_career</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>14143226</td>\n",
       "      <td>286874</td>\n",
       "      <td>2011</td>\n",
       "      <td>2011-03-31</td>\n",
       "      <td>108</td>\n",
       "      <td>118</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>RL</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>14143227</td>\n",
       "      <td>286874</td>\n",
       "      <td>2011</td>\n",
       "      <td>2011-03-31</td>\n",
       "      <td>108</td>\n",
       "      <td>118</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>B</td>\n",
       "      <td>RL</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>14143228</td>\n",
       "      <td>286874</td>\n",
       "      <td>2011</td>\n",
       "      <td>2011-03-31</td>\n",
       "      <td>108</td>\n",
       "      <td>118</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>B</td>\n",
       "      <td>RL</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>14143229</td>\n",
       "      <td>286874</td>\n",
       "      <td>2011</td>\n",
       "      <td>2011-03-31</td>\n",
       "      <td>108</td>\n",
       "      <td>118</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "      <td>RL</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>14143230</td>\n",
       "      <td>286874</td>\n",
       "      <td>2011</td>\n",
       "      <td>2011-03-31</td>\n",
       "      <td>108</td>\n",
       "      <td>118</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>X</td>\n",
       "      <td>RR</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>718956</th>\n",
       "      <td>19838192</td>\n",
       "      <td>317073</td>\n",
       "      <td>2011</td>\n",
       "      <td>2011-10-28</td>\n",
       "      <td>140</td>\n",
       "      <td>138</td>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>72</td>\n",
       "      <td>3</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>FF</td>\n",
       "      <td>B</td>\n",
       "      <td>RR</td>\n",
       "      <td>-4</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>94.525118</td>\n",
       "      <td>97.06</td>\n",
       "      <td>0.026817</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>718957</th>\n",
       "      <td>19838193</td>\n",
       "      <td>317073</td>\n",
       "      <td>2011</td>\n",
       "      <td>2011-10-28</td>\n",
       "      <td>140</td>\n",
       "      <td>138</td>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>72</td>\n",
       "      <td>4</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>FF</td>\n",
       "      <td>S</td>\n",
       "      <td>RR</td>\n",
       "      <td>-4</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>94.527782</td>\n",
       "      <td>97.10</td>\n",
       "      <td>0.027211</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>718958</th>\n",
       "      <td>19838194</td>\n",
       "      <td>317073</td>\n",
       "      <td>2011</td>\n",
       "      <td>2011-10-28</td>\n",
       "      <td>140</td>\n",
       "      <td>138</td>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>72</td>\n",
       "      <td>5</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>FC</td>\n",
       "      <td>B</td>\n",
       "      <td>RR</td>\n",
       "      <td>-4</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>94.525315</td>\n",
       "      <td>96.04</td>\n",
       "      <td>0.016024</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>718959</th>\n",
       "      <td>19838195</td>\n",
       "      <td>317073</td>\n",
       "      <td>2011</td>\n",
       "      <td>2011-10-28</td>\n",
       "      <td>140</td>\n",
       "      <td>138</td>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>73</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>FF</td>\n",
       "      <td>X</td>\n",
       "      <td>RL</td>\n",
       "      <td>-4</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>94.527502</td>\n",
       "      <td>96.16</td>\n",
       "      <td>0.017270</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>718960</th>\n",
       "      <td>19838196</td>\n",
       "      <td>317073</td>\n",
       "      <td>2011</td>\n",
       "      <td>2011-10-28</td>\n",
       "      <td>140</td>\n",
       "      <td>138</td>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>73</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>FT</td>\n",
       "      <td>S</td>\n",
       "      <td>RL</td>\n",
       "      <td>-4</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>94.526378</td>\n",
       "      <td>95.46</td>\n",
       "      <td>0.009877</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>718961 rows × 135 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             uid  game_pk  year        date  team_id_b  team_id_p  inning  \\\n",
       "0       14143226   286874  2011  2011-03-31        108        118       1   \n",
       "1       14143227   286874  2011  2011-03-31        108        118       1   \n",
       "2       14143228   286874  2011  2011-03-31        108        118       1   \n",
       "3       14143229   286874  2011  2011-03-31        108        118       1   \n",
       "4       14143230   286874  2011  2011-03-31        108        118       1   \n",
       "...          ...      ...   ...         ...        ...        ...     ...   \n",
       "718956  19838192   317073  2011  2011-10-28        140        138       9   \n",
       "718957  19838193   317073  2011  2011-10-28        140        138       9   \n",
       "718958  19838194   317073  2011  2011-10-28        140        138       9   \n",
       "718959  19838195   317073  2011  2011-10-28        140        138       9   \n",
       "718960  19838196   317073  2011  2011-10-28        140        138       9   \n",
       "\n",
       "        top  at_bat_num  pcount_at_bat  ...   RISP  previous_pitch  \\\n",
       "0         1           1              1  ...  False             NaN   \n",
       "1         1           1              2  ...  False             NaN   \n",
       "2         1           1              3  ...  False             NaN   \n",
       "3         1           1              4  ...  False             NaN   \n",
       "4         1           2              1  ...  False             NaN   \n",
       "...     ...         ...            ...  ...    ...             ...   \n",
       "718956    1          72              3  ...  False              FF   \n",
       "718957    1          72              4  ...  False              FF   \n",
       "718958    1          72              5  ...  False              FC   \n",
       "718959    1          73              1  ...  False              FF   \n",
       "718960    1          73              2  ...  False              FT   \n",
       "\n",
       "        previous_outcome  handedness  run_diff  leading  trailing  \\\n",
       "0                    NaN          RL         0        0         0   \n",
       "1                      B          RL         0        0         0   \n",
       "2                      B          RL         0        0         0   \n",
       "3                      S          RL         0        0         0   \n",
       "4                      X          RR         0        0         0   \n",
       "...                  ...         ...       ...      ...       ...   \n",
       "718956                 B          RR        -4        0         1   \n",
       "718957                 S          RR        -4        0         1   \n",
       "718958                 B          RR        -4        0         1   \n",
       "718959                 X          RL        -4        0         1   \n",
       "718960                 S          RL        -4        0         1   \n",
       "\n",
       "        mean_career_velocity  l5_velocity  %_diff_career  \n",
       "0                        NaN          NaN            NaN  \n",
       "1                        NaN          NaN            NaN  \n",
       "2                        NaN          NaN            NaN  \n",
       "3                        NaN          NaN            NaN  \n",
       "4                        NaN          NaN            NaN  \n",
       "...                      ...          ...            ...  \n",
       "718956             94.525118        97.06       0.026817  \n",
       "718957             94.527782        97.10       0.027211  \n",
       "718958             94.525315        96.04       0.016024  \n",
       "718959             94.527502        96.16       0.017270  \n",
       "718960             94.526378        95.46       0.009877  \n",
       "\n",
       "[718961 rows x 135 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Typical Features\n",
    "pitches['RISP'] = pitches[['on_2b','on_3b']].any(axis = 1) # Bool for is there a runner in scoring position\n",
    "pitches['previous_pitch'] = pitches.groupby(['game_pk','inning','pitcher_id'])['pitch_type'].shift(1) # what was the previous pitch\n",
    "pitches['previous_outcome'] = pitches.groupby(['game_pk','inning','pitcher_id'])['type'].shift(1) # was the previous pitch a ball, strike, out, etc\n",
    "pitches['handedness'] = pitches['p_throws'] + pitches['stand'] # string to OH encode for pitcher batter matchup\n",
    "pitches['run_diff'] = np.where(pitches['top'] == 1, pitches['away_team_runs'] - pitches['home_team_runs'], pitches['home_team_runs'] - pitches['away_team_runs']) # run differential\n",
    "pitches['leading'] = np.where(pitches['run_diff'] > 0, 1,0) # bool for team leading\n",
    "pitches['trailing'] = np.where(pitches['run_diff'] < 0, 1,0) # bool for team trailing\n",
    "\n",
    "# Pitcher Specific \n",
    "pitches['mean_career_velocity'] = pitches.groupby('pitcher_id')['start_speed'].rolling(999999, min_periods=100, closed='left').mean().reset_index().set_index('level_1').sort_index()['start_speed'] # What is the pitchers mean velo in his career? min 100 pitches.\n",
    "pitches['l5_velocity'] = pitches.groupby('pitcher_id')['start_speed'].rolling(5, min_periods=5, closed='left').mean().reset_index().set_index('level_1').sort_index()['start_speed'] # What is the pitchers mean velo over his last 5 pitches?\n",
    "pitches['%_diff_career'] = pitches['l5_velocity']  / pitches['mean_career_velocity'] - 1 # % difference for the pitchers last 5 velo vs career.\n",
    "pitches"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>uid</th>\n",
       "      <th>game_pk</th>\n",
       "      <th>year</th>\n",
       "      <th>date</th>\n",
       "      <th>team_id_b</th>\n",
       "      <th>team_id_p</th>\n",
       "      <th>inning</th>\n",
       "      <th>top</th>\n",
       "      <th>at_bat_num</th>\n",
       "      <th>pcount_at_bat</th>\n",
       "      <th>...</th>\n",
       "      <th>previous_outcome_S</th>\n",
       "      <th>previous_outcome_X</th>\n",
       "      <th>previous_pitch_CH</th>\n",
       "      <th>previous_pitch_CU</th>\n",
       "      <th>previous_pitch_FC</th>\n",
       "      <th>previous_pitch_FF</th>\n",
       "      <th>previous_pitch_FT</th>\n",
       "      <th>previous_pitch_SI</th>\n",
       "      <th>previous_pitch_SL</th>\n",
       "      <th>previous_pitch_other</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>14143226</td>\n",
       "      <td>286874</td>\n",
       "      <td>2011</td>\n",
       "      <td>2011-03-31</td>\n",
       "      <td>108</td>\n",
       "      <td>118</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>14143227</td>\n",
       "      <td>286874</td>\n",
       "      <td>2011</td>\n",
       "      <td>2011-03-31</td>\n",
       "      <td>108</td>\n",
       "      <td>118</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>14143228</td>\n",
       "      <td>286874</td>\n",
       "      <td>2011</td>\n",
       "      <td>2011-03-31</td>\n",
       "      <td>108</td>\n",
       "      <td>118</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>14143229</td>\n",
       "      <td>286874</td>\n",
       "      <td>2011</td>\n",
       "      <td>2011-03-31</td>\n",
       "      <td>108</td>\n",
       "      <td>118</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>14143230</td>\n",
       "      <td>286874</td>\n",
       "      <td>2011</td>\n",
       "      <td>2011-03-31</td>\n",
       "      <td>108</td>\n",
       "      <td>118</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>718956</th>\n",
       "      <td>19838192</td>\n",
       "      <td>317073</td>\n",
       "      <td>2011</td>\n",
       "      <td>2011-10-28</td>\n",
       "      <td>140</td>\n",
       "      <td>138</td>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>72</td>\n",
       "      <td>3</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>718957</th>\n",
       "      <td>19838193</td>\n",
       "      <td>317073</td>\n",
       "      <td>2011</td>\n",
       "      <td>2011-10-28</td>\n",
       "      <td>140</td>\n",
       "      <td>138</td>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>72</td>\n",
       "      <td>4</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>718958</th>\n",
       "      <td>19838194</td>\n",
       "      <td>317073</td>\n",
       "      <td>2011</td>\n",
       "      <td>2011-10-28</td>\n",
       "      <td>140</td>\n",
       "      <td>138</td>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>72</td>\n",
       "      <td>5</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>718959</th>\n",
       "      <td>19838195</td>\n",
       "      <td>317073</td>\n",
       "      <td>2011</td>\n",
       "      <td>2011-10-28</td>\n",
       "      <td>140</td>\n",
       "      <td>138</td>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>73</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>718960</th>\n",
       "      <td>19838196</td>\n",
       "      <td>317073</td>\n",
       "      <td>2011</td>\n",
       "      <td>2011-10-28</td>\n",
       "      <td>140</td>\n",
       "      <td>138</td>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>73</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>718961 rows × 150 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             uid  game_pk  year        date  team_id_b  team_id_p  inning  \\\n",
       "0       14143226   286874  2011  2011-03-31        108        118       1   \n",
       "1       14143227   286874  2011  2011-03-31        108        118       1   \n",
       "2       14143228   286874  2011  2011-03-31        108        118       1   \n",
       "3       14143229   286874  2011  2011-03-31        108        118       1   \n",
       "4       14143230   286874  2011  2011-03-31        108        118       1   \n",
       "...          ...      ...   ...         ...        ...        ...     ...   \n",
       "718956  19838192   317073  2011  2011-10-28        140        138       9   \n",
       "718957  19838193   317073  2011  2011-10-28        140        138       9   \n",
       "718958  19838194   317073  2011  2011-10-28        140        138       9   \n",
       "718959  19838195   317073  2011  2011-10-28        140        138       9   \n",
       "718960  19838196   317073  2011  2011-10-28        140        138       9   \n",
       "\n",
       "        top  at_bat_num  pcount_at_bat  ...  previous_outcome_S  \\\n",
       "0         1           1              1  ...                   0   \n",
       "1         1           1              2  ...                   0   \n",
       "2         1           1              3  ...                   0   \n",
       "3         1           1              4  ...                   1   \n",
       "4         1           2              1  ...                   0   \n",
       "...     ...         ...            ...  ...                 ...   \n",
       "718956    1          72              3  ...                   0   \n",
       "718957    1          72              4  ...                   1   \n",
       "718958    1          72              5  ...                   0   \n",
       "718959    1          73              1  ...                   0   \n",
       "718960    1          73              2  ...                   1   \n",
       "\n",
       "        previous_outcome_X  previous_pitch_CH  previous_pitch_CU  \\\n",
       "0                        0                  0                  0   \n",
       "1                        0                  0                  0   \n",
       "2                        0                  0                  0   \n",
       "3                        0                  0                  0   \n",
       "4                        1                  0                  0   \n",
       "...                    ...                ...                ...   \n",
       "718956                   0                  0                  0   \n",
       "718957                   0                  0                  0   \n",
       "718958                   0                  0                  0   \n",
       "718959                   1                  0                  0   \n",
       "718960                   0                  0                  0   \n",
       "\n",
       "        previous_pitch_FC  previous_pitch_FF  previous_pitch_FT  \\\n",
       "0                       0                  0                  0   \n",
       "1                       0                  0                  0   \n",
       "2                       0                  0                  0   \n",
       "3                       0                  0                  0   \n",
       "4                       0                  0                  0   \n",
       "...                   ...                ...                ...   \n",
       "718956                  0                  1                  0   \n",
       "718957                  0                  1                  0   \n",
       "718958                  1                  0                  0   \n",
       "718959                  0                  1                  0   \n",
       "718960                  0                  0                  1   \n",
       "\n",
       "        previous_pitch_SI  previous_pitch_SL  previous_pitch_other  \n",
       "0                       0                  0                     0  \n",
       "1                       0                  0                     0  \n",
       "2                       0                  0                     0  \n",
       "3                       0                  0                     0  \n",
       "4                       0                  0                     0  \n",
       "...                   ...                ...                   ...  \n",
       "718956                  0                  0                     0  \n",
       "718957                  0                  0                     0  \n",
       "718958                  0                  0                     0  \n",
       "718959                  0                  0                     0  \n",
       "718960                  0                  0                     0  \n",
       "\n",
       "[718961 rows x 150 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dummies = pd.get_dummies(pitches[['handedness','previous_outcome','previous_pitch']]) # Encoding categorical variables.\n",
    "pitches[dummies.columns] = dummies # Setting the column names\n",
    "pitches"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['mean_career_velocity',\n",
       " 'l5_velocity',\n",
       " '%_diff_career',\n",
       " 'RISP',\n",
       " 'run_diff',\n",
       " 'leading',\n",
       " 'trailing',\n",
       " 'handedness_LL',\n",
       " 'handedness_LR',\n",
       " 'handedness_RL',\n",
       " 'handedness_RR',\n",
       " 'previous_outcome_B',\n",
       " 'previous_outcome_S',\n",
       " 'previous_outcome_X',\n",
       " 'previous_pitch_CH',\n",
       " 'previous_pitch_CU',\n",
       " 'previous_pitch_FC',\n",
       " 'previous_pitch_FF',\n",
       " 'previous_pitch_FT',\n",
       " 'previous_pitch_SI',\n",
       " 'previous_pitch_SL',\n",
       " 'previous_pitch_other']"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "available_features = metadata[metadata['available_prior_to_pitch'] == 'Yes']['column_name'].to_list() # These are all of the provided columns that are available before a pitch\n",
    "available_features += ['previous_pitch','previous_outcome','handedness','mean_career_velocity','l5_velocity','%_diff_career', 'RISP','run_diff','leading','trailing'] # These are the columns I've created that I want to use\n",
    "available_features += dummies.columns.to_list() # These are the columns that are encoded\n",
    "unwanted_features = ['uid', 'game_pk', 'year', 'date', 'top', 'at_bat_num','start_tfs','start_tfs_zulu','team_id_b', 'team_id_p', 'inning', 'pcount_at_bat', 'pcount_pitcher', 'balls',\n",
    "       'strikes', 'fouls', 'outs', 'pitch_id', 'on_1b', 'on_2b', 'on_3b','batter_id','stand','b_height','pitcher_id','p_throws','away_team_runs','home_team_runs','previous_pitch', 'previous_outcome', 'handedness',] ## These are the features that are provided but are either encoded or not selected during feature testing\n",
    "for feat in unwanted_features:\n",
    "    available_features.remove(feat) # Removing unwated features. I think there's a better way to do this but this was pretty fast.\n",
    "available_features "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "response_var = 'pitch_type' # y var\n",
    "df = pitches[available_features + [response_var]].dropna(subset = [response_var]) # creating DF of wanted features\n",
    "\n",
    "# Train Test Split\n",
    "train_size = .8 # setting split size to 80% \n",
    "train_idx = round(len(df)*train_size) # Train idx to stop on\n",
    "train_df = df[:train_idx] # train data\n",
    "test_df = df[train_idx:] # test data\n",
    "\n",
    "# X Y \n",
    "train_X = train_df.drop(response_var, axis = 1) # training X\n",
    "train_y = train_df[response_var] # training y\n",
    "\n",
    "test_X = test_df.drop(response_var, axis = 1) # test X\n",
    "test_y = test_df[response_var] # Test y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Imputing missing data\n",
    "from sklearn.impute import SimpleImputer\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "imputer = SimpleImputer()\n",
    "train_X = imputer.fit_transform(train_X) # technically there's a bit of data leakage but it's confined to my training set, which is fine given the sample size and test set (which won't have leakage).\n",
    "test_X = imputer.transform(test_X) # imputing test set based on train data\n",
    "\n",
    "scaler = StandardScaler()\n",
    "train_X = scaler.fit_transform(train_X)\n",
    "test_X = scaler.transform(test_X) # scaling test set based on train data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "## My goal is to build a regressor with an accuracy > .34 with weighted f1 > .13. \n",
    "# To be honest, this shouldn't be too hard, so I'll do a simple classification task in the best interest of time and simplicity.\n",
    "# Basically, I'm trying to do better than randomly guessing\n",
    "\n",
    "## This is my goal from the dummy_results notebook."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "from sklearn.linear_model import LogisticRegression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/benwolbransky/Library/Python/3.8/lib/python/site-packages/sklearn/linear_model/_logistic.py:444: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-1 {color: black;background-color: white;}#sk-container-id-1 pre{padding: 0;}#sk-container-id-1 div.sk-toggleable {background-color: white;}#sk-container-id-1 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-1 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-1 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-1 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-1 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-1 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-1 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-1 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-1 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-1 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-1 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-1 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-1 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-1 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-1 div.sk-item {position: relative;z-index: 1;}#sk-container-id-1 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-1 div.sk-item::before, #sk-container-id-1 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-1 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-1 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-1 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-1 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-1 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-1 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-1 div.sk-label-container {text-align: center;}#sk-container-id-1 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-1 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-1\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>LogisticRegression()</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-1\" type=\"checkbox\" checked><label for=\"sk-estimator-id-1\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">LogisticRegression</label><div class=\"sk-toggleable__content\"><pre>LogisticRegression()</pre></div></div></div></div></div>"
      ],
      "text/plain": [
       "LogisticRegression()"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## Simple logistic regression to gauge performance\n",
    "clf = LogisticRegression() \n",
    "clf.fit(train_X, train_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "          CH       0.27      0.07      0.11     14024\n",
      "          CU       0.23      0.04      0.07     11452\n",
      "          FC       0.34      0.24      0.28      8649\n",
      "          FF       0.42      0.76      0.54     49124\n",
      "          FT       0.38      0.35      0.36     16598\n",
      "          SI       0.52      0.48      0.50     16005\n",
      "          SL       0.36      0.15      0.22     21661\n",
      "       other       0.48      0.28      0.35      5823\n",
      "\n",
      "    accuracy                           0.41    143336\n",
      "   macro avg       0.38      0.30      0.31    143336\n",
      "weighted avg       0.39      0.41      0.36    143336\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import classification_report\n",
    "print(classification_report(test_y, clf.predict(test_X))) ## I like using these metrics for multiclass classification problems. There are multi-label ROC AUC metrics but this is fine."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "## super easy to beat my goal with a simple logit model... but I'm disappointed in the changeup and cutter recall. \n",
    "# I'm going to see if I can improve those numbers with a few more features and then hypertune a classifer."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# I'm going to add pitcher tendencies. Basically, what has each pitcher historically thrown? Pitchers who throw more sliders than league average are likely to continue to do so.\n",
    "\n",
    "pitches[['total_CH','total_CU','total_FC','total_FF','total_pitch_FT','total_SI','total_SL','total_other']] = pitches.groupby('pitcher_id')[['previous_pitch_CH','previous_pitch_CU','previous_pitch_FC','previous_pitch_FF','previous_pitch_FT','previous_pitch_SI','previous_pitch_SL','previous_pitch_other']].apply('cumsum') # Total count of each pitch_type thrown\n",
    "pitches['total_pitches'] = pitches.groupby('pitcher_id')['pitch_id'].rolling(999999, min_periods=100, closed='left').count().reset_index().set_index('level_1').sort_index()['pitch_id'] #total pitches thrown\n",
    "pitches[['percent_CH','percent_CU','percent_FC','percent_FF','percent_FT','percent_SI','percent_SL','percent_other']] = pitches[['total_CH','total_CU','total_FC','total_FF','total_pitch_FT','total_SI','total_SL','total_other']].div(pitches['total_pitches'],axis = 0) #percent of each pitch thrown"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "response_var = 'pitch_type'\n",
    "new_features = ['percent_CH','percent_CU','percent_FC','percent_FF','percent_FT','percent_SI','percent_SL','percent_other']\n",
    "df = pitches[available_features + new_features + [response_var]].dropna(subset = [response_var])\n",
    "df.select_dtypes(include=[\"number\",\"bool_\"])\n",
    "\n",
    "# Train Test Split\n",
    "train_size = .8\n",
    "train_idx = round(len(df)*train_size)\n",
    "train_df = df[:train_idx]\n",
    "test_df = df[train_idx:]\n",
    "\n",
    "# X Y \n",
    "train_X = train_df.drop(response_var, axis = 1)\n",
    "train_y = train_df[response_var]\n",
    "\n",
    "test_X = test_df.drop(response_var, axis = 1)\n",
    "test_y = test_df[response_var]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Imputing missing data\n",
    "from sklearn.impute import SimpleImputer\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "imputer = SimpleImputer()\n",
    "train_X = imputer.fit_transform(train_X)\n",
    "test_X = imputer.transform(test_X) # imputing test set based on train data\n",
    "\n",
    "scaler = StandardScaler()\n",
    "train_X = scaler.fit_transform(train_X)\n",
    "test_X = scaler.transform(test_X) # scaling test set based on train data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/benwolbransky/Library/Python/3.8/lib/python/site-packages/sklearn/linear_model/_logistic.py:444: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-2 {color: black;background-color: white;}#sk-container-id-2 pre{padding: 0;}#sk-container-id-2 div.sk-toggleable {background-color: white;}#sk-container-id-2 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-2 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-2 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-2 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-2 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-2 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-2 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-2 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-2 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-2 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-2 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-2 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-2 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-2 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-2 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-2 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-2 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-2 div.sk-item {position: relative;z-index: 1;}#sk-container-id-2 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-2 div.sk-item::before, #sk-container-id-2 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-2 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-2 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-2 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-2 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-2 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-2 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-2 div.sk-label-container {text-align: center;}#sk-container-id-2 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-2 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-2\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>LogisticRegression()</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-2\" type=\"checkbox\" checked><label for=\"sk-estimator-id-2\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">LogisticRegression</label><div class=\"sk-toggleable__content\"><pre>LogisticRegression()</pre></div></div></div></div></div>"
      ],
      "text/plain": [
       "LogisticRegression()"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## Simple logistic regression to gauge new feauture performance\n",
    "\n",
    "clf = LogisticRegression()\n",
    "clf.fit(train_X, train_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "          CH       0.31      0.13      0.18     14024\n",
      "          CU       0.33      0.14      0.19     11452\n",
      "          FC       0.48      0.33      0.39      8649\n",
      "          FF       0.48      0.73      0.58     49124\n",
      "          FT       0.43      0.35      0.39     16598\n",
      "          SI       0.53      0.71      0.61     16005\n",
      "          SL       0.40      0.25      0.31     21661\n",
      "       other       0.51      0.27      0.35      5823\n",
      "\n",
      "    accuracy                           0.46    143336\n",
      "   macro avg       0.43      0.36      0.38    143336\n",
      "weighted avg       0.44      0.46      0.43    143336\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(test_y, clf.predict(test_X)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Way better results. Going to see if a base random forest performs better and then play around with model types/hyper tuning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "          CH       0.26      0.12      0.16     14024\n",
      "          CU       0.28      0.12      0.17     11452\n",
      "          FC       0.40      0.33      0.36      8649\n",
      "          FF       0.47      0.67      0.56     49124\n",
      "          FT       0.38      0.36      0.37     16598\n",
      "          SI       0.51      0.72      0.60     16005\n",
      "          SL       0.36      0.23      0.28     21661\n",
      "       other       0.53      0.33      0.40      5823\n",
      "\n",
      "    accuracy                           0.44    143336\n",
      "   macro avg       0.40      0.36      0.36    143336\n",
      "weighted avg       0.41      0.44      0.41    143336\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "clf = RandomForestClassifier()\n",
    "clf.fit(train_X, train_y)\n",
    "print(classification_report(test_y, clf.predict(test_X)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "## I'm actually a bit surprised this performed worse. \n",
    "# I'm going to try xgb next as thats typically the best performing model out there"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.34      0.07      0.11     14024\n",
      "           1       0.38      0.09      0.15     11452\n",
      "           2       0.49      0.34      0.40      8649\n",
      "           3       0.48      0.77      0.59     49124\n",
      "           4       0.42      0.40      0.41     16598\n",
      "           5       0.53      0.78      0.63     16005\n",
      "           6       0.41      0.21      0.28     21661\n",
      "           7       0.62      0.32      0.43      5823\n",
      "\n",
      "    accuracy                           0.48    143336\n",
      "   macro avg       0.46      0.37      0.38    143336\n",
      "weighted avg       0.45      0.48      0.43    143336\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import xgboost as xgb\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "\n",
    "# I need to label encode pitch type \n",
    "le = LabelEncoder()\n",
    "train_yle = le.fit_transform(train_y) # _yle is \"Y Label Encoded\"\n",
    "test_yle = le.transform(test_y)\n",
    "\n",
    "clf = xgb.XGBClassifier(learning_rate = .1)\n",
    "clf.fit(train_X, train_yle)\n",
    "print(classification_report(test_yle, clf.predict(test_X)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Best performing model yet, no suprise. I'll do some typer parameter tuning to finish."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 3 candidates, totalling 15 fits\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-7 {color: black;background-color: white;}#sk-container-id-7 pre{padding: 0;}#sk-container-id-7 div.sk-toggleable {background-color: white;}#sk-container-id-7 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-7 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-7 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-7 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-7 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-7 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-7 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-7 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-7 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-7 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-7 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-7 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-7 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-7 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-7 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-7 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-7 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-7 div.sk-item {position: relative;z-index: 1;}#sk-container-id-7 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-7 div.sk-item::before, #sk-container-id-7 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-7 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-7 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-7 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-7 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-7 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-7 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-7 div.sk-label-container {text-align: center;}#sk-container-id-7 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-7 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-7\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>GridSearchCV(cv=5,\n",
       "             estimator=XGBClassifier(base_score=None, booster=None,\n",
       "                                     callbacks=None, colsample_bylevel=None,\n",
       "                                     colsample_bynode=None,\n",
       "                                     colsample_bytree=None,\n",
       "                                     early_stopping_rounds=None,\n",
       "                                     enable_categorical=False, eval_metric=None,\n",
       "                                     gamma=None, gpu_id=None, grow_policy=None,\n",
       "                                     importance_type=None,\n",
       "                                     interaction_constraints=None,\n",
       "                                     learning_rate=None, max_bin=None,\n",
       "                                     max_cat_to_onehot=None,\n",
       "                                     max_delta_step=None, max_depth=None,\n",
       "                                     max_leaves=None, min_child_weight=None,\n",
       "                                     missing=nan, monotone_constraints=None,\n",
       "                                     n_estimators=100, n_jobs=None,\n",
       "                                     num_parallel_tree=None, predictor=None,\n",
       "                                     random_state=None, reg_alpha=None,\n",
       "                                     reg_lambda=None, ...),\n",
       "             param_grid={&#x27;max_depth&#x27;: range(5, 10, 2)}, scoring=&#x27;f1_weighted&#x27;,\n",
       "             verbose=True)</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-13\" type=\"checkbox\" ><label for=\"sk-estimator-id-13\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">GridSearchCV</label><div class=\"sk-toggleable__content\"><pre>GridSearchCV(cv=5,\n",
       "             estimator=XGBClassifier(base_score=None, booster=None,\n",
       "                                     callbacks=None, colsample_bylevel=None,\n",
       "                                     colsample_bynode=None,\n",
       "                                     colsample_bytree=None,\n",
       "                                     early_stopping_rounds=None,\n",
       "                                     enable_categorical=False, eval_metric=None,\n",
       "                                     gamma=None, gpu_id=None, grow_policy=None,\n",
       "                                     importance_type=None,\n",
       "                                     interaction_constraints=None,\n",
       "                                     learning_rate=None, max_bin=None,\n",
       "                                     max_cat_to_onehot=None,\n",
       "                                     max_delta_step=None, max_depth=None,\n",
       "                                     max_leaves=None, min_child_weight=None,\n",
       "                                     missing=nan, monotone_constraints=None,\n",
       "                                     n_estimators=100, n_jobs=None,\n",
       "                                     num_parallel_tree=None, predictor=None,\n",
       "                                     random_state=None, reg_alpha=None,\n",
       "                                     reg_lambda=None, ...),\n",
       "             param_grid={&#x27;max_depth&#x27;: range(5, 10, 2)}, scoring=&#x27;f1_weighted&#x27;,\n",
       "             verbose=True)</pre></div></div></div><div class=\"sk-parallel\"><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-14\" type=\"checkbox\" ><label for=\"sk-estimator-id-14\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">estimator: XGBClassifier</label><div class=\"sk-toggleable__content\"><pre>XGBClassifier(base_score=None, booster=None, callbacks=None,\n",
       "              colsample_bylevel=None, colsample_bynode=None,\n",
       "              colsample_bytree=None, early_stopping_rounds=None,\n",
       "              enable_categorical=False, eval_metric=None, gamma=None,\n",
       "              gpu_id=None, grow_policy=None, importance_type=None,\n",
       "              interaction_constraints=None, learning_rate=None, max_bin=None,\n",
       "              max_cat_to_onehot=None, max_delta_step=None, max_depth=None,\n",
       "              max_leaves=None, min_child_weight=None, missing=nan,\n",
       "              monotone_constraints=None, n_estimators=100, n_jobs=None,\n",
       "              num_parallel_tree=None, predictor=None, random_state=None,\n",
       "              reg_alpha=None, reg_lambda=None, ...)</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-15\" type=\"checkbox\" ><label for=\"sk-estimator-id-15\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">XGBClassifier</label><div class=\"sk-toggleable__content\"><pre>XGBClassifier(base_score=None, booster=None, callbacks=None,\n",
       "              colsample_bylevel=None, colsample_bynode=None,\n",
       "              colsample_bytree=None, early_stopping_rounds=None,\n",
       "              enable_categorical=False, eval_metric=None, gamma=None,\n",
       "              gpu_id=None, grow_policy=None, importance_type=None,\n",
       "              interaction_constraints=None, learning_rate=None, max_bin=None,\n",
       "              max_cat_to_onehot=None, max_delta_step=None, max_depth=None,\n",
       "              max_leaves=None, min_child_weight=None, missing=nan,\n",
       "              monotone_constraints=None, n_estimators=100, n_jobs=None,\n",
       "              num_parallel_tree=None, predictor=None, random_state=None,\n",
       "              reg_alpha=None, reg_lambda=None, ...)</pre></div></div></div></div></div></div></div></div></div></div>"
      ],
      "text/plain": [
       "GridSearchCV(cv=5,\n",
       "             estimator=XGBClassifier(base_score=None, booster=None,\n",
       "                                     callbacks=None, colsample_bylevel=None,\n",
       "                                     colsample_bynode=None,\n",
       "                                     colsample_bytree=None,\n",
       "                                     early_stopping_rounds=None,\n",
       "                                     enable_categorical=False, eval_metric=None,\n",
       "                                     gamma=None, gpu_id=None, grow_policy=None,\n",
       "                                     importance_type=None,\n",
       "                                     interaction_constraints=None,\n",
       "                                     learning_rate=None, max_bin=None,\n",
       "                                     max_cat_to_onehot=None,\n",
       "                                     max_delta_step=None, max_depth=None,\n",
       "                                     max_leaves=None, min_child_weight=None,\n",
       "                                     missing=nan, monotone_constraints=None,\n",
       "                                     n_estimators=100, n_jobs=None,\n",
       "                                     num_parallel_tree=None, predictor=None,\n",
       "                                     random_state=None, reg_alpha=None,\n",
       "                                     reg_lambda=None, ...),\n",
       "             param_grid={'max_depth': range(5, 10, 2)}, scoring='f1_weighted',\n",
       "             verbose=True)"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## I don't want to overfit -- and since I don't have a holdout set, I'll stick to basic tuning parameters. \n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "parameters = {\n",
    "\n",
    "    'learning_rate': [0.1,.01],\n",
    "    'max_depth': range(5,10,2)\n",
    "}\n",
    "\n",
    "clf = xgb.XGBClassifier()\n",
    "\n",
    "grid_search = GridSearchCV(\n",
    "    estimator=clf,\n",
    "    param_grid=parameters,\n",
    "    scoring = 'f1_weighted',\n",
    "    cv = 5,\n",
    "    verbose=True\n",
    ")\n",
    "\n",
    "grid_search.fit(train_X, train_yle)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.33      0.08      0.12     14024\n",
      "           1       0.38      0.10      0.15     11452\n",
      "           2       0.47      0.35      0.40      8649\n",
      "           3       0.48      0.76      0.59     49124\n",
      "           4       0.42      0.41      0.41     16598\n",
      "           5       0.53      0.77      0.63     16005\n",
      "           6       0.41      0.21      0.28     21661\n",
      "           7       0.61      0.32      0.42      5823\n",
      "\n",
      "    accuracy                           0.48    143336\n",
      "   macro avg       0.45      0.38      0.38    143336\n",
      "weighted avg       0.45      0.48      0.43    143336\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(test_yle, grid_search.predict(test_X)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-8 {color: black;background-color: white;}#sk-container-id-8 pre{padding: 0;}#sk-container-id-8 div.sk-toggleable {background-color: white;}#sk-container-id-8 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-8 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-8 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-8 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-8 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-8 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-8 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-8 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-8 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-8 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-8 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-8 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-8 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-8 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-8 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-8 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-8 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-8 div.sk-item {position: relative;z-index: 1;}#sk-container-id-8 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-8 div.sk-item::before, #sk-container-id-8 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-8 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-8 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-8 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-8 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-8 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-8 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-8 div.sk-label-container {text-align: center;}#sk-container-id-8 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-8 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-8\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>XGBClassifier(base_score=0.5, booster=&#x27;gbtree&#x27;, callbacks=None,\n",
       "              colsample_bylevel=1, colsample_bynode=1, colsample_bytree=1,\n",
       "              early_stopping_rounds=None, enable_categorical=False,\n",
       "              eval_metric=None, gamma=0, gpu_id=-1, grow_policy=&#x27;depthwise&#x27;,\n",
       "              importance_type=None, interaction_constraints=&#x27;&#x27;,\n",
       "              learning_rate=0.300000012, max_bin=256, max_cat_to_onehot=4,\n",
       "              max_delta_step=0, max_depth=5, max_leaves=0, min_child_weight=1,\n",
       "              missing=nan, monotone_constraints=&#x27;()&#x27;, n_estimators=100,\n",
       "              n_jobs=0, num_parallel_tree=1, objective=&#x27;multi:softprob&#x27;,\n",
       "              predictor=&#x27;auto&#x27;, random_state=0, reg_alpha=0, ...)</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-16\" type=\"checkbox\" checked><label for=\"sk-estimator-id-16\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">XGBClassifier</label><div class=\"sk-toggleable__content\"><pre>XGBClassifier(base_score=0.5, booster=&#x27;gbtree&#x27;, callbacks=None,\n",
       "              colsample_bylevel=1, colsample_bynode=1, colsample_bytree=1,\n",
       "              early_stopping_rounds=None, enable_categorical=False,\n",
       "              eval_metric=None, gamma=0, gpu_id=-1, grow_policy=&#x27;depthwise&#x27;,\n",
       "              importance_type=None, interaction_constraints=&#x27;&#x27;,\n",
       "              learning_rate=0.300000012, max_bin=256, max_cat_to_onehot=4,\n",
       "              max_delta_step=0, max_depth=5, max_leaves=0, min_child_weight=1,\n",
       "              missing=nan, monotone_constraints=&#x27;()&#x27;, n_estimators=100,\n",
       "              n_jobs=0, num_parallel_tree=1, objective=&#x27;multi:softprob&#x27;,\n",
       "              predictor=&#x27;auto&#x27;, random_state=0, reg_alpha=0, ...)</pre></div></div></div></div></div>"
      ],
      "text/plain": [
       "XGBClassifier(base_score=0.5, booster='gbtree', callbacks=None,\n",
       "              colsample_bylevel=1, colsample_bynode=1, colsample_bytree=1,\n",
       "              early_stopping_rounds=None, enable_categorical=False,\n",
       "              eval_metric=None, gamma=0, gpu_id=-1, grow_policy='depthwise',\n",
       "              importance_type=None, interaction_constraints='',\n",
       "              learning_rate=0.300000012, max_bin=256, max_cat_to_onehot=4,\n",
       "              max_delta_step=0, max_depth=5, max_leaves=0, min_child_weight=1,\n",
       "              missing=nan, monotone_constraints='()', n_estimators=100,\n",
       "              n_jobs=0, num_parallel_tree=1, objective='multi:softprob',\n",
       "              predictor='auto', random_state=0, reg_alpha=0, ...)"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid_search.best_estimator_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Next Steps"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "I learned here at the end that pitcher specific tendencies improve the model a good bit. Some things I'd like to do if I continued playing with this dataset:\n",
    "\n",
    "1. Pitcher clustering. Understanding which pitchers have similar tendencies to see if one pitchers decision might be similar to another pitchers given a game situation.\n",
    "2. More historical data. Using 2010 and further back data to see how trends might change over time and use league average numbers/decisions to bolster decision making.\n",
    "3. Current batter data. There's no way a pitcher is throwing the same pitch sequence to lefty Bryce Harper and righty Aaron Judge. Similarly, pitchers throw to the same batter dozens of times per game. We should be using that data to help us predict as well (and potentially using the cluster data I was talking about earlier!)\n",
    "\n",
    "I also think a simple classification task was great here for my goal, but to improve model performance beyond 50% accuracy I'd like to try the following:\n",
    "\n",
    "1. Bootstrapping/Monte Carlo sims. I've found incredible success simulating outcomes in the NFL and NBA given historical trends. I'd imagine it works the same with MLB, but those models typically take longer to build, test and run.\n",
    "2. Sequence encoding. Building a neural network would have taken more time than alotted, but I've messed around with LTSM (long-term short-term memory) neural networks. I think this would do even better at predicting what a pitcher might do based on historical and recent trends for different types of pitchers.\n",
    "3. Pitcher pipelining. Using pitcher specific data (like % of previous throws that were fast-balls, sliders etc) boosted performance, so I'd wager that building a pipeline that is fit on specific to individual pitchers would perform even better. That said, you'd run the risk of overfitting with small samples."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.8.9 64-bit",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.9"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "31f2aee4e71d21fbe5cf8b01ff0e069b9275f58929596ceb00d14d90e3e16cd6"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}