checkpoint_2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import xgboost\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import tqdm\n",
    "import copy\n",
    "from sklearn.model_selection import RepeatedKFold\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "pd.set_option('display.max_rows', 500)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv(\"data/train_competition_2023.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(432600, 18)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Same preprocessing as in checkpoint 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['time']=pd.to_datetime(df['time']) \n",
    "df['hour']=df['time'].dt.hour\n",
    "df['month']=df['time'].dt.month\n",
    "df['year']=df['time'].dt.year"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.sort_values(['obs','time'])\n",
    "mean_df=df.groupby('obs').first()[['num_0','num_1','num_2','cat_0','cat_1','cat_2','cat_3','cat_4','t_0','t_1','t_2','t_3','t_4','year','month','hour','y_1','y_2']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/rp/w4jh5qxd1gd5qhqv_rsk4syw0000gn/T/ipykernel_31992/2945922554.py:2: FutureWarning: The default value of numeric_only in DataFrameGroupBy.mean is deprecated. In a future version, numeric_only will default to False. Either specify numeric_only or select only columns which should be valid for the function.\n",
      "  mean_df_mean=df.groupby('obs').mean()[['t_0','t_1','t_2','t_3','t_4']]\n",
      "/var/folders/rp/w4jh5qxd1gd5qhqv_rsk4syw0000gn/T/ipykernel_31992/2945922554.py:3: FutureWarning: The default value of numeric_only in DataFrameGroupBy.var is deprecated. In a future version, numeric_only will default to False. Either specify numeric_only or select only columns which should be valid for the function.\n",
      "  mean_df_var=df.groupby('obs').var()[['t_0','t_1','t_2','t_3','t_4']]\n",
      "/var/folders/rp/w4jh5qxd1gd5qhqv_rsk4syw0000gn/T/ipykernel_31992/2945922554.py:4: FutureWarning: The default value of numeric_only in DataFrameGroupBy.median is deprecated. In a future version, numeric_only will default to False. Either specify numeric_only or select only columns which should be valid for the function.\n",
      "  mean_df_med=df.groupby('obs').median()[['t_0','t_1','t_2','t_3','t_4']]\n"
     ]
    }
   ],
   "source": [
    "mean_df_last=df.groupby('obs').last()[['t_0','t_1','t_2','t_3','t_4']]\n",
    "mean_df_mean=df.groupby('obs').mean()[['t_0','t_1','t_2','t_3','t_4']]\n",
    "mean_df_var=df.groupby('obs').var()[['t_0','t_1','t_2','t_3','t_4']]\n",
    "mean_df_med=df.groupby('obs').median()[['t_0','t_1','t_2','t_3','t_4']]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "mean_df[['t_0d','t_1d','t_2d','t_3d','t_4d']]=mean_df_last\n",
    "mean_df[['t_0m','t_1m','t_2m','t_3m','t_4m']]=mean_df_mean\n",
    "mean_df[['t_0v','t_1v','t_2v','t_3v','t_4v']]=mean_df_var\n",
    "mean_df[['t_0med','t_1med','t_2med','t_3med','t_4med']]=mean_df_med"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train, x_test, y_train, y_test = train_test_split(mean_df[['t_0','t_1','t_2','t_3','t_4','t_0d','t_1d','t_2d','t_3d','t_4d','t_0m','t_1m','t_2m','t_3m','t_4m','t_0med','t_1med','t_2med','t_3med','t_4med']], mean_df[['y_1','y_2']], test_size=0.2, random_state=13)\n",
    "x_train, x_val, y_train, y_val = train_test_split(x_train, y_train, test_size=0.25, random_state=13)\n",
    "                                                    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Pytorch "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train = torch.tensor(x_train.values, dtype=torch.float32)\n",
    "x_val = torch.tensor(x_val.values, dtype=torch.float32)\n",
    "y_train = torch.tensor(y_train.values, dtype=torch.float32)\n",
    "y_val = torch.tensor(y_val.values, dtype=torch.float32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(torch.Size([8652, 20]), torch.Size([8652, 2]))"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_train.shape, y_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = nn.Sequential(\n",
    "    nn.Linear(20, 160),\n",
    "    nn.ReLU(),\n",
    "    nn.Linear(160, 80),\n",
    "    nn.ReLU(),\n",
    "    nn.Linear(80, 40),\n",
    "    nn.ReLU(),\n",
    "    nn.Linear(40, 20),\n",
    "    nn.ReLU(),\n",
    "    nn.Linear(20, 2)\n",
    ")\n",
    "\n",
    "\n",
    "loss_fn = nn.L1Loss()  # mean absolute error\n",
    "optimizer = optim.Adam(model.parameters(), lr=0.0005, weight_decay=1e-5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_epochs = 1500   # number of epochs to run\n",
    "batch_size = 90  # size of each batch\n",
    "batch_start = torch.arange(0, len(x_train), batch_size)\n",
    "\n",
    "best_mae = np.inf   # init to infinity\n",
    "best_weights = None\n",
    "history = []"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train and Validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "for epoch in range(n_epochs):\n",
    "    model.train()\n",
    "    with tqdm.tqdm(batch_start, unit=\"batch\", mininterval=0, disable=True) as bar: # Can set disable=False if you want to see the progress\n",
    "        bar.set_description(f\"Epoch {epoch}\")\n",
    "        for start in bar:\n",
    "            # take a batch\n",
    "            X_batch = x_train[start:start+batch_size]\n",
    "            y_batch = y_train[start:start+batch_size]\n",
    "            # forward pass\n",
    "            y_pred = model(X_batch)\n",
    "            loss = loss_fn(y_pred, y_batch)\n",
    "            # backward pass\n",
    "            optimizer.zero_grad()\n",
    "            loss.backward()\n",
    "            # update weights\n",
    "            optimizer.step()\n",
    "            # print progress\n",
    "            bar.set_postfix(mae=float(loss))\n",
    "    # evaluate accuracy at end of each epoch\n",
    "    model.eval()\n",
    "    y_pred = model(x_val)\n",
    "    mae = loss_fn(y_pred, y_val)\n",
    "    mae = float(mae)\n",
    "    history.append(mae)\n",
    "    if mae < best_mae:\n",
    "        best_mae = mae\n",
    "        best_weights = copy.deepcopy(model.state_dict())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MAE: 4.17\n"
     ]
    }
   ],
   "source": [
    "model.load_state_dict(best_weights)\n",
    "print(\"MAE: %.2f\" % best_mae)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test MAE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.eval()\n",
    "y_pred_test = model(torch.tensor(x_test.values, dtype=torch.float32))\n",
    "y_test = torch.tensor(y_test.values, dtype=torch.float32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(4.1873)"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "loss_fn(y_pred_test, y_test).detach()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Using the Test Dataset to produce second batch of inferences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/rp/w4jh5qxd1gd5qhqv_rsk4syw0000gn/T/ipykernel_31992/3838813641.py:9: FutureWarning: The default value of numeric_only in DataFrameGroupBy.mean is deprecated. In a future version, numeric_only will default to False. Either specify numeric_only or select only columns which should be valid for the function.\n",
      "  mean_df_test_mean=df_test.groupby('obs').mean()[['t_0','t_1','t_2','t_3','t_4']]\n",
      "/var/folders/rp/w4jh5qxd1gd5qhqv_rsk4syw0000gn/T/ipykernel_31992/3838813641.py:10: FutureWarning: The default value of numeric_only in DataFrameGroupBy.var is deprecated. In a future version, numeric_only will default to False. Either specify numeric_only or select only columns which should be valid for the function.\n",
      "  mean_df_test_var=df_test.groupby('obs').var()[['t_0','t_1','t_2','t_3','t_4']]\n",
      "/var/folders/rp/w4jh5qxd1gd5qhqv_rsk4syw0000gn/T/ipykernel_31992/3838813641.py:11: FutureWarning: The default value of numeric_only in DataFrameGroupBy.median is deprecated. In a future version, numeric_only will default to False. Either specify numeric_only or select only columns which should be valid for the function.\n",
      "  mean_df_test_med=df_test.groupby('obs').median()[['t_0','t_1','t_2','t_3','t_4']]\n"
     ]
    }
   ],
   "source": [
    "df_test = pd.read_csv(\"data/test_no_outcome.csv\")\n",
    "df_test['time']=pd.to_datetime(df_test['time']) \n",
    "df_test['hour']=df_test['time'].dt.hour\n",
    "df_test['month']=df_test['time'].dt.month\n",
    "df_test['year']=df_test['time'].dt.year\n",
    "df_test.sort_values(['obs','time'],inplace=True)\n",
    "mean_df_test=df_test.groupby('obs').first()[['num_0','num_1','num_2','cat_0','cat_1','cat_2','cat_3','cat_4','t_0','t_1','t_2','t_3','t_4','year','month','hour']]\n",
    "mean_df_test_last=df_test.groupby('obs').last()[['t_0','t_1','t_2','t_3','t_4']]\n",
    "mean_df_test_mean=df_test.groupby('obs').mean()[['t_0','t_1','t_2','t_3','t_4']]\n",
    "mean_df_test_var=df_test.groupby('obs').var()[['t_0','t_1','t_2','t_3','t_4']]\n",
    "mean_df_test_med=df_test.groupby('obs').median()[['t_0','t_1','t_2','t_3','t_4']]\n",
    "\n",
    "mean_df_test[['t_0d','t_1d','t_2d','t_3d','t_4d']]=mean_df_test_last\n",
    "mean_df_test[['t_0m','t_1m','t_2m','t_3m','t_4m']]=mean_df_test_mean\n",
    "mean_df_test[['t_0med','t_1med','t_2med','t_3med','t_4med']]=mean_df_test_med"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([3450, 20])"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# make predictions using neural network\n",
    "kaggle_x = mean_df_test[['t_0','t_1','t_2','t_3','t_4','t_0d','t_1d','t_2d','t_3d','t_4d','t_0m','t_1m','t_2m','t_3m','t_4m','t_0med','t_1med','t_2med','t_3med','t_4med']]\n",
    "kaggle_x = torch.tensor(kaggle_x.values, dtype=torch.float32)\n",
    "kaggle_x.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 40.9478, 105.1709],\n",
       "        [ 31.8946, 100.5568],\n",
       "        [ 34.9711,  95.5676],\n",
       "        ...,\n",
       "        [ 50.1896, 112.6436],\n",
       "        [ 46.6133,  99.8016],\n",
       "        [ 56.1247, 105.5437]], grad_fn=<AddmmBackward0>)"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.eval()\n",
    "y_pred = model(kaggle_x)\n",
    "y_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred_df = pd.DataFrame(y_pred.detach().numpy(),index=mean_df_test.index,columns=['y_1','y_2'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>y_1</th>\n",
       "      <th>y_2</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>obs</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>40.947823</td>\n",
       "      <td>105.170876</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>31.894579</td>\n",
       "      <td>100.556824</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>34.971069</td>\n",
       "      <td>95.567619</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>34.118431</td>\n",
       "      <td>94.770287</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>33.065632</td>\n",
       "      <td>94.224350</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17850</th>\n",
       "      <td>55.067841</td>\n",
       "      <td>87.783844</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17862</th>\n",
       "      <td>58.339146</td>\n",
       "      <td>103.796722</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17863</th>\n",
       "      <td>50.189644</td>\n",
       "      <td>112.643623</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17864</th>\n",
       "      <td>46.613251</td>\n",
       "      <td>99.801582</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17865</th>\n",
       "      <td>56.124718</td>\n",
       "      <td>105.543701</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3450 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             y_1         y_2\n",
       "obs                         \n",
       "18     40.947823  105.170876\n",
       "19     31.894579  100.556824\n",
       "20     34.971069   95.567619\n",
       "21     34.118431   94.770287\n",
       "22     33.065632   94.224350\n",
       "...          ...         ...\n",
       "17850  55.067841   87.783844\n",
       "17862  58.339146  103.796722\n",
       "17863  50.189644  112.643623\n",
       "17864  46.613251   99.801582\n",
       "17865  56.124718  105.543701\n",
       "\n",
       "[3450 rows x 2 columns]"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred_df.to_csv('submission_nn.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### XGboost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/rp/w4jh5qxd1gd5qhqv_rsk4syw0000gn/T/ipykernel_31992/2940504030.py:11: FutureWarning: The default value of numeric_only in DataFrameGroupBy.mean is deprecated. In a future version, numeric_only will default to False. Either specify numeric_only or select only columns which should be valid for the function.\n",
      "  mean_df_mean=df.groupby('obs').mean()[['t_0','t_1','t_2','t_3','t_4']]\n",
      "/var/folders/rp/w4jh5qxd1gd5qhqv_rsk4syw0000gn/T/ipykernel_31992/2940504030.py:12: FutureWarning: The default value of numeric_only in DataFrameGroupBy.var is deprecated. In a future version, numeric_only will default to False. Either specify numeric_only or select only columns which should be valid for the function.\n",
      "  mean_df_var=df.groupby('obs').var()[['t_0','t_1','t_2','t_3','t_4']]\n",
      "/var/folders/rp/w4jh5qxd1gd5qhqv_rsk4syw0000gn/T/ipykernel_31992/2940504030.py:13: FutureWarning: The default value of numeric_only in DataFrameGroupBy.median is deprecated. In a future version, numeric_only will default to False. Either specify numeric_only or select only columns which should be valid for the function.\n",
      "  mean_df_med=df.groupby('obs').median()[['t_0','t_1','t_2','t_3','t_4']]\n"
     ]
    }
   ],
   "source": [
    "df = pd.read_csv(\"data/train_competition_2023.csv\")\n",
    "df['time']=pd.to_datetime(df['time']) \n",
    "df['hour']=df['time'].dt.hour\n",
    "df['month']=df['time'].dt.month\n",
    "df['year']=df['time'].dt.year\n",
    "df.sort_values(['obs','time'])\n",
    "#check how many sub_id's per obs\n",
    "# mean_df=df.groupby('obs').mean()[['num_0','num_1','num_2','cat_0','cat_1_2','cat_1_1','cat_1_5','cat_1_6','cat_2','cat_3','cat_4','t_0','t_1','t_2','t_3','t_4','y_1','y_2']]\n",
    "mean_df=df.groupby('obs').first()[['num_0','num_1','num_2','cat_0','cat_1','cat_2','cat_3','cat_4','t_0','t_1','t_2','t_3','t_4','year','month','hour','y_1','y_2']]\n",
    "mean_df_last=df.groupby('obs').last()[['t_0','t_1','t_2','t_3','t_4']]\n",
    "mean_df_mean=df.groupby('obs').mean()[['t_0','t_1','t_2','t_3','t_4']]\n",
    "mean_df_var=df.groupby('obs').var()[['t_0','t_1','t_2','t_3','t_4']]\n",
    "mean_df_med=df.groupby('obs').median()[['t_0','t_1','t_2','t_3','t_4']]\n",
    "mean_df[['t_0d','t_1d','t_2d','t_3d','t_4d']]=mean_df_last\n",
    "mean_df[['t_0m','t_1m','t_2m','t_3m','t_4m']]=mean_df_mean\n",
    "mean_df[['t_0v','t_1v','t_2v','t_3v','t_4v']]=mean_df_var\n",
    "mean_df[['t_0med','t_1med','t_2med','t_3med','t_4med']]=mean_df_med\n",
    "x_train, x_test, y_train, y_test = train_test_split(mean_df[['t_0','t_1','t_2','t_3','t_4','t_0d','t_1d','t_2d','t_3d','t_4d','t_0m','t_1m','t_2m','t_3m','t_4m','t_0v','t_1v','t_2v','t_3v','t_4v','t_0med','t_1med','t_2med','t_3med','t_4med']], mean_df[['y_1','y_2']], test_size=0.2, random_state=13)\n",
    "x_train, x_val, y_train, y_val = train_test_split(x_train, y_train, test_size=0.25, random_state=13)\n",
    "                                                    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['t_0', 't_1', 't_2', 't_3', 't_4', 't_0d', 't_1d', 't_2d', 't_3d',\n",
       "       't_4d', 't_0m', 't_1m', 't_2m', 't_3m', 't_4m', 't_0v', 't_1v', 't_2v',\n",
       "       't_3v', 't_4v', 't_0med', 't_1med', 't_2med', 't_3med', 't_4med'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_train.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(8652, 25)"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "params = {\n",
    "        'n_estimators': [500, 2000, 2500, 5000],\n",
    "        'eta': [0.05, 0.1],\n",
    "        'subsample': [0.6, 0.8, 1.0],\n",
    "        'colsample_bytree': [0.6, 0.8, 1.0],\n",
    "        'max_depth': [3, 5, 7]\n",
    "        }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = xgboost.XGBRegressor()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "grid_cv = GridSearchCV(estimator=model,\n",
    "            param_grid=params,\n",
    "            cv=3,\n",
    "            scoring = 'neg_mean_absolute_error',\n",
    "            n_jobs = 4,\n",
    "            verbose = 1, \n",
    "            return_train_score = True)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "I used grid search to find the best parameters for the XGBoost model "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 216 candidates, totalling 648 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/kevin_yang/opt/anaconda3/lib/python3.8/site-packages/joblib/externals/loky/process_executor.py:702: UserWarning: A worker stopped while some jobs were given to the executor. This can be caused by a too short worker timeout or by a memory leak.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'colsample_bytree': 0.6,\n",
       " 'eta': 0.05,\n",
       " 'max_depth': 3,\n",
       " 'n_estimators': 500,\n",
       " 'subsample': 0.8}"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid_cv.fit(x_train, y_train)\n",
    "grid_cv.best_params_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "# best model\n",
    "model = xgboost.XGBRegressor(n_estimators=500, max_depth=3, eta=0.05, subsample=0.8, colsample_bytree=1, alpha= 5, gamma=5, reg_lambda=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "cv = RepeatedKFold(n_splits=10, n_repeats=3, random_state=1)\n",
    "scores = cross_val_score(model, x_train, y_train, scoring='neg_mean_absolute_error', cv=cv, n_jobs=-1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "XGBoost performed worse than the neural network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean MAE: 4.299 (0.128)\n"
     ]
    }
   ],
   "source": [
    "scores = np.absolute(scores)\n",
    "print('Mean MAE: %.3f (%.3f)' % (scores.mean(), scores.std()) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/rp/w4jh5qxd1gd5qhqv_rsk4syw0000gn/T/ipykernel_31992/571052759.py:9: FutureWarning: The default value of numeric_only in DataFrameGroupBy.mean is deprecated. In a future version, numeric_only will default to False. Either specify numeric_only or select only columns which should be valid for the function.\n",
      "  mean_df_test_mean=df_test.groupby('obs').mean()[['t_0','t_1','t_2','t_3','t_4']]\n",
      "/var/folders/rp/w4jh5qxd1gd5qhqv_rsk4syw0000gn/T/ipykernel_31992/571052759.py:10: FutureWarning: The default value of numeric_only in DataFrameGroupBy.var is deprecated. In a future version, numeric_only will default to False. Either specify numeric_only or select only columns which should be valid for the function.\n",
      "  mean_df_test_var=df.groupby('obs').var()[['t_0','t_1','t_2','t_3','t_4']]\n",
      "/var/folders/rp/w4jh5qxd1gd5qhqv_rsk4syw0000gn/T/ipykernel_31992/571052759.py:11: FutureWarning: The default value of numeric_only in DataFrameGroupBy.median is deprecated. In a future version, numeric_only will default to False. Either specify numeric_only or select only columns which should be valid for the function.\n",
      "  mean_df_test_med=df.groupby('obs').median()[['t_0','t_1','t_2','t_3','t_4']]\n"
     ]
    }
   ],
   "source": [
    "df_test = pd.read_csv(\"data/test_no_outcome.csv\")\n",
    "df_test['time']=pd.to_datetime(df_test['time']) \n",
    "df_test['hour']=df_test['time'].dt.hour\n",
    "df_test['month']=df_test['time'].dt.month\n",
    "df_test['year']=df_test['time'].dt.year\n",
    "df_test.sort_values(['obs','time'],inplace=True)\n",
    "mean_df_test=df_test.groupby('obs').first()[['num_0','num_1','num_2','cat_0','cat_1','cat_2','cat_3','cat_4','t_0','t_1','t_2','t_3','t_4','year','month','hour']]\n",
    "mean_df_test_last=df_test.groupby('obs').last()[['t_0','t_1','t_2','t_3','t_4']]\n",
    "mean_df_test_mean=df_test.groupby('obs').mean()[['t_0','t_1','t_2','t_3','t_4']]\n",
    "mean_df_test_var=df.groupby('obs').var()[['t_0','t_1','t_2','t_3','t_4']]\n",
    "mean_df_test_med=df.groupby('obs').median()[['t_0','t_1','t_2','t_3','t_4']]\n",
    "\n",
    "\n",
    "mean_df_test[['t_0d','t_1d','t_2d','t_3d','t_4d']]=mean_df_test_last\n",
    "mean_df_test[['t_0m','t_1m','t_2m','t_3m','t_4m']]=mean_df_test_mean\n",
    "mean_df_test[['t_0v','t_1v','t_2v','t_3v','t_4v']]=mean_df_test_var\n",
    "mean_df_test[['t_0med','t_1med','t_2med','t_3med','t_4med']]=mean_df_test_med\n",
    "# make predictions using neural network\n",
    "kaggle_x = mean_df_test[['t_0','t_1','t_2','t_3','t_4','t_0d','t_1d','t_2d','t_3d','t_4d','t_0m','t_1m','t_2m','t_3m','t_4m','t_0v','t_1v','t_2v','t_3v','t_4v','t_0med','t_1med','t_2med','t_3med','t_4med']]\n",
    "# kaggle_x = torch.tensor(kaggle_x.values, dtype=torch.float32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "XGBRegressor(alpha=5, base_score=0.5, booster='gbtree', callbacks=None,\n",
       "             colsample_bylevel=1, colsample_bynode=1, colsample_bytree=1,\n",
       "             early_stopping_rounds=None, enable_categorical=False, eta=0.05,\n",
       "             eval_metric=None, feature_types=None, gamma=5, gpu_id=-1,\n",
       "             grow_policy='depthwise', importance_type=None,\n",
       "             interaction_constraints='', learning_rate=0.0500000007,\n",
       "             max_bin=256, max_cat_threshold=64, max_cat_to_onehot=4,\n",
       "             max_delta_step=0, max_depth=3, max_leaves=0, min_child_weight=1,\n",
       "             missing=nan, monotone_constraints='()', n_estimators=500, n_jobs=0,\n",
       "             num_parallel_tree=1, ...)"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(x_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "yhat = model.predict(kaggle_x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "xg_df = pd.DataFrame(yhat,index=mean_df_test.index,columns=['y_1','y_2'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>y_1</th>\n",
       "      <th>y_2</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>obs</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>47.632801</td>\n",
       "      <td>102.961861</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>34.066547</td>\n",
       "      <td>99.767471</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>39.727699</td>\n",
       "      <td>94.508415</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>38.666443</td>\n",
       "      <td>94.296005</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>40.546028</td>\n",
       "      <td>93.643608</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17850</th>\n",
       "      <td>60.875542</td>\n",
       "      <td>85.320419</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17862</th>\n",
       "      <td>61.855297</td>\n",
       "      <td>100.711723</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17863</th>\n",
       "      <td>56.507080</td>\n",
       "      <td>109.092285</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17864</th>\n",
       "      <td>54.376888</td>\n",
       "      <td>99.590324</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17865</th>\n",
       "      <td>60.877430</td>\n",
       "      <td>104.006187</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3450 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             y_1         y_2\n",
       "obs                         \n",
       "18     47.632801  102.961861\n",
       "19     34.066547   99.767471\n",
       "20     39.727699   94.508415\n",
       "21     38.666443   94.296005\n",
       "22     40.546028   93.643608\n",
       "...          ...         ...\n",
       "17850  60.875542   85.320419\n",
       "17862  61.855297  100.711723\n",
       "17863  56.507080  109.092285\n",
       "17864  54.376888   99.590324\n",
       "17865  60.877430  104.006187\n",
       "\n",
       "[3450 rows x 2 columns]"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xg_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch_df = pd.DataFrame(y_pred.detach().numpy(),index=mean_df_test.index,columns=['y_1','y_2'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Output from the ensembling of neural network and XGBoost predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "fin_df = (xg_df + torch_df)/2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "fin_df.to_csv('submission_nn_xgb.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "b36cb969838aa7efc71662dcae87ad2e5df476bc74b903cea8549a6450c3f723"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}