04 - Other Models.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "0",
   "metadata": {},
   "source": [
    "If you're curious, here's one of the better logistic regression models I was able to come up with just 4 features (not counting ^3 transformations) -- `mate_eval`, `regular_eval`, `white_adjusted_clock_usage`, and `black_adjusted_clock_usage`. Of course, blinding throwing everything into XGBoost, as we did for our first model, still performs better."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1",
   "metadata": {},
   "outputs": [],
   "source": [
    "NUM_MOVES = 10\n",
    "\n",
    "feature_suffixes = (\"eval\", \"adjusted_clock_usage\")\n",
    "# Not including \"ply\" and \"elo_diff\" makes the coefficients for mate_eval^3 more interpretable\n",
    "# without sacrificing much accuracy.\n",
    "\n",
    "lr_steps = (\n",
    "    ml.Mutate(\n",
    "        adjusted_base_time=_.base_time + _.increment * NUM_MOVES,\n",
    "        white_adjusted_clock=_.white_clock + _.increment * NUM_MOVES,\n",
    "        black_adjusted_clock=_.black_clock + _.increment * NUM_MOVES,\n",
    "    ),\n",
    "    ml.Mutate(\n",
    "        white_adjusted_clock_usage=(_.adjusted_base_time - _.white_adjusted_clock)\n",
    "        / _.adjusted_base_time,\n",
    "        black_adjusted_clock_usage=(_.adjusted_base_time - _.black_adjusted_clock)\n",
    "        / _.adjusted_base_time,\n",
    "    ),\n",
    "    ml.Mutate(elo_diff=_.white_elo - _.black_elo),\n",
    "    # in case you want to play with adding \"elo_diff\" to feature_suffixes above\n",
    "    ml.Drop(~ml.endswith(feature_suffixes)),\n",
    "    ml.FillNA(ml.numeric(), fill_value=0),\n",
    "    ml.MutateAt(ml.endswith(\"eval\"), pow3=_**3),\n",
    "    ml.ScaleStandard(~ml.contains(\"adjusted_clock_usage\")),\n",
    ")\n",
    "\n",
    "lr_pipe = Pipeline(\n",
    "    [\n",
    "        (\"lr_recipe\", ml.Recipe(*(basic_steps + lr_steps))),\n",
    "        (\"lr_model\", LogisticRegression()),\n",
    "    ]\n",
    ")\n",
    "lr_pipe.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(f\"Training score: {lr_pipe.score(X_train, y_train)}\")\n",
    "print(f\"Test score: {lr_pipe.score(X_test, y_test)}\")\n",
    "\n",
    "X_fit_transformed = lr_pipe[\"lr_recipe\"].to_ibis(X_train)\n",
    "coef_df = pd.DataFrame(\n",
    "    lr_pipe[\"lr_model\"].coef_,\n",
    "    columns=X_fit_transformed.columns,\n",
    "    index=[\"black win\", \"draw\", \"white win\"],\n",
    ")\n",
    "coef_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Example of game with titled player: 5vD7WOT9\n",
    "# test_data[_.game_id == \"5vD7WOT9\"]\n",
    "test_data_df.loc[test_data_df.game_id == \"5vD7WOT9\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4",
   "metadata": {},
   "source": [
    "## Clipped XGB regressor for use with LETSQL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# ordered_titles_list = [\"BOT\", \"WCM\", \"WFM\", \"NM\", \"CM\", \"WIM\", \"FM\", \"WGM\", \"IM\", \"LM\", \"GM\"]\n",
    "# expression = \"_.case()\"\n",
    "\n",
    "# for i, title in enumerate(ordered_titles_list):\n",
    "#     expression += f\".when('{title}', {i+1})\"\n",
    "\n",
    "# expression += \".end()\"\n",
    "# expression\n",
    "\n",
    "# xgb_steps_plus = xgb_steps + (\n",
    "#     ml.MutateAt(ml.endswith(\"title\"), eval(expression)),\n",
    "# )\n",
    "\n",
    "X_train = train_data.drop(\"target\")\n",
    "y_train = train_data.target\n",
    "\n",
    "xgb_reg_pipe = Pipeline(\n",
    "    [\n",
    "        (\"xgb_recipe\", ml.Recipe(*(xgb_steps + basic_steps))),\n",
    "        (\"xgb_reg\", xgb.XGBRegressor(n_estimators=10)),\n",
    "    ]\n",
    ")\n",
    "# The regressor seems to overfit much more quickly than the classifier\n",
    "# With the default eta=0.3, try n_estimators=10\n",
    "# With eta=0.1, try n_estimators=20\n",
    "xgb_reg_pipe.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Score probably isn't meaningful for us for a regressor\n",
    "print(f\"Training score: {xgb_reg_pipe.score(X_train, y_train)}\")\n",
    "print(f\"Test score: {xgb_reg_pipe.score(X_test, y_test)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7",
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred_win_proba = xgb_reg_pipe.predict(X_test).clip(0, 1)\n",
    "\n",
    "xgb_reg_test_results_df = test_results_df.copy(deep=True)\n",
    "xgb_reg_test_results_df[\"y_pred_win\"] = y_pred_win_proba"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Out of ~1,000,000 total rows\n",
    "xgb_reg_test_results_df[xgb_reg_test_results_df.y_pred_win > 0.99]\n",
    "xgb_reg_test_results_df[xgb_reg_test_results_df.y_pred_win < 0.01]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9",
   "metadata": {},
   "outputs": [],
   "source": [
    "print_losses(\n",
    "    xgb_reg_test_results_df.target,\n",
    "    xgb_reg_test_results_df.y_pred_win,\n",
    "    train_results_df.target,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "10",
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, axs = plt.subplots(1, 3, figsize=(15, 4))\n",
    "plot_losses(\n",
    "    xgb_test_results_df,\n",
    "    train_results_df,\n",
    "    ax=axs[0],\n",
    "    fmt=\"c\",\n",
    "    title=\"XGBoost with Added Features\",\n",
    ")\n",
    "plot_losses(\n",
    "    xgb_reg_test_results_df,\n",
    "    train_results_df,\n",
    "    ax=axs[1],\n",
    "    fmt=\"b\",\n",
    "    title=\"Clipped XGBoost Regressor\",\n",
    ")\n",
    "\n",
    "plot_losses(xgb_test_results_df, train_results_df, ax=axs[2], fmt=\"c\")\n",
    "plot_losses(\n",
    "    xgb_reg_test_results_df, train_results_df, ax=axs[2], fmt=\"b\", title=\"Comparison\"\n",
    ")\n",
    "axs[2].legend(\n",
    "    [\n",
    "        \"XGB with added features\",\n",
    "        \"Predicting mean of y_train\",\n",
    "        \"Clipped XGBoost regressor\",\n",
    "    ]\n",
    ");"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "11",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_fit_transformed = xgb_reg_pipe[\"xgb_recipe\"].to_ibis(X_train)\n",
    "xgb_reg_pipe[\"xgb_reg\"].get_booster().feature_names = X_fit_transformed.columns\n",
    "\n",
    "xgb.plot_importance(\n",
    "    xgb_reg_pipe[\"xgb_reg\"],\n",
    "    importance_type=\"gain\",\n",
    "    xlabel=\"Average Gain\",\n",
    "    show_values=False,\n",
    ")\n",
    "xgb.plot_importance(\n",
    "    xgb_reg_pipe[\"xgb_reg\"],\n",
    "    importance_type=\"cover\",\n",
    "    xlabel=\"Average Coverage (# of samples impacted)\",\n",
    "    show_values=False,\n",
    ");"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}