Chapter 9. Hyper-Parameter Tuning with Cross-Validation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# imports\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import seaborn as sns\n",
    "sns.set(palette='Set2')\n",
    "import plotly.graph_objects as go\n",
    "# import plotly.io as pio\n",
    "# pio.renderers.default = \"svg\"\n",
    "\n",
    "from scipy.stats import rv_continuous, kstest\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV, RandomizedSearchCV\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.svm import SVC\n",
    "\n",
    "from typing import Any, Dict, Optional\n",
    "\n",
    "from timeit import default_timer\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "import nbimporter\n",
    "from Functions import PurgedKFold, cvScore, get_test_data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Auxilary Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MyPipeline(Pipeline):\n",
    "    '''\n",
    "    Augmentation of sklearn Pipeline class that allows to pass 'sample_weight' to 'fit' method.\n",
    "    '''\n",
    "    def fit(\n",
    "        self, X: pd.DataFrame, y: pd.Series, sample_weight: Optional[pd.Series] = None, **fit_params\n",
    "    ) -> 'MyPipeline':\n",
    "        if sample_weight is not None:\n",
    "            fit_params[self.steps[-1][0] + '__sample_weight'] = sample_weight\n",
    "        return super().fit(X, y, **fit_params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def clf_hyper_fit_base(\n",
    "    feat: pd.DataFrame, lbl: pd.Series, t1: pd.Series, pipe_clf: Any, param_grid: Dict[str, list],\n",
    "    cv: int = 3, bagging: list = [0, None, 1.0], n_jobs: int = -1, pctEmbargo: float = 0.0, **fit_params\n",
    ") -> Any:\n",
    "    '''\n",
    "    Implements purged GridSearchCV with a possibility of fitting bagging of tuned estimator.\n",
    "    \n",
    "        Parameters:\n",
    "            feat (pd.DataFrame): features dataset\n",
    "            lbl (pd.Series): labels\n",
    "            t1 (pd.Series): start timestamps (t1.index) and end timestamps (t1.values) of observations\n",
    "            pipe_clf (Any): classififer to fit\n",
    "            param_grid (Dict[str, list]): dictionary with parameters values\n",
    "            cv (int): number of splits\n",
    "            bagging (list): bagging parameters (used when bagging[1] is not None)\n",
    "            n_jobs (int): number of jobs to run in parallel\n",
    "            pctEmbargo (float): share of observations to drop after train\n",
    "        \n",
    "        Returns:\n",
    "            gs (Any): fitted best estimator found by grid search\n",
    "    '''\n",
    "    if set(lbl.values) == {0, 1}:\n",
    "        scoring='f1'    # f1 for meta-labeling\n",
    "    else:\n",
    "        scoring='neg_log_loss'    # symmetric towards all cases\n",
    "    inner_cv = PurgedKFold(n_splits=cv, t1=t1, pctEmbargo=pctEmbargo)    # purged\n",
    "    gs=GridSearchCV(estimator=pipe_clf ,param_grid=param_grid, scoring=scoring, cv=inner_cv, n_jobs=n_jobs)\n",
    "    gs = gs.fit(feat, lbl, **fit_params).best_estimator_    # pipeline\n",
    "    if bagging[1] is not None and bagging[1] > 0:\n",
    "        gs = BaggingClassifier(base_estimator=MyPipeline(gs.steps), n_estimators=int(bagging[0]),\n",
    "                               max_samples=float(bagging[1]), max_features=float(bagging[2]), n_jobs=n_jobs)\n",
    "        gs = gs.fit(feat, lbl, sample_weight=fit_params[gs.base_estimator.steps[-1][0]+'__sample_weight'])\n",
    "        gs = Pipeline([('bag', gs)])\n",
    "    return gs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# expand clf_hyper_fit_base to incorporate option to use randomized grid search\n",
    "def clf_hyper_fit(\n",
    "    feat: pd.DataFrame, lbl: pd.Series, t1: pd.Series, pipe_clf: Any, param_grid: Dict[str, list],\n",
    "    cv: int = 3, bagging: list = [0, None, 1.0], rndSearchIter: int = 0,\n",
    "    n_jobs: int = -1, pctEmbargo: float = 0.0, **fit_params\n",
    ") -> Any:\n",
    "    '''\n",
    "    Implements purged GridSearchCV with a possibility of fitting bagging of tuned estimator.\n",
    "    \n",
    "        Parameters:\n",
    "            feat (pd.DataFrame): features dataset\n",
    "            lbl (pd.Series): labels\n",
    "            t1 (pd.Series): start timestamps (t1.index) and end timestamps (t1.values) of observations\n",
    "            pipe_clf (Any): classififer to fit\n",
    "            param_grid (Dict[str, list]): dictionary with parameters values\n",
    "            cv (int): number of splits\n",
    "            bagging (list): bagging parameters (used when bagging[1] is not None)\n",
    "            rndSearchIter (int): number of iterations to use in randomized GS (if 0 then apply standard GS)\n",
    "            n_jobs (int): number of jobs to run in parallel\n",
    "            pctEmbargo (float): share of observations to drop after train\n",
    "        \n",
    "        Returns:\n",
    "            gs (Any): fitted best estimator found by grid search\n",
    "    '''\n",
    "    if set(lbl.values) == {0, 1}:\n",
    "        scoring='f1'    # f1 for meta-labeling\n",
    "    else:\n",
    "        scoring='neg_log_loss'    # symmetric towards all cases\n",
    "    inner_cv = PurgedKFold(n_splits=cv, t1=t1, pctEmbargo=pctEmbargo)    # purged\n",
    "    \n",
    "    if rndSearchIter == 0:\n",
    "        gs = GridSearchCV(estimator=pipe_clf, param_grid=param_grid, scoring=scoring, cv=inner_cv, n_jobs=n_jobs)\n",
    "    else:\n",
    "        gs = RandomizedSearchCV(estimator=pipe_clf, param_distributions=param_grid, scoring=scoring,\n",
    "                                cv=inner_cv, n_jobs=n_jobs, n_iter=rndSearchIter)\n",
    "    gs = gs.fit(feat, lbl, **fit_params).best_estimator_    # pipeline\n",
    "    \n",
    "    if bagging[1] is not None and bagging[1] > 0:\n",
    "        gs = BaggingClassifier(base_estimator=MyPipeline(gs.steps), n_estimators=int(bagging[0]),\n",
    "                               max_samples=float(bagging[1]), max_features=float(bagging[2]), n_jobs=n_jobs)\n",
    "        gs = gs.fit(feat, lbl, sample_weight=fit_params[gs.base_estimator.steps[-1][0]+'__sample_weight'])\n",
    "        gs = Pipeline([('bag', gs)])\n",
    "    return gs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "class logUniform_gen(rv_continuous):\n",
    "    '''\n",
    "    Implements generator of log-uniform random variables.\n",
    "    '''\n",
    "    def _cdf(self, x: float) -> float:\n",
    "        return np.log(x / self.a) / np.log(self.b / self.a)\n",
    "\n",
    "\n",
    "def log_uniform(a: float = 1.0, b: float = np.exp(1.0)) -> 'logUniform_gen':\n",
    "    return logUniform_gen(a=a, b=b, name='log_uniform')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Example:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "KstestResult(statistic=0.008573147129436381, pvalue=0.4542750397879601)\n",
      "count    10000.000000\n",
      "mean        69.665528\n",
      "std        172.653504\n",
      "min          0.001000\n",
      "25%          0.034666\n",
      "50%          1.055682\n",
      "75%         30.094845\n",
      "max        999.756637\n",
      "dtype: float64\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAApYAAAExCAYAAADGA+LBAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3dfVSUd37//9fAjESDW4uZWShxyc3apUezumdJNmzyHWJOVgaRmp01pwZacnbPRuWkajgnpARYqG0smOVI2mMxaU+anGWTtmgUDAeHpGvixmAT5XTjcdU9aVeIAYsDmsiNIMxcvz/ycyrBG26uuWF8Ps7JIddnLrjeby+u4TXzmeu6LIZhGAIAAACmKSbcBQAAACA6ECwBAABgCoIlAAAATEGwBAAAgCkIlgAAADAFwRIAAACmIFgCAADAFNZwFzBR588PyO8P7SU358+PV29vf0i3eT2RVo9ETRMRafVI1CRJMTEW/eEf3hqy7YXLZJ87I/F3YyqipQ+JXiJVtPQy2T5u9Nw5Y4Kl32+EPFhe3m4kibR6JGqaiEirR6Kmm8VUnjujZT9ESx8SvUSqaOnFzD6YCgcAAIApCJYAAAAwBcESAAAApiBYAgAAwBQESwAAAJiCYAkAAABTECwBAABgCoIlAAAATEGwBAAAgClmzJ13gIka8ftkt88N6jaGR0d04fxQULcBhAPHD4DpIFgi6thiYrXu/TeCuo2X/1+uJP4wIvpw/ACYDqbCAQAAYAqCJQAAAExBsAQAAIAp+IwlJElf+8NbFGe1Tel7J/pBfz6wDwBAdCNYQpIUZ7XxgX0AADAtBEvgJjedd6sninerAeDmQLAEbnK8Ww0AMAsn7wBAEDQ2Nio7O1vZ2dnaunWrJOnEiRNyu93KzMxUaWmpRkdHJUldXV3Ky8uTy+VSQUGBBgYGJEkXLlzQ2rVrlZWVpby8PHm93rD1AwATQbAEAJNdvHhRW7ZsUV1dnRobG3XkyBG1traqqKhI5eXlamlpkWEYqq+vlyRt3rxZubm58ng8Wrx4sWprayVJL774otLS0rRv3z499thj2rJlSzjbAoAbIlgCgMl8Pp/8fr8uXryo0dFRjY6Oymq1amhoSEuXLpUkud1ueTwejYyM6PDhw8rMzBwzLknvvfeecnJyJEkrV67Ur3/9a42MjISnKQCYgAl9xrK/v19r1qzRSy+9pNtvv12tra2qrKzU8PCwsrKyVFhYKOnLaZ7S0lINDAwoLS1NmzdvltVqVVdXl4qKitTb26s777xT1dXVuvXWW4PaGDDTBfukmmDfD/pmFh8fr02bNikrK0uzZ8/WvffeK5vNJrvdHljHbreru7tb58+fV3x8vKxW65hxSTp79mzge6xWq+Lj43Xu3Dl9/etfn3At8+fHm9iZeULx+xdNv+P0EpmipRcz+7hhsPz4449VVlam9vZ2SdLQ0JBKSkpUV1enpKQkrVu3TgcOHFBGRoaKior0/PPPa+nSpSopKVF9fb1yc3MD0zzZ2dn6x3/8R9XW1qqoqMi0JoBoFIqTaqTLJ9bATCdPntSbb76pd999V3PnztUzzzyjDz74QBaLJbCOYRiyWCyBr1f66vKV3xMTM7mJpt7efvn9xoTXD9UfSq+3L6g/326fG/RthAq9RKZo6WWyfcTEWK77gvWGz1D19fWqqKiQw+GQJB09elQpKSlasGCBrFarcnJy5PF41NnZOelpHgCIRgcPHlR6errmz5+vWbNmye1268MPPxxz8k1PT48cDocSEhLU19cnn88nSfJ6vYHnW4fDoZ6eHknS6OioBgYGNG/evNA3BAATdMN3LL/6YfErp2akL5/4uru7x41PZJpnMsI1nRNpb3NHWj2TNdPrv9JkeommvqdqIv8G0fLvlJqaqp///OcaHBzU7NmztX//ft13331qaWlRW1ubvvvd76qxsVFOp1M2m01paWlqbm5WTk6OGhoa5HQ6JUkZGRlqaGjQ+vXr1dzcrLS0NNlswb3mKABMx6SvY+n3+686nXOt8clM81zPZKdzzBBpb3MHs55omf6SIq+Xqe63aAlZl93o3yDUx9uNpnOm48EHH9Tx48fldrtls9l0zz33aO3atfrBD36gsrIy9ff3a9GiRcrPz5ckVVRUqLi4WDt27FBSUpK2bdsmSdq0aZOKi4uVnZ2tuXPnqrq6Oij1AoBZJh0sExMTx0znXJ62+er41aZ5YmNjx0zzYGKuPIljJoeNEb9vRtcPTMbatWu1du3aMWOpqanatWvXuHWTk5NVV1c3bnzevHl66aWXglYjAJht0sFyyZIlOnXqlDo6OnT77berqalJP/rRj5ScnKy4uLhJTfNgYkJ3Z5TgssXERs3JKJMNyQRqAMDNYNLBMi4uTlVVVdqwYYOGh4eVkZEhl8slSaqurp7UNA8wU4UiJHO2NgBgpplwsNy/f3/g/9PT07V3795x60x2mgcAAADRgzvvAAAAwBSTngrH/wn2nVEAAABmEoLlNHBnFAAAgP/DVDgAAABMQbAEAACAKaJ2Ktyszz9y/UEAAICJidpgGS0XFQcAAJgpmAoHAACAKQiWAAAAMEXUToUDiBwTvbf6dD/TPDw6ogvnh6b1MwAAU0ewBBB0obi3unT5c88ESwAIF6bCAQAAYAqCJQAAAExBsAQAAIApCJYAAAAwBcESAAAApiBYAgAAwBQESwAAAJiCYAkAAABTcIF0AAiCnTt36pe//GVg+bPPPtOqVav0yCOPqLKyUsPDw8rKylJhYaEk6cSJEyotLdXAwIDS0tK0efNmWa1WdXV1qaioSL29vbrzzjtVXV2tW2+9NVxtAcB18Y4lAATBY489psbGRjU2Nqq6ulrz58/Xk08+qZKSEtXW1qq5uVnHjh3TgQMHJElFRUUqLy9XS0uLDMNQfX29JGnz5s3Kzc2Vx+PR4sWLVVtbG862AOC6CJYAEGR//dd/rcLCQp0+fVopKSlasGCBrFarcnJy5PF41NnZqaGhIS1dulSS5Ha75fF4NDIyosOHDyszM3PMOABEKqbCASCIWltbNTQ0pKysLDU1Nclutwceczgc6u7u1tmzZ8eM2+12dXd36/z584qPj5fVah0zPhnz58eb04jJ7Pa5UbGNUKGXyBQtvZjZB8ESAILo3/7t3/TjH/9YkuT3+2WxWAKPGYYhi8VyzfHLX6/01eUb6e3tl99vTHj9UP2h9Hr7gvrz7fa5Qd9GqNBLZIqWXibbR0yM5bovWJkKB4AguXTpkg4fPqyHH35YkpSYmCiv1xt43Ov1yuFwjBvv6emRw+FQQkKC+vr65PP5xqwPAJGKYAkAQfK73/1Od9xxh+bMmSNJWrJkiU6dOqWOjg75fD41NTXJ6XQqOTlZcXFxamtrkyQ1NjbK6XTKZrMpLS1Nzc3NkqSGhgY5nc6w9QMAN8JUOAAEyenTp5WYmBhYjouLU1VVlTZs2KDh4WFlZGTI5XJJkqqrq1VWVqb+/n4tWrRI+fn5kqSKigoVFxdrx44dSkpK0rZt28LSCwBMBMESAIJkxYoVWrFixZix9PR07d27d9y6qamp2rVr17jx5ORk1dXVBa1GADATU+EAAAAwBcESAAAApiBYAgAAwBQESwAAAJiCYAkAAABTECwBAABgCoIlAAAATEGwBAAAgCkIlgAAADAFwRIAAACmIFgCAADAFARLAAAAmGJawbKxsVHZ2dnKzs7W1q1bJUknTpyQ2+1WZmamSktLNTo6Kknq6upSXl6eXC6XCgoKNDAwMP3qAQAAEDGmHCwvXryoLVu2qK6uTo2NjTpy5IhaW1tVVFSk8vJytbS0yDAM1dfXS5I2b96s3NxceTweLV68WLW1taY1AQAAgPCbcrD0+Xzy+/26ePGiRkdHNTo6KqvVqqGhIS1dulSS5Ha75fF4NDIyosOHDyszM3PMOAAAAKKHdarfGB8fr02bNikrK0uzZ8/WvffeK5vNJrvdHljHbreru7tb58+fV3x8vKxW65jxyZg/P36qpQK4idjtc8NdAgDctKYcLE+ePKk333xT7777rubOnatnnnlGH3zwgSwWS2AdwzBksVgCX6/01eUb6e3tl99vTHh9/rgANyevt29C68XEWHjBCgAmm/JU+MGDB5Wenq758+dr1qxZcrvd+vDDD+X1egPr9PT0yOFwKCEhQX19ffL5fJIkr9crh8Mx/eoBAAAQMaYcLFNTU9Xa2qrBwUEZhqH9+/frvvvuU1xcnNra2iR9eda40+mUzWZTWlqampubJUkNDQ1yOp3mdAAAAICIMOWp8AcffFDHjx+X2+2WzWbTPffco7Vr1+oHP/iBysrK1N/fr0WLFik/P1+SVFFRoeLiYu3YsUNJSUnatm2baU0AAAAg/KYcLCVp7dq1Wrt27Zix1NRU7dq1a9y6ycnJqqurm87mAAAAEMG48w4AAABMQbAEgCDYv3+/3G63srKy9Pzzz0uSWltblZOTo+XLl6umpiawLncsAxAtCJYAYLLTp0+roqJCtbW12rt3r44fP64DBw6opKREtbW1am5u1rFjx3TgwAFJ4o5lAKIGwRIATPbOO+9oxYoVSkxMlM1mU01NjWbPnq2UlBQtWLBAVqtVOTk58ng86uzs5I5lAKLGtE7eAQCM19HRIZvNpvXr1+vMmTN66KGHtHDhwjF3JnM4HOru7tbZs2eDdscyKXLvWhaKm1hE040y6CUyRUsvZvZBsAQAk/l8Ph05ckR1dXWaM2eOCgoKdMstt1z1zmR+vz9odyyTIveuZRO9Q9JU2e1zg76NUKGXyBQtvUy2jxvdtYxgCQAmu+2225Senq6EhARJ0iOPPCKPx6PY2NjAOpfvQJaYmHjDO5bFxsZyxzIAMwKfsQQAky1btkwHDx7UhQsX5PP59P7778vlcunUqVPq6OiQz+dTU1OTnE6nkpOTuWMZgKjBO5YAYLIlS5bopz/9qXJzczUyMqIHHnhAjz/+uO666y5t2LBBw8PDysjIkMvlkiRVV1dzxzIAUYFgCQBBsHr1aq1evXrMWHp6uvbu3TtuXe5YBiBaMBUOAAAAUxAsAQAAYAqCJQAAAExBsAQAAIApCJYAAAAwBcESAAAApiBYAgAAwBQESwAAAJiCYAkAAABTECwBAABgCoIlAAAATEGwBAAAgCkIlgAAADAFwRIAAACmIFgCAADAFARLAAAAmIJgCQAAAFMQLAEAAGAKgiUAAABMQbAEAACAKQiWAAAAMAXBEgAAAKawhrsAAIhWf/EXf6Fz587Jav3yqfZv/uZv9Omnn2rHjh0aHR3VE088oby8PElSa2urKisrNTw8rKysLBUWFkqSTpw4odLSUg0MDCgtLU2bN28O/DwAiDS8YwkAQWAYhtrb29XY2Bj4LzExUTU1NXrjjTfU0NCgf//3f9d///d/a2hoSCUlJaqtrVVzc7OOHTumAwcOSJKKiopUXl6ulpYWGYah+vr6MHcGANdGsASAIPj9738vSfrJT36iP/3TP9Uvf/lLtba26v7779e8efM0Z84cZWZmyuPx6OjRo0pJSdGCBQtktVqVk5Mjj8ejzs5ODQ0NaenSpZIkt9stj8cTzrYA4LqYTwGAILhw4YLS09P1s5/9TCMjI8rPz1dWVpbsdntgHYfDoaNHj+rs2bPjxru7u8eN2+12dXd3T6qO+fPjp99MENjtc6NiG6FCL5EpWnoxsw+CJQAEwXe+8x195zvfCSyvXr1alZWVKigoCIwZhiGLxSK/3y+LxTLh8cno7e2X329MeP1Q/aH0evuC+vPt9rlB30ao0EtkipZeJttHTIzlui9YmQoHgCA4cuSIDh06FFg2DEPJycnyer2BMa/XK4fDocTExAmN9/T0yOFwhKYBAJgCgiUABEFfX59eeOEFDQ8Pq7+/X3v27NHPf/5zHTp0SOfOndPFixf19ttvy+l0asmSJTp16pQ6Ojrk8/nU1NQkp9Op5ORkxcXFqa2tTZLU2Ngop9MZ5s4A4NqYCgeAIFi2bJk+/vhjPfroo/L7/crNzdV3v/tdFRYWKj8/XyMjI1q9erW+/e1vS5Kqqqq0YcMGDQ8PKyMjQy6XS5JUXV2tsrIy9ff3a9GiRcrPzw9nWwBwXdMKlvv379f27dt18eJFPfDAAyorK+NabADw/3v66af19NNPjxnLyclRTk7OuHXT09O1d+/eceOpqanatWtX0GoEADNNeSr89OnTqqioUG1trfbu3avjx4/rwIEDXIsNAADgJjXlYPnOO+9oxYoVSkxMlM1mU01NjWbPns212AAAAG5SU56L7ujokM1m0/r163XmzBk99NBDWrhw4U13LTYAkSVarisHADPRlIOlz+fTkSNHVFdXpzlz5qigoEC33HLLTXctNgCRZaLXY7vRtdgAAJM35WB52223KT09XQkJCZKkRx55RB6PR7GxsYF1uBYbAADAzWPKn7FctmyZDh48qAsXLsjn8+n999+Xy+XiWmwAAAA3qSm/Y7lkyRL99Kc/VW5urkZGRvTAAw/o8ccf11133cW12AAAAG5C07qQ5OrVq7V69eoxY1yLDQAA4ObELR0BAABgCoIlAAAATEGwBAAAgCkIlgAAADAFwRIAAACmIFgCAADAFARLAAAAmIJgCQAAAFMQLAEAAGAKgiUAAABMQbAEAACAKQiWAAAAMAXBEgAAAKYgWAJAEG3dulXFxcWSpBMnTsjtdiszM1OlpaUaHR2VJHV1dSkvL08ul0sFBQUaGBiQJF24cEFr165VVlaW8vLy5PV6w9YHAEwEwRIAguTQoUPas2dPYLmoqEjl5eVqaWmRYRiqr6+XJG3evFm5ubnyeDxavHixamtrJUkvvvii0tLStG/fPj322GPasmVLWPoAgIkiWAJAEHz++eeqqanR+vXrJUmdnZ0aGhrS0qVLJUlut1sej0cjIyM6fPiwMjMzx4xL0nvvvaecnBxJ0sqVK/XrX/9aIyMjYegGACaGYAkAQVBeXq7CwkJ97WtfkySdPXtWdrs98Ljdbld3d7fOnz+v+Ph4Wa3WMeNf/R6r1ar4+HidO3cuxJ0AwMRZw10AAESbnTt3KikpSenp6dq9e7ckye/3y2KxBNYxDEMWiyXw9UpfXb7ye2JiJvd+wPz58ZOsPjTs9rlRsY1QoZfIFC29mNkHwRIATNbc3Cyv16tVq1bpiy++0ODgoCwWy5iTb3p6euRwOJSQkKC+vj75fD7FxsbK6/XK4XBIkhwOh3p6epSYmKjR0VENDAxo3rx5k6qlt7dffr8x4fVD9YfS6+0L6s+32+cGfRuhQi+RKVp6mWwfMTGW675gZSocAEz26quvqqmpSY2Njdq4caMefvhhVVZWKi4uTm1tbZKkxsZGOZ1O2Ww2paWlqbm5WZLU0NAgp9MpScrIyFBDQ4OkL8NqWlqabDZbeJoCgAkgWAJAiFRXV6uyslIul0uDg4PKz8+XJFVUVKi+vl4rVqzQkSNH9PTTT0uSNm3apN/85jfKzs7WG2+8ofLy8nCWDwA3xFQ4AASR2+2W2+2WJKWmpmrXrl3j1klOTlZdXd248Xnz5umll14Keo0AYBbesQQAAIApCJYAAAAwBcESAAAApiBYAgAAwBQESwAAAJiCYAkAAABTECwBAABgCoIlAAAATEGwBAAAgCkIlgAAADAFwRIAAACmIFgCAADAFARLAAAAmIJgCQAAAFMQLAEAAGAKgiUAAABMQbAEAACAKaYdLLdu3ari4mJJ0okTJ+R2u5WZmanS0lKNjo5Kkrq6upSXlyeXy6WCggINDAxMd7MAAACIMNMKlocOHdKePXsCy0VFRSovL1dLS4sMw1B9fb0kafPmzcrNzZXH49HixYtVW1s7vaoBAAAQcaYcLD///HPV1NRo/fr1kqTOzk4NDQ1p6dKlkiS32y2Px6ORkREdPnxYmZmZY8YBAAAQXaYcLMvLy1VYWKivfe1rkqSzZ8/KbrcHHrfb7eru7tb58+cVHx8vq9U6ZhwAAADRxTqVb9q5c6eSkpKUnp6u3bt3S5L8fr8sFktgHcMwZLFYAl+v9NXliZg/P34qpQK4ydjtc8NdAgDctKYULJubm+X1erVq1Sp98cUXGhwclMVikdfrDazT09Mjh8OhhIQE9fX1yefzKTY2Vl6vVw6HY9Lb7O3tl99vTHh9/rgANyevt29C68XEWIL6gvXv//7v1dLSIovFotWrV+vHP/6xWltbVVlZqeHhYWVlZamwsFDSlyc+lpaWamBgQGlpadq8ebOsVqu6urpUVFSk3t5e3Xnnnaqurtatt94atJoBYLqmNBX+6quvqqmpSY2Njdq4caMefvhhVVZWKi4uTm1tbZKkxsZGOZ1O2Ww2paWlqbm5WZLU0NAgp9NpXgcAEGE++ugj/ed//qf27t2rN998U3V1dTp58qRKSkpUW1ur5uZmHTt2TAcOHJDEiY8Aooep17Gsrq5WZWWlXC6XBgcHlZ+fL0mqqKhQfX29VqxYoSNHjujpp582c7MAEFHuu+8+/eIXv5DValVvb698Pp8uXLiglJQULViwQFarVTk5OfJ4PJz4CCCqTGkq/Eput1tut1uSlJqaql27do1bJzk5WXV1ddPdFADMGDabTf/wD/+gf/mXf5HL5Rp3gqPD4VB3dzcnPgKIKtMOlgCAq9u4caOefPJJrV+/Xu3t7Vc9wfFmPfExFJ+Dj6bP2tNLZIqWXszsg2AJACb7n//5H126dEl/8id/otmzZ2v58uXyeDyKjY0NrHP5RMbExMSb8sTHiZ5kNVV2+9ygbyNU6CUyRUsvk+3jRic+cq9wADDZZ599prKyMl26dEmXLl3Sr371K61Zs0anTp1SR0eHfD6fmpqa5HQ6lZyczImPAKIG71gCgMkyMjJ09OhRPfroo4qNjdXy5cuVnZ2thIQEbdiwQcPDw8rIyJDL5ZL05YmPZWVl6u/v16JFi8ac+FhcXKwdO3YoKSlJ27ZtC2dbAHBDBEsACIINGzZow4YNY8bS09O1d+/ecety4iOAaMFUOAAAAExBsAQAAIApCJYAAAAwBcESAAAApiBYAgAAwBQESwAAAJiCYAkAAABTECwBAABgCoIlAAAATEGwBAAAgCkIlgAAADAFwRIAAACmIFgCAADAFARLAAAAmIJgCQAAAFMQLAEAAGAKgiUAAABMQbAEAACAKQiWAAAAMAXBEgAAAKYgWAIAAMAUBEsAAACYgmAJAEGwfft2ZWdnKzs7Wy+88IIkqbW1VTk5OVq+fLlqamoC6544cUJut1uZmZkqLS3V6OioJKmrq0t5eXlyuVwqKCjQwMBAWHoBgIkiWAKAyVpbW3Xw4EHt2bNHDQ0N+u1vf6umpiaVlJSotrZWzc3NOnbsmA4cOCBJKioqUnl5uVpaWmQYhurr6yVJmzdvVm5urjwejxYvXqza2tpwtgUAN0SwBACT2e12FRcXa9asWbLZbLr77rvV3t6ulJQULViwQFarVTk5OfJ4POrs7NTQ0JCWLl0qSXK73fJ4PBoZGdHhw4eVmZk5ZhwAIhnBEgBMtnDhwkBQbG9v1759+2SxWGS32wPrOBwOdXd36+zZs2PG7Xa7uru7df78ecXHx8tqtY4ZB4BIZg13AQAQrT755BOtW7dOzz77rGJjY9Xe3h54zDAMWSwW+f1+WSyWceOXv17pq8sTMX9+/JTrDya7fW5UbCNU6CUyRUsvZvZBsASAIGhra9PGjRtVUlKi7OxsffTRR/J6vYHHvV6vHA6HEhMTx4z39PTI4XAoISFBfX198vl8io2NDaw/Wb29/fL7jQmvH6o/lF5vX1B/vt0+N+jbCBV6iUzR0stk+4iJsVz3BStT4QBgsjNnzuipp55SdXW1srOzJUlLlizRqVOn1NHRIZ/Pp6amJjmdTiUnJysuLk5tbW2SpMbGRjmdTtlsNqWlpam5uVmS1NDQIKfTGbaeAGAieMcSAEz2yiuvaHh4WFVVVYGxNWvWqKqqShs2bNDw8LAyMjLkcrkkSdXV1SorK1N/f78WLVqk/Px8SVJFRYWKi4u1Y8cOJSUladu2bWHpBwAmimAJACYrKytTWVnZVR/bu3fvuLHU1FTt2rVr3HhycrLq6upMrw8AgoWpcAAAAJiCYAkAAABTECwBAABgCoIlAAAATDGtYLl9+3ZlZ2crOztbL7zwgqQv75Gbk5Oj5cuXq6amJrDuiRMn5Ha7lZmZqdLSUo2Ojk6vcgAAAESUKQfL1tZWHTx4UHv27FFDQ4N++9vfqqmpSSUlJaqtrVVzc7OOHTumAwcOSJKKiopUXl6ulpYWGYah+vp605oAAABA+E05WNrtdhUXF2vWrFmy2Wy6++671d7erpSUFC1YsEBWq1U5OTnyeDzq7OzU0NBQ4N65brdbHo/HtCYAAAAQflMOlgsXLgwExfb2du3bt08Wi0V2uz2wjsPhUHd3t86ePTtm3G63q7u7explAwAAINJM+wLpn3zyidatW6dnn31WsbGxam9vDzxmGIYsFov8fr8sFsu48cm43n0pAeCyUN3rGgAw3rSCZVtbmzZu3KiSkhJlZ2fro48+ktfrDTzu9XrlcDiUmJg4Zrynp0cOh2NS2+rt7Zffb0x4ff64ADcnr7dvQuvFxFh4wQoAJpvyVPiZM2f01FNPqbq6WtnZ2ZKkJUuW6NSpU+ro6JDP51NTU5OcTqeSk5MVFxentrY2SVJjY6OcTqc5HQAAACAiTPkdy1deeUXDw8OqqqoKjK1Zs0ZVVVXasGGDhoeHlZGRIZfLJUmqrq5WWVmZ+vv7tWjRIuXn50+/egAAAESMKQfLsrIylZWVXfWxvXv3jhtLTU3Vrl27pro5AAAARDjuvAMAAABTECwBAABgCoIlAAAATEGwBAAAgCkIlgAAADAFwRIAAACmIFgCAADAFARLAAAAmIJgCQAAAFMQLAEgSPr7+7Vy5Up99tlnkqTW1lbl5ORo+fLlqqmpCax34sQJud1uZWZmqrS0VKOjo5Kkrq4u5eXlyeVyqaCgQAMDA2HpAwAmimAJAEHw8ccf6/HHH1d7e7skaWhoSCUlJaqtrVVzc7OOHTumAwcOSJKKiopUXl6ulpYWGYah+vp6SdLmzQv2mAUAAAzCSURBVJuVm5srj8ejxYsXq7a2NlztAMCEECwBIAjq6+tVUVEhh8MhSTp69KhSUlK0YMECWa1W5eTkyOPxqLOzU0NDQ1q6dKkkye12y+PxaGRkRIcPH1ZmZuaYcQCIZNZwFwAA0WjLli1jls+ePSu73R5Ydjgc6u7uHjdut9vV3d2t8+fPKz4+Xlardcz4ZM2fHz/FDoLLbp8bFdsIFXqJTNHSi5l9ECwBIAT8fr8sFktg2TAMWSyWa45f/nqlry5PRG9vv/x+Y8Lrh+oPpdfbF9Sfb7fPDfo2QoVeIlO09DLZPmJiLNd9wcpUOACEQGJiorxeb2DZ6/XK4XCMG+/p6ZHD4VBCQoL6+vrk8/nGrA8AkYxgCQAhsGTJEp06dUodHR3y+XxqamqS0+lUcnKy4uLi1NbWJklqbGyU0+mUzWZTWlqampubJUkNDQ1yOp3hbAEAboipcAAIgbi4OFVVVWnDhg0aHh5WRkaGXC6XJKm6ulplZWXq7+/XokWLlJ+fL0mqqKhQcXGxduzYoaSkJG3bti2cLQDADREsASCI9u/fH/j/9PR07d27d9w6qamp2rVr17jx5ORk1dXVBbU+ADATU+EAAAAwBcESAAAApiBYAgAAwBQESwAAAJiCYAkAAABTECwBAABgCoIlAAAATEGwBAAAgCkIlgAAADAFd94BAITUiN8nu31u0LcBIPQIlgCAkLLFxGrd+28EdRsv/7/coP58AFfHVDgAAABMQbAEAACAKQiWAAAAMAXBEgAAAKYgWAIAAMAUBEsAAACYgssNAQCiTiiulSlJw6MjunB+KOjbAWYKgiUAIOqE4lqZ0uXrZRIsgcsIlgAATBF3EQLGIlgCADBF3EUIGItgCQBABAvV50Uv+UY1Kza4sYB3X6NfSIPlW2+9pR07dmh0dFRPPPGE8vLyQrl5AJiReO68uYXy86LB3s72B/4s6CGZE6rCK2TBsru7WzU1Ndq9e7dmzZqlNWvW6Hvf+56++c1vhqoEAJhxeO5ENAlFSA5FeJW+fIc32NuZie8ihyxYtra26v7779e8efMkSZmZmfJ4PPrLv/zLCX1/TIxl0tucH3frpL8nErcRqu1EyzZCtZ1o2UaothOqXib6XDGV55Rw4LkzsrcRqu3Qy8TZYmJV8lFjULchSX9336qgbydU25jM88SN1rUYhmFMt6iJePnllzU4OKjCwkJJ0s6dO3X06FH97d/+bSg2DwAzEs+dAGaSkN15x+/3y2L5v5RrGMaYZQDAeDx3AphJQhYsExMT5fV6A8ter1cOhyNUmweAGYnnTgAzSciC5fe//30dOnRI586d08WLF/X222/L6XSGavMAMCPx3AlgJgnZyTtf//rXVVhYqPz8fI2MjGj16tX69re/HarNA8CMxHMngJkkZCfvAAAAILqFbCocAAAA0Y1gCQAAAFMQLAEAAGAKgiUAAABMQbAEAACAKQiW13H27FmtXbtWjz76qNasWaPPPvss3CUFHD9+XIsXLw53GWpra9Pq1au1atUqPfHEE+rs7AxbLW+99ZZWrFih5cuX6/XXXw9bHVfavn27srOzlZ2drRdeeCHc5QRs3bpVxcXF4S5DkrR//3653W5lZWXp+eefD3c5N61IPH5u5GrHV2trq3JycrR8+XLV1NQE1j1x4oTcbrcyMzNVWlqq0dHRcJV9TVcel9eqt6urS3l5eXK5XCooKNDAwEA4Sx7nasfzTN0njY2Ngd+vrVu3SppZ+6W/v18rV64MZJfJ7ocp92Tgmp544gnjjTfeMAzDMN544w1j06ZNYa7oS4ODg8aaNWuMP/7jPw53KcayZcuMEydOGIZhGDt37jTWr18fljr+93//11i2bJlx/vx5Y2BgwMjJyTE++eSTsNRy2QcffGD82Z/9mTE8PGxcunTJyM/PN95+++2w1mQYhtHa2mp873vfM/7qr/4q3KUYn376qfHggw8aZ86cMS5dumQ8/vjjxnvvvRfusm46kXj83MjVjq+33nrLyMjIMD799FNjZGTE+MlPfhL4fcrOzjb+67/+yzAMw3juueeM119/PZzlj/PV4/Ja9a5du9ZoamoyDMMwtm/fbrzwwgvhKfgqrnU8z8R9Mjg4aNx7771Gb2+vMTIyYqxevdr44IMPZsx++c1vfmOsXLnSWLRokXH69Gnj4sWLk94PU+2Jdyyv4dy5czp58qTWrFkjSfrRj36kp59+OsxVfamqqkpPPPFEuMvQpUuXtGnTJqWmpkqSvvWtb+nMmTNhqaW1tVX333+/5s2bpzlz5igzM1MejycstVxmt9tVXFysWbNmyWaz6e6771ZXV1dYa/r8889VU1Oj9evXh7WOy9555x2tWLFCiYmJstlsqqmp0ZIlS8Jd1k0nEo+fG7na8dXe3q6UlBQtWLBAVqtVOTk58ng86uzs1NDQkJYuXSpJcrvdEdXfV4/La9U7MjKiw4cPKzMzc8x4pLja8Tx79uwZuU98Pp/8fr8uXryo0dFRjY6Oymq1zpj9Ul9fr4qKisDtX48ePTqp/TCdnkJ2552Z5vTp0/qjP/ojVVVV6ciRI7Lb7frZz34W7rL0q1/9SkNDQ3K5XOEuRbNmzdKqVaskSX6/X9u3b9cjjzwSllrOnj0ru90eWHY4HDp69GhYarls4cKFgf9vb2/Xvn379K//+q9hrEgqLy9XYWFh2F4AfFVHR4dsNpvWr1+vM2fO6KGHHoqYF3A3k0g8fm7kasfXn//5n4/ro7u7e1x/drtd3d3dIa33er56XF6r3vPnzys+Pl5Wq3XMeKS42vG8cOHCGblP4uPjtWnTJmVlZWn27Nm69957ZbPZZsx+2bJly5jlqx3j19sP0+mJYClp3759qqysHDOWkpKi48ePa8OGDXruuee0c+dOFRcXq66uLmw13XXXXerv79drr70WkhomUs9rr72mS5cuqbi4WKOjo1q3bl3Ia5O+DLYWiyWwbBjGmOVw+uSTT7Ru3To9++yzuuOOO8JWx86dO5WUlKT09HTt3r07bHVcyefz6ciRI6qrq9OcOXNUUFCgPXv2yO12h7u0m0okHz83cuXxFRsbq/b29sBjl/uI5P6udlxeq96r1R0pfUhXP55vueWWq/YSyftEkk6ePKk333xT7777rubOnatnnnlGH3zwwYzcL9K1f6eC8btGsJSUlZWlrKysMWOffvqpfvjDH2rZsmWSpJUrV4b0xIKr1bRz5069/PLLysvLC4ytWrVKr7/+uuLj40NejyQNDAyooKBA8+bN044dO2Sz2YJax7UkJibqyJEjgWWv1xuYAgintrY2bdy4USUlJcrOzg5rLc3NzfJ6vVq1apW++OILDQ4O6u/+7u9UUlIStppuu+02paenKyEhQZL0yCOP6OjRowTLEIvU4+dGvnp8ffTRR/J6vYHHL/eRmJg4Zrynpydi+rvacWmxWK5ab0JCgvr6+uTz+RQbGxtx++lqx7PH41FsbGxgnZmwTyTp4MGDSk9P1/z58yV9ORX8yiuvzMj9Imncv/eN9sN0euIzltfwjW98Q4mJiTpw4IAk6d1339WiRYvCWtNjjz2m//iP/1BjY6MaGxslfXnWWrBD5fUUFRUpJSVFL774ombNmhW2Or7//e/r0KFDOnfunC5evKi3335bTqczbPVI0pkzZ/TUU0+puro67KFSkl599VU1NTWpsbFRGzdu1MMPPxzWUClJy5Yt08GDB3XhwgX5fD69//77YT/ObkaRePzcyNWOryVLlujUqVPq6OiQz+dTU1OTnE6nkpOTFRcXp7a2NklfPm9GSn9XOy4rKyuvWq/NZlNaWpqam5slSQ0NDRHTh3T149nlcs24fSJJqampam1t1eDgoAzD0P79+3XffffNyP0iTf7YmE5PFsMwjKB1MsP9/ve/V0VFReCzBlVVVWGdyvyqb33rW/rd734Xtu0fP35cP/zhD/XNb34z8DkMh8Ohf/7nfw5LPW+99ZZefvlljYyMaPXq1XryySfDUsdlzz//vN5880194xvfCIytWbNGjz/+eBir+tLu3bv10UcfqaqqKtylaNeuXXrttdc0MjKiBx54QGVlZYqJ4TVvqEXa8XMj1zq+7rjjDlVWVmp4eFgZGRl67rnnZLFYdPLkSZWVlam/v1+LFi1SZWVlWF8MX82Vx+W16u3s7FRxcbF6e3uVlJSkbdu26Q/+4A/CXXrA1Y7nDz/8cEbuk3/6p3/S7t27ZbPZdM8996iiokKnTp2aUfvl4Ycf1i9+8QvdfvvtOnTo0KT2w1R7IlgCAADAFLwtAAAAAFMQLAEAAGAKgiUAAABMQbAEAACAKQiWAAAAMAXBEgAAAKYgWAIAAMAU/x+N2NjOoCT0rQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 792x360 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "a, b, size = 1e-3, 1e3, 10000\n",
    "vals = log_uniform(a=a, b=b).rvs(size=size)\n",
    "print(kstest(rvs=np.log(vals), cdf='uniform', args=(np.log(a), np.log(b / a)), N=size))\n",
    "print(pd.Series(vals).describe())\n",
    "plt.figure(figsize=(11, 5))\n",
    "plt.subplot(121)\n",
    "pd.Series(np.log(vals)).hist()\n",
    "plt.subplot(122)\n",
    "pd.Series(vals).hist()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Exercises"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1.GridSearch on Synthetic Data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Here I apply GridSearch to a synthetic dataset. I use `timeit.default_timer` to measure time spent on fitting."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total fitting time: 405.809 seconds\n"
     ]
    }
   ],
   "source": [
    "X, cont = get_test_data(n_features=10, n_informative=5, n_redundant=0, n_samples=2000)\n",
    "\n",
    "clf = SVC(kernel='rbf', probability=True)\n",
    "param_grid = {'C': [1e2, 1e-1, 1, 10, 100], 'gamma': [1e-2, 1e-1, 1, 10, 100]}\n",
    "\n",
    "start_time = default_timer()\n",
    "gs = GridSearchCV(estimator=clf, param_grid=param_grid, scoring='neg_log_loss', n_jobs=-1, cv=10)\n",
    "gs.fit(X, cont['bin'], sample_weight=cont['w'])\n",
    "end_time = default_timer()\n",
    "print(f\"Total fitting time: {end_time - start_time:.3f} seconds\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Best parameters:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'C': 100.0, 'gamma': 0.1}"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gs.best_params_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Best score:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean CV score: -0.2913886473171636\n"
     ]
    }
   ],
   "source": [
    "print(f'Mean CV score: {gs.best_score_}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. Randomized GridSearch on Synthetic Data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Here I apply Randomized GridSearch via the same function. The number of iterations is 25 which is the same as the number of nodes in the previous task."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total fitting time: 379.966 seconds\n"
     ]
    }
   ],
   "source": [
    "clf = SVC(kernel='rbf', probability=True)\n",
    "param_distributions = {'C': log_uniform(a=1e-2, b=1e2), 'gamma': log_uniform(a=1e-2, b=1e2)}\n",
    "\n",
    "start_time = default_timer()\n",
    "gs_rand = RandomizedSearchCV(estimator=clf, param_distributions=param_distributions, n_iter=25,\n",
    "                             scoring='neg_log_loss', n_jobs=-1, cv=10)\n",
    "gs_rand.fit(X, cont['bin'], sample_weight=cont['w'])\n",
    "end_time = default_timer()\n",
    "print(f\"Total fitting time: {end_time - start_time:.3f} seconds\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Best parameters:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'C': 32.222054950379814, 'gamma': 0.05494236104173599}"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gs_rand.best_params_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Best score:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean CV score: -0.33877779710671935\n"
     ]
    }
   ],
   "source": [
    "print(f'Mean CV score: {gs_rand.best_score_}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The best parameters of randomized GridSearch are quite close to those of the regular GridSearch, however, the score is lower."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3. Sharpe Ratio, Regular GridSearch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_IS_sharpe_ratio(clf: Any) -> float:\n",
    "    '''\n",
    "    Given a fitted gridsearch classifier, returns Sharpe ratio of the best estimator's in-sample forecasts.\n",
    "    '''\n",
    "    best_estimator_ind = np.argmin(clf.cv_results_['rank_test_score'])\n",
    "    mean_score = clf.cv_results_['mean_test_score'][best_estimator_ind]\n",
    "    std_score = clf.cv_results_['std_test_score'][best_estimator_ind]\n",
    "    if mean_score < 0:\n",
    "        return -mean_score / std_score\n",
    "    else:\n",
    "        return mean_score / std_score"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Sharpe ratio of regular GridSearch, `neg_log_loss` scoring:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sharpe ratio: 2.0717487544063835\n"
     ]
    }
   ],
   "source": [
    "print(f'Sharpe ratio: {get_IS_sharpe_ratio(gs)}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Sharpe ratio of regular GridSearch, `accuracy` scoring (need to refit GridSearch):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total fitting time: 305.060 seconds\n",
      "Sharpe ratio: 11.700428101519837\n"
     ]
    }
   ],
   "source": [
    "start_time = default_timer()\n",
    "gs_acc = GridSearchCV(estimator=clf, param_grid=param_grid, scoring='accuracy', n_jobs=-1, cv=10)\n",
    "gs_acc.fit(X, cont['bin'], sample_weight=cont['w'])\n",
    "end_time = default_timer()\n",
    "print(f\"Total fitting time: {end_time - start_time:.3f} seconds\")\n",
    "\n",
    "print(f'Sharpe ratio: {get_IS_sharpe_ratio(gs_acc)}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "As we see, `accuracy` scoring method gives much higher Sharpe ratio."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4. Sharpe Ratio, Randomized GridSearch"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Sharpe ratio of randomized GridSearch, `neg_log_loss` scoring:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sharpe ratio: 2.005028594036192\n"
     ]
    }
   ],
   "source": [
    "print(f'Sharpe ratio: {get_IS_sharpe_ratio(gs_rand)}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Sharpe ratio of randomized GridSearch, `accuracy` scoring (need to refit GridSearch):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total fitting time: 268.132 seconds\n",
      "Sharpe ratio: 11.8319056411605\n"
     ]
    }
   ],
   "source": [
    "start_time = default_timer()\n",
    "gs_rand_acc = RandomizedSearchCV(estimator=clf, param_distributions=param_distributions, n_iter=25,\n",
    "                                 scoring='accuracy', n_jobs=-1, cv=10)\n",
    "gs_rand_acc.fit(X, cont['bin'], sample_weight=cont['w'])\n",
    "end_time = default_timer()\n",
    "print(f\"Total fitting time: {end_time - start_time:.3f} seconds\")\n",
    "\n",
    "print(f'Sharpe ratio: {get_IS_sharpe_ratio(gs_rand_acc)}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In case of randomized GS, `accuracy` scoring method also gives much higher Sharpe ratio."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}