1_classical_classifiers.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VyWFQyvcVql2"
      },
      "source": [
        "# Machine learning: exam assignment\n",
        "### Predicting Marvel vs DC\n",
        "\n",
        "#### Student: Jeroen Van Sweeveldt\n",
        "##### Assignment: binary classification, predicting the creator of the respective superhero using the 'history_text' feature.\n",
        "##### Part 2: training classical algorithms"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HoZr58mnVql3"
      },
      "source": [
        "### 1 Model preparation"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "P4jJBWH4Vql3"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Sklearn == 1.3.2\n"
          ]
        }
      ],
      "source": [
        "# imports for data exploration\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "\n",
        "# imports for data visualisation\n",
        "import matplotlib.pyplot as plt\n",
        "%matplotlib inline\n",
        "import seaborn as sns\n",
        "import pprint as pp\n",
        "\n",
        "# imports for classifiers\n",
        "import sklearn\n",
        "from sklearn.model_selection import train_test_split, cross_val_score, GridSearchCV\n",
        "from sklearn.pipeline import make_pipeline\n",
        "from sklearn.feature_extraction.text import TfidfVectorizer\n",
        "from sklearn.preprocessing import RobustScaler\n",
        "\n",
        "from sklearn import set_config\n",
        "\n",
        "from sklearn.dummy import DummyClassifier\n",
        "from sklearn.linear_model import LogisticRegression\n",
        "from sklearn.neighbors import KNeighborsClassifier\n",
        "from sklearn.svm import SVC\n",
        "from sklearn.naive_bayes import MultinomialNB\n",
        "\n",
        "# imports for evaluation\n",
        "from sklearn.metrics import classification_report, accuracy_score, ConfusionMatrixDisplay\n",
        "\n",
        "print(f\"Sklearn == {sklearn.__version__}\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "P7EEag0-Vql4"
      },
      "outputs": [],
      "source": [
        "# open the dataset as a comma separated values file\n",
        "heroes = pd.read_csv(\"data/heroes_preprocessed.csv\",\n",
        "                     encoding=\"UTF-8\",\n",
        "                     usecols=[\"creator\",\n",
        "                              \"hist_text_classic\",\n",
        "                              \"label\"\n",
        "                              ])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "id": "kTpdcEy7Vql4",
        "outputId": "2775d934-e2da-4682-8103-e258125143ff"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>hist_text_classic</th>\n",
              "      <th>creator</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>1513</th>\n",
              "      <td>xavi father make statue magneto professor x gi...</td>\n",
              "      <td>Marvel Comics</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1358</th>\n",
              "      <td>2006 \" civil war \" storyline , misty knight co...</td>\n",
              "      <td>Marvel Comics</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>55</th>\n",
              "      <td>aaron helzinger giant man prone fit homicidal ...</td>\n",
              "      <td>DC Comics</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1607</th>\n",
              "      <td>soldier fail kill vicki vale , ra al ghul deci...</td>\n",
              "      <td>DC Comics</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>63</th>\n",
              "      <td>aldrif odinsdottir firstborn odin freyja , hei...</td>\n",
              "      <td>Marvel Comics</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>189</th>\n",
              "      <td>daughter david cain sandra wu - san ( lady shi...</td>\n",
              "      <td>DC Comics</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1275</th>\n",
              "      <td>john jameson , son newspaper mogul j. jonah ja...</td>\n",
              "      <td>Marvel Comics</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1297</th>\n",
              "      <td>appear feeling wonder girl , claim \" hate wond...</td>\n",
              "      <td>DC Comics</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>680</th>\n",
              "      <td>early life n'jadaka father press work villain ...</td>\n",
              "      <td>Marvel Comics</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>570</th>\n",
              "      <td>note read , \" - . \"       deathstroke found me...</td>\n",
              "      <td>DC Comics</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                      hist_text_classic        creator  label\n",
              "1513  xavi father make statue magneto professor x gi...  Marvel Comics      1\n",
              "1358  2006 \" civil war \" storyline , misty knight co...  Marvel Comics      1\n",
              "55    aaron helzinger giant man prone fit homicidal ...      DC Comics      0\n",
              "1607  soldier fail kill vicki vale , ra al ghul deci...      DC Comics      0\n",
              "63    aldrif odinsdottir firstborn odin freyja , hei...  Marvel Comics      1\n",
              "189   daughter david cain sandra wu - san ( lady shi...      DC Comics      0\n",
              "1275  john jameson , son newspaper mogul j. jonah ja...  Marvel Comics      1\n",
              "1297  appear feeling wonder girl , claim \" hate wond...      DC Comics      0\n",
              "680   early life n'jadaka father press work villain ...  Marvel Comics      1\n",
              "570   note read , \" - . \"       deathstroke found me...      DC Comics      0"
            ]
          },
          "execution_count": 3,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "heroes.sample(10)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {},
      "outputs": [],
      "source": [
        "# account for the data imbalance\n",
        "c_weights = {0: 1.45, 1: 1.0}"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FvpyX3SmXUMX"
      },
      "source": [
        "Given the size of the dataset, I will perform an 80/10/10 split. With classical algorithms, we were taught to split the dataset in two sets, a train and test set, but since a neural network comes into play, I will split the data in three sets, a train, development, and test set.\n",
        "\n",
        "We were never shown how to apply this in a situation in which both a classical and neural algorithm were used, but my reasoning, based on what we were taught in the lectures, is that you can only perform a single train-test-split. (That means, no split with two sets for the classical and another split with three sets for the neural, as I imagine some people would do). Otherwise the results of our training will be unreliable, as we'll end up with different data distributions.\n",
        "\n",
        "Since the development set is used in the neural network as a substitute for the classical ML's cross-validation, I will perform the grid search for the classical algorithms on the dev set."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EFxAKteTl2lC",
        "outputId": "b65fe857-2ad4-4add-ceb0-161e0b6263fd"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "((1853,), (232,), (232,), (1853,), (232,), (232,))"
            ]
          },
          "execution_count": 5,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "X_text = heroes.hist_text_classic\n",
        "y_label = heroes.label\n",
        "\n",
        "\n",
        "X_train, X_dev_test, y_train, y_dev_test = train_test_split(X_text, y_label, test_size=0.2,\n",
        "                                                    random_state=256, stratify=y_label)\n",
        "\n",
        "X_dev, X_test, y_dev, y_test = train_test_split(X_dev_test, y_dev_test, test_size=0.5,\n",
        "                                                    random_state=256, stratify=y_dev_test)\n",
        "\n",
        "X_train.shape, X_dev.shape, X_test.shape, y_train.shape, y_dev.shape, y_test.shape"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "id": "FCAfhyDmVql-"
      },
      "outputs": [],
      "source": [
        "set_config(display=\"diagram\")\n",
        "\n",
        "vect = TfidfVectorizer(analyzer=\"word\", max_features=512)\n",
        "# we're going to use the RobustScaler because neither the\n",
        "# StandardScaler nor MinMaxScaler support sparse matrices\n",
        "# (the former cannot center the data in sparse matrices)\n",
        "# moreover, it yields better performance than the MaxAbsScaler\n",
        "# on the Logistic Regression model tests\n",
        "# this is probably due to the RobustScaler's ability to\n",
        "# handle outliers\n",
        "scaler = RobustScaler(with_centering=False)\n",
        "\n",
        "logreg = LogisticRegression(max_iter=5120, random_state=256)\n",
        "knn = KNeighborsClassifier()\n",
        "svm = SVC(random_state=256)\n",
        "nb = MultinomialNB()\n",
        "\n",
        "logreg_pipe = make_pipeline(vect, scaler, logreg)\n",
        "knn_pipe = make_pipeline(vect, scaler, knn)\n",
        "svm_pipe = make_pipeline(vect, scaler, svm)\n",
        "nb_pipe = make_pipeline(vect, scaler, nb)\n",
        "\n",
        "# baseline classifier:\n",
        "dummy = DummyClassifier(strategy=\"stratified\", random_state=256)\n",
        "\n",
        "dummy_pipe = make_pipeline(vect, scaler, dummy)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aqJtbxCvOBvY"
      },
      "source": [
        "#### 1.2 Baseline"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 161
        },
        "id": "xr3iD2-yM4V4",
        "outputId": "5a18bf4e-e6e0-40d1-e98b-cb536a670a79"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<style>#sk-container-id-1 {color: black;}#sk-container-id-1 pre{padding: 0;}#sk-container-id-1 div.sk-toggleable {background-color: white;}#sk-container-id-1 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-1 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-1 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-1 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-1 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-1 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-1 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-1 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-1 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-1 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-1 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-1 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-1 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-1 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-1 div.sk-item {position: relative;z-index: 1;}#sk-container-id-1 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-1 div.sk-item::before, #sk-container-id-1 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-1 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-1 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-1 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-1 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-1 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-1 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-1 div.sk-label-container {text-align: center;}#sk-container-id-1 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-1 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-1\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>Pipeline(steps=[(&#x27;tfidfvectorizer&#x27;, TfidfVectorizer(max_features=512)),\n",
              "                (&#x27;robustscaler&#x27;, RobustScaler(with_centering=False)),\n",
              "                (&#x27;dummyclassifier&#x27;,\n",
              "                 DummyClassifier(random_state=256, strategy=&#x27;stratified&#x27;))])</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-1\" type=\"checkbox\" ><label for=\"sk-estimator-id-1\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">Pipeline</label><div class=\"sk-toggleable__content\"><pre>Pipeline(steps=[(&#x27;tfidfvectorizer&#x27;, TfidfVectorizer(max_features=512)),\n",
              "                (&#x27;robustscaler&#x27;, RobustScaler(with_centering=False)),\n",
              "                (&#x27;dummyclassifier&#x27;,\n",
              "                 DummyClassifier(random_state=256, strategy=&#x27;stratified&#x27;))])</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-2\" type=\"checkbox\" ><label for=\"sk-estimator-id-2\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">TfidfVectorizer</label><div class=\"sk-toggleable__content\"><pre>TfidfVectorizer(max_features=512)</pre></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-3\" type=\"checkbox\" ><label for=\"sk-estimator-id-3\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">RobustScaler</label><div class=\"sk-toggleable__content\"><pre>RobustScaler(with_centering=False)</pre></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-4\" type=\"checkbox\" ><label for=\"sk-estimator-id-4\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">DummyClassifier</label><div class=\"sk-toggleable__content\"><pre>DummyClassifier(random_state=256, strategy=&#x27;stratified&#x27;)</pre></div></div></div></div></div></div></div>"
            ],
            "text/plain": [
              "Pipeline(steps=[('tfidfvectorizer', TfidfVectorizer(max_features=512)),\n",
              "                ('robustscaler', RobustScaler(with_centering=False)),\n",
              "                ('dummyclassifier',\n",
              "                 DummyClassifier(random_state=256, strategy='stratified'))])"
            ]
          },
          "execution_count": 7,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "dummy_pipe.fit(X_train, y_train)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KbwmF7EENlNk",
        "outputId": "72c66951-0994-4db9-a881-10366a1bfe36"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.41      0.38      0.40        95\n",
            "           1       0.59      0.63      0.61       137\n",
            "\n",
            "    accuracy                           0.53       232\n",
            "   macro avg       0.50      0.50      0.50       232\n",
            "weighted avg       0.52      0.53      0.52       232\n",
            "\n"
          ]
        }
      ],
      "source": [
        "y_test_pred_dummy = dummy_pipe.predict(X_test)\n",
        "print(classification_report(y_test, y_test_pred_dummy))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### 2 Model training\n",
        "#### 2.1 Logistic Regression"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "['logisticregression',\n",
            " 'logisticregression__C',\n",
            " 'logisticregression__class_weight',\n",
            " 'logisticregression__dual',\n",
            " 'logisticregression__fit_intercept',\n",
            " 'logisticregression__intercept_scaling',\n",
            " 'logisticregression__l1_ratio',\n",
            " 'logisticregression__max_iter',\n",
            " 'logisticregression__multi_class',\n",
            " 'logisticregression__n_jobs',\n",
            " 'logisticregression__penalty',\n",
            " 'logisticregression__random_state',\n",
            " 'logisticregression__solver',\n",
            " 'logisticregression__tol',\n",
            " 'logisticregression__verbose',\n",
            " 'logisticregression__warm_start',\n",
            " 'memory',\n",
            " 'robustscaler',\n",
            " 'robustscaler__copy',\n",
            " 'robustscaler__quantile_range',\n",
            " 'robustscaler__unit_variance',\n",
            " 'robustscaler__with_centering',\n",
            " 'robustscaler__with_scaling',\n",
            " 'steps',\n",
            " 'tfidfvectorizer',\n",
            " 'tfidfvectorizer__analyzer',\n",
            " 'tfidfvectorizer__binary',\n",
            " 'tfidfvectorizer__decode_error',\n",
            " 'tfidfvectorizer__dtype',\n",
            " 'tfidfvectorizer__encoding',\n",
            " 'tfidfvectorizer__input',\n",
            " 'tfidfvectorizer__lowercase',\n",
            " 'tfidfvectorizer__max_df',\n",
            " 'tfidfvectorizer__max_features',\n",
            " 'tfidfvectorizer__min_df',\n",
            " 'tfidfvectorizer__ngram_range',\n",
            " 'tfidfvectorizer__norm',\n",
            " 'tfidfvectorizer__preprocessor',\n",
            " 'tfidfvectorizer__smooth_idf',\n",
            " 'tfidfvectorizer__stop_words',\n",
            " 'tfidfvectorizer__strip_accents',\n",
            " 'tfidfvectorizer__sublinear_tf',\n",
            " 'tfidfvectorizer__token_pattern',\n",
            " 'tfidfvectorizer__tokenizer',\n",
            " 'tfidfvectorizer__use_idf',\n",
            " 'tfidfvectorizer__vocabulary',\n",
            " 'verbose']\n"
          ]
        }
      ],
      "source": [
        "pp.pprint(sorted(logreg_pipe.get_params().keys()))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {},
      "outputs": [],
      "source": [
        "logreg_params = [{\n",
        "    \"tfidfvectorizer__ngram_range\": [(1,1), (1,2), (2,3)],\n",
        "    \"logisticregression__C\": [0.01, 0.1, 1.0, 10, 100],\n",
        "    \"logisticregression__class_weight\": [None, \"balanced\", c_weights],\n",
        "    \"logisticregression__solver\": [\"newton-cg\", \"saga\"]\n",
        "    }]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Fitting 10 folds for each of 90 candidates, totalling 900 fits\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "<style>#sk-container-id-2 {color: black;}#sk-container-id-2 pre{padding: 0;}#sk-container-id-2 div.sk-toggleable {background-color: white;}#sk-container-id-2 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-2 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-2 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-2 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-2 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-2 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-2 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-2 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-2 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-2 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-2 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-2 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-2 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-2 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-2 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-2 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-2 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-2 div.sk-item {position: relative;z-index: 1;}#sk-container-id-2 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-2 div.sk-item::before, #sk-container-id-2 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-2 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-2 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-2 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-2 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-2 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-2 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-2 div.sk-label-container {text-align: center;}#sk-container-id-2 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-2 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-2\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>GridSearchCV(cv=10,\n",
              "             estimator=Pipeline(steps=[(&#x27;tfidfvectorizer&#x27;,\n",
              "                                        TfidfVectorizer(max_features=512)),\n",
              "                                       (&#x27;robustscaler&#x27;,\n",
              "                                        RobustScaler(with_centering=False)),\n",
              "                                       (&#x27;logisticregression&#x27;,\n",
              "                                        LogisticRegression(max_iter=5120,\n",
              "                                                           random_state=256))]),\n",
              "             n_jobs=-1,\n",
              "             param_grid=[{&#x27;logisticregression__C&#x27;: [0.01, 0.1, 1.0, 10, 100],\n",
              "                          &#x27;logisticregression__class_weight&#x27;: [None, &#x27;balanced&#x27;,\n",
              "                                                               {0: 1.45,\n",
              "                                                                1: 1.0}],\n",
              "                          &#x27;logisticregression__solver&#x27;: [&#x27;newton-cg&#x27;, &#x27;saga&#x27;],\n",
              "                          &#x27;tfidfvectorizer__ngram_range&#x27;: [(1, 1), (1, 2),\n",
              "                                                           (2, 3)]}],\n",
              "             scoring=&#x27;f1_macro&#x27;, verbose=1)</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-5\" type=\"checkbox\" ><label for=\"sk-estimator-id-5\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">GridSearchCV</label><div class=\"sk-toggleable__content\"><pre>GridSearchCV(cv=10,\n",
              "             estimator=Pipeline(steps=[(&#x27;tfidfvectorizer&#x27;,\n",
              "                                        TfidfVectorizer(max_features=512)),\n",
              "                                       (&#x27;robustscaler&#x27;,\n",
              "                                        RobustScaler(with_centering=False)),\n",
              "                                       (&#x27;logisticregression&#x27;,\n",
              "                                        LogisticRegression(max_iter=5120,\n",
              "                                                           random_state=256))]),\n",
              "             n_jobs=-1,\n",
              "             param_grid=[{&#x27;logisticregression__C&#x27;: [0.01, 0.1, 1.0, 10, 100],\n",
              "                          &#x27;logisticregression__class_weight&#x27;: [None, &#x27;balanced&#x27;,\n",
              "                                                               {0: 1.45,\n",
              "                                                                1: 1.0}],\n",
              "                          &#x27;logisticregression__solver&#x27;: [&#x27;newton-cg&#x27;, &#x27;saga&#x27;],\n",
              "                          &#x27;tfidfvectorizer__ngram_range&#x27;: [(1, 1), (1, 2),\n",
              "                                                           (2, 3)]}],\n",
              "             scoring=&#x27;f1_macro&#x27;, verbose=1)</pre></div></div></div><div class=\"sk-parallel\"><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-6\" type=\"checkbox\" ><label for=\"sk-estimator-id-6\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">estimator: Pipeline</label><div class=\"sk-toggleable__content\"><pre>Pipeline(steps=[(&#x27;tfidfvectorizer&#x27;, TfidfVectorizer(max_features=512)),\n",
              "                (&#x27;robustscaler&#x27;, RobustScaler(with_centering=False)),\n",
              "                (&#x27;logisticregression&#x27;,\n",
              "                 LogisticRegression(max_iter=5120, random_state=256))])</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-7\" type=\"checkbox\" ><label for=\"sk-estimator-id-7\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">TfidfVectorizer</label><div class=\"sk-toggleable__content\"><pre>TfidfVectorizer(max_features=512)</pre></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-8\" type=\"checkbox\" ><label for=\"sk-estimator-id-8\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">RobustScaler</label><div class=\"sk-toggleable__content\"><pre>RobustScaler(with_centering=False)</pre></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-9\" type=\"checkbox\" ><label for=\"sk-estimator-id-9\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">LogisticRegression</label><div class=\"sk-toggleable__content\"><pre>LogisticRegression(max_iter=5120, random_state=256)</pre></div></div></div></div></div></div></div></div></div></div></div></div>"
            ],
            "text/plain": [
              "GridSearchCV(cv=10,\n",
              "             estimator=Pipeline(steps=[('tfidfvectorizer',\n",
              "                                        TfidfVectorizer(max_features=512)),\n",
              "                                       ('robustscaler',\n",
              "                                        RobustScaler(with_centering=False)),\n",
              "                                       ('logisticregression',\n",
              "                                        LogisticRegression(max_iter=5120,\n",
              "                                                           random_state=256))]),\n",
              "             n_jobs=-1,\n",
              "             param_grid=[{'logisticregression__C': [0.01, 0.1, 1.0, 10, 100],\n",
              "                          'logisticregression__class_weight': [None, 'balanced',\n",
              "                                                               {0: 1.45,\n",
              "                                                                1: 1.0}],\n",
              "                          'logisticregression__solver': ['newton-cg', 'saga'],\n",
              "                          'tfidfvectorizer__ngram_range': [(1, 1), (1, 2),\n",
              "                                                           (2, 3)]}],\n",
              "             scoring='f1_macro', verbose=1)"
            ]
          },
          "execution_count": 11,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "logreg_gs = GridSearchCV(logreg_pipe,\n",
        "                         logreg_params,\n",
        "                         cv=10,\n",
        "                         scoring='f1_macro',\n",
        "                         n_jobs=-1,\n",
        "                         refit=True,\n",
        "                         verbose=1)\n",
        "logreg_gs.fit(X_train, y_train)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/html": [
              "<style>#sk-container-id-3 {color: black;}#sk-container-id-3 pre{padding: 0;}#sk-container-id-3 div.sk-toggleable {background-color: white;}#sk-container-id-3 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-3 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-3 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-3 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-3 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-3 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-3 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-3 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-3 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-3 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-3 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-3 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-3 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-3 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-3 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-3 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-3 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-3 div.sk-item {position: relative;z-index: 1;}#sk-container-id-3 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-3 div.sk-item::before, #sk-container-id-3 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-3 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-3 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-3 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-3 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-3 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-3 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-3 div.sk-label-container {text-align: center;}#sk-container-id-3 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-3 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-3\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>Pipeline(steps=[(&#x27;tfidfvectorizer&#x27;, TfidfVectorizer(max_features=512)),\n",
              "                (&#x27;robustscaler&#x27;, RobustScaler(with_centering=False)),\n",
              "                (&#x27;logisticregression&#x27;,\n",
              "                 LogisticRegression(C=10, class_weight={0: 1.45, 1: 1.0},\n",
              "                                    max_iter=5120, random_state=256,\n",
              "                                    solver=&#x27;saga&#x27;))])</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-10\" type=\"checkbox\" ><label for=\"sk-estimator-id-10\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">Pipeline</label><div class=\"sk-toggleable__content\"><pre>Pipeline(steps=[(&#x27;tfidfvectorizer&#x27;, TfidfVectorizer(max_features=512)),\n",
              "                (&#x27;robustscaler&#x27;, RobustScaler(with_centering=False)),\n",
              "                (&#x27;logisticregression&#x27;,\n",
              "                 LogisticRegression(C=10, class_weight={0: 1.45, 1: 1.0},\n",
              "                                    max_iter=5120, random_state=256,\n",
              "                                    solver=&#x27;saga&#x27;))])</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-11\" type=\"checkbox\" ><label for=\"sk-estimator-id-11\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">TfidfVectorizer</label><div class=\"sk-toggleable__content\"><pre>TfidfVectorizer(max_features=512)</pre></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-12\" type=\"checkbox\" ><label for=\"sk-estimator-id-12\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">RobustScaler</label><div class=\"sk-toggleable__content\"><pre>RobustScaler(with_centering=False)</pre></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-13\" type=\"checkbox\" ><label for=\"sk-estimator-id-13\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">LogisticRegression</label><div class=\"sk-toggleable__content\"><pre>LogisticRegression(C=10, class_weight={0: 1.45, 1: 1.0}, max_iter=5120,\n",
              "                   random_state=256, solver=&#x27;saga&#x27;)</pre></div></div></div></div></div></div></div>"
            ],
            "text/plain": [
              "Pipeline(steps=[('tfidfvectorizer', TfidfVectorizer(max_features=512)),\n",
              "                ('robustscaler', RobustScaler(with_centering=False)),\n",
              "                ('logisticregression',\n",
              "                 LogisticRegression(C=10, class_weight={0: 1.45, 1: 1.0},\n",
              "                                    max_iter=5120, random_state=256,\n",
              "                                    solver='saga'))])"
            ]
          },
          "execution_count": 12,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "logreg_gs.best_estimator_"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 13,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/plain": [
              "0.9336180959651946"
            ]
          },
          "execution_count": 13,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "logreg_gs.best_score_"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 14,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.88      0.89      0.89        95\n",
            "           1       0.93      0.91      0.92       137\n",
            "\n",
            "    accuracy                           0.91       232\n",
            "   macro avg       0.90      0.90      0.90       232\n",
            "weighted avg       0.91      0.91      0.91       232\n",
            "\n"
          ]
        },
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAfsAAAGwCAYAAACuFMx9AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/NK7nSAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAzJUlEQVR4nO3deXhU9dn/8c9kmywkgQCZEA0QJAgKIgZEcAGKgCgIpS1SsKKChUbFPKigRSUuJEJbRKHgWuBRqfiooPXnQlwAEakQwAUoFo0QhBiskezbzPn9ERk7hiWTM8kwc96v6zpXPevcQ3Plzn1/v+ccm2EYhgAAQNAK8XcAAACgeZHsAQAIciR7AACCHMkeAIAgR7IHACDIkewBAAhyJHsAAIJcmL8DMMPlcunQoUOKjY2VzWbzdzgAAC8ZhqHS0lIlJycrJKT56s+qqirV1NSYvk5ERIQiIyN9EFHLCuhkf+jQIaWkpPg7DACASQUFBTrzzDOb5dpVVVVK7dRKhUVO09dKSkpSfn5+wCX8gE72sbGxkqQey29VaLTdz9EAzaPdNf/2dwhAs6lTrTbpDffv8+ZQU1OjwiKn9ud1Vlxs07sHJaUudUr/WjU1NST7lnSsdR8abSfZI2iF2cL9HQLQfH58YHtLDMW2irWpVWzTP8elwB0uDuhkDwBAYzkNl5wm3gbjNFy+C6aFkewBAJbgkiGXmp7tzZzrb9x6BwBAkKOyBwBYgksumWnEmzvbv0j2AABLcBqGnEbTW/FmzvU32vgAAAQ5KnsAgCVYeYIeyR4AYAkuGXJaNNnTxgcAIMhR2QMALMHKbXwqewCAJRybjW9m8cbGjRs1evRoJScny2azae3ate59tbW1mj17tnr16qWYmBglJyfruuuu06FDhzyuUV1drVtvvVXt2rVTTEyMrr76ah08eNDr706yBwCgGZSXl6t3795asmRJg30VFRXavn277r33Xm3fvl2vvPKKvvjiC1199dUex2VmZmrNmjV64YUXtGnTJpWVlWnUqFFyOr17gx9tfACAJbh+XMycL0klJSUe2+12u+z2hi9jGzlypEaOHHnca8XHxys3N9dj2+LFi3XhhRfqwIED6tixo44ePapnnnlGzz77rC6//HJJ0nPPPaeUlBS98847GjFiRKNjp7IHAFiC88fZ+GYWSUpJSVF8fLx7ycnJ8Ul8R48elc1mU+vWrSVJeXl5qq2t1fDhw93HJCcnq2fPntq8ebNX16ayBwBYgtOQybfe1f9vQUGB4uLi3NuPV9V7q6qqSnfddZcmTpzovnZhYaEiIiLUpk0bj2MdDocKCwu9uj7JHgAAL8TFxXkke7Nqa2s1YcIEuVwuLV269JTHG4Yhm83m1WfQxgcAWILLB4uv1dbWavz48crPz1dubq7HHxFJSUmqqalRcXGxxzlFRUVyOBxefQ7JHgBgCS7Z5DSxuORdNX0qxxL9v//9b73zzjtq27atx/709HSFh4d7TOQ7fPiwPv/8cw0cONCrz6KNDwBAMygrK9O+ffvc6/n5+dq5c6cSEhKUnJysX//619q+fbtef/11OZ1O9zh8QkKCIiIiFB8frylTpuj2229X27ZtlZCQoDvuuEO9evVyz85vLJI9AMASXEb9YuZ8b2zbtk1Dhgxxr8+cOVOSNHnyZGVlZem1116TJJ1//vke573//vsaPHiwJOmRRx5RWFiYxo8fr8rKSg0dOlQrVqxQaGioV7GQ7AEAlnCsHW/mfG8MHjxYxkmeuneyfcdERkZq8eLFWrx4sVef/XOM2QMAEOSo7AEAltDSlf3phGQPALAEl2GTy2h6wjZzrr/RxgcAIMhR2QMALIE2PgAAQc6pEDlNNLS9e6ns6YVkDwCwBMPkmL3BmD0AADhdUdkDACyBMXsAAIKc0wiR0zAxZm/iUbv+RhsfAIAgR2UPALAEl2xymahxXQrc0p5kDwCwBCuP2dPGBwAgyFHZAwAswfwEPdr4AACc1urH7E28CIc2PgAAOF1R2QMALMFl8tn4zMYHAOA0x5g9AABBzqUQy95nz5g9AABBjsoeAGAJTsMmp4nX1Jo5199I9gAAS3CanKDnpI0PAABOV1T2AABLcBkhcpmYje9iNj4AAKc32vgAACBoUdkDACzBJXMz6l2+C6XFkewBAJZg/qE6gdsMD9zIAQBAo1DZAwAswfyz8QO3PibZAwAswcrvsyfZAwAswcqVfeBGDgAAGoXKHgBgCeYfqhO49THJHgBgCS7DJpeZ++wD+K13gftnCgAAaBQqewCAJbhMtvED+aE6JHsAgCWYf+td4Cb7wI0cAAA0CpU9AMASnLLJaeLBOGbO9TeSPQDAEmjjAwCAoEVlDwCwBKfMteKdvgulxZHsAQCWYOU2PskeAGAJvAgHAAAELSp7AIAlGCbfZ29w6x0AAKc32vgAACBoUdkDACzByq+4JdkDACzBafKtd2bO9bfAjRwAADQKyR4AYAnH2vhmFm9s3LhRo0ePVnJysmw2m9auXeux3zAMZWVlKTk5WVFRURo8eLB27drlcUx1dbVuvfVWtWvXTjExMbr66qt18OBBr787yR4AYAkuhZhevFFeXq7evXtryZIlx92/YMECLVy4UEuWLNHWrVuVlJSkYcOGqbS01H1MZmam1qxZoxdeeEGbNm1SWVmZRo0aJafTu4f3MmYPAIAXSkpKPNbtdrvsdnuD40aOHKmRI0ce9xqGYWjRokWaM2eOxo0bJ0lauXKlHA6HVq1apWnTpuno0aN65pln9Oyzz+ryyy+XJD333HNKSUnRO++8oxEjRjQ6Zip7AIAlOA2b6UWSUlJSFB8f715ycnK8jiU/P1+FhYUaPny4e5vdbtegQYO0efNmSVJeXp5qa2s9jklOTlbPnj3dxzQWlT0AwBJ8detdQUGB4uLi3NuPV9WfSmFhoSTJ4XB4bHc4HNq/f7/7mIiICLVp06bBMcfObyySPQDAEgyTb70zfjw3Li7OI9mbYbN5/vFhGEaDbQ3jOPUxP0cbHwCAFpaUlCRJDSr0oqIid7WflJSkmpoaFRcXn/CYxiLZAwAswSmb6cVXUlNTlZSUpNzcXPe2mpoabdiwQQMHDpQkpaenKzw83OOYw4cP6/PPP3cf01i08QEAluAyzD3y1mV4d3xZWZn27dvnXs/Pz9fOnTuVkJCgjh07KjMzU9nZ2UpLS1NaWpqys7MVHR2tiRMnSpLi4+M1ZcoU3X777Wrbtq0SEhJ0xx13qFevXu7Z+Y1FsgcAoBls27ZNQ4YMca/PnDlTkjR58mStWLFCs2bNUmVlpTIyMlRcXKz+/ftr3bp1io2NdZ/zyCOPKCwsTOPHj1dlZaWGDh2qFStWKDQ01KtYbIZhePm3yumjpKRE8fHx6rn6DoVGez8bEifgNBT99+8Uub5UIT/UydUmTFVD41Qxvq0UUv9Xceyiw4p8z/Ne09pukfrhz538EXFQa3/1Xn+HEHR69i/TbzKOKK1Xhdom1Snrxs766K34/zrC0LW3f6srJ/1HreKd+teOaP31j2dq/xeRfos5WNUZtVqvV3X06FGfTXr7uWO5YvL7ExTRKqLJ16kpq9HKIS80a6zNhcoeDUS//L2i3jyq0swk1XW0K2xflWIfOywjOlSVV/90C0jNBTEquS3ppxPDAveNULCWyGiXvtoVqXUvtNF9z+xvsH/8zUc07vdH9JfMFB38yq6JmUXKeeFLTbm0uyrLvauocPpwySaXiXF3M+f6m98n6C1dulSpqamKjIxUenq6PvjgA3+HZHlh/6pUdf9WqunXSi5HuGoujlXt+TEK21flcZwRbpPRJuynJZZfgggM296P08oFHfThm62Ps9fQ2KlH9MJjDn34Zmvt3xulP9+WInuUS0N++UMLRwr4hl+T/erVq5WZmak5c+Zox44duvTSSzVy5EgdOHDAn2FZXu05UYr4tFyh39RIkkLzqxS+u1I16TEex4V/XqG2v9unNtO/UqslhbL9UOePcAGfSupYo7aOOuVtaOXeVlsTos+2tNI5fcv9GBnM8tUT9AKRX9v4Cxcu1JQpUzR16lRJ0qJFi/T2229r2bJlTXr8IHyj8lcJCil3qU1Gfv2fgy6p/Np2qh700xhVTXqMqi+OlTMxXKHf1irm+e/U+p4CFT/SSQr3e8MIaLKExPo/WouPhHtsLz4SpsQza/wREnzEZfKhOmbO9Te/Jfuamhrl5eXprrvu8tg+fPjwEz7zt7q6WtXV1e71n7+MAL5h/6BU9g0lKr29Q/2YfX6VWj1dJFdCmKqH1k9iqr70p8Tv7GRXXddIJUz9UhFby1UzMPZElwYCx8+mLttskgK4soO1+e3PlO+++05Op/O4zwU+0TN/c3JyPF4+kJKS0hKhWk7MiiOq+FWCqi+Lk7OzXdVD4lV5dYKiX/r+hOe4EsLkbB+u0MNUPghs3xfV10BtEms9trduV6fiI8xpDmQumXyfPRP0ms6b5wLffffdOnr0qHspKChoiRAtx1bt0s9/po0QSSe5S9NW4lTod/W36QGBrPBAhP7zbZguuKzMvS0s3KVeF5Vp97aYk5yJ053x42z8pi5GACd7v/1mbteunUJDQ0/6XOCfO9E7g+FbNf1aKfr/vperfXh9G/+rKkW/Wqyqy3+8D7nSpZi/f6fqgbFytQlTaFGtYp49IldcqGouooWP019ktFPJqT91oZJSatTl3EqV/hCqI99EaO3T7TXh1m/1zVd2fZMfod/OKFJ1ZYjeX9Paf0HDNF+99S4Q+S3ZR0REKD09Xbm5ufrlL3/p3p6bm6sxY8b4KyxIKvu9Q9HPf6dWj3+rkKNOuRLCVHlFvCquaVd/QIgUtr9ake+XyFbulKtNmGp7RavkzmQZ0X5vFgGn1K13pf708pfu9en3H5IkrVvdRn/5n4568a/tFRHp0i05BxX740N17v5tF+6xR8Dya8915syZ+t3vfqe+fftqwIABevLJJ3XgwAFNnz7dn2FZnhEdovKbElV+U+LxD7CH6Oj9zJdA4Pr0o1Yakdz7JEfY9NxfkvTcX5JOcgwCDbPx/eSaa67Rf/7zHz3wwAM6fPiwevbsqTfeeEOdOvHIVQCAb9HG96OMjAxlZGT4OwwAAIKW35M9AAAtwcrPxifZAwAswcpt/MCdbQAAABqFyh4AYAlWruxJ9gAAS7BysqeNDwBAkKOyBwBYgpUre5I9AMASDJm7fe7ErwI7/ZHsAQCWYOXKnjF7AACCHJU9AMASrFzZk+wBAJZg5WRPGx8AgCBHZQ8AsAQrV/YkewCAJRiGTYaJhG3mXH+jjQ8AQJCjsgcAWALvswcAIMhZecyeNj4AAEGOyh4AYAlWnqBHsgcAWIKV2/gkewCAJVi5smfMHgCAIEdlDwCwBMNkGz+QK3uSPQDAEgxJhmHu/EBFGx8AgCBHZQ8AsASXbLLxBD0AAIIXs/EBAEDQorIHAFiCy7DJxkN1AAAIXoZhcjZ+AE/Hp40PAECQo7IHAFiClSfokewBAJZAsgcAIMhZeYIeY/YAAAQ5KnsAgCVYeTY+yR4AYAn1yd7MmL0Pg2lhtPEBAAhyVPYAAEtgNj4AAEHOkLl30gdwF582PgAAzaGurk733HOPUlNTFRUVpS5duuiBBx6Qy+VyH2MYhrKyspScnKyoqCgNHjxYu3bt8nksJHsAgCUca+ObWbwxf/58Pf7441qyZIn27NmjBQsW6E9/+pMWL17sPmbBggVauHChlixZoq1btyopKUnDhg1TaWmpT787yR4AYA2GDxYvfPTRRxozZoyuuuoqde7cWb/+9a81fPhwbdu2rT4cw9CiRYs0Z84cjRs3Tj179tTKlStVUVGhVatW+eAL/4RkDwCwBrNV/Y+VfUlJicdSXV193I+75JJL9O677+qLL76QJH3yySfatGmTrrzySklSfn6+CgsLNXz4cPc5drtdgwYN0ubNm3361ZmgBwCAF1JSUjzW586dq6ysrAbHzZ49W0ePHlX37t0VGhoqp9OpefPm6be//a0kqbCwUJLkcDg8znM4HNq/f79PYybZAwAswVdP0CsoKFBcXJx7u91uP+7xq1ev1nPPPadVq1bp3HPP1c6dO5WZmank5GRNnjzZfZzN5jkXwDCMBtvMItkDACzBV/fZx8XFeST7E7nzzjt11113acKECZKkXr16af/+/crJydHkyZOVlJQkqb7C79Chg/u8oqKiBtW+WYzZAwDQDCoqKhQS4plmQ0ND3bfepaamKikpSbm5ue79NTU12rBhgwYOHOjTWKjsAQDW8F+T7Jp8vhdGjx6tefPmqWPHjjr33HO1Y8cOLVy4UDfeeKOk+vZ9ZmamsrOzlZaWprS0NGVnZys6OloTJ05sepzHQbIHAFhCS7/1bvHixbr33nuVkZGhoqIiJScna9q0abrvvvvcx8yaNUuVlZXKyMhQcXGx+vfvr3Xr1ik2NrbpgR6HzTAC9z0+JSUlio+PV8/Vdyg0+vgTJIBA1/7qvf4OAWg2dUat1utVHT16tFHj4E1xLFd0evpehURHNvk6rooq7Z/6YLPG2lyo7AEA1mDhh+OT7AEAlsBb707hsccea/QFZ8yY0eRgAACA7zUq2T/yyCONupjNZiPZAwBOXwHcijejUck+Pz+/ueMAAKBZWbmN3+SH6tTU1Gjv3r2qq6vzZTwAADSPFn7r3enE62RfUVGhKVOmKDo6Wueee64OHDggqX6s/uGHH/Z5gAAAwByvk/3dd9+tTz75ROvXr1dk5E/3K15++eVavXq1T4MDAMB3bD5YApPXt96tXbtWq1ev1kUXXeTxVp5zzjlHX375pU+DAwDAZyx8n73Xlf2RI0eUmJjYYHt5ebnPX8kHAADM8zrZ9+vXT//v//0/9/qxBP/UU09pwIABvosMAABfsvAEPa/b+Dk5Obriiiu0e/du1dXV6dFHH9WuXbv00UcfacOGDc0RIwAA5rXwW+9OJ15X9gMHDtSHH36oiooKnXXWWVq3bp0cDoc++ugjpaenN0eMAADAhCY9G79Xr15auXKlr2MBAKDZtPQrbk8nTUr2TqdTa9as0Z49e2Sz2dSjRw+NGTNGYWG8VwcAcJqy8Gx8r7Pz559/rjFjxqiwsFBnn322JOmLL75Q+/bt9dprr6lXr14+DxIAADSd12P2U6dO1bnnnquDBw9q+/bt2r59uwoKCnTeeefp97//fXPECACAeccm6JlZApTXlf0nn3yibdu2qU2bNu5tbdq00bx589SvXz+fBgcAgK/YjPrFzPmByuvK/uyzz9a3337bYHtRUZG6du3qk6AAAPA5C99n36hkX1JS4l6ys7M1Y8YMvfTSSzp48KAOHjyol156SZmZmZo/f35zxwsAALzUqDZ+69atPR6FaxiGxo8f795m/Hg/wujRo+V0OpshTAAATLLwQ3Ualezff//95o4DAIDmxa13Jzdo0KDmjgMAADSTJj8Fp6KiQgcOHFBNTY3H9vPOO890UAAA+ByVfeMdOXJEN9xwg958883j7mfMHgBwWrJwsvf61rvMzEwVFxdry5YtioqK0ltvvaWVK1cqLS1Nr732WnPECAAATPC6sn/vvff06quvql+/fgoJCVGnTp00bNgwxcXFKScnR1dddVVzxAkAgDkWno3vdWVfXl6uxMRESVJCQoKOHDkiqf5NeNu3b/dtdAAA+MixJ+iZWQJVk56gt3fvXknS+eefryeeeELffPONHn/8cXXo0MHnAQIAAHO8buNnZmbq8OHDkqS5c+dqxIgRev755xUREaEVK1b4Oj4AAHzDwhP0vE72kyZNcv93nz599PXXX+tf//qXOnbsqHbt2vk0OAAAYF6T77M/Jjo6WhdccIEvYgEAoNnYZPKtdz6LpOU1KtnPnDmz0RdcuHBhk4MBAAC+16hkv2PHjkZd7L9fltOS2l+7X2G2cL98NtDc3jq0098hAM2mpNSlNt1a6MMsfOsdL8IBAFiDhSfoeX3rHQAACCymJ+gBABAQLFzZk+wBAJZg9il4lnqCHgAACCxU9gAAa7BwG79Jlf2zzz6riy++WMnJydq/f78kadGiRXr11Vd9GhwAAD5j+GAJUF4n+2XLlmnmzJm68sor9cMPP8jpdEqSWrdurUWLFvk6PgAAYJLXyX7x4sV66qmnNGfOHIWGhrq39+3bV5999plPgwMAwFes/Ipbr8fs8/Pz1adPnwbb7Xa7ysvLfRIUAAA+Z+En6Hld2aempmrnzp0Ntr/55ps655xzfBETAAC+Z+Exe68r+zvvvFM333yzqqqqZBiGPv74Y/39739XTk6Onn766eaIEQAAmOB1sr/hhhtUV1enWbNmqaKiQhMnTtQZZ5yhRx99VBMmTGiOGAEAMM3KD9Vp0n32N910k2666SZ99913crlcSkxM9HVcAAD4loXvszf1UJ127dr5Kg4AANBMvE72qampJ31v/VdffWUqIAAAmoXZ2+esVNlnZmZ6rNfW1mrHjh166623dOedd/oqLgAAfIs2fuPddtttx93+17/+Vdu2bTMdEAAA8C2fvfVu5MiRevnll311OQAAfMvC99n7LNm/9NJLSkhI8NXlAADwKX88Lvebb77Rtddeq7Zt2yo6Olrnn3++8vLy3PsNw1BWVpaSk5MVFRWlwYMHa9euXT781vW8buP36dPHY4KeYRgqLCzUkSNHtHTpUp8GBwBAoCouLtbFF1+sIUOG6M0331RiYqK+/PJLtW7d2n3MggULtHDhQq1YsULdunXTQw89pGHDhmnv3r2KjY31WSxeJ/uxY8d6rIeEhKh9+/YaPHiwunfv7qu4AAAIaPPnz1dKSoqWL1/u3ta5c2f3fxuGoUWLFmnOnDkaN26cJGnlypVyOBxatWqVpk2b5rNYvEr2dXV16ty5s0aMGKGkpCSfBQEAQLPz0Wz8kpISj812u112u73B4a+99ppGjBih3/zmN9qwYYPOOOMMZWRk6KabbpJU/2K5wsJCDR8+3ONagwYN0ubNm32a7L0asw8LC9Mf/vAHVVdX+ywAAABagq/G7FNSUhQfH+9ecnJyjvt5X331lZYtW6a0tDS9/fbbmj59umbMmKH//d//lSQVFhZKkhwOh8d5DofDvc9XvG7j9+/fXzt27FCnTp18GggAAIGgoKBAcXFx7vXjVfWS5HK51LdvX2VnZ0uqn/O2a9cuLVu2TNddd537uJ8/qM4wjJM+vK4pvE72GRkZuv3223Xw4EGlp6crJibGY/95553ns+AAAPApH9w+FxcX55HsT6RDhw4NXv3eo0cP923qx4bDCwsL1aFDB/cxRUVFDap9sxqd7G+88UYtWrRI11xzjSRpxowZ7n02m839l4jT6fRpgAAA+EQLP0Hv4osv1t69ez22ffHFF+7OeGpqqpKSkpSbm6s+ffpIkmpqarRhwwbNnz/fRKANNTrZr1y5Ug8//LDy8/N9GgAAAMHof/7nfzRw4EBlZ2dr/Pjx+vjjj/Xkk0/qySeflFRfKGdmZio7O1tpaWlKS0tTdna2oqOjNXHiRJ/G0uhkbxj1f9IwVg8ACEQt/T77fv36ac2aNbr77rv1wAMPKDU1VYsWLdKkSZPcx8yaNUuVlZXKyMhQcXGx+vfvr3Xr1vn0HnvJyzF7X08YAACgxfjhRTijRo3SqFGjTrjfZrMpKytLWVlZTY+rEbxK9t26dTtlwv/+++9NBQQAAHzLq2R///33Kz4+vrliAQCg2bR0G/904lWynzBhghITE5srFgAAmo+F32ff6CfoMV4PAEBg8no2PgAAAcnClX2jk73L5WrOOAAAaFaM2QMAEOwsXNl79dY7AAAQeKjsAQDWYOHKnmQPALAEK4/Z08YHACDIUdkDAKyBNj4AAMGNNj4AAAhaVPYAAGugjQ8AQJCzcLKnjQ8AQJCjsgcAWILtx8XM+YGKZA8AsAYLt/FJ9gAAS+DWOwAAELSo7AEA1kAbHwAACwjghG0GbXwAAIIclT0AwBKsPEGPZA8AsAYLj9nTxgcAIMhR2QMALIE2PgAAwY42PgAACFZU9gAAS6CNDwBAsLNwG59kDwCwBgsne8bsAQAIclT2AABLYMweAIBgRxsfAAAEKyp7AIAl2AxDNqPp5bmZc/2NZA8AsAba+AAAIFhR2QMALIHZ+AAABDva+AAAIFhR2QMALIE2PgAAwc7CbXySPQDAEqxc2TNmDwBAkKOyBwBYA218AACCXyC34s2gjQ8AQJCjsgcAWINh1C9mzg9QJHsAgCUwGx8AAAQtKnsAgDVYeDY+lT0AwBJsLvNLU+Xk5MhmsykzM9O9zTAMZWVlKTk5WVFRURo8eLB27dpl/oseB8keAIBmtHXrVj355JM677zzPLYvWLBACxcu1JIlS7R161YlJSVp2LBhKi0t9XkMtPHRQM8LS/XraYeV1qtCbR21uv+mrvpoXRtJUmiYS5Pv+Eb9hhxVh47VKi8N1Y5Ncfrbw2fq+6IIP0cOHN9nW2L0f0sT9e/PovX9t+Ga+0y+Bo48Kkmqq5VWzO+gre/F6fD+CMXEudTn0lJN+eMhtU2qc1/jzl911acftfK47qCri/XHx/e36HeBCT5q45eUlHhsttvtstvtxz2lrKxMkyZN0lNPPaWHHnrop0sZhhYtWqQ5c+Zo3LhxkqSVK1fK4XBo1apVmjZtmolAG6KyRwOR0U7l74nW0vs6Nthnj3Kpa88KrXosWbdcdY4enNZVZ6RWKeuZf/shUqBxqipC1OXcSt0872CDfdWVIdr3WbQmZn6rv779he57Ol/ffGXX3Ou7NDh25KTv9Pedn7uX2xYUtET48JFjs/HNLJKUkpKi+Ph495KTk3PCz7z55pt11VVX6fLLL/fYnp+fr8LCQg0fPty9zW63a9CgQdq8ebPPv7tfK/uNGzfqT3/6k/Ly8nT48GGtWbNGY8eO9WdIkLRtfWttW9/6x7UvPfZVlIbpj9ee7bFt2dyOeuwfe9Q+uVpHDh3/r1vAn/r9olT9fnH81mhMnEsPr/b8Oc946KBmXHm2ig6GK/HMWvd2e5ShhMS6n18CgcJH99kXFBQoLi7OvflEVf0LL7yg7du3a+vWrQ32FRYWSpIcDofHdofDof37fd8t8muyLy8vV+/evXXDDTfoV7/6lT9DgQkxsU65XFJ5CaNCCA7lJaGy2QzFxDs9tr//Shu993IbtW5fq35DSnXt7YWKbmVi1hYCUlxcnEeyP56CggLddtttWrdunSIjI094nM1m81g3DKPBNl/w62/nkSNHauTIkY0+vrq6WtXV1e71n4+boOWF21264a6DWv9qgirKQv0dDmBaTZVNf8tO1pBfFism9qdEPmTc90pKqVFCYp2+/lek/pbTQV/tjmrQFcDpqyUfqpOXl6eioiKlp6e7tzmdTm3cuFFLlizR3r17JdVX+B06dHAfU1RU1KDa94WAKsVycnJ0//33+zsM/Cg0zKW7F3+pkBBpyT2d/R0OYFpdrZT9h84yXNItOZ7j+1dO+t793527V+mMLtW65Yqz9e9Po5R2XmVLh4qmaMH77IcOHarPPvvMY9sNN9yg7t27a/bs2erSpYuSkpKUm5urPn36SJJqamq0YcMGzZ8/30SQxxdQyf7uu+/WzJkz3eslJSVKSUnxY0TWFRrm0h//+qWSUqo1+7fdqeoR8OpqpXnTOquwIEILXtznUdUfT9delQoLd+mbfDvJHg3ExsaqZ8+eHttiYmLUtm1b9/bMzExlZ2crLS1NaWlpys7OVnR0tCZOnOjzeAIq2Z/s9ga0nGOJ/ozUas2ecLZKfwioHyOggWOJ/pt8uxa8tE9xCc5TnrN/b6TqakPU1lF7ymNxejjdno0/a9YsVVZWKiMjQ8XFxerfv7/WrVun2NhY336QAizZo2VERjuV3PmnuRFJKdXqck6FSn8I1X++jdA9y75U157luu/GbgoJldq0r/9lV/pDqOpquZsTp5/K8hAdyv+pUCgsiNCXn0cptnWd2ibV6sGbUrXvsyg98L9fyeW06fui+l+Nsa2dCo8wdOjrCL33ShtdOLREcQlOHfjCrifvP0Nde1bonH7l/vpa8Jaf33q3fv16j3WbzaasrCxlZWWZum5jkOzRQLfzyrVg9V73+rT76u8lzv2/tnpu0RkaMPwHSdKytzwf6zjrmrP16ZaTz1AF/OGLT6I169dd3etPZJ0hSRo2/ntde3uhtqyLlyRlDOvucd6Cl/ap98AyhYUb2rkpVmufaa+q8hC1S65V/6ElmjSzUKGMYCEA+DXZl5WVad++fe71/Px87dy5UwkJCerYseEDXdAyPt0Spys69Tvh/pPtA05HvQeW6e1DO0+4/2T7JCnxjFr9+ZV9Jz0Gp7/TrY3fkvya7Ldt26YhQ4a4149Nvps8ebJWrFjhp6gAAEHJwm+982uyHzx4sAyTYyAAAODkGLMHAFgCbXwAAIKdy6hfzJwfoEj2AABrsPCYPTdFAwAQ5KjsAQCWYJPJMXufRdLySPYAAGvw8xP0/Ik2PgAAQY7KHgBgCdx6BwBAsGM2PgAACFZU9gAAS7AZhmwmJtmZOdffSPYAAGtw/biYOT9A0cYHACDIUdkDACyBNj4AAMHOwrPxSfYAAGvgCXoAACBYUdkDACyBJ+gBABDsaOMDAIBgRWUPALAEm6t+MXN+oCLZAwCsgTY+AAAIVlT2AABr4KE6AAAENys/Lpc2PgAAQY7KHgBgDRaeoEeyBwBYgyFz76QP3FxPsgcAWANj9gAAIGhR2QMArMGQyTF7n0XS4kj2AABrsPAEPdr4AAAEOSp7AIA1uCTZTJ4foEj2AABLYDY+AAAIWlT2AABrsPAEPZI9AMAaLJzsaeMDABDkqOwBANZg4cqeZA8AsAZuvQMAILhx6x0AAAhaVPYAAGtgzB4AgCDnMiSbiYTtCtxkTxsfAIAgR2UPALAG2vgAAAQ7k8legZvsaeMDABDkSPYAAGs41sY3s3ghJydH/fr1U2xsrBITEzV27Fjt3bv3ZyEZysrKUnJysqKiojR48GDt2rXLl99aEskeAGAVLsP84oUNGzbo5ptv1pYtW5Sbm6u6ujoNHz5c5eXl7mMWLFighQsXasmSJdq6dauSkpI0bNgwlZaW+vSrM2YPAEAzeOuttzzWly9frsTEROXl5emyyy6TYRhatGiR5syZo3HjxkmSVq5cKYfDoVWrVmnatGk+i4XKHgBgDYbL/CKppKTEY6murm7Uxx89elSSlJCQIEnKz89XYWGhhg8f7j7Gbrdr0KBB2rx5s0+/OskeAGANPhqzT0lJUXx8vHvJyclpxEcbmjlzpi655BL17NlTklRYWChJcjgcHsc6HA73Pl+hjQ8AsAaXIVO3z/04Zl9QUKC4uDj3ZrvdfspTb7nlFn366afatGlTg302m+er+AzDaLDNLJI9AABeiIuL80j2p3Lrrbfqtdde08aNG3XmmWe6tyclJUmqr/A7dOjg3l5UVNSg2jeLNj4AwBpa+NY7wzB0yy236JVXXtF7772n1NRUj/2pqalKSkpSbm6ue1tNTY02bNiggQMH+uQrH0NlDwCwBkMmH5fr3eE333yzVq1apVdffVWxsbHucfj4+HhFRUXJZrMpMzNT2dnZSktLU1pamrKzsxUdHa2JEyc2Pc7jINkDANAMli1bJkkaPHiwx/bly5fr+uuvlyTNmjVLlZWVysjIUHFxsfr3769169YpNjbWp7GQ7AEA1tDCL8IxGnG8zWZTVlaWsrKymhhU45DsAQDW4HJJcpk8PzAxQQ8AgCBHZQ8AsAbeZw8AQJCzcLKnjQ8AQJCjsgcAWIOPHpcbiEj2AABLMAyXDKPpM+rNnOtvJHsAgDUYhrnqnDF7AABwuqKyBwBYg2FyzD6AK3uSPQDAGlwuyWZi3D2Ax+xp4wMAEOSo7AEA1kAbHwCA4Ga4XDJMtPED+dY72vgAAAQ5KnsAgDXQxgcAIMi5DMlmzWRPGx8AgCBHZQ8AsAbDkGTmPvvArexJ9gAASzBchgwTbXyDZA8AwGnOcMlcZc+tdwAA4DRFZQ8AsATa+AAABDsLt/EDOtkf+yurzqj1cyRA8ykpDdxfMMCplJTV/3y3RNVcp1pTz9SpU+DmmoBO9qWlpZKkD+rW+jcQoBm16ebvCIDmV1paqvj4+Ga5dkREhJKSkrSp8A3T10pKSlJERIQPompZNiOAByFcLpcOHTqk2NhY2Ww2f4djCSUlJUpJSVFBQYHi4uL8HQ7gU/x8tzzDMFRaWqrk5GSFhDTfnPGqqirV1NSYvk5ERIQiIyN9EFHLCujKPiQkRGeeeaa/w7CkuLg4fhkiaPHz3bKaq6L/b5GRkQGZpH2FW+8AAAhyJHsAAIIcyR5esdvtmjt3rux2u79DAXyOn28Eq4CeoAcAAE6Nyh4AgCBHsgcAIMiR7AEACHIkewAAghzJHo22dOlSpaamKjIyUunp6frggw/8HRLgExs3btTo0aOVnJwsm82mtWvX+jskwKdI9miU1atXKzMzU3PmzNGOHTt06aWXauTIkTpw4IC/QwNMKy8vV+/evbVkyRJ/hwI0C269Q6P0799fF1xwgZYtW+be1qNHD40dO1Y5OTl+jAzwLZvNpjVr1mjs2LH+DgXwGSp7nFJNTY3y8vI0fPhwj+3Dhw/X5s2b/RQVAKCxSPY4pe+++05Op1MOh8Nju8PhUGFhoZ+iAgA0Fskejfbz1wgbhsGrhQEgAJDscUrt2rVTaGhogyq+qKioQbUPADj9kOxxShEREUpPT1dubq7H9tzcXA0cONBPUQEAGivM3wEgMMycOVO/+93v1LdvXw0YMEBPPvmkDhw4oOnTp/s7NMC0srIy7du3z72en5+vnTt3KiEhQR07dvRjZIBvcOsdGm3p0qVasGCBDh8+rJ49e+qRRx7RZZdd5u+wANPWr1+vIUOGNNg+efJkrVixouUDAnyMZA8AQJBjzB4AgCBHsgcAIMiR7AEACHIkewAAghzJHgCAIEeyBwAgyJHsAQAIciR7AACCHMkeMCkrK0vnn3++e/3666/X2LFjWzyOr7/+WjabTTt37jzhMZ07d9aiRYsafc0VK1aodevWpmOz2Wxau3at6esAaBqSPYLS9ddfL5vNJpvNpvDwcHXp0kV33HGHysvLm/2zH3300UY/YrUxCRoAzOJFOAhaV1xxhZYvX67a2lp98MEHmjp1qsrLy7Vs2bIGx9bW1io8PNwnnxsfH++T6wCAr1DZI2jZ7XYlJSUpJSVFEydO1KRJk9yt5GOt97/97W/q0qWL7Ha7DMPQ0aNH9fvf/16JiYmKi4vTL37xC33yySce13344YflcDgUGxurKVOmqKqqymP/z9v4LpdL8+fPV9euXWW329WxY0fNmzdPkpSamipJ6tOnj2w2mwYPHuw+b/ny5erRo4ciIyPVvXt3LV261ONzPv74Y/Xp00eRkZHq27evduzY4fW/0cKFC9WrVy/FxMQoJSVFGRkZKisra3Dc2rVr1a1bN0VGRmrYsGEqKCjw2P+Pf/xD6enpioyMVJcuXXT//ferrq7O63gANA+SPSwjKipKtbW17vV9+/bpxRdf1Msvv+xuo1911VUqLCzUG2+8oby8PF1wwQUaOnSovv/+e0nSiy++qLlz52revHnatm2bOnTo0CAJ/9zdd9+t+fPn695779Xu3bu1atUqORwOSfUJW5LeeecdHT58WK+88ook6amnntKcOXM0b9487dmzR9nZ2br33nu1cuVKSVJ5eblGjRqls88+W3l5ecrKytIdd9zh9b9JSEiIHnvsMX3++edauXKl3nvvPc2aNcvjmIqKCs2bN08rV67Uhx9+qJKSEk2YMMG9/+2339a1116rGTNmaPfu3XriiSe0YsUK9x80AE4DBhCEJk+ebIwZM8a9/s9//tNo27atMX78eMMwDGPu3LlGeHi4UVRU5D7m3XffNeLi4oyqqiqPa5111lnGE088YRiGYQwYMMCYPn26x/7+/fsbvXv3Pu5nl5SUGHa73XjqqaeOG2d+fr4hydixY4fH9pSUFGPVqlUe2x588EFjwIABhmEYxhNPPGEkJCQY5eXl7v3Lli077rX+W6dOnYxHHnnkhPtffPFFo23btu715cuXG5KMLVu2uLft2bPHkGT885//NAzDMC699FIjOzvb4zrPPvus0aFDB/e6JGPNmjUn/FwAzYsxewSt119/Xa1atVJdXZ1qa2s1ZswYLV682L2/U6dOat++vXs9Ly9PZWVlatu2rcd1Kisr9eWXX0qS9uzZo+nTp3vsHzBggN5///3jxrBnzx5VV1dr6NChjY77yJEjKigo0JQpU3TTTTe5t9fV1bnnA+zZs0e9e/dWdHS0Rxzeev/995Wdna3du3erpKREdXV1qqqqUnl5uWJiYiRJYWFh6tu3r/uc7t27q3Xr1tqzZ48uvPBC5eXlaevWrR6VvNPpVFVVlSoqKjxiBOAfJHsErSFDhmjZsmUKDw9XcnJygwl4x5LZMS6XSx06dND69esbXKupt59FRUV5fY7L5ZJU38rv37+/x77Q0FBJkmEYTYrnv+3fv19XXnmlpk+frgcffFAJCQnatGmTpkyZ4jHcIdXfOvdzx7a5XC7df//9GjduXINjIiMjTccJwDySPYJWTEyMunbt2ujjL7jgAhUWFiosLEydO3c+7jE9evTQli1bdN1117m3bdmy5YTXTEtLU1RUlN59911NnTq1wf6IiAhJ9ZXwMQ6HQ2eccYa++uorTZo06bjXPeecc/Tss8+qsrLS/QfFyeI4nm3btqmurk5/+ctfFBJSP33nxRdfbHBcXV2dtm3bpgsvvFCStHfvXv3www/q3r27pPp/t71793r1bw2gZZHsgR9dfvnlGjBggMaOHav58+fr7LPP1qFDh/TGG29o7Nix6tu3r2677TZNnjxZffv21SWXXKLnn39eu3btUpcuXY57zcjISM2ePVuzZs1SRESELr74Yh05ckS7du3SlClTlJiYqKioKL311ls688wzFRkZqfj4eGVlZWnGjBmKi4vTyJEjVV1drW3btqm4uFgzZ87UxIkTNWfOHE2ZMkX33HOPvv76a/35z3/26vueddZZqqur0+LFizV69Gh9+OGHevzxxxscFx4erltvvVWPPfaYwsPDdcstt+iiiy5yJ//77rtPo0aNUkpKin7zm98oJCREn376qT777DM99NBD3v8fAcDnmI0P/Mhms+mNN97QZZddphtvvFHdunXThAkT9PXXX7tnz19zzTW67777NHv2bKWnp2v//v36wx/+cNLr3nvvvbr99tt13333qUePHrrmmmtUVFQkqX48/LHHHtMTTzyh5ORkjRkzRpI0depUPf3001qxYoV69eqlQYMGacWKFe5b9Vq1aqV//OMf2r17t/r06aM5c+Zo/vz5Xn3f888/XwsXLtT8+fPVs2dPPf/888rJyWlwXHR0tGbPnq2JEydqwIABioqK0gsvvODeP2LECL3++uvKzc1Vv379dNFFF2nhwoXq1KmTV/EAaD42wxeDfwAA4LRFZQ8AQJAj2QMAEORI9gAABDmSPQAAQY5kDwBAkCPZAwAQ5Ej2AAAEOZI9AABBjmQPAECQI9kDABDkSPYAAAS5/w9Jeuaq7vICGAAAAABJRU5ErkJggg==",
            "text/plain": [
              "<Figure size 640x480 with 2 Axes>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "y_test_pred_logreg = logreg_gs.predict(X_test)\n",
        "print(classification_report(y_test, y_test_pred_logreg))\n",
        "ConfusionMatrixDisplay.from_predictions(y_test, y_test_pred_logreg);"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "metadata": {},
      "outputs": [],
      "source": [
        "# create a new dataframe that shows the performance of the model\n",
        "# the true and predicted labels will be shown side-to-side,\n",
        "# and a third column will display whether the model did a correct prediction,\n",
        "# displaying \"incorrect\" if the values of the \"true label\" and \"predicted label\"\n",
        "# differ from one another\n",
        "# for readability, the encoded labels will be reverted back to their original\n",
        "# values (DC Comics and Marvel Comics)\n",
        "\n",
        "results_logreg = pd.DataFrame({\"hist_text_classic\": X_test, \n",
        "                              \"true_label\": y_test,\n",
        "                              \"predicted_label\": y_test_pred_logreg,\n",
        "                              \"result\": [\"Correct\" if true == pred else \"Incorrect\" for true, pred in zip(y_test, y_test_pred_logreg)],\n",
        "                              \"n_words\": [len(text.split()) for text in X_test]\n",
        "                              })\n",
        "\n",
        "results_logreg.true_label.replace({0: \"DC Comics\", 1: \"Marvel Comics\"}, inplace=True)\n",
        "results_logreg.predicted_label.replace({0: \"DC Comics\", 1: \"Marvel Comics\"}, inplace=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 16,
      "metadata": {},
      "outputs": [],
      "source": [
        "results_logreg.to_csv(\"data/error_logreg.csv\",\n",
        "                      index=False)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "#### 2.2 K-Nearest Neighbours"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 17,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-0GQpTfQGebp",
        "outputId": "1c6ddfef-6fcc-480e-b820-ea003f9e544b"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "['kneighborsclassifier',\n",
            " 'kneighborsclassifier__algorithm',\n",
            " 'kneighborsclassifier__leaf_size',\n",
            " 'kneighborsclassifier__metric',\n",
            " 'kneighborsclassifier__metric_params',\n",
            " 'kneighborsclassifier__n_jobs',\n",
            " 'kneighborsclassifier__n_neighbors',\n",
            " 'kneighborsclassifier__p',\n",
            " 'kneighborsclassifier__weights',\n",
            " 'memory',\n",
            " 'robustscaler',\n",
            " 'robustscaler__copy',\n",
            " 'robustscaler__quantile_range',\n",
            " 'robustscaler__unit_variance',\n",
            " 'robustscaler__with_centering',\n",
            " 'robustscaler__with_scaling',\n",
            " 'steps',\n",
            " 'tfidfvectorizer',\n",
            " 'tfidfvectorizer__analyzer',\n",
            " 'tfidfvectorizer__binary',\n",
            " 'tfidfvectorizer__decode_error',\n",
            " 'tfidfvectorizer__dtype',\n",
            " 'tfidfvectorizer__encoding',\n",
            " 'tfidfvectorizer__input',\n",
            " 'tfidfvectorizer__lowercase',\n",
            " 'tfidfvectorizer__max_df',\n",
            " 'tfidfvectorizer__max_features',\n",
            " 'tfidfvectorizer__min_df',\n",
            " 'tfidfvectorizer__ngram_range',\n",
            " 'tfidfvectorizer__norm',\n",
            " 'tfidfvectorizer__preprocessor',\n",
            " 'tfidfvectorizer__smooth_idf',\n",
            " 'tfidfvectorizer__stop_words',\n",
            " 'tfidfvectorizer__strip_accents',\n",
            " 'tfidfvectorizer__sublinear_tf',\n",
            " 'tfidfvectorizer__token_pattern',\n",
            " 'tfidfvectorizer__tokenizer',\n",
            " 'tfidfvectorizer__use_idf',\n",
            " 'tfidfvectorizer__vocabulary',\n",
            " 'verbose']\n"
          ]
        }
      ],
      "source": [
        "pp.pprint(sorted(knn_pipe.get_params().keys()))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 18,
      "metadata": {
        "id": "jFsSpyI8Glx9"
      },
      "outputs": [],
      "source": [
        "knn_params = [{\n",
        "    \"tfidfvectorizer__ngram_range\": [(1, 1), (1, 2), (2,3)],\n",
        "    \"kneighborsclassifier__metric\": [\"cosine\", \"euclidean\", \"manhattan\"],\n",
        "    \"kneighborsclassifier__n_neighbors\": [1, 2, 3, 5, 7],\n",
        "    \"kneighborsclassifier__weights\": [\"uniform\", \"distance\"]\n",
        "    }]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 19,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 180
        },
        "id": "kv9ft50-I8sq",
        "outputId": "0ac3e640-db0d-4b84-99e2-96ff342498f5"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Fitting 10 folds for each of 90 candidates, totalling 900 fits\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "<style>#sk-container-id-4 {color: black;}#sk-container-id-4 pre{padding: 0;}#sk-container-id-4 div.sk-toggleable {background-color: white;}#sk-container-id-4 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-4 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-4 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-4 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-4 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-4 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-4 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-4 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-4 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-4 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-4 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-4 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-4 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-4 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-4 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-4 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-4 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-4 div.sk-item {position: relative;z-index: 1;}#sk-container-id-4 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-4 div.sk-item::before, #sk-container-id-4 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-4 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-4 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-4 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-4 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-4 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-4 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-4 div.sk-label-container {text-align: center;}#sk-container-id-4 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-4 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-4\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>GridSearchCV(cv=10,\n",
              "             estimator=Pipeline(steps=[(&#x27;tfidfvectorizer&#x27;,\n",
              "                                        TfidfVectorizer(max_features=512)),\n",
              "                                       (&#x27;robustscaler&#x27;,\n",
              "                                        RobustScaler(with_centering=False)),\n",
              "                                       (&#x27;kneighborsclassifier&#x27;,\n",
              "                                        KNeighborsClassifier())]),\n",
              "             n_jobs=-1,\n",
              "             param_grid=[{&#x27;kneighborsclassifier__metric&#x27;: [&#x27;cosine&#x27;,\n",
              "                                                           &#x27;euclidean&#x27;,\n",
              "                                                           &#x27;manhattan&#x27;],\n",
              "                          &#x27;kneighborsclassifier__n_neighbors&#x27;: [1, 2, 3, 5, 7],\n",
              "                          &#x27;kneighborsclassifier__weights&#x27;: [&#x27;uniform&#x27;,\n",
              "                                                            &#x27;distance&#x27;],\n",
              "                          &#x27;tfidfvectorizer__ngram_range&#x27;: [(1, 1), (1, 2),\n",
              "                                                           (2, 3)]}],\n",
              "             scoring=&#x27;f1_macro&#x27;, verbose=1)</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-14\" type=\"checkbox\" ><label for=\"sk-estimator-id-14\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">GridSearchCV</label><div class=\"sk-toggleable__content\"><pre>GridSearchCV(cv=10,\n",
              "             estimator=Pipeline(steps=[(&#x27;tfidfvectorizer&#x27;,\n",
              "                                        TfidfVectorizer(max_features=512)),\n",
              "                                       (&#x27;robustscaler&#x27;,\n",
              "                                        RobustScaler(with_centering=False)),\n",
              "                                       (&#x27;kneighborsclassifier&#x27;,\n",
              "                                        KNeighborsClassifier())]),\n",
              "             n_jobs=-1,\n",
              "             param_grid=[{&#x27;kneighborsclassifier__metric&#x27;: [&#x27;cosine&#x27;,\n",
              "                                                           &#x27;euclidean&#x27;,\n",
              "                                                           &#x27;manhattan&#x27;],\n",
              "                          &#x27;kneighborsclassifier__n_neighbors&#x27;: [1, 2, 3, 5, 7],\n",
              "                          &#x27;kneighborsclassifier__weights&#x27;: [&#x27;uniform&#x27;,\n",
              "                                                            &#x27;distance&#x27;],\n",
              "                          &#x27;tfidfvectorizer__ngram_range&#x27;: [(1, 1), (1, 2),\n",
              "                                                           (2, 3)]}],\n",
              "             scoring=&#x27;f1_macro&#x27;, verbose=1)</pre></div></div></div><div class=\"sk-parallel\"><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-15\" type=\"checkbox\" ><label for=\"sk-estimator-id-15\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">estimator: Pipeline</label><div class=\"sk-toggleable__content\"><pre>Pipeline(steps=[(&#x27;tfidfvectorizer&#x27;, TfidfVectorizer(max_features=512)),\n",
              "                (&#x27;robustscaler&#x27;, RobustScaler(with_centering=False)),\n",
              "                (&#x27;kneighborsclassifier&#x27;, KNeighborsClassifier())])</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-16\" type=\"checkbox\" ><label for=\"sk-estimator-id-16\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">TfidfVectorizer</label><div class=\"sk-toggleable__content\"><pre>TfidfVectorizer(max_features=512)</pre></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-17\" type=\"checkbox\" ><label for=\"sk-estimator-id-17\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">RobustScaler</label><div class=\"sk-toggleable__content\"><pre>RobustScaler(with_centering=False)</pre></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-18\" type=\"checkbox\" ><label for=\"sk-estimator-id-18\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">KNeighborsClassifier</label><div class=\"sk-toggleable__content\"><pre>KNeighborsClassifier()</pre></div></div></div></div></div></div></div></div></div></div></div></div>"
            ],
            "text/plain": [
              "GridSearchCV(cv=10,\n",
              "             estimator=Pipeline(steps=[('tfidfvectorizer',\n",
              "                                        TfidfVectorizer(max_features=512)),\n",
              "                                       ('robustscaler',\n",
              "                                        RobustScaler(with_centering=False)),\n",
              "                                       ('kneighborsclassifier',\n",
              "                                        KNeighborsClassifier())]),\n",
              "             n_jobs=-1,\n",
              "             param_grid=[{'kneighborsclassifier__metric': ['cosine',\n",
              "                                                           'euclidean',\n",
              "                                                           'manhattan'],\n",
              "                          'kneighborsclassifier__n_neighbors': [1, 2, 3, 5, 7],\n",
              "                          'kneighborsclassifier__weights': ['uniform',\n",
              "                                                            'distance'],\n",
              "                          'tfidfvectorizer__ngram_range': [(1, 1), (1, 2),\n",
              "                                                           (2, 3)]}],\n",
              "             scoring='f1_macro', verbose=1)"
            ]
          },
          "execution_count": 19,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "knn_gs = GridSearchCV(knn_pipe,\n",
        "                      knn_params,\n",
        "                      cv=10,\n",
        "                      scoring='f1_macro',\n",
        "                      n_jobs=-1,\n",
        "                      refit=True,\n",
        "                      verbose=1)\n",
        "knn_gs.fit(X_train, y_train)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 20,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 161
        },
        "id": "qcTVoWrTKPuz",
        "outputId": "6ec7ec4f-4979-4b7e-a5f2-8cbe635fe219"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<style>#sk-container-id-5 {color: black;}#sk-container-id-5 pre{padding: 0;}#sk-container-id-5 div.sk-toggleable {background-color: white;}#sk-container-id-5 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-5 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-5 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-5 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-5 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-5 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-5 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-5 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-5 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-5 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-5 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-5 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-5 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-5 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-5 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-5 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-5 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-5 div.sk-item {position: relative;z-index: 1;}#sk-container-id-5 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-5 div.sk-item::before, #sk-container-id-5 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-5 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-5 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-5 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-5 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-5 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-5 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-5 div.sk-label-container {text-align: center;}#sk-container-id-5 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-5 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-5\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>Pipeline(steps=[(&#x27;tfidfvectorizer&#x27;,\n",
              "                 TfidfVectorizer(max_features=512, ngram_range=(2, 3))),\n",
              "                (&#x27;robustscaler&#x27;, RobustScaler(with_centering=False)),\n",
              "                (&#x27;kneighborsclassifier&#x27;,\n",
              "                 KNeighborsClassifier(metric=&#x27;cosine&#x27;, n_neighbors=7))])</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-19\" type=\"checkbox\" ><label for=\"sk-estimator-id-19\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">Pipeline</label><div class=\"sk-toggleable__content\"><pre>Pipeline(steps=[(&#x27;tfidfvectorizer&#x27;,\n",
              "                 TfidfVectorizer(max_features=512, ngram_range=(2, 3))),\n",
              "                (&#x27;robustscaler&#x27;, RobustScaler(with_centering=False)),\n",
              "                (&#x27;kneighborsclassifier&#x27;,\n",
              "                 KNeighborsClassifier(metric=&#x27;cosine&#x27;, n_neighbors=7))])</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-20\" type=\"checkbox\" ><label for=\"sk-estimator-id-20\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">TfidfVectorizer</label><div class=\"sk-toggleable__content\"><pre>TfidfVectorizer(max_features=512, ngram_range=(2, 3))</pre></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-21\" type=\"checkbox\" ><label for=\"sk-estimator-id-21\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">RobustScaler</label><div class=\"sk-toggleable__content\"><pre>RobustScaler(with_centering=False)</pre></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-22\" type=\"checkbox\" ><label for=\"sk-estimator-id-22\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">KNeighborsClassifier</label><div class=\"sk-toggleable__content\"><pre>KNeighborsClassifier(metric=&#x27;cosine&#x27;, n_neighbors=7)</pre></div></div></div></div></div></div></div>"
            ],
            "text/plain": [
              "Pipeline(steps=[('tfidfvectorizer',\n",
              "                 TfidfVectorizer(max_features=512, ngram_range=(2, 3))),\n",
              "                ('robustscaler', RobustScaler(with_centering=False)),\n",
              "                ('kneighborsclassifier',\n",
              "                 KNeighborsClassifier(metric='cosine', n_neighbors=7))])"
            ]
          },
          "execution_count": 20,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "knn_gs.best_estimator_"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 21,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IdjWfuQrKbsM",
        "outputId": "0d55df28-d4d0-49d3-9194-aa74575fa8f2"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "0.89171749127019"
            ]
          },
          "execution_count": 21,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "knn_gs.best_score_"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 22,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WB-KcutQNaBB",
        "outputId": "3402f630-20bc-4f49-fb99-11a90a8a415f"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "{'kneighborsclassifier__metric': 'cosine',\n",
              " 'kneighborsclassifier__n_neighbors': 7,\n",
              " 'kneighborsclassifier__weights': 'uniform',\n",
              " 'tfidfvectorizer__ngram_range': (2, 3)}"
            ]
          },
          "execution_count": 22,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "knn_gs.best_params_"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 23,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 444
        },
        "id": "tyd9UfRPKrQQ",
        "outputId": "7657e79e-82b0-4e6f-87eb-34312d36841b"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.88      0.93      0.90        95\n",
            "           1       0.95      0.91      0.93       137\n",
            "\n",
            "    accuracy                           0.92       232\n",
            "   macro avg       0.91      0.92      0.92       232\n",
            "weighted avg       0.92      0.92      0.92       232\n",
            "\n"
          ]
        },
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAfsAAAGwCAYAAACuFMx9AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/NK7nSAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAyhElEQVR4nO3de3hU5bn38d/kNDmQBAJmQiRA0CgoiBgQwSpQIBQFYbNbZEMrVrRgVMxGRSlV4oFE6N4RhYJKLVCViq8VtG4PxBOKSIUAikCxaIAgpMESExJynFnvH5HRMaCZrJkMM+v7ua511XWce6xX7rnv51lr2QzDMAQAAEJWWKADAAAA/kWyBwAgxJHsAQAIcSR7AABCHMkeAIAQR7IHACDEkewBAAhxEYEOwAyXy6XDhw8rPj5eNpst0OEAALxkGIaOHz+u1NRUhYX5r/6sra1VfX296etERUUpOjraBxG1raBO9ocPH1ZaWlqgwwAAmFRSUqIuXbr45dq1tbVK79ZOpWVO09dKSUlRcXFx0CX8oE728fHxkqQBq6crIjYqwNEA/mEffzDQIQB+06gGbdSr7r/n/lBfX6/SMqcOFHVXQnzruweVx13qlrlf9fX1JPu2dLJ1HxEbpYg4e4CjAfwjwhYZ6BAA//nmge1tMRTbLt6mdvGt/xyXgne4OKiTPQAALeU0XHKaeBuM03D5Lpg2RrIHAFiCS4Zcan22N3NuoHHrHQAAIY7KHgBgCS65ZKYRb+7swCLZAwAswWkYchqtb8WbOTfQaOMDABDiqOwBAJZg5Ql6JHsAgCW4ZMhp0WRPGx8AgBBHsgcAWMLJNr6ZxRvvvfeexo4dq9TUVNlsNq1bt869r6GhQXfffbf69OmjuLg4paam6rrrrtPhw4c9rlFXV6fbbrtNnTp1UlxcnK655hodOnTI6+9OsgcAWMLJ2fhmFm9UV1erb9++WrJkSbN9J06c0LZt23Tvvfdq27ZtevHFF/XZZ5/pmmuu8TguJydHa9eu1XPPPaeNGzeqqqpKY8aMkdPp3Ut9GLMHAMAPRo8erdGjR59yX2JiogoLCz22LV68WJdeeqkOHjyorl27qqKiQk899ZSefvppjRgxQpL0zDPPKC0tTW+++aZGjRrV4lio7AEAluDywSJJlZWVHktdXZ1P4quoqJDNZlP79u0lSUVFRWpoaFBWVpb7mNTUVPXu3VubNm3y6tokewCAJTi/mY1vZpGktLQ0JSYmupf8/HzTsdXW1uqee+7R5MmTlZCQIEkqLS1VVFSUOnTo4HGsw+FQaWmpV9enjQ8AsASnIZNvvWv635KSEndCliS73dwr1hsaGjRp0iS5XC4tXbr0R483DMPrVwJT2QMA4IWEhASPxUyyb2ho0MSJE1VcXKzCwkKPHxEpKSmqr69XeXm5xzllZWVyOBxefQ7JHgBgCb4as/eVk4n+n//8p95880117NjRY39mZqYiIyM9JvIdOXJEn376qQYPHuzVZ9HGBwBYgks2OeVd+/v753ujqqpK+/btc68XFxdrx44dSkpKUmpqqn7+859r27ZteuWVV+R0Ot3j8ElJSYqKilJiYqKmTZumO+64Qx07dlRSUpLuvPNO9enTxz07v6VI9gAA+MHWrVs1bNgw9/qsWbMkSVOnTlVubq5efvllSdLFF1/scd4777yjoUOHSpIeeeQRRUREaOLEiaqpqdHw4cO1cuVKhYeHexULyR4AYAkuo2kxc743hg4dKuMHHsTzQ/tOio6O1uLFi7V48WLvPvx7SPYAAEtwmmzjmzk30JigBwBAiKOyBwBYgpUre5I9AMASXIZNLsPEbHwT5wYabXwAAEIclT0AwBJo4wMAEOKcCpPTREPbuzfIn1lI9gAASzBMjtkbjNkDAIAzFZU9AMASGLMHACDEOY0wOQ0TY/YmHrUbaLTxAQAIcVT2AABLcMkml4ka16XgLe1J9gAAS7DymD1tfAAAQhyVPQDAEsxP0KONDwDAGa1pzN7Ei3Bo4wMAgDMVlT0AwBJcJp+Nz2x8AADOcIzZAwAQ4lwKs+x99ozZAwAQ4qjsAQCW4DRscpp4Ta2ZcwONZA8AsASnyQl6Ttr4AADgTEVlDwCwBJcRJpeJ2fguZuMDAHBmo40PAABCFpU9AMASXDI3o97lu1DaHMkeAGAJ5h+qE7zN8OCNHAAAtAiVPQDAEsw/Gz9462OSPQDAEqz8PnuSPQDAEqxc2Qdv5AAAoEWo7AEAlmD+oTrBWx+T7AEAluAybHKZuc8+iN96F7w/UwAAQItQ2QMALMFlso0fzA/VIdkDACzB/FvvgjfZB2/kAACgRajsAQCW4JRNThMPxjFzbqCR7AEAlkAbHwAAhCwqewCAJThlrhXv9F0obY5kDwCwBCu38Un2AABL4EU4AAAgZFHZAwAswTD5PnuDW+8AADiz0cYHAAAhi8oeAGAJvOIWAIAQ5/zmrXdmFm+89957Gjt2rFJTU2Wz2bRu3TqP/YZhKDc3V6mpqYqJidHQoUO1a9cuj2Pq6up02223qVOnToqLi9M111yjQ4cOef3dSfYAAPhBdXW1+vbtqyVLlpxy/8KFC1VQUKAlS5Zoy5YtSklJ0ciRI3X8+HH3MTk5OVq7dq2ee+45bdy4UVVVVRozZoycTu8e8UMbHwBgCb5q41dWVnpst9vtstvtzY4fPXq0Ro8efcprGYahRYsWae7cuZowYYIkadWqVXI4HFq9erWmT5+uiooKPfXUU3r66ac1YsQISdIzzzyjtLQ0vfnmmxo1alSLY6eyBwBYgkthphdJSktLU2JionvJz8/3Opbi4mKVlpYqKyvLvc1ut2vIkCHatGmTJKmoqEgNDQ0ex6Smpqp3797uY1qKyh4AAC+UlJQoISHBvX6qqv7HlJaWSpIcDofHdofDoQMHDriPiYqKUocOHZodc/L8liLZAwAswWnY5DTRxj95bkJCgkeyN8Nm84zHMIxm276vJcd8H218AIAlnByzN7P4SkpKiiQ1q9DLysrc1X5KSorq6+tVXl5+2mNaimQPALAE45u33rV2MXz4BL309HSlpKSosLDQva2+vl4bNmzQ4MGDJUmZmZmKjIz0OObIkSP69NNP3ce0FG18AAD8oKqqSvv27XOvFxcXa8eOHUpKSlLXrl2Vk5OjvLw8ZWRkKCMjQ3l5eYqNjdXkyZMlSYmJiZo2bZruuOMOdezYUUlJSbrzzjvVp08f9+z8liLZAwAswSmbnCZeZuPtuVu3btWwYcPc67NmzZIkTZ06VStXrtTs2bNVU1Oj7OxslZeXa+DAgVq/fr3i4+Pd5zzyyCOKiIjQxIkTVVNTo+HDh2vlypUKDw/3KhabYRiGV2ecQSorK5WYmKhB625TRJz3syGBYGDP2h/oEAC/aTQa9K5eUkVFhc8mvX3fyVzx63cnKqpdVKuvU19VrxVDn/drrP7CmD0AACGONj6acxoK//PXCn+7Wip3Sknhcma1k3NyohT2TRurxqWIp8oVtumEVOmS4YiQc3y8XGOD69cuIEmr/r5bKWkNzba/vLKj/vDbLgGICP5wcqKdmfODFckezYSvqVD4/x1X412d5OoWqbDP6hXxv19JcWFy/kdTMo94/JjCPq5Vw92dZDgiFFZUq4jF/1Zjxwi5BscG+BsA3pk5+jyFhX87otm9Z60eXvOF3v9b+8AFBZ9zySaXiTF7M+cGWsB/pixdulTp6emKjo5WZmam3n///UCHZHm2PXVyDYqVa2CslBIp15VxcmXGyPZZ3bfH7K6Tc0Q7GX1jmo65Ol5GjyiPY4BgUXEsQuVHI93LwBGVOlwcpU8+jAt0aIBPBDTZr1mzRjk5OZo7d662b9+uK664QqNHj9bBgwcDGZblGRfaFbajRrZDTW1N2+f1Cvu0Vq5LY749prddYZtPSF81SoYh244a2b5skKt/zOkuCwSFiEiXfvqf5XrjuSQpiCs5NHfyCXpmlmAV0DZ+QUGBpk2bphtvvFGStGjRIr3xxhtatmxZq14sAN9wXpsoVRuKnPZl089Bl+S8vr1cw9q5j2nM7qiIR76SffIhGeGSwqTG/+4ko3d0wOIGfGHwzyrVLsGp9c8nBToU+Bhj9gFQX1+voqIi3XPPPR7bs7KyTvs2n7q6OtXVfdsm/v5rBuEbYe9WK/ytKjXe00lG9yjZPq9XxLJjMjpGyJXVlPDD11XK9o86NdyfLMMRIdvOpjH7hqRwGZdQ3SN4jfqvf2vLOwk69q/IQIcC+EzAfqZ89dVXcjqdp3zjz+ne5pOfn+/xWsG0tLS2CNVyIpaXq3FSolzD2slIj5JrRDs5JyQo/Lmvmw6ocyl8Rbmc05PkGhQro0eUXOMS5BoSp4gXKgIaO2BG8tn16ndFlV5fTVUfilwy+Wz8IB7WCXhPwps3/syZM0cVFRXupaSkpC1CtJ46o/lQZZikk5OVGyVbo059jMvfwQH+kzXpmL7+KkJ/f5NbSEOR8c1s/NYuRhAn+4C18Tt16qTw8PAffOPP99nt9la9NxjecV0Wo4i/VKgxOaLp1rt99Qp/sVLOUd+M2ceFyXWRXeHLy2XYbTKSIxS2s1Zhb1arcXqHH744cIay2QxlXXtMb/6/DnI5g/ePOk7P7JvrfPnWu7YWsMo+KipKmZmZHm/zkaTCwkKv3+YD32q8paOcV8QqYvG/FXXjYUUsL5fzqng5p36byBt+e5aM8+2KfPgrRd10WOHPVTRN4hsT/wNXBs5c/a6skqNLg954rmOgQwF8LqCz8WfNmqVf/epX6t+/vwYNGqQnn3xSBw8e1IwZMwIZFmLD5Ly5o5w3/8AfvaQINd7Zqe1iAvxs24Z4jUrtG+gw4EfMxg+Qa6+9Vv/+97/1wAMP6MiRI+rdu7deffVVdevWLZBhAQBCkJXb+AF/XG52drays7MDHQYAACEr4MkeAIC2YOVn45PsAQCWYOU2fvDONgAAAC1CZQ8AsAQrV/YkewCAJVg52dPGBwAgxFHZAwAswcqVPckeAGAJhszdPmf8+CFnLJI9AMASrFzZM2YPAECIo7IHAFiClSt7kj0AwBKsnOxp4wMAEOKo7AEAlmDlyp5kDwCwBMOwyTCRsM2cG2i08QEACHFU9gAAS+B99gAAhDgrj9nTxgcAIMRR2QMALMHKE/RI9gAAS7ByG59kDwCwBCtX9ozZAwAQ4qjsAQCWYJhs4wdzZU+yBwBYgiHJMMydH6xo4wMAEOKo7AEAluCSTTaeoAcAQOhiNj4AAAhZVPYAAEtwGTbZeKgOAAChyzBMzsYP4un4tPEBAAhxVPYAAEuw8gQ9kj0AwBJI9gAAhDgrT9BjzB4AgBBHZQ8AsARm4wMAEOKakr3NxOLd5zU2Nup3v/ud0tPTFRMTox49euiBBx6Qy+X6TkyGcnNzlZqaqpiYGA0dOlS7du3y8Tcn2QMA4BcLFizQ448/riVLlmjPnj1auHChfv/732vx4sXuYxYuXKiCggItWbJEW7ZsUUpKikaOHKnjx4/7NBba+AAAS2jr2fgffvihxo0bp6uvvlqS1L17d/3lL3/R1q1bv7meoUWLFmnu3LmaMGGCJGnVqlVyOBxavXq1pk+f3upYv4/KHgBgCYYPFkmqrKz0WOrq6k75eT/5yU/01ltv6bPPPpMkffzxx9q4caOuuuoqSVJxcbFKS0uVlZXlPsdut2vIkCHatGmTT787lT0AAF5IS0vzWJ83b55yc3ObHXf33XeroqJCPXv2VHh4uJxOp+bPn6//+q//kiSVlpZKkhwOh8d5DodDBw4c8GnMJHsAgCX4qo1fUlKihIQE93a73X7K49esWaNnnnlGq1ev1oUXXqgdO3YoJydHqampmjp1qvs4m80zJsMwmm0zi2QPALCG7/biW3u+pISEBI9kfzp33XWX7rnnHk2aNEmS1KdPHx04cED5+fmaOnWqUlJSJDVV+J07d3afV1ZW1qzaN4sxewCANZi67c4medkVOHHihMLCPNNseHi4+9a79PR0paSkqLCw0L2/vr5eGzZs0ODBg81/3++gsgcAwA/Gjh2r+fPnq2vXrrrwwgu1fft2FRQU6IYbbpDU1L7PyclRXl6eMjIylJGRoby8PMXGxmry5Mk+jYVkDwCwhLZ+gt7ixYt17733Kjs7W2VlZUpNTdX06dN13333uY+ZPXu2ampqlJ2drfLycg0cOFDr169XfHx86wM9BZthBO8DACsrK5WYmKhB625TRNypJ0gAwc6etT/QIQB+02g06F29pIqKihaNg7fGyVzR/U+/U1hsdKuv4zpRq/03POTXWP2FMXsAAEIcbXwAgDW0YpJds/ODFMkeAGAJvPUOAACELCp7AIA1+OihOsGIZA8AsIS2fuvdmaRFyf6xxx5r8QVnzpzZ6mAAAIDvtSjZP/LIIy26mM1mI9kDAM5cQdyKN6NFyb64uNjfcQAA4FdWbuO3ejZ+fX299u7dq8bGRl/GAwCAfxg+WIKU18n+xIkTmjZtmmJjY3XhhRfq4MGDkprG6h9++GGfBwgAAMzxOtnPmTNHH3/8sd59911FR3/7jOERI0ZozZo1Pg0OAADfsflgCU5e33q3bt06rVmzRpdddplstm+/+AUXXKDPP//cp8EBAOAzFr7P3uvK/ujRo0pOTm62vbq62iP5AwCAM4PXyX7AgAH6v//7P/f6yQS/fPlyDRo0yHeRAQDgSxaeoOd1Gz8/P18/+9nPtHv3bjU2NurRRx/Vrl279OGHH2rDhg3+iBEAAPMs/NY7ryv7wYMH64MPPtCJEyd0zjnnaP369XI4HPrwww+VmZnpjxgBAIAJrXo2fp8+fbRq1SpfxwIAgN9Y+RW3rUr2TqdTa9eu1Z49e2Sz2dSrVy+NGzdOERG8VwcAcIay8Gx8r7Pzp59+qnHjxqm0tFTnn3++JOmzzz7TWWedpZdffll9+vTxeZAAAKD1vB6zv/HGG3XhhRfq0KFD2rZtm7Zt26aSkhJddNFF+s1vfuOPGAEAMO/kBD0zS5DyurL/+OOPtXXrVnXo0MG9rUOHDpo/f74GDBjg0+AAAPAVm9G0mDk/WHld2Z9//vn617/+1Wx7WVmZzj33XJ8EBQCAz1n4PvsWJfvKykr3kpeXp5kzZ+qFF17QoUOHdOjQIb3wwgvKycnRggUL/B0vAADwUova+O3bt/d4FK5hGJo4caJ7m/HN/Qhjx46V0+n0Q5gAAJhk4YfqtCjZv/POO/6OAwAA/+LWux82ZMgQf8cBAAD8pNVPwTlx4oQOHjyo+vp6j+0XXXSR6aAAAPA5KvuWO3r0qH7961/rtddeO+V+xuwBAGckCyd7r2+9y8nJUXl5uTZv3qyYmBi9/vrrWrVqlTIyMvTyyy/7I0YAAGCC15X922+/rZdeekkDBgxQWFiYunXrppEjRyohIUH5+fm6+uqr/REnAADmWHg2vteVfXV1tZKTkyVJSUlJOnr0qKSmN+Ft27bNt9EBAOAjJ5+gZ2YJVq16gt7evXslSRdffLGeeOIJffnll3r88cfVuXNnnwcIAADM8bqNn5OToyNHjkiS5s2bp1GjRunZZ59VVFSUVq5c6ev4AADwDQtP0PM62U+ZMsX9z/369dP+/fv1j3/8Q127dlWnTp18GhwAADCv1ffZnxQbG6tLLrnEF7EAAOA3Npl8653PIml7LUr2s2bNavEFCwoKWh0MAADwvRYl++3bt7foYt99WU5bir2uXBG2qIB8NuBvrx7eEegQAL+pPO5Sh/Pa6MMsfOsdL8IBAFiDhSfoeX3rHQAACC6mJ+gBABAULFzZk+wBAJZg9il4lnqCHgAACC5U9gAAa7BwG79Vlf3TTz+tyy+/XKmpqTpw4IAkadGiRXrppZd8GhwAAD5j+GAJUl4n+2XLlmnWrFm66qqr9PXXX8vpdEqS2rdvr0WLFvk6PgAAYJLXyX7x4sVavny55s6dq/DwcPf2/v37a+fOnT4NDgAAX7HyK269HrMvLi5Wv379mm232+2qrq72SVAAAPichZ+g53Vln56erh07djTb/tprr+mCCy7wRUwAAPiehcfsva7s77rrLt1yyy2qra2VYRj66KOP9Je//EX5+fn64x//6I8YAQCACV4n+1//+tdqbGzU7NmzdeLECU2ePFlnn322Hn30UU2aNMkfMQIAYJqVH6rTqvvsb7rpJt1000366quv5HK5lJyc7Ou4AADwLe6zb51OnTqR6AEAOI0vv/xSv/zlL9WxY0fFxsbq4osvVlFRkXu/YRjKzc1VamqqYmJiNHToUO3atcvncXhd2aenp//ge+u/+OILUwEBAOAXZm+f8/Lc8vJyXX755Ro2bJhee+01JScn6/PPP1f79u3dxyxcuFAFBQVauXKlzjvvPD300EMaOXKk9u7dq/j4eBPBevI62efk5HisNzQ0aPv27Xr99dd11113+SouAAB8q43b+AsWLFBaWppWrFjh3ta9e/dvL2cYWrRokebOnasJEyZIklatWiWHw6HVq1dr+vTpJoL15HWyv/3220+5/Q9/+IO2bt1qOiAAAM5klZWVHut2u112u73ZcS+//LJGjRqlX/ziF9qwYYPOPvtsZWdn66abbpLU9Nya0tJSZWVleVxryJAh2rRpk0+Tvc/eejd69Gj99a9/9dXlAADwLR/dZ5+WlqbExET3kp+ff8qP++KLL7Rs2TJlZGTojTfe0IwZMzRz5kz9+c9/liSVlpZKkhwOh8d5DofDvc9XfPbWuxdeeEFJSUm+uhwAAD7lq1vvSkpKlJCQ4N5+qqpeklwul/r376+8vDxJUr9+/bRr1y4tW7ZM11133bfX/d48OMMwfnBuXGt4nez79evnEYRhGCotLdXRo0e1dOlSnwYHAMCZJiEhwSPZn07nzp2bPVm2V69e7i54SkqKpKYKv3Pnzu5jysrKmlX7Znmd7MePH++xHhYWprPOOktDhw5Vz549fRUXAABB7fLLL9fevXs9tn322Wfq1q2bpKa721JSUlRYWOh+50x9fb02bNigBQsW+DQWr5J9Y2OjunfvrlGjRrl/kQAAEBTaeDb+f//3f2vw4MHKy8vTxIkT9dFHH+nJJ5/Uk08+KampfZ+Tk6O8vDxlZGQoIyNDeXl5io2N1eTJk00E2pxXyT4iIkI333yz9uzZ49MgAADwt7Z+XO6AAQO0du1azZkzRw888IDS09O1aNEiTZkyxX3M7NmzVVNTo+zsbJWXl2vgwIFav369T++xl1rRxh84cKC2b9/ubkMAAIBTGzNmjMaMGXPa/TabTbm5ucrNzfVrHF4n++zsbN1xxx06dOiQMjMzFRcX57H/oosu8llwAAD4VBA/396MFif7G264QYsWLdK1114rSZo5c6Z7n81mc98q4HQ6fR8lAABmWfhFOC1O9qtWrdLDDz+s4uJif8YDAAB8rMXJ3jCaftIwVg8ACEa8z76FfP1EHwAA2gxt/JY577zzfjThHzt2zFRAAADAt7xK9vfff78SExP9FQsAAH5DG7+FJk2apOTkZH/FAgCA/1i4jd/iV9wyXg8AQHDyejY+AABBycKVfYuTvcvl8mccAAD4FWP2AACEOgtX9i0eswcAAMGJyh4AYA0WruxJ9gAAS7DymD1tfAAAQhyVPQDAGmjjAwAQ2mjjAwCAkEVlDwCwBtr4AACEOAsne9r4AACEOCp7AIAl2L5ZzJwfrEj2AABrsHAbn2QPALAEbr0DAAAhi8oeAGANtPEBALCAIE7YZtDGBwAgxFHZAwAswcoT9Ej2AABrsPCYPW18AABCHJU9AMASaOMDABDqaOMDAIBQRWUPALAE2vgAAIQ6C7fxSfYAAGuwcLJnzB4AgBBHZQ8AsATG7AEACHW08QEAQKiisgcAWILNMGQzWl+emzk30Ej2AABroI0PAABCFZU9AMASmI0PAECoo40PAABCFZU9AMASaOMDABDqLNzGJ9kDACzBypU9Y/YAAIQ4kj0AwBoMHyytlJ+fL5vNppycnG/DMQzl5uYqNTVVMTExGjp0qHbt2tX6D/kBJHsAgGWcbOW3ZmmtLVu26Mknn9RFF13ksX3hwoUqKCjQkiVLtGXLFqWkpGjkyJE6fvy4yW/ZHMkeAAAvVFZWeix1dXWnPbaqqkpTpkzR8uXL1aFDB/d2wzC0aNEizZ07VxMmTFDv3r21atUqnThxQqtXr/Z5zCR7AIA1GIb5RVJaWpoSExPdS35+/mk/8pZbbtHVV1+tESNGeGwvLi5WaWmpsrKy3NvsdruGDBmiTZs2+fyrMxsfAGAJvpqNX1JSooSEBPd2u91+yuOfe+45bdu2TVu2bGm2r7S0VJLkcDg8tjscDh04cKD1QZ4GyR4AAC8kJCR4JPtTKSkp0e23367169crOjr6tMfZbDaPdcMwmm3zBdr4AABraMPZ+EVFRSorK1NmZqYiIiIUERGhDRs26LHHHlNERIS7oj9Z4Z9UVlbWrNr3BZI9AMASbC7zS0sNHz5cO3fu1I4dO9xL//79NWXKFO3YsUM9evRQSkqKCgsL3efU19drw4YNGjx4sM+/O218AAB8LD4+Xr179/bYFhcXp44dO7q35+TkKC8vTxkZGcrIyFBeXp5iY2M1efJkn8dDskczvftX6D+nHdK5F1apY3K9Hryllz58q5MkKTzCpetuP6ABQ44ppUutqqsitGNTe60o6K5jZaeepAIE2s7Ncfp/S5P1z52xOvavSM17qliDR1dIkhobpJULOmvL2wk6ciBKcQku9bviuKb99rA6pjS6r3HXf56rTz5s53HdIdeU67eP+34yFfzkDHs2/uzZs1VTU6Ps7GyVl5dr4MCBWr9+veLj4337QSLZ4xSiY5wq/kecCl906HeL93jss0e7dO4FVfrL0q76Ym+c2iU0avqcLzRv6W7d/vN+AYoY+GG1J8LU48IaZU06pgdvTPfYV1cTpn07YzU551/qcUGNqirC9fi8szXv+h5a8vpnHseOnvKVrrvr2zFWe7QXfV0EXKCfjf/uu+96Xs9mU25urnJzc81duAUCmuzfe+89/f73v1dRUZGOHDmitWvXavz48YEMCZK2vp+kre8nnXLfiaoIzZ3Wx2PbsofO0aMv7NBZnWt19MjpZ50CgTLgp8c14KenfipZXIJLD6/53GNb9kOHNPOq81V2KFLJXRrc2+0xhpKSG79/CQSL79wr3+rzg1RAJ+hVV1erb9++WrJkSSDDgElx8Y1yuaSqShpFCA3VleGy2QzFJTo9tr/zYgf94sLeumno+Xry/lSdqGKOM4JDQP86jx49WqNHj27x8XV1dR6PJaysrPRHWPBCZJRLv75jv9595SzVVJPsEfzqa236U16qhv1HueLiv23TD5twTClp9UpKbtT+f0TrT/md9cXumGZdAZy5At3GD6Sg+uucn5+v+++/P9Bh4BvhES7dU/AP2WyG/nD/uYEOBzCtsUHKu7m7DJd0a/4hj31XTTnm/ufuPWt1do863fqz8/XPT2KUcVFNW4eK1jjDJui1paDqQc2ZM0cVFRXupaSkJNAhWVZ4hEtzHvmHHF1qNXdaH6p6BL3GBmn+9O4qLYlS/nOfe1T1p3JunxpFRLr0ZTF3oeDMF1R/oe12+2mfQYy2czLRp3ar0T1T++j415GBDgkw5WSi/7LYroUv7FNCkvNHzzmwN1qNDWHq6Gj40WNxZqCND3xHdKxTqV2/bUs6utSpR88qHa+I0L/L7Prto3t07gVVyp1xocLDpQ6d6iVJxysi1NgQVM0iWERNdZgOf6cCLy2J0uefxii+faM6pjTowZvStW9njB748xdyOW06Vtb0pzG+vVORUYYO74/S2y920KXDK5WQ5NTBz+x68v6zdW7vE7pgQHWgvha8ZeHZ+CR7NJPR+7gW/Hmne/03c76QJBWuTdazS7pp0PCmscs/vLTd47y7r+ujnR+1b7M4gZb67ONYzf75t/NKnsg9W5I0cuIx/fKOUm1enyhJyh7Z0+O8hS/sU9/BVYqINLRjY7zWPXWWaqvD1Cm1QQOHV2rKrFKFh7fd9wBaK6DJvqqqSvv27XOvFxcXa8eOHUpKSlLXrl0DGJm17fyova7qecVp9//QPuBM1Hdwld44vOO0+39onyQln92g/3lx3w8egzMfbfwA2bp1q4YNG+ZenzVrliRp6tSpWrlyZYCiAgCEJAvPxg9osh86dKiMIB4DAQAgGDBmDwCwBNr4AACEOpfRtJg5P0iR7AEA1mDhMXtuigYAIMRR2QMALMEmk2P2Pouk7ZHsAQDWYOEn6NHGBwAgxFHZAwAsgVvvAAAIdczGBwAAoYrKHgBgCTbDkM3EJDsz5wYayR4AYA2ubxYz5wcp2vgAAIQ4KnsAgCXQxgcAINRZeDY+yR4AYA08QQ8AAIQqKnsAgCXwBD0AAEIdbXwAABCqqOwBAJZgczUtZs4PViR7AIA10MYHAAChisoeAGANPFQHAIDQZuXH5dLGBwAgxFHZAwCswcIT9Ej2AABrMGTunfTBm+tJ9gAAa2DMHgAAhCwqewCANRgyOWbvs0jaHMkeAGANFp6gRxsfAIAQR2UPALAGlySbyfODFMkeAGAJzMYHAAAhi8oeAGANFp6gR7IHAFiDhZM9bXwAAEIclT0AwBqo7AEACHEuHyxeyM/P14ABAxQfH6/k5GSNHz9ee/fu9TjGMAzl5uYqNTVVMTExGjp0qHbt2mXiS54ayR4AYAknb70zs3hjw4YNuuWWW7R582YVFhaqsbFRWVlZqq6udh+zcOFCFRQUaMmSJdqyZYtSUlI0cuRIHT9+3KffnTY+AAB+8Prrr3usr1ixQsnJySoqKtKVV14pwzC0aNEizZ07VxMmTJAkrVq1Sg6HQ6tXr9b06dN9FguVPQDAGk6O2ZtZJFVWVnosdXV1Lfr4iooKSVJSUpIkqbi4WKWlpcrKynIfY7fbNWTIEG3atMmnX51kDwCwBpdhfpGUlpamxMRE95Kfn/+jH20YhmbNmqWf/OQn6t27tySptLRUkuRwODyOdTgc7n2+QhsfAAAvlJSUKCEhwb1ut9t/9Jxbb71Vn3zyiTZu3Nhsn83m+cB+wzCabTOLZA8AsAYf3XqXkJDgkex/zG233aaXX35Z7733nrp06eLenpKSIqmpwu/cubN7e1lZWbNq3yza+AAAizA7Xu/dDwXDMHTrrbfqxRdf1Ntvv6309HSP/enp6UpJSVFhYaF7W319vTZs2KDBgwf74gu7UdkDAOAHt9xyi1avXq2XXnpJ8fHx7nH4xMRExcTEyGazKScnR3l5ecrIyFBGRoby8vIUGxuryZMn+zQWkj0AwBra+Al6y5YtkyQNHTrUY/uKFSt0/fXXS5Jmz56tmpoaZWdnq7y8XAMHDtT69esVHx/f+jhPgWQPALAGl/et+Obnt5zRgh8HNptNubm5ys3NbWVQLcOYPQAAIY7KHgBgDYaraTFzfpAi2QMArMHCb70j2QMArKGNx+zPJIzZAwAQ4qjsAQDWQBsfAIAQZ8hksvdZJG2ONj4AACGOyh4AYA208QEACHEulyQT98q7gvc+e9r4AACEOCp7AIA10MYHACDEWTjZ08YHACDEUdkDAKzBwo/LJdkDACzBMFwyTLy5zsy5gUayBwBYg2GYq84ZswcAAGcqKnsAgDUYJsfsg7iyJ9kDAKzB5ZJsJsbdg3jMnjY+AAAhjsoeAGANtPEBAAhthsslw0QbP5hvvaONDwBAiKOyBwBYA218AABCnMuQbNZM9rTxAQAIcVT2AABrMAxJZu6zD97KnmQPALAEw2XIMNHGN0j2AACc4QyXzFX23HoHAADOUFT2AABLoI0PAECos3AbP6iT/clfWY1GfYAjAfyn8njw/oEBfkxlVdN/321RNTeqwdQzdRrV4Ltg2lhQJ/vjx49LkjYcfz7AkQD+0+G8QEcA+N/x48eVmJjol2tHRUUpJSVFG0tfNX2tlJQURUVF+SCqtmUzgngQwuVy6fDhw4qPj5fNZgt0OJZQWVmptLQ0lZSUKCEhIdDhAD7Ff99tzzAMHT9+XKmpqQoL89+c8draWtXXm+8CR0VFKTo62gcRta2gruzDwsLUpUuXQIdhSQkJCfwxRMjiv++25a+K/ruio6ODMkn7CrfeAQAQ4kj2AACEOJI9vGK32zVv3jzZ7fZAhwL4HP99I1QF9QQ9AADw46jsAQAIcSR7AABCHMkeAIAQR7IHACDEkezRYkuXLlV6erqio6OVmZmp999/P9AhAT7x3nvvaezYsUpNTZXNZtO6desCHRLgUyR7tMiaNWuUk5OjuXPnavv27briiis0evRoHTx4MNChAaZVV1erb9++WrJkSaBDAfyCW+/QIgMHDtQll1yiZcuWubf16tVL48ePV35+fgAjA3zLZrNp7dq1Gj9+fKBDAXyGyh4/qr6+XkVFRcrKyvLYnpWVpU2bNgUoKgBAS5Hs8aO++uorOZ1OORwOj+0Oh0OlpaUBigoA0FIke7TY918jbBgGrxYGgCBAsseP6tSpk8LDw5tV8WVlZc2qfQDAmYdkjx8VFRWlzMxMFRYWemwvLCzU4MGDAxQVAKClIgIdAILDrFmz9Ktf/Ur9+/fXoEGD9OSTT+rgwYOaMWNGoEMDTKuqqtK+ffvc68XFxdqxY4eSkpLUtWvXAEYG+Aa33qHFli5dqoULF+rIkSPq3bu3HnnkEV155ZWBDgsw7d1339WwYcOabZ86dapWrlzZ9gEBPkayBwAgxDFmDwBAiCPZAwAQ4kj2AACEOJI9AAAhjmQPAECII9kDABDiSPYAAIQ4kj0AACGOZA+YlJubq4svvti9fv3112v8+PFtHsf+/ftls9m0Y8eO0x7TvXt3LVq0qMXXXLlypdq3b286NpvNpnXr1pm+DoDWIdkjJF1//fWy2Wyy2WyKjIxUjx49dOedd6q6utrvn/3oo4+2+BGrLUnQAGAWL8JByPrZz36mFStWqKGhQe+//75uvPFGVVdXa9myZc2ObWhoUGRkpE8+NzEx0SfXAQBfobJHyLLb7UpJSVFaWpomT56sKVOmuFvJJ1vvf/rTn9SjRw/Z7XYZhqGKigr95je/UXJyshISEvTTn/5UH3/8scd1H374YTkcDsXHx2vatGmqra312P/9Nr7L5dKCBQt07rnnym63q2vXrpo/f74kKT09XZLUr18/2Ww2DR061H3eihUr1KtXL0VHR6tnz55aunSpx+d89NFH6tevn6Kjo9W/f39t377d639HBQUF6tOnj+Li4pSWlqbs7GxVVVU1O27dunU677zzFB0drZEjR6qkpMRj/9/+9jdlZmYqOjpaPXr00P3336/Gxkav4wHgHyR7WEZMTIwaGhrc6/v27dPzzz+vv/71r+42+tVXX63S0lK9+uqrKioq0iWXXKLhw4fr2LFjkqTnn39e8+bN0/z587V161Z17ty5WRL+vjlz5mjBggW69957tXv3bq1evVoOh0NSU8KWpDfffFNHjhzRiy++KElavny55s6dq/nz52vPnj3Ky8vTvffeq1WrVkmSqqurNWbMGJ1//vkqKipSbm6u7rzzTq//nYSFhemxxx7Tp59+qlWrVuntt9/W7NmzPY45ceKE5s+fr1WrVumDDz5QZWWlJk2a5N7/xhtv6Je//KVmzpyp3bt364knntDKlSvdP2gAnAEMIARNnTrVGDdunHv973//u9GxY0dj4sSJhmEYxrx584zIyEijrKzMfcxbb71lJCQkGLW1tR7XOuecc4wnnnjCMAzDGDRokDFjxgyP/QMHDjT69u17ys+urKw07Ha7sXz58lPGWVxcbEgytm/f7rE9LS3NWL16tce2Bx980Bg0aJBhGIbxxBNPGElJSUZ1dbV7/7Jly055re/q1q2b8cgjj5x2//PPP2907NjRvb5ixQpDkrF582b3tj179hiSjL///e+GYRjGFVdcYeTl5Xlc5+mnnzY6d+7sXpdkrF279rSfC8C/GLNHyHrllVfUrl07NTY2qqGhQePGjdPixYvd+7t166azzjrLvV5UVKSqqip17NjR4zo1NTX6/PPPJUl79uzRjBkzPPYPGjRI77zzzilj2LNnj+rq6jR8+PAWx3306FGVlJRo2rRpuummm9zbGxsb3fMB9uzZo759+yo2NtYjDm+98847ysvL0+7du1VZWanGxkbV1taqurpacXFxkqSIiAj179/ffU7Pnj3Vvn177dmzR5deeqmKioq0ZcsWj0re6XSqtrZWJ06c8IgRQGCQ7BGyhg0bpmXLlikyMlKpqanNJuCdTGYnuVwude7cWe+++26za7X29rOYmBivz3G5XJKaWvkDBw702BceHi5JMgyjVfF814EDB3TVVVdpxowZevDBB5WUlKSNGzdq2rRpHsMdUtOtc993cpvL5dL999+vCRMmNDsmOjradJwAzCPZI2TFxcXp3HPPbfHxl1xyiUpLSxUREaHu3buf8phevXpp8+bNuu6669zbNm/efNprZmRkKCYmRm+99ZZuvPHGZvujoqIkNVXCJzkcDp199tn64osvNGXKlFNe94ILLtDTTz+tmpoa9w+KH4rjVLZu3arGxkb97//+r8LCmqbvPP/8882Oa2xs1NatW3XppZdKkvbu3auvv/5aPXv2lNT0723v3r1e/bsG0LZI9sA3RowYoUGDBmn8+PFasGCBzj//fB0+fFivvvqqxo8fr/79++v222/X1KlT1b9/f/3kJz/Rs88+q127dqlHjx6nvGZ0dLTuvvtuzZ49W1FRUbr88st19OhR7dq1S9OmTVNycrJiYmL0+uuvq0uXLoqOjlZiYqJyc3M1c+ZMJSQkaPTo0aqrq9PWrVtVXl6uWbNmafLkyZo7d66mTZum3/3ud9q/f7/+53/+x6vve84556ixsVGLFy/W2LFj9cEHH+jxxx9vdlxkZKRuu+02PfbYY4qMjNStt96qyy67zJ3877vvPo0ZM0ZpaWn6xS9+obCwMH3yySfauXOnHnroIe//jwDgc8zGB75hs9n06quv6sorr9QNN9yg8847T5MmTdL+/fvds+evvfZa3Xfffbr77ruVmZmpAwcO6Oabb/7B695777264447dN9996lXr1669tprVVZWJqlpPPyxxx7TE088odTUVI0bN06SdOONN+qPf/yjVq5cqT59+mjIkCFauXKl+1a9du3a6W9/+5t2796tfv36ae7cuVqwYIFX3/fiiy9WQUGBFixYoN69e+vZZ59Vfn5+s+NiY2N19913a/LkyRo0aJBiYmL03HPPufePGjVKr7zyigoLCzVgwABddtllKigoULdu3byKB4D/2AxfDP4BAIAzFpU9AAAhjmQPAECII9kDABDiSPYAAIQ4kj0AACGOZA8AQIgj2QMAEOJI9gAAhDiSPQAAIY5kDwBAiCPZAwAQ4v4/qh/0lvU8wUsAAAAASUVORK5CYII=",
            "text/plain": [
              "<Figure size 640x480 with 2 Axes>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "y_test_pred_knn = knn_gs.predict(X_test)\n",
        "print(classification_report(y_test, y_test_pred_knn))\n",
        "ConfusionMatrixDisplay.from_predictions(y_test, y_test_pred_knn);"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 24,
      "metadata": {},
      "outputs": [],
      "source": [
        "results_knn = pd.DataFrame({\"hist_text_classic\": X_test, \n",
        "                            \"true_label\": y_test,\n",
        "                            \"predicted_label\": y_test_pred_knn,\n",
        "                            \"result\": [\"Correct\" if true == pred else \"Incorrect\" for true, pred in zip(y_test, y_test_pred_knn)],\n",
        "                            \"n_words\": [len(text.split()) for text in X_test]\n",
        "                            })\n",
        "\n",
        "results_knn.true_label.replace({0: \"DC Comics\", 1: \"Marvel Comics\"}, inplace=True)\n",
        "results_knn.predicted_label.replace({0: \"DC Comics\", 1: \"Marvel Comics\"}, inplace=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 25,
      "metadata": {},
      "outputs": [],
      "source": [
        "results_knn.to_csv(\"data/error_knn.csv\",\n",
        "                    index=False)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jUYz4NIJ0aBI"
      },
      "source": [
        "#### 2.3 Support Vector Machines"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 26,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4QbZTINo0iWf",
        "outputId": "51583615-4a57-4172-f2c5-3e8c3e1b5d32"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "['memory',\n",
            " 'robustscaler',\n",
            " 'robustscaler__copy',\n",
            " 'robustscaler__quantile_range',\n",
            " 'robustscaler__unit_variance',\n",
            " 'robustscaler__with_centering',\n",
            " 'robustscaler__with_scaling',\n",
            " 'steps',\n",
            " 'svc',\n",
            " 'svc__C',\n",
            " 'svc__break_ties',\n",
            " 'svc__cache_size',\n",
            " 'svc__class_weight',\n",
            " 'svc__coef0',\n",
            " 'svc__decision_function_shape',\n",
            " 'svc__degree',\n",
            " 'svc__gamma',\n",
            " 'svc__kernel',\n",
            " 'svc__max_iter',\n",
            " 'svc__probability',\n",
            " 'svc__random_state',\n",
            " 'svc__shrinking',\n",
            " 'svc__tol',\n",
            " 'svc__verbose',\n",
            " 'tfidfvectorizer',\n",
            " 'tfidfvectorizer__analyzer',\n",
            " 'tfidfvectorizer__binary',\n",
            " 'tfidfvectorizer__decode_error',\n",
            " 'tfidfvectorizer__dtype',\n",
            " 'tfidfvectorizer__encoding',\n",
            " 'tfidfvectorizer__input',\n",
            " 'tfidfvectorizer__lowercase',\n",
            " 'tfidfvectorizer__max_df',\n",
            " 'tfidfvectorizer__max_features',\n",
            " 'tfidfvectorizer__min_df',\n",
            " 'tfidfvectorizer__ngram_range',\n",
            " 'tfidfvectorizer__norm',\n",
            " 'tfidfvectorizer__preprocessor',\n",
            " 'tfidfvectorizer__smooth_idf',\n",
            " 'tfidfvectorizer__stop_words',\n",
            " 'tfidfvectorizer__strip_accents',\n",
            " 'tfidfvectorizer__sublinear_tf',\n",
            " 'tfidfvectorizer__token_pattern',\n",
            " 'tfidfvectorizer__tokenizer',\n",
            " 'tfidfvectorizer__use_idf',\n",
            " 'tfidfvectorizer__vocabulary',\n",
            " 'verbose']\n"
          ]
        }
      ],
      "source": [
        "pp.pprint(sorted(svm_pipe.get_params().keys()))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 27,
      "metadata": {
        "id": "XcjihYNN0sZL"
      },
      "outputs": [],
      "source": [
        "svm_params = [{\n",
        "    \"tfidfvectorizer__ngram_range\": [(1, 1), (1, 2), (2,3)],\n",
        "    \"svc__C\": [0.1, 1, 10, 100],\n",
        "    \"svc__class_weight\": [None, \"balanced\", c_weights],\n",
        "    \"svc__gamma\": [0.0001, 0.001, 0.01, 0.1, 1],\n",
        "    \"svc__kernel\": [\"rbf\", \"linear\"]\n",
        "    }]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 28,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 180
        },
        "id": "AOmrnBe56Jxq",
        "outputId": "72d15f1e-bec6-41b4-b093-e9d2c723820d"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Fitting 10 folds for each of 360 candidates, totalling 3600 fits\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "<style>#sk-container-id-6 {color: black;}#sk-container-id-6 pre{padding: 0;}#sk-container-id-6 div.sk-toggleable {background-color: white;}#sk-container-id-6 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-6 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-6 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-6 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-6 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-6 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-6 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-6 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-6 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-6 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-6 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-6 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-6 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-6 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-6 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-6 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-6 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-6 div.sk-item {position: relative;z-index: 1;}#sk-container-id-6 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-6 div.sk-item::before, #sk-container-id-6 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-6 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-6 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-6 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-6 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-6 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-6 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-6 div.sk-label-container {text-align: center;}#sk-container-id-6 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-6 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-6\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>GridSearchCV(cv=10,\n",
              "             estimator=Pipeline(steps=[(&#x27;tfidfvectorizer&#x27;,\n",
              "                                        TfidfVectorizer(max_features=512)),\n",
              "                                       (&#x27;robustscaler&#x27;,\n",
              "                                        RobustScaler(with_centering=False)),\n",
              "                                       (&#x27;svc&#x27;, SVC(random_state=256))]),\n",
              "             n_jobs=-1,\n",
              "             param_grid=[{&#x27;svc__C&#x27;: [0.1, 1, 10, 100],\n",
              "                          &#x27;svc__class_weight&#x27;: [None, &#x27;balanced&#x27;,\n",
              "                                                {0: 1.45, 1: 1.0}],\n",
              "                          &#x27;svc__gamma&#x27;: [0.0001, 0.001, 0.01, 0.1, 1],\n",
              "                          &#x27;svc__kernel&#x27;: [&#x27;rbf&#x27;, &#x27;linear&#x27;],\n",
              "                          &#x27;tfidfvectorizer__ngram_range&#x27;: [(1, 1), (1, 2),\n",
              "                                                           (2, 3)]}],\n",
              "             scoring=&#x27;f1_macro&#x27;, verbose=1)</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-23\" type=\"checkbox\" ><label for=\"sk-estimator-id-23\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">GridSearchCV</label><div class=\"sk-toggleable__content\"><pre>GridSearchCV(cv=10,\n",
              "             estimator=Pipeline(steps=[(&#x27;tfidfvectorizer&#x27;,\n",
              "                                        TfidfVectorizer(max_features=512)),\n",
              "                                       (&#x27;robustscaler&#x27;,\n",
              "                                        RobustScaler(with_centering=False)),\n",
              "                                       (&#x27;svc&#x27;, SVC(random_state=256))]),\n",
              "             n_jobs=-1,\n",
              "             param_grid=[{&#x27;svc__C&#x27;: [0.1, 1, 10, 100],\n",
              "                          &#x27;svc__class_weight&#x27;: [None, &#x27;balanced&#x27;,\n",
              "                                                {0: 1.45, 1: 1.0}],\n",
              "                          &#x27;svc__gamma&#x27;: [0.0001, 0.001, 0.01, 0.1, 1],\n",
              "                          &#x27;svc__kernel&#x27;: [&#x27;rbf&#x27;, &#x27;linear&#x27;],\n",
              "                          &#x27;tfidfvectorizer__ngram_range&#x27;: [(1, 1), (1, 2),\n",
              "                                                           (2, 3)]}],\n",
              "             scoring=&#x27;f1_macro&#x27;, verbose=1)</pre></div></div></div><div class=\"sk-parallel\"><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-24\" type=\"checkbox\" ><label for=\"sk-estimator-id-24\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">estimator: Pipeline</label><div class=\"sk-toggleable__content\"><pre>Pipeline(steps=[(&#x27;tfidfvectorizer&#x27;, TfidfVectorizer(max_features=512)),\n",
              "                (&#x27;robustscaler&#x27;, RobustScaler(with_centering=False)),\n",
              "                (&#x27;svc&#x27;, SVC(random_state=256))])</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-25\" type=\"checkbox\" ><label for=\"sk-estimator-id-25\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">TfidfVectorizer</label><div class=\"sk-toggleable__content\"><pre>TfidfVectorizer(max_features=512)</pre></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-26\" type=\"checkbox\" ><label for=\"sk-estimator-id-26\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">RobustScaler</label><div class=\"sk-toggleable__content\"><pre>RobustScaler(with_centering=False)</pre></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-27\" type=\"checkbox\" ><label for=\"sk-estimator-id-27\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">SVC</label><div class=\"sk-toggleable__content\"><pre>SVC(random_state=256)</pre></div></div></div></div></div></div></div></div></div></div></div></div>"
            ],
            "text/plain": [
              "GridSearchCV(cv=10,\n",
              "             estimator=Pipeline(steps=[('tfidfvectorizer',\n",
              "                                        TfidfVectorizer(max_features=512)),\n",
              "                                       ('robustscaler',\n",
              "                                        RobustScaler(with_centering=False)),\n",
              "                                       ('svc', SVC(random_state=256))]),\n",
              "             n_jobs=-1,\n",
              "             param_grid=[{'svc__C': [0.1, 1, 10, 100],\n",
              "                          'svc__class_weight': [None, 'balanced',\n",
              "                                                {0: 1.45, 1: 1.0}],\n",
              "                          'svc__gamma': [0.0001, 0.001, 0.01, 0.1, 1],\n",
              "                          'svc__kernel': ['rbf', 'linear'],\n",
              "                          'tfidfvectorizer__ngram_range': [(1, 1), (1, 2),\n",
              "                                                           (2, 3)]}],\n",
              "             scoring='f1_macro', verbose=1)"
            ]
          },
          "execution_count": 28,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "svm_gs = GridSearchCV(svm_pipe,\n",
        "                      svm_params,\n",
        "                      cv=10,\n",
        "                      scoring='f1_macro',\n",
        "                      n_jobs=-1,\n",
        "                      refit=True,\n",
        "                      verbose=1)\n",
        "svm_gs.fit(X_train, y_train)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 29,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2jt6nxCe6VYw",
        "outputId": "11be3baa-b8b5-4e07-e793-455065f427b8"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "0.9277006235561804"
            ]
          },
          "execution_count": 29,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "svm_gs.best_score_"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 30,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "c_8vpwGm6U99",
        "outputId": "cb615b6e-6153-4825-c5a1-880c081c0a0b"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "{'svc__C': 1,\n",
              " 'svc__class_weight': {0: 1.45, 1: 1.0},\n",
              " 'svc__gamma': 0.0001,\n",
              " 'svc__kernel': 'linear',\n",
              " 'tfidfvectorizer__ngram_range': (1, 1)}"
            ]
          },
          "execution_count": 30,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "svm_gs.best_params_"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 31,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 444
        },
        "id": "Hpags3L261py",
        "outputId": "c488bcb1-bec8-4257-e66d-753cd004a01c"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.88      0.93      0.90        95\n",
            "           1       0.95      0.91      0.93       137\n",
            "\n",
            "    accuracy                           0.92       232\n",
            "   macro avg       0.91      0.92      0.92       232\n",
            "weighted avg       0.92      0.92      0.92       232\n",
            "\n"
          ]
        },
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAfsAAAGwCAYAAACuFMx9AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/NK7nSAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAyhElEQVR4nO3de3hU5bn38d/kNDmQBAJmQiRA0CgoiBgQwSpQIBQFYbNbZEMrVrRgVMxGRSlV4oFE6N4RhYJKLVCViq8VtG4PxBOKSIUAikCxaIAgpMESExJynFnvH5HRMaCZrJkMM+v7ua511XWce6xX7rnv51lr2QzDMAQAAEJWWKADAAAA/kWyBwAgxJHsAQAIcSR7AABCHMkeAIAQR7IHACDEkewBAAhxEYEOwAyXy6XDhw8rPj5eNpst0OEAALxkGIaOHz+u1NRUhYX5r/6sra1VfX296etERUUpOjraBxG1raBO9ocPH1ZaWlqgwwAAmFRSUqIuXbr45dq1tbVK79ZOpWVO09dKSUlRcXFx0CX8oE728fHxkqQBq6crIjYqwNEA/mEffzDQIQB+06gGbdSr7r/n/lBfX6/SMqcOFHVXQnzruweVx13qlrlf9fX1JPu2dLJ1HxEbpYg4e4CjAfwjwhYZ6BAA//nmge1tMRTbLt6mdvGt/xyXgne4OKiTPQAALeU0XHKaeBuM03D5Lpg2RrIHAFiCS4Zcan22N3NuoHHrHQAAIY7KHgBgCS65ZKYRb+7swCLZAwAswWkYchqtb8WbOTfQaOMDABDiqOwBAJZg5Ql6JHsAgCW4ZMhp0WRPGx8AgBBHsgcAWMLJNr6ZxRvvvfeexo4dq9TUVNlsNq1bt869r6GhQXfffbf69OmjuLg4paam6rrrrtPhw4c9rlFXV6fbbrtNnTp1UlxcnK655hodOnTI6+9OsgcAWMLJ2fhmFm9UV1erb9++WrJkSbN9J06c0LZt23Tvvfdq27ZtevHFF/XZZ5/pmmuu8TguJydHa9eu1XPPPaeNGzeqqqpKY8aMkdPp3Ut9GLMHAMAPRo8erdGjR59yX2JiogoLCz22LV68WJdeeqkOHjyorl27qqKiQk899ZSefvppjRgxQpL0zDPPKC0tTW+++aZGjRrV4lio7AEAluDywSJJlZWVHktdXZ1P4quoqJDNZlP79u0lSUVFRWpoaFBWVpb7mNTUVPXu3VubNm3y6tokewCAJTi/mY1vZpGktLQ0JSYmupf8/HzTsdXW1uqee+7R5MmTlZCQIEkqLS1VVFSUOnTo4HGsw+FQaWmpV9enjQ8AsASnIZNvvWv635KSEndCliS73dwr1hsaGjRp0iS5XC4tXbr0R483DMPrVwJT2QMA4IWEhASPxUyyb2ho0MSJE1VcXKzCwkKPHxEpKSmqr69XeXm5xzllZWVyOBxefQ7JHgBgCb4as/eVk4n+n//8p95880117NjRY39mZqYiIyM9JvIdOXJEn376qQYPHuzVZ9HGBwBYgks2OeVd+/v753ujqqpK+/btc68XFxdrx44dSkpKUmpqqn7+859r27ZteuWVV+R0Ot3j8ElJSYqKilJiYqKmTZumO+64Qx07dlRSUpLuvPNO9enTxz07v6VI9gAA+MHWrVs1bNgw9/qsWbMkSVOnTlVubq5efvllSdLFF1/scd4777yjoUOHSpIeeeQRRUREaOLEiaqpqdHw4cO1cuVKhYeHexULyR4AYAkuo2kxc743hg4dKuMHHsTzQ/tOio6O1uLFi7V48WLvPvx7SPYAAEtwmmzjmzk30JigBwBAiKOyBwBYgpUre5I9AMASXIZNLsPEbHwT5wYabXwAAEIclT0AwBJo4wMAEOKcCpPTREPbuzfIn1lI9gAASzBMjtkbjNkDAIAzFZU9AMASGLMHACDEOY0wOQ0TY/YmHrUbaLTxAQAIcVT2AABLcMkml4ka16XgLe1J9gAAS7DymD1tfAAAQhyVPQDAEsxP0KONDwDAGa1pzN7Ei3Bo4wMAgDMVlT0AwBJcJp+Nz2x8AADOcIzZAwAQ4lwKs+x99ozZAwAQ4qjsAQCW4DRscpp4Ta2ZcwONZA8AsASnyQl6Ttr4AADgTEVlDwCwBJcRJpeJ2fguZuMDAHBmo40PAABCFpU9AMASXDI3o97lu1DaHMkeAGAJ5h+qE7zN8OCNHAAAtAiVPQDAEsw/Gz9462OSPQDAEqz8PnuSPQDAEqxc2Qdv5AAAoEWo7AEAlmD+oTrBWx+T7AEAluAybHKZuc8+iN96F7w/UwAAQItQ2QMALMFlso0fzA/VIdkDACzB/FvvgjfZB2/kAACgRajsAQCW4JRNThMPxjFzbqCR7AEAlkAbHwAAhCwqewCAJThlrhXv9F0obY5kDwCwBCu38Un2AABL4EU4AAAgZFHZAwAswTD5PnuDW+8AADiz0cYHAAAhi8oeAGAJvOIWAIAQ5/zmrXdmFm+89957Gjt2rFJTU2Wz2bRu3TqP/YZhKDc3V6mpqYqJidHQoUO1a9cuj2Pq6up02223qVOnToqLi9M111yjQ4cOef3dSfYAAPhBdXW1+vbtqyVLlpxy/8KFC1VQUKAlS5Zoy5YtSklJ0ciRI3X8+HH3MTk5OVq7dq2ee+45bdy4UVVVVRozZoycTu8e8UMbHwBgCb5q41dWVnpst9vtstvtzY4fPXq0Ro8efcprGYahRYsWae7cuZowYYIkadWqVXI4HFq9erWmT5+uiooKPfXUU3r66ac1YsQISdIzzzyjtLQ0vfnmmxo1alSLY6eyBwBYgkthphdJSktLU2JionvJz8/3Opbi4mKVlpYqKyvLvc1ut2vIkCHatGmTJKmoqEgNDQ0ex6Smpqp3797uY1qKyh4AAC+UlJQoISHBvX6qqv7HlJaWSpIcDofHdofDoQMHDriPiYqKUocOHZodc/L8liLZAwAswWnY5DTRxj95bkJCgkeyN8Nm84zHMIxm276vJcd8H218AIAlnByzN7P4SkpKiiQ1q9DLysrc1X5KSorq6+tVXl5+2mNaimQPALAE45u33rV2MXz4BL309HSlpKSosLDQva2+vl4bNmzQ4MGDJUmZmZmKjIz0OObIkSP69NNP3ce0FG18AAD8oKqqSvv27XOvFxcXa8eOHUpKSlLXrl2Vk5OjvLw8ZWRkKCMjQ3l5eYqNjdXkyZMlSYmJiZo2bZruuOMOdezYUUlJSbrzzjvVp08f9+z8liLZAwAswSmbnCZeZuPtuVu3btWwYcPc67NmzZIkTZ06VStXrtTs2bNVU1Oj7OxslZeXa+DAgVq/fr3i4+Pd5zzyyCOKiIjQxIkTVVNTo+HDh2vlypUKDw/3KhabYRiGV2ecQSorK5WYmKhB625TRJz3syGBYGDP2h/oEAC/aTQa9K5eUkVFhc8mvX3fyVzx63cnKqpdVKuvU19VrxVDn/drrP7CmD0AACGONj6acxoK//PXCn+7Wip3Sknhcma1k3NyohT2TRurxqWIp8oVtumEVOmS4YiQc3y8XGOD69cuIEmr/r5bKWkNzba/vLKj/vDbLgGICP5wcqKdmfODFckezYSvqVD4/x1X412d5OoWqbDP6hXxv19JcWFy/kdTMo94/JjCPq5Vw92dZDgiFFZUq4jF/1Zjxwi5BscG+BsA3pk5+jyFhX87otm9Z60eXvOF3v9b+8AFBZ9zySaXiTF7M+cGWsB/pixdulTp6emKjo5WZmam3n///UCHZHm2PXVyDYqVa2CslBIp15VxcmXGyPZZ3bfH7K6Tc0Q7GX1jmo65Ol5GjyiPY4BgUXEsQuVHI93LwBGVOlwcpU8+jAt0aIBPBDTZr1mzRjk5OZo7d662b9+uK664QqNHj9bBgwcDGZblGRfaFbajRrZDTW1N2+f1Cvu0Vq5LY749prddYZtPSF81SoYh244a2b5skKt/zOkuCwSFiEiXfvqf5XrjuSQpiCs5NHfyCXpmlmAV0DZ+QUGBpk2bphtvvFGStGjRIr3xxhtatmxZq14sAN9wXpsoVRuKnPZl089Bl+S8vr1cw9q5j2nM7qiIR76SffIhGeGSwqTG/+4ko3d0wOIGfGHwzyrVLsGp9c8nBToU+Bhj9gFQX1+voqIi3XPPPR7bs7KyTvs2n7q6OtXVfdsm/v5rBuEbYe9WK/ytKjXe00lG9yjZPq9XxLJjMjpGyJXVlPDD11XK9o86NdyfLMMRIdvOpjH7hqRwGZdQ3SN4jfqvf2vLOwk69q/IQIcC+EzAfqZ89dVXcjqdp3zjz+ne5pOfn+/xWsG0tLS2CNVyIpaXq3FSolzD2slIj5JrRDs5JyQo/Lmvmw6ocyl8Rbmc05PkGhQro0eUXOMS5BoSp4gXKgIaO2BG8tn16ndFlV5fTVUfilwy+Wz8IB7WCXhPwps3/syZM0cVFRXupaSkpC1CtJ46o/lQZZikk5OVGyVbo059jMvfwQH+kzXpmL7+KkJ/f5NbSEOR8c1s/NYuRhAn+4C18Tt16qTw8PAffOPP99nt9la9NxjecV0Wo4i/VKgxOaLp1rt99Qp/sVLOUd+M2ceFyXWRXeHLy2XYbTKSIxS2s1Zhb1arcXqHH744cIay2QxlXXtMb/6/DnI5g/ePOk7P7JvrfPnWu7YWsMo+KipKmZmZHm/zkaTCwkKv3+YD32q8paOcV8QqYvG/FXXjYUUsL5fzqng5p36byBt+e5aM8+2KfPgrRd10WOHPVTRN4hsT/wNXBs5c/a6skqNLg954rmOgQwF8LqCz8WfNmqVf/epX6t+/vwYNGqQnn3xSBw8e1IwZMwIZFmLD5Ly5o5w3/8AfvaQINd7Zqe1iAvxs24Z4jUrtG+gw4EfMxg+Qa6+9Vv/+97/1wAMP6MiRI+rdu7deffVVdevWLZBhAQBCkJXb+AF/XG52drays7MDHQYAACEr4MkeAIC2YOVn45PsAQCWYOU2fvDONgAAAC1CZQ8AsAQrV/YkewCAJVg52dPGBwAgxFHZAwAswcqVPckeAGAJhszdPmf8+CFnLJI9AMASrFzZM2YPAECIo7IHAFiClSt7kj0AwBKsnOxp4wMAEOKo7AEAlmDlyp5kDwCwBMOwyTCRsM2cG2i08QEACHFU9gAAS+B99gAAhDgrj9nTxgcAIMRR2QMALMHKE/RI9gAAS7ByG59kDwCwBCtX9ozZAwAQ4qjsAQCWYJhs4wdzZU+yBwBYgiHJMMydH6xo4wMAEOKo7AEAluCSTTaeoAcAQOhiNj4AAAhZVPYAAEtwGTbZeKgOAAChyzBMzsYP4un4tPEBAAhxVPYAAEuw8gQ9kj0AwBJI9gAAhDgrT9BjzB4AgBBHZQ8AsARm4wMAEOKakr3NxOLd5zU2Nup3v/ud0tPTFRMTox49euiBBx6Qy+X6TkyGcnNzlZqaqpiYGA0dOlS7du3y8Tcn2QMA4BcLFizQ448/riVLlmjPnj1auHChfv/732vx4sXuYxYuXKiCggItWbJEW7ZsUUpKikaOHKnjx4/7NBba+AAAS2jr2fgffvihxo0bp6uvvlqS1L17d/3lL3/R1q1bv7meoUWLFmnu3LmaMGGCJGnVqlVyOBxavXq1pk+f3upYv4/KHgBgCYYPFkmqrKz0WOrq6k75eT/5yU/01ltv6bPPPpMkffzxx9q4caOuuuoqSVJxcbFKS0uVlZXlPsdut2vIkCHatGmTT787lT0AAF5IS0vzWJ83b55yc3ObHXf33XeroqJCPXv2VHh4uJxOp+bPn6//+q//kiSVlpZKkhwOh8d5DodDBw4c8GnMJHsAgCX4qo1fUlKihIQE93a73X7K49esWaNnnnlGq1ev1oUXXqgdO3YoJydHqampmjp1qvs4m80zJsMwmm0zi2QPALCG7/biW3u+pISEBI9kfzp33XWX7rnnHk2aNEmS1KdPHx04cED5+fmaOnWqUlJSJDVV+J07d3afV1ZW1qzaN4sxewCANZi67c4medkVOHHihMLCPNNseHi4+9a79PR0paSkqLCw0L2/vr5eGzZs0ODBg81/3++gsgcAwA/Gjh2r+fPnq2vXrrrwwgu1fft2FRQU6IYbbpDU1L7PyclRXl6eMjIylJGRoby8PMXGxmry5Mk+jYVkDwCwhLZ+gt7ixYt17733Kjs7W2VlZUpNTdX06dN13333uY+ZPXu2ampqlJ2drfLycg0cOFDr169XfHx86wM9BZthBO8DACsrK5WYmKhB625TRNypJ0gAwc6etT/QIQB+02g06F29pIqKihaNg7fGyVzR/U+/U1hsdKuv4zpRq/03POTXWP2FMXsAAEIcbXwAgDW0YpJds/ODFMkeAGAJvPUOAACELCp7AIA1+OihOsGIZA8AsIS2fuvdmaRFyf6xxx5r8QVnzpzZ6mAAAIDvtSjZP/LIIy26mM1mI9kDAM5cQdyKN6NFyb64uNjfcQAA4FdWbuO3ejZ+fX299u7dq8bGRl/GAwCAfxg+WIKU18n+xIkTmjZtmmJjY3XhhRfq4MGDkprG6h9++GGfBwgAAMzxOtnPmTNHH3/8sd59911FR3/7jOERI0ZozZo1Pg0OAADfsflgCU5e33q3bt06rVmzRpdddplstm+/+AUXXKDPP//cp8EBAOAzFr7P3uvK/ujRo0pOTm62vbq62iP5AwCAM4PXyX7AgAH6v//7P/f6yQS/fPlyDRo0yHeRAQDgSxaeoOd1Gz8/P18/+9nPtHv3bjU2NurRRx/Vrl279OGHH2rDhg3+iBEAAPMs/NY7ryv7wYMH64MPPtCJEyd0zjnnaP369XI4HPrwww+VmZnpjxgBAIAJrXo2fp8+fbRq1SpfxwIAgN9Y+RW3rUr2TqdTa9eu1Z49e2Sz2dSrVy+NGzdOERG8VwcAcIay8Gx8r7Pzp59+qnHjxqm0tFTnn3++JOmzzz7TWWedpZdffll9+vTxeZAAAKD1vB6zv/HGG3XhhRfq0KFD2rZtm7Zt26aSkhJddNFF+s1vfuOPGAEAMO/kBD0zS5DyurL/+OOPtXXrVnXo0MG9rUOHDpo/f74GDBjg0+AAAPAVm9G0mDk/WHld2Z9//vn617/+1Wx7WVmZzj33XJ8EBQCAz1n4PvsWJfvKykr3kpeXp5kzZ+qFF17QoUOHdOjQIb3wwgvKycnRggUL/B0vAADwUova+O3bt/d4FK5hGJo4caJ7m/HN/Qhjx46V0+n0Q5gAAJhk4YfqtCjZv/POO/6OAwAA/+LWux82ZMgQf8cBAAD8pNVPwTlx4oQOHjyo+vp6j+0XXXSR6aAAAPA5KvuWO3r0qH7961/rtddeO+V+xuwBAGckCyd7r2+9y8nJUXl5uTZv3qyYmBi9/vrrWrVqlTIyMvTyyy/7I0YAAGCC15X922+/rZdeekkDBgxQWFiYunXrppEjRyohIUH5+fm6+uqr/REnAADmWHg2vteVfXV1tZKTkyVJSUlJOnr0qKSmN+Ft27bNt9EBAOAjJ5+gZ2YJVq16gt7evXslSRdffLGeeOIJffnll3r88cfVuXNnnwcIAADM8bqNn5OToyNHjkiS5s2bp1GjRunZZ59VVFSUVq5c6ev4AADwDQtP0PM62U+ZMsX9z/369dP+/fv1j3/8Q127dlWnTp18GhwAADCv1ffZnxQbG6tLLrnEF7EAAOA3Npl8653PIml7LUr2s2bNavEFCwoKWh0MAADwvRYl++3bt7foYt99WU5bir2uXBG2qIB8NuBvrx7eEegQAL+pPO5Sh/Pa6MMsfOsdL8IBAFiDhSfoeX3rHQAACC6mJ+gBABAULFzZk+wBAJZg9il4lnqCHgAACC5U9gAAa7BwG79Vlf3TTz+tyy+/XKmpqTpw4IAkadGiRXrppZd8GhwAAD5j+GAJUl4n+2XLlmnWrFm66qqr9PXXX8vpdEqS2rdvr0WLFvk6PgAAYJLXyX7x4sVavny55s6dq/DwcPf2/v37a+fOnT4NDgAAX7HyK269HrMvLi5Wv379mm232+2qrq72SVAAAPichZ+g53Vln56erh07djTb/tprr+mCCy7wRUwAAPiehcfsva7s77rrLt1yyy2qra2VYRj66KOP9Je//EX5+fn64x//6I8YAQCACV4n+1//+tdqbGzU7NmzdeLECU2ePFlnn322Hn30UU2aNMkfMQIAYJqVH6rTqvvsb7rpJt1000366quv5HK5lJyc7Ou4AADwLe6zb51OnTqR6AEAOI0vv/xSv/zlL9WxY0fFxsbq4osvVlFRkXu/YRjKzc1VamqqYmJiNHToUO3atcvncXhd2aenp//ge+u/+OILUwEBAOAXZm+f8/Lc8vJyXX755Ro2bJhee+01JScn6/PPP1f79u3dxyxcuFAFBQVauXKlzjvvPD300EMaOXKk9u7dq/j4eBPBevI62efk5HisNzQ0aPv27Xr99dd11113+SouAAB8q43b+AsWLFBaWppWrFjh3ta9e/dvL2cYWrRokebOnasJEyZIklatWiWHw6HVq1dr+vTpJoL15HWyv/3220+5/Q9/+IO2bt1qOiAAAM5klZWVHut2u112u73ZcS+//LJGjRqlX/ziF9qwYYPOPvtsZWdn66abbpLU9Nya0tJSZWVleVxryJAh2rRpk0+Tvc/eejd69Gj99a9/9dXlAADwLR/dZ5+WlqbExET3kp+ff8qP++KLL7Rs2TJlZGTojTfe0IwZMzRz5kz9+c9/liSVlpZKkhwOh8d5DofDvc9XfPbWuxdeeEFJSUm+uhwAAD7lq1vvSkpKlJCQ4N5+qqpeklwul/r376+8vDxJUr9+/bRr1y4tW7ZM11133bfX/d48OMMwfnBuXGt4nez79evnEYRhGCotLdXRo0e1dOlSnwYHAMCZJiEhwSPZn07nzp2bPVm2V69e7i54SkqKpKYKv3Pnzu5jysrKmlX7Znmd7MePH++xHhYWprPOOktDhw5Vz549fRUXAABB7fLLL9fevXs9tn322Wfq1q2bpKa721JSUlRYWOh+50x9fb02bNigBQsW+DQWr5J9Y2OjunfvrlGjRrl/kQAAEBTaeDb+f//3f2vw4MHKy8vTxIkT9dFHH+nJJ5/Uk08+KampfZ+Tk6O8vDxlZGQoIyNDeXl5io2N1eTJk00E2pxXyT4iIkI333yz9uzZ49MgAADwt7Z+XO6AAQO0du1azZkzRw888IDS09O1aNEiTZkyxX3M7NmzVVNTo+zsbJWXl2vgwIFav369T++xl1rRxh84cKC2b9/ubkMAAIBTGzNmjMaMGXPa/TabTbm5ucrNzfVrHF4n++zsbN1xxx06dOiQMjMzFRcX57H/oosu8llwAAD4VBA/396MFif7G264QYsWLdK1114rSZo5c6Z7n81mc98q4HQ6fR8lAABmWfhFOC1O9qtWrdLDDz+s4uJif8YDAAB8rMXJ3jCaftIwVg8ACEa8z76FfP1EHwAA2gxt/JY577zzfjThHzt2zFRAAADAt7xK9vfff78SExP9FQsAAH5DG7+FJk2apOTkZH/FAgCA/1i4jd/iV9wyXg8AQHDyejY+AABBycKVfYuTvcvl8mccAAD4FWP2AACEOgtX9i0eswcAAMGJyh4AYA0WruxJ9gAAS7DymD1tfAAAQhyVPQDAGmjjAwAQ2mjjAwCAkEVlDwCwBtr4AACEOAsne9r4AACEOCp7AIAl2L5ZzJwfrEj2AABrsHAbn2QPALAEbr0DAAAhi8oeAGANtPEBALCAIE7YZtDGBwAgxFHZAwAswcoT9Ej2AABrsPCYPW18AABCHJU9AMASaOMDABDqaOMDAIBQRWUPALAE2vgAAIQ6C7fxSfYAAGuwcLJnzB4AgBBHZQ8AsATG7AEACHW08QEAQKiisgcAWILNMGQzWl+emzk30Ej2AABroI0PAABCFZU9AMASmI0PAECoo40PAABCFZU9AMASaOMDABDqLNzGJ9kDACzBypU9Y/YAAIQ4kj0AwBoMHyytlJ+fL5vNppycnG/DMQzl5uYqNTVVMTExGjp0qHbt2tX6D/kBJHsAgGWcbOW3ZmmtLVu26Mknn9RFF13ksX3hwoUqKCjQkiVLtGXLFqWkpGjkyJE6fvy4yW/ZHMkeAAAvVFZWeix1dXWnPbaqqkpTpkzR8uXL1aFDB/d2wzC0aNEizZ07VxMmTFDv3r21atUqnThxQqtXr/Z5zCR7AIA1GIb5RVJaWpoSExPdS35+/mk/8pZbbtHVV1+tESNGeGwvLi5WaWmpsrKy3NvsdruGDBmiTZs2+fyrMxsfAGAJvpqNX1JSooSEBPd2u91+yuOfe+45bdu2TVu2bGm2r7S0VJLkcDg8tjscDh04cKD1QZ4GyR4AAC8kJCR4JPtTKSkp0e23367169crOjr6tMfZbDaPdcMwmm3zBdr4AABraMPZ+EVFRSorK1NmZqYiIiIUERGhDRs26LHHHlNERIS7oj9Z4Z9UVlbWrNr3BZI9AMASbC7zS0sNHz5cO3fu1I4dO9xL//79NWXKFO3YsUM9evRQSkqKCgsL3efU19drw4YNGjx4sM+/O218AAB8LD4+Xr179/bYFhcXp44dO7q35+TkKC8vTxkZGcrIyFBeXp5iY2M1efJkn8dDskczvftX6D+nHdK5F1apY3K9Hryllz58q5MkKTzCpetuP6ABQ44ppUutqqsitGNTe60o6K5jZaeepAIE2s7Ncfp/S5P1z52xOvavSM17qliDR1dIkhobpJULOmvL2wk6ciBKcQku9bviuKb99rA6pjS6r3HXf56rTz5s53HdIdeU67eP+34yFfzkDHs2/uzZs1VTU6Ps7GyVl5dr4MCBWr9+veLj4337QSLZ4xSiY5wq/kecCl906HeL93jss0e7dO4FVfrL0q76Ym+c2iU0avqcLzRv6W7d/vN+AYoY+GG1J8LU48IaZU06pgdvTPfYV1cTpn07YzU551/qcUGNqirC9fi8szXv+h5a8vpnHseOnvKVrrvr2zFWe7QXfV0EXKCfjf/uu+96Xs9mU25urnJzc81duAUCmuzfe+89/f73v1dRUZGOHDmitWvXavz48YEMCZK2vp+kre8nnXLfiaoIzZ3Wx2PbsofO0aMv7NBZnWt19MjpZ50CgTLgp8c14KenfipZXIJLD6/53GNb9kOHNPOq81V2KFLJXRrc2+0xhpKSG79/CQSL79wr3+rzg1RAJ+hVV1erb9++WrJkSSDDgElx8Y1yuaSqShpFCA3VleGy2QzFJTo9tr/zYgf94sLeumno+Xry/lSdqGKOM4JDQP86jx49WqNHj27x8XV1dR6PJaysrPRHWPBCZJRLv75jv9595SzVVJPsEfzqa236U16qhv1HueLiv23TD5twTClp9UpKbtT+f0TrT/md9cXumGZdAZy5At3GD6Sg+uucn5+v+++/P9Bh4BvhES7dU/AP2WyG/nD/uYEOBzCtsUHKu7m7DJd0a/4hj31XTTnm/ufuPWt1do863fqz8/XPT2KUcVFNW4eK1jjDJui1paDqQc2ZM0cVFRXupaSkJNAhWVZ4hEtzHvmHHF1qNXdaH6p6BL3GBmn+9O4qLYlS/nOfe1T1p3JunxpFRLr0ZTF3oeDMF1R/oe12+2mfQYy2czLRp3ar0T1T++j415GBDgkw5WSi/7LYroUv7FNCkvNHzzmwN1qNDWHq6Gj40WNxZqCND3xHdKxTqV2/bUs6utSpR88qHa+I0L/L7Prto3t07gVVyp1xocLDpQ6d6iVJxysi1NgQVM0iWERNdZgOf6cCLy2J0uefxii+faM6pjTowZvStW9njB748xdyOW06Vtb0pzG+vVORUYYO74/S2y920KXDK5WQ5NTBz+x68v6zdW7vE7pgQHWgvha8ZeHZ+CR7NJPR+7gW/Hmne/03c76QJBWuTdazS7pp0PCmscs/vLTd47y7r+ujnR+1b7M4gZb67ONYzf75t/NKnsg9W5I0cuIx/fKOUm1enyhJyh7Z0+O8hS/sU9/BVYqINLRjY7zWPXWWaqvD1Cm1QQOHV2rKrFKFh7fd9wBaK6DJvqqqSvv27XOvFxcXa8eOHUpKSlLXrl0DGJm17fyova7qecVp9//QPuBM1Hdwld44vOO0+39onyQln92g/3lx3w8egzMfbfwA2bp1q4YNG+ZenzVrliRp6tSpWrlyZYCiAgCEJAvPxg9osh86dKiMIB4DAQAgGDBmDwCwBNr4AACEOpfRtJg5P0iR7AEA1mDhMXtuigYAIMRR2QMALMEmk2P2Pouk7ZHsAQDWYOEn6NHGBwAgxFHZAwAsgVvvAAAIdczGBwAAoYrKHgBgCTbDkM3EJDsz5wYayR4AYA2ubxYz5wcp2vgAAIQ4KnsAgCXQxgcAINRZeDY+yR4AYA08QQ8AAIQqKnsAgCXwBD0AAEIdbXwAABCqqOwBAJZgczUtZs4PViR7AIA10MYHAAChisoeAGANPFQHAIDQZuXH5dLGBwAgxFHZAwCswcIT9Ej2AABrMGTunfTBm+tJ9gAAa2DMHgAAhCwqewCANRgyOWbvs0jaHMkeAGANFp6gRxsfAIAQR2UPALAGlySbyfODFMkeAGAJzMYHAAAhi8oeAGANFp6gR7IHAFiDhZM9bXwAAEIclT0AwBqo7AEACHEuHyxeyM/P14ABAxQfH6/k5GSNHz9ee/fu9TjGMAzl5uYqNTVVMTExGjp0qHbt2mXiS54ayR4AYAknb70zs3hjw4YNuuWWW7R582YVFhaqsbFRWVlZqq6udh+zcOFCFRQUaMmSJdqyZYtSUlI0cuRIHT9+3KffnTY+AAB+8Prrr3usr1ixQsnJySoqKtKVV14pwzC0aNEizZ07VxMmTJAkrVq1Sg6HQ6tXr9b06dN9FguVPQDAGk6O2ZtZJFVWVnosdXV1Lfr4iooKSVJSUpIkqbi4WKWlpcrKynIfY7fbNWTIEG3atMmnX51kDwCwBpdhfpGUlpamxMRE95Kfn/+jH20YhmbNmqWf/OQn6t27tySptLRUkuRwODyOdTgc7n2+QhsfAAAvlJSUKCEhwb1ut9t/9Jxbb71Vn3zyiTZu3Nhsn83m+cB+wzCabTOLZA8AsAYf3XqXkJDgkex/zG233aaXX35Z7733nrp06eLenpKSIqmpwu/cubN7e1lZWbNq3yza+AAAizA7Xu/dDwXDMHTrrbfqxRdf1Ntvv6309HSP/enp6UpJSVFhYaF7W319vTZs2KDBgwf74gu7UdkDAOAHt9xyi1avXq2XXnpJ8fHx7nH4xMRExcTEyGazKScnR3l5ecrIyFBGRoby8vIUGxuryZMn+zQWkj0AwBra+Al6y5YtkyQNHTrUY/uKFSt0/fXXS5Jmz56tmpoaZWdnq7y8XAMHDtT69esVHx/f+jhPgWQPALAGl/et+Obnt5zRgh8HNptNubm5ys3NbWVQLcOYPQAAIY7KHgBgDYaraTFzfpAi2QMArMHCb70j2QMArKGNx+zPJIzZAwAQ4qjsAQDWQBsfAIAQZ8hksvdZJG2ONj4AACGOyh4AYA208QEACHEulyQT98q7gvc+e9r4AACEOCp7AIA10MYHACDEWTjZ08YHACDEUdkDAKzBwo/LJdkDACzBMFwyTLy5zsy5gUayBwBYg2GYq84ZswcAAGcqKnsAgDUYJsfsg7iyJ9kDAKzB5ZJsJsbdg3jMnjY+AAAhjsoeAGANtPEBAAhthsslw0QbP5hvvaONDwBAiKOyBwBYA218AABCnMuQbNZM9rTxAQAIcVT2AABrMAxJZu6zD97KnmQPALAEw2XIMNHGN0j2AACc4QyXzFX23HoHAADOUFT2AABLoI0PAECos3AbP6iT/clfWY1GfYAjAfyn8njw/oEBfkxlVdN/321RNTeqwdQzdRrV4Ltg2lhQJ/vjx49LkjYcfz7AkQD+0+G8QEcA+N/x48eVmJjol2tHRUUpJSVFG0tfNX2tlJQURUVF+SCqtmUzgngQwuVy6fDhw4qPj5fNZgt0OJZQWVmptLQ0lZSUKCEhIdDhAD7Ff99tzzAMHT9+XKmpqQoL89+c8draWtXXm+8CR0VFKTo62gcRta2gruzDwsLUpUuXQIdhSQkJCfwxRMjiv++25a+K/ruio6ODMkn7CrfeAQAQ4kj2AACEOJI9vGK32zVv3jzZ7fZAhwL4HP99I1QF9QQ9AADw46jsAQAIcSR7AABCHMkeAIAQR7IHACDEkezRYkuXLlV6erqio6OVmZmp999/P9AhAT7x3nvvaezYsUpNTZXNZtO6desCHRLgUyR7tMiaNWuUk5OjuXPnavv27briiis0evRoHTx4MNChAaZVV1erb9++WrJkSaBDAfyCW+/QIgMHDtQll1yiZcuWubf16tVL48ePV35+fgAjA3zLZrNp7dq1Gj9+fKBDAXyGyh4/qr6+XkVFRcrKyvLYnpWVpU2bNgUoKgBAS5Hs8aO++uorOZ1OORwOj+0Oh0OlpaUBigoA0FIke7TY918jbBgGrxYGgCBAsseP6tSpk8LDw5tV8WVlZc2qfQDAmYdkjx8VFRWlzMxMFRYWemwvLCzU4MGDAxQVAKClIgIdAILDrFmz9Ktf/Ur9+/fXoEGD9OSTT+rgwYOaMWNGoEMDTKuqqtK+ffvc68XFxdqxY4eSkpLUtWvXAEYG+Aa33qHFli5dqoULF+rIkSPq3bu3HnnkEV155ZWBDgsw7d1339WwYcOabZ86dapWrlzZ9gEBPkayBwAgxDFmDwBAiCPZAwAQ4kj2AACEOJI9AAAhjmQPAECII9kDABDiSPYAAIQ4kj0AACGOZA+YlJubq4svvti9fv3112v8+PFtHsf+/ftls9m0Y8eO0x7TvXt3LVq0qMXXXLlypdq3b286NpvNpnXr1pm+DoDWIdkjJF1//fWy2Wyy2WyKjIxUjx49dOedd6q6utrvn/3oo4+2+BGrLUnQAGAWL8JByPrZz36mFStWqKGhQe+//75uvPFGVVdXa9myZc2ObWhoUGRkpE8+NzEx0SfXAQBfobJHyLLb7UpJSVFaWpomT56sKVOmuFvJJ1vvf/rTn9SjRw/Z7XYZhqGKigr95je/UXJyshISEvTTn/5UH3/8scd1H374YTkcDsXHx2vatGmqra312P/9Nr7L5dKCBQt07rnnym63q2vXrpo/f74kKT09XZLUr18/2Ww2DR061H3eihUr1KtXL0VHR6tnz55aunSpx+d89NFH6tevn6Kjo9W/f39t377d639HBQUF6tOnj+Li4pSWlqbs7GxVVVU1O27dunU677zzFB0drZEjR6qkpMRj/9/+9jdlZmYqOjpaPXr00P3336/Gxkav4wHgHyR7WEZMTIwaGhrc6/v27dPzzz+vv/71r+42+tVXX63S0lK9+uqrKioq0iWXXKLhw4fr2LFjkqTnn39e8+bN0/z587V161Z17ty5WRL+vjlz5mjBggW69957tXv3bq1evVoOh0NSU8KWpDfffFNHjhzRiy++KElavny55s6dq/nz52vPnj3Ky8vTvffeq1WrVkmSqqurNWbMGJ1//vkqKipSbm6u7rzzTq//nYSFhemxxx7Tp59+qlWrVuntt9/W7NmzPY45ceKE5s+fr1WrVumDDz5QZWWlJk2a5N7/xhtv6Je//KVmzpyp3bt364knntDKlSvdP2gAnAEMIARNnTrVGDdunHv973//u9GxY0dj4sSJhmEYxrx584zIyEijrKzMfcxbb71lJCQkGLW1tR7XOuecc4wnnnjCMAzDGDRokDFjxgyP/QMHDjT69u17ys+urKw07Ha7sXz58lPGWVxcbEgytm/f7rE9LS3NWL16tce2Bx980Bg0aJBhGIbxxBNPGElJSUZ1dbV7/7Jly055re/q1q2b8cgjj5x2//PPP2907NjRvb5ixQpDkrF582b3tj179hiSjL///e+GYRjGFVdcYeTl5Xlc5+mnnzY6d+7sXpdkrF279rSfC8C/GLNHyHrllVfUrl07NTY2qqGhQePGjdPixYvd+7t166azzjrLvV5UVKSqqip17NjR4zo1NTX6/PPPJUl79uzRjBkzPPYPGjRI77zzzilj2LNnj+rq6jR8+PAWx3306FGVlJRo2rRpuummm9zbGxsb3fMB9uzZo759+yo2NtYjDm+98847ysvL0+7du1VZWanGxkbV1taqurpacXFxkqSIiAj179/ffU7Pnj3Vvn177dmzR5deeqmKioq0ZcsWj0re6XSqtrZWJ06c8IgRQGCQ7BGyhg0bpmXLlikyMlKpqanNJuCdTGYnuVwude7cWe+++26za7X29rOYmBivz3G5XJKaWvkDBw702BceHi5JMgyjVfF814EDB3TVVVdpxowZevDBB5WUlKSNGzdq2rRpHsMdUtOtc993cpvL5dL999+vCRMmNDsmOjradJwAzCPZI2TFxcXp3HPPbfHxl1xyiUpLSxUREaHu3buf8phevXpp8+bNuu6669zbNm/efNprZmRkKCYmRm+99ZZuvPHGZvujoqIkNVXCJzkcDp199tn64osvNGXKlFNe94ILLtDTTz+tmpoa9w+KH4rjVLZu3arGxkb97//+r8LCmqbvPP/8882Oa2xs1NatW3XppZdKkvbu3auvv/5aPXv2lNT0723v3r1e/bsG0LZI9sA3RowYoUGDBmn8+PFasGCBzj//fB0+fFivvvqqxo8fr/79++v222/X1KlT1b9/f/3kJz/Rs88+q127dqlHjx6nvGZ0dLTuvvtuzZ49W1FRUbr88st19OhR7dq1S9OmTVNycrJiYmL0+uuvq0uXLoqOjlZiYqJyc3M1c+ZMJSQkaPTo0aqrq9PWrVtVXl6uWbNmafLkyZo7d66mTZum3/3ud9q/f7/+53/+x6vve84556ixsVGLFy/W2LFj9cEHH+jxxx9vdlxkZKRuu+02PfbYY4qMjNStt96qyy67zJ3877vvPo0ZM0ZpaWn6xS9+obCwMH3yySfauXOnHnroIe//jwDgc8zGB75hs9n06quv6sorr9QNN9yg8847T5MmTdL+/fvds+evvfZa3Xfffbr77ruVmZmpAwcO6Oabb/7B695777264447dN9996lXr1669tprVVZWJqlpPPyxxx7TE088odTUVI0bN06SdOONN+qPf/yjVq5cqT59+mjIkCFauXKl+1a9du3a6W9/+5t2796tfv36ae7cuVqwYIFX3/fiiy9WQUGBFixYoN69e+vZZ59Vfn5+s+NiY2N19913a/LkyRo0aJBiYmL03HPPufePGjVKr7zyigoLCzVgwABddtllKigoULdu3byKB4D/2AxfDP4BAIAzFpU9AAAhjmQPAECII9kDABDiSPYAAIQ4kj0AACGOZA8AQIgj2QMAEOJI9gAAhDiSPQAAIY5kDwBAiCPZAwAQ4v4/qh/0lvU8wUsAAAAASUVORK5CYII=",
            "text/plain": [
              "<Figure size 640x480 with 2 Axes>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "y_test_pred_svm = svm_gs.predict(X_test)\n",
        "print(classification_report(y_test, y_test_pred_svm))\n",
        "ConfusionMatrixDisplay.from_predictions(y_test, y_test_pred_svm);"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 32,
      "metadata": {},
      "outputs": [],
      "source": [
        "results_svm = pd.DataFrame({\"hist_text_classic\": X_test, \n",
        "                            \"true_label\": y_test,\n",
        "                            \"predicted_label\": y_test_pred_svm,\n",
        "                            \"result\": [\"Correct\" if true == pred else \"Incorrect\" for true, pred in zip(y_test, y_test_pred_svm)],\n",
        "                            \"n_words\": [len(text.split()) for text in X_test]\n",
        "                            })\n",
        "\n",
        "results_svm.true_label.replace({0: \"DC Comics\", 1: \"Marvel Comics\"}, inplace=True)\n",
        "results_svm.predicted_label.replace({0: \"DC Comics\", 1: \"Marvel Comics\"}, inplace=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 33,
      "metadata": {},
      "outputs": [],
      "source": [
        "results_svm.to_csv(\"data/error_svm.csv\",\n",
        "                    index=False)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "e__U2oo372zI"
      },
      "source": [
        "#### 2.4 Multinominal Naive Bayes\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 34,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cZ8X7xBx7rV7",
        "outputId": "0acb1d34-2b5d-48af-91b2-9d26c037aca3"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "['memory',\n",
            " 'multinomialnb',\n",
            " 'multinomialnb__alpha',\n",
            " 'multinomialnb__class_prior',\n",
            " 'multinomialnb__fit_prior',\n",
            " 'multinomialnb__force_alpha',\n",
            " 'robustscaler',\n",
            " 'robustscaler__copy',\n",
            " 'robustscaler__quantile_range',\n",
            " 'robustscaler__unit_variance',\n",
            " 'robustscaler__with_centering',\n",
            " 'robustscaler__with_scaling',\n",
            " 'steps',\n",
            " 'tfidfvectorizer',\n",
            " 'tfidfvectorizer__analyzer',\n",
            " 'tfidfvectorizer__binary',\n",
            " 'tfidfvectorizer__decode_error',\n",
            " 'tfidfvectorizer__dtype',\n",
            " 'tfidfvectorizer__encoding',\n",
            " 'tfidfvectorizer__input',\n",
            " 'tfidfvectorizer__lowercase',\n",
            " 'tfidfvectorizer__max_df',\n",
            " 'tfidfvectorizer__max_features',\n",
            " 'tfidfvectorizer__min_df',\n",
            " 'tfidfvectorizer__ngram_range',\n",
            " 'tfidfvectorizer__norm',\n",
            " 'tfidfvectorizer__preprocessor',\n",
            " 'tfidfvectorizer__smooth_idf',\n",
            " 'tfidfvectorizer__stop_words',\n",
            " 'tfidfvectorizer__strip_accents',\n",
            " 'tfidfvectorizer__sublinear_tf',\n",
            " 'tfidfvectorizer__token_pattern',\n",
            " 'tfidfvectorizer__tokenizer',\n",
            " 'tfidfvectorizer__use_idf',\n",
            " 'tfidfvectorizer__vocabulary',\n",
            " 'verbose']\n"
          ]
        }
      ],
      "source": [
        "pp.pprint(sorted(nb_pipe.get_params().keys()))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 35,
      "metadata": {
        "id": "fs0C6mq99Aep"
      },
      "outputs": [],
      "source": [
        "nb_params = [{\n",
        "    \"tfidfvectorizer__ngram_range\": [(1, 1), (1, 2), (2,3)],\n",
        "    \"multinomialnb__alpha\": [000.1, 00.1, 0.1, 0.5, 1, 10]\n",
        "    }]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 36,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 180
        },
        "id": "VhXR3TSWZLUu",
        "outputId": "a54f0774-e15a-4d23-ee8c-5b439130c2c4"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Fitting 10 folds for each of 18 candidates, totalling 180 fits\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "<style>#sk-container-id-7 {color: black;}#sk-container-id-7 pre{padding: 0;}#sk-container-id-7 div.sk-toggleable {background-color: white;}#sk-container-id-7 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-7 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-7 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-7 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-7 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-7 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-7 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-7 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-7 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-7 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-7 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-7 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-7 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-7 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-7 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-7 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-7 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-7 div.sk-item {position: relative;z-index: 1;}#sk-container-id-7 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-7 div.sk-item::before, #sk-container-id-7 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-7 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-7 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-7 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-7 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-7 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-7 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-7 div.sk-label-container {text-align: center;}#sk-container-id-7 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-7 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-7\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>GridSearchCV(cv=10,\n",
              "             estimator=Pipeline(steps=[(&#x27;tfidfvectorizer&#x27;,\n",
              "                                        TfidfVectorizer(max_features=512)),\n",
              "                                       (&#x27;robustscaler&#x27;,\n",
              "                                        RobustScaler(with_centering=False)),\n",
              "                                       (&#x27;multinomialnb&#x27;, MultinomialNB())]),\n",
              "             n_jobs=-1,\n",
              "             param_grid=[{&#x27;multinomialnb__alpha&#x27;: [0.1, 0.1, 0.1, 0.5, 1, 10],\n",
              "                          &#x27;tfidfvectorizer__ngram_range&#x27;: [(1, 1), (1, 2),\n",
              "                                                           (2, 3)]}],\n",
              "             scoring=&#x27;f1_macro&#x27;, verbose=1)</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-28\" type=\"checkbox\" ><label for=\"sk-estimator-id-28\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">GridSearchCV</label><div class=\"sk-toggleable__content\"><pre>GridSearchCV(cv=10,\n",
              "             estimator=Pipeline(steps=[(&#x27;tfidfvectorizer&#x27;,\n",
              "                                        TfidfVectorizer(max_features=512)),\n",
              "                                       (&#x27;robustscaler&#x27;,\n",
              "                                        RobustScaler(with_centering=False)),\n",
              "                                       (&#x27;multinomialnb&#x27;, MultinomialNB())]),\n",
              "             n_jobs=-1,\n",
              "             param_grid=[{&#x27;multinomialnb__alpha&#x27;: [0.1, 0.1, 0.1, 0.5, 1, 10],\n",
              "                          &#x27;tfidfvectorizer__ngram_range&#x27;: [(1, 1), (1, 2),\n",
              "                                                           (2, 3)]}],\n",
              "             scoring=&#x27;f1_macro&#x27;, verbose=1)</pre></div></div></div><div class=\"sk-parallel\"><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-29\" type=\"checkbox\" ><label for=\"sk-estimator-id-29\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">estimator: Pipeline</label><div class=\"sk-toggleable__content\"><pre>Pipeline(steps=[(&#x27;tfidfvectorizer&#x27;, TfidfVectorizer(max_features=512)),\n",
              "                (&#x27;robustscaler&#x27;, RobustScaler(with_centering=False)),\n",
              "                (&#x27;multinomialnb&#x27;, MultinomialNB())])</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-30\" type=\"checkbox\" ><label for=\"sk-estimator-id-30\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">TfidfVectorizer</label><div class=\"sk-toggleable__content\"><pre>TfidfVectorizer(max_features=512)</pre></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-31\" type=\"checkbox\" ><label for=\"sk-estimator-id-31\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">RobustScaler</label><div class=\"sk-toggleable__content\"><pre>RobustScaler(with_centering=False)</pre></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-32\" type=\"checkbox\" ><label for=\"sk-estimator-id-32\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">MultinomialNB</label><div class=\"sk-toggleable__content\"><pre>MultinomialNB()</pre></div></div></div></div></div></div></div></div></div></div></div></div>"
            ],
            "text/plain": [
              "GridSearchCV(cv=10,\n",
              "             estimator=Pipeline(steps=[('tfidfvectorizer',\n",
              "                                        TfidfVectorizer(max_features=512)),\n",
              "                                       ('robustscaler',\n",
              "                                        RobustScaler(with_centering=False)),\n",
              "                                       ('multinomialnb', MultinomialNB())]),\n",
              "             n_jobs=-1,\n",
              "             param_grid=[{'multinomialnb__alpha': [0.1, 0.1, 0.1, 0.5, 1, 10],\n",
              "                          'tfidfvectorizer__ngram_range': [(1, 1), (1, 2),\n",
              "                                                           (2, 3)]}],\n",
              "             scoring='f1_macro', verbose=1)"
            ]
          },
          "execution_count": 36,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "nb_gs = GridSearchCV(nb_pipe,\n",
        "                     nb_params,\n",
        "                     cv=10,\n",
        "                     scoring='f1_macro',\n",
        "                     n_jobs=-1,\n",
        "                     refit=True,\n",
        "                     verbose=1)\n",
        "nb_gs.fit(X_train, y_train)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 37,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "RRWI1iyA7E0B",
        "outputId": "87aa5de8-adbc-417e-ce8e-86bbd420c142"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "0.9076078032110857"
            ]
          },
          "execution_count": 37,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "nb_gs.best_score_"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 38,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jyMdfGbV7MFw",
        "outputId": "05421b83-5a4e-48ec-e133-48d263486e04"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "{'multinomialnb__alpha': 0.1, 'tfidfvectorizer__ngram_range': (2, 3)}"
            ]
          },
          "execution_count": 38,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "nb_gs.best_params_"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 39,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 444
        },
        "id": "IhKMmM9z7Sdi",
        "outputId": "791d3ac9-3a7e-478c-a46b-acf2722cc5a6"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.98      0.85      0.91        95\n",
            "           1       0.91      0.99      0.94       137\n",
            "\n",
            "    accuracy                           0.93       232\n",
            "   macro avg       0.94      0.92      0.93       232\n",
            "weighted avg       0.93      0.93      0.93       232\n",
            "\n"
          ]
        },
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAfsAAAGwCAYAAACuFMx9AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/NK7nSAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAyUElEQVR4nO3de3gU9dn/8c/mtAmYBAKyIRIgSAQUBARE8AAUCUVBeHgsUmhFxQNFxRQV9UeVoJIIbTEKBcX6AA9K1ccKWuuBeEIRqRDAKlAsGiAIaUBCzsfd+f2RsnUNaDazybIz79d1zXW5M9+ZvYNc3Lnv73dmHIZhGAIAAJYVFuwAAABA8yLZAwBgcSR7AAAsjmQPAIDFkewBALA4kj0AABZHsgcAwOIigh2AGR6PR4cPH1ZsbKwcDkewwwEA+MkwDJWWliopKUlhYc1Xf1ZVVammpsb0daKiohQdHR2AiFpWSCf7w4cPKzk5OdhhAABMys/PV6dOnZrl2lVVVUrpcpYKCt2mr5WYmKi8vLyQS/ghnexjY2MlSclL71VYjDPI0QDNI/XhomCHADSbOk+NPsh/xvvveXOoqalRQaFbB3K7Ki626d2DklKPugzYr5qaGpJ9SzrZug+LcSqsVWj9wQONFRHGL7KwvpaYij0r1qGzYpv+PR6F7nRxSCd7AAAay2145DbxNhi34QlcMC2MZA8AsAWPDHnU9Gxv5txg49Y7AAAsjsoeAGALHnlkphFv7uzgItkDAGzBbRhyG01vxZs5N9ho4wMAYHFU9gAAW7DzAj2SPQDAFjwy5LZpsqeNDwCAxVHZAwBsgTY+AAAWx2p8AABgWVT2AABb8Px7M3N+qCLZAwBswW1yNb6Zc4ONZA8AsAW3IZNvvQtcLC2NOXsAACyOyh4AYAvM2QMAYHEeOeSWw9T5oYo2PgAAFkdlDwCwBY9Rv5k5P1SR7AEAtuA22cY3c26w0cYHAMDiqOwBALZg58qeZA8AsAWP4ZDHMLEa38S5wUYbHwAAi6OyBwDYAm18AAAszq0wuU00tN0BjKWlkewBALZgmJyzN5izBwAAZyoqewCALTBnDwCAxbmNMLkNE3P2Ify4XNr4AABYHJU9AMAWPHLIY6LG9Sh0S3uSPQDAFuw8Z08bHwAAiyPZAwBs4eQCPTObPz788EONGzdOSUlJcjgcWr9+vfdYbW2t7rvvPvXp00etW7dWUlKSrr/+eh0+fNjnGtXV1brzzjvVvn17tW7dWtdcc40OHTrk989OsgcA2EL9nL25zR/l5eXq27evli5d2uBYRUWFtm/frgcffFDbt2/XK6+8oi+//FLXXHONz7j09HStW7dOL7zwgjZt2qSysjKNHTtWbrd/z/Njzh4AgGYwZswYjRkz5pTH4uPjlZOT47NvyZIluvjii3Xw4EF17txZxcXFevbZZ7VmzRpdeeWVkqTnnntOycnJeueddzR69OhGx0JlDwCwBc+/n43f1O3kSv6SkhKfrbq6OiDxFRcXy+FwqE2bNpKk3Nxc1dbWKi0tzTsmKSlJvXv31ubNm/26NskeAGALgZqzT05OVnx8vHfLysoyHVtVVZXuv/9+TZkyRXFxcZKkgoICRUVFqW3btj5jXS6XCgoK/Lo+bXwAgC14vlOdN+38+vvs8/PzvQlZkpxOp6m4amtrNXnyZHk8Hi1btuxHxxuGIYfDv/UDVPYAAPghLi7OZzOT7GtrazVp0iTl5eUpJyfH55eIxMRE1dTUqKioyOecwsJCuVwuv76HZA8AsAW34TC9BdLJRP/Pf/5T77zzjtq1a+dzfMCAAYqMjPRZyHfkyBF98cUXGjp0qF/fRRsfAGALJxfaNf18/x6XW1ZWpn379nk/5+XlaefOnUpISFBSUpKuvfZabd++Xa+//rrcbrd3Hj4hIUFRUVGKj4/X9OnTdffdd6tdu3ZKSEjQPffcoz59+nhX5zcWyR4AgGawbds2jRgxwvt59uzZkqRp06YpIyNDr732miSpX79+Pue9//77Gj58uCTp8ccfV0REhCZNmqTKykqNHDlSq1atUnh4uF+xkOwBALbgMcLkMfGKW4/hX2U/fPhwGT9wzg8dOyk6OlpLlizRkiVL/Pru7yPZAwBsoaXb+GcSFugBAGBxVPYAAFvwSKZW1HsCF0qLI9kDAGzB/EN1QrcZHrqRAwCARqGyBwDYQlPeSf/980MVyR4AYAtNeSf9988PVSR7AIAt2LmyD93IAQBAo1DZAwBswfxDdUK3PibZAwBswWM45DFzn32A33rXkkL31xQAANAoVPYAAFvwmGzjh/JDdUj2AABbMP/Wu9BN9qEbOQAAaBQqewCALbjlkNvEg3HMnBtsJHsAgC3QxgcAAJZFZQ8AsAW3zLXi3YELpcWR7AEAtmDnNj7JHgBgC7wIBwAAWBaVPQDAFgyT77M3uPUOAIAzG218AABgWVT2AABbsPMrbkn2AABbcJt8652Zc4MtdCMHAACNQmUPALAF2vgAAFicR2HymGhomzk32EI3cgAA0ChU9gAAW3AbDrlNtOLNnBtsJHsAgC0wZw8AgMUZJt96Z/AEPQAAcKaisgcA2IJbDrlNvMzGzLnBRrIHANiCxzA37+4xAhhMC6ONDwCAxVHZoyG3oYQ/F+isj4sUfqJW7jaRKh2WoKIJLims/rfi1p+eUNy738qZV6HwMrfyM89TTddWQQ4caJwL+n2r/57ylbr3OKF2Z1frkfsHasuHHU859o45n2nMhINakX2BXn2pWwtHikDymFygZ+bcYAvdyNFs2vzlX4p755iO3XCO8n/XU99OSVKb1wsV//Yx7xhHtUdVPVrr258nBTFSoGmio+uUty9OTy3u84PjLrniiHqcf0LHjka3UGRoTh45TG+hKujJftmyZUpJSVF0dLQGDBigjz76KNgh2V70PytUPjBeFf3jVXe2U+WD26iiT6yceRXeMWWXJ6hoYqIqe58VxEiBpsnd4tKaFT21eeOpq3lJate+Ur+a/YV+O/8iuetC9x95QApysn/xxReVnp6uuXPnaseOHbr88ss1ZswYHTx4MJhh2V5Vj9aK+aJUkUeqJElRByoVvbdcFf3ighwZ0DIcDkN3z9uhP689VwfzYoMdDgLk5BP0zGyhKqhz9osXL9b06dN18803S5Kys7P19ttva/ny5crKygpmaLZ2YlwHhVW4lXzPP+p/HfRIxyd1VNnQtsEODWgR1/5in9xuh157KSXYoSCA7DxnH7RkX1NTo9zcXN1///0++9PS0rR58+ZTnlNdXa3q6mrv55KSkmaN0a7O+uSEYjcVqfD2LqrpFK2oA5Vqv+YbudtGqvSKhGCHBzSr7j1OaPykPM268QophOdoge8KWrI/duyY3G63XC6Xz36Xy6WCgoJTnpOVlaX58+e3RHi21m7tYRVd08Fbydd0jlHEsRq1efVfJHtY3gV9jyu+bbVWvfKOd194hKHpd+7S+Ou+1k3/fWUQo4MZHpl8Nn4I//IX9J6Ew+H7h2cYRoN9Jz3wwAMqLi72bvn5+S0Rou04ajzS9/8fhDmkEH6gBNBY773VSXdcP0x33nCFdzt2NFqvrO2uB399SbDDgwmGyZX4hp/J/sMPP9S4ceOUlJQkh8Oh9evX+8ZjGMrIyFBSUpJiYmI0fPhw7dq1y2dMdXW17rzzTrVv316tW7fWNddco0OHDvn9swct2bdv317h4eENqvjCwsIG1f5JTqdTcXFxPhsCr/yiOLV99V9qtaNYEUer1XrrCbV5o1Dlg+K9Y8LK6hS1v0JRh+qnVSKPVCtqf4XCT9QGK2yg0aJj6tQttVjdUoslSYkdK9QttVhnuypUWhKlA1/H+WzuOoeKvnXqm4PcfRLKTr71zszmj/LycvXt21dLly495fFFixZp8eLFWrp0qbZu3arExESNGjVKpaWl3jHp6elat26dXnjhBW3atEllZWUaO3as3G63X7EErY0fFRWlAQMGKCcnR//1X//l3Z+Tk6Px48cHKyxIOjatkxL+74jarzyk8OI6udtGqmRkex2f+J9fwlrnFqvD0//prCQuOSBJOj7RpaJrT387E3AmSO15Qo/94RPv51vu2i1JeuevnfT4gv7BCgsh4vvrxZxOp5xOZ4NxY8aM0ZgxY055DcMwlJ2drblz52rixImSpNWrV8vlcmnt2rW67bbbVFxcrGeffVZr1qzRlVfWTx8999xzSk5O1jvvvKPRo0c3OuagrsafPXu2fvnLX2rgwIEaMmSIVqxYoYMHD2rGjBnBDMv2jJhwfXt9J317fafTjikd1k6lw9q1YFRA4Hy+o72uHjqu0eOZp7eGQK3GT05O9tk/b948ZWRk+HWtvLw8FRQUKC0tzbvP6XRq2LBh2rx5s2677Tbl5uaqtrbWZ0xSUpJ69+6tzZs3h06yv+666/Ttt9/q4Ycf1pEjR9S7d2+98cYb6tKlSzDDAgBYUFNa8d8/X5Ly8/N9ppFPVdX/mJNT2KdapH7gwAHvmKioKLVt27bBmNMtZD+doD8bf+bMmZo5c2awwwAAoFECuWbMn0Xq/oz5vqCvxgcAoCWcSc/GT0xMlKQfXKSemJiompoaFRUVnXZMY5HsAQC20NKr8X9ISkqKEhMTlZOT491XU1OjjRs3aujQoZKkAQMGKDIy0mfMkSNH9MUXX3jHNFbQ2/gAAFhRWVmZ9u3b5/2cl5ennTt3KiEhQZ07d1Z6eroyMzOVmpqq1NRUZWZmqlWrVpoyZYokKT4+XtOnT9fdd9+tdu3aKSEhQffcc4/69OnjXZ3fWCR7AIAtBGqBXmNt27ZNI0aM8H6ePXu2JGnatGlatWqV5syZo8rKSs2cOVNFRUUaPHiwNmzYoNjY/7x86fHHH1dERIQmTZqkyspKjRw5UqtWrVJ4eLhfsTgMwwjZ56KVlJQoPj5eXZ79jcJa8b5pWFOP/3c82CEAzabOU613DvxBxcXFzfagtJO5YvSbtyqydVSTr1NbXqO3x6xo1libC3P2AABYHG18AIAttHQb/0xCsgcA2IIhc2+uC9k5b5HsAQA2YefKnjl7AAAsjsoeAGALdq7sSfYAAFuwc7KnjQ8AgMVR2QMAbMHOlT3JHgBgC4bhkGEiYZs5N9ho4wMAYHFU9gAAWzD7TvpAvs++pZHsAQC2YOc5e9r4AABYHJU9AMAW7LxAj2QPALAFO7fxSfYAAFuwc2XPnD0AABZHZQ8AsAXDZBs/lCt7kj0AwBYMSYZh7vxQRRsfAACLo7IHANiCRw45eIIeAADWxWp8AABgWVT2AABb8BgOOXioDgAA1mUYJlfjh/ByfNr4AABYHJU9AMAW7LxAj2QPALAFkj0AABZn5wV6zNkDAGBxVPYAAFuw82p8kj0AwBbqk72ZOfsABtPCaOMDAGBxVPYAAFtgNT4AABZnyNw76UO4i08bHwAAq6OyBwDYAm18AACszsZ9fJI9AMAeTFb2CuHKnjl7AAAsjsoeAGALPEEPAACLs/MCPdr4AABYHMkeAGAPhsP85oe6ujr95je/UUpKimJiYtStWzc9/PDD8ng8/wnJMJSRkaGkpCTFxMRo+PDh2rVrV6B/cpI9AMAeTs7Zm9n8sXDhQj311FNaunSp9uzZo0WLFum3v/2tlixZ4h2zaNEiLV68WEuXLtXWrVuVmJioUaNGqbS0NKA/O8keAIBm8Mknn2j8+PG6+uqr1bVrV1177bVKS0vTtm3bJNVX9dnZ2Zo7d64mTpyo3r17a/Xq1aqoqNDatWsDGgvJHgBgD0YANkklJSU+W3V19Sm/7rLLLtO7776rL7/8UpL02WefadOmTbrqqqskSXl5eSooKFBaWpr3HKfTqWHDhmnz5s0B/dFZjQ8AsIVArcZPTk722T9v3jxlZGQ0GH/fffepuLhYPXv2VHh4uNxutxYsWKCf//znkqSCggJJksvl8jnP5XLpwIEDTY7zVBqV7J988slGX3DWrFlNDgYAgDNdfn6+4uLivJ+dTucpx7344ot67rnntHbtWl1wwQXauXOn0tPTlZSUpGnTpnnHORy+v4AYhtFgn1mNSvaPP/54oy7mcDhI9gCAM1cAHowTFxfnk+xP595779X999+vyZMnS5L69OmjAwcOKCsrS9OmTVNiYqKk+gq/Y8eO3vMKCwsbVPtmNSrZ5+XlBfRLAQBoaS39UJ2KigqFhfkujQsPD/feepeSkqLExETl5OSof//+kqSamhpt3LhRCxcubHKcp9LkOfuamhrl5eXp3HPPVUQEU/8AgDNcC7/1bty4cVqwYIE6d+6sCy64QDt27NDixYt10003SarvhqenpyszM1OpqalKTU1VZmamWrVqpSlTppgItCG/s3RFRYXuvPNOrV69WpL05Zdfqlu3bpo1a5aSkpJ0//33BzRAAABC0ZIlS/Tggw9q5syZKiwsVFJSkm677TY99NBD3jFz5sxRZWWlZs6cqaKiIg0ePFgbNmxQbGxsQGPx+9a7Bx54QJ999pk++OADRUdHe/dfeeWVevHFFwMaHAAAgeMIwNZ4sbGxys7O1oEDB1RZWamvvvpKjz76qKKiov4TkcOhjIwMHTlyRFVVVdq4caN69+5t9gdtwO/Kfv369XrxxRd1ySWX+KwWPP/88/XVV18FNDgAAAKmhdv4ZxK/K/ujR4+qQ4cODfaXl5cH/FYBAABgnt/JftCgQfrrX//q/XwywT/zzDMaMmRI4CIDACCQAvQEvVDkdxs/KytLP/3pT7V7927V1dXpiSee0K5du/TJJ59o48aNzREjAADmNeHNdQ3OD1F+V/ZDhw7Vxx9/rIqKCp177rnasGGDXC6XPvnkEw0YMKA5YgQAACY06Qb5Pn36eG+9AwAgFDTlNbXfPz9UNSnZu91urVu3Tnv27JHD4VCvXr00fvx4Hq4DADhz2Xg1vt/Z+YsvvtD48eNVUFCgHj16SKp/sM7ZZ5+t1157TX369Al4kAAAoOn8nrO/+eabdcEFF+jQoUPavn27tm/frvz8fF144YW69dZbmyNGAADMO7lAz8wWovyu7D/77DNt27ZNbdu29e5r27atFixYoEGDBgU0OAAAAsVh1G9mzg9Vflf2PXr00L/+9a8G+wsLC9W9e/eABAUAQMDZ+D77RiX7kpIS75aZmalZs2bp5Zdf1qFDh3To0CG9/PLLSk9PD/gr+QAAgHmNauO3adPG51G4hmFo0qRJ3n3Gv+9HGDdunNxudzOECQCASTZ+qE6jkv3777/f3HEAANC8uPXuhw0bNqy54wAAAM2kyU/Bqaio0MGDB1VTU+Oz/8ILLzQdFAAAAUdl33hHjx7VjTfeqDfffPOUx5mzBwCckWyc7P2+9S49PV1FRUXasmWLYmJi9NZbb2n16tVKTU3Va6+91hwxAgAAE/yu7N977z29+uqrGjRokMLCwtSlSxeNGjVKcXFxysrK0tVXX90ccQIAYI6NV+P7XdmXl5erQ4cOkqSEhAQdPXpUUv2b8LZv3x7Y6AAACJCTT9Azs4WqJj1Bb+/evZKkfv366emnn9Y333yjp556Sh07dgx4gAAAwBy/2/jp6ek6cuSIJGnevHkaPXq0nn/+eUVFRWnVqlWBjg8AgMCw8QI9v5P91KlTvf/dv39/7d+/X//4xz/UuXNntW/fPqDBAQAA85p8n/1JrVq10kUXXRSIWAAAaDYOmXzrXcAiaXmNSvazZ89u9AUXL17c5GAAAEDgNSrZ79ixo1EX++7LclpSyvTPFeGIDMp3A83tr4d3BjsEoNmUlHrU9rwW+jIb33rHi3AAAPZg4wV6ft96BwAAQovpBXoAAIQEG1f2JHsAgC2YfQqerZ6gBwAAQguVPQDAHmzcxm9SZb9mzRpdeumlSkpK0oEDByRJ2dnZevXVVwMaHAAAAWMEYAtRfif75cuXa/bs2brqqqt04sQJud1uSVKbNm2UnZ0d6PgAAIBJfif7JUuW6JlnntHcuXMVHh7u3T9w4EB9/vnnAQ0OAIBAsfMrbv2es8/Ly1P//v0b7Hc6nSovLw9IUAAABJyNn6Dnd2WfkpKinTt3Ntj/5ptv6vzzzw9ETAAABJ6N5+z9ruzvvfde3X777aqqqpJhGPr000/1pz/9SVlZWfrjH//YHDECAAAT/E72N954o+rq6jRnzhxVVFRoypQpOuecc/TEE09o8uTJzREjAACm2fmhOk26z/6WW27RLbfcomPHjsnj8ahDhw6BjgsAgMCy8X32ph6q0759+0DFAQAAmonfyT4lJeUH31v/9ddfmwoIAIBmYfb2OTtV9unp6T6fa2trtWPHDr311lu69957AxUXAACBRRu/8e66665T7v/DH/6gbdu2mQ4IAAAEVsDeejdmzBj9+c9/DtTlAAAILBvfZx+wZP/yyy8rISEhUJcDACCggvG43G+++Ua/+MUv1K5dO7Vq1Ur9+vVTbm6u97hhGMrIyFBSUpJiYmI0fPhw7dq1K4A/dT2/2/j9+/f3WaBnGIYKCgp09OhRLVu2LKDBAQAQqoqKinTppZdqxIgRevPNN9WhQwd99dVXatOmjXfMokWLtHjxYq1atUrnnXeeHn30UY0aNUp79+5VbGxswGLxO9lPmDDB53NYWJjOPvtsDR8+XD179gxUXAAAhLSFCxcqOTlZK1eu9O7r2rWr978Nw1B2drbmzp2riRMnSpJWr14tl8ultWvX6rbbbgtYLH4l+7q6OnXt2lWjR49WYmJiwIIAAKDZBWg1fklJic9up9Mpp9PZYPhrr72m0aNH62c/+5k2btyoc845RzNnztQtt9wiqf7FcgUFBUpLS/O51rBhw7R58+aAJnu/5uwjIiL0q1/9StXV1QELAACAlhCoOfvk5GTFx8d7t6ysrFN+39dff63ly5crNTVVb7/9tmbMmKFZs2bpf//3fyVJBQUFkiSXy+Vznsvl8h4LFL/b+IMHD9aOHTvUpUuXgAYCAEAoyM/PV1xcnPfzqap6SfJ4PBo4cKAyMzMl1a9527Vrl5YvX67rr7/eO+77D6ozDOMHH17XFH4n+5kzZ+ruu+/WoUOHNGDAALVu3drn+IUXXhiw4AAACKgA3D4XFxfnk+xPp2PHjg1e/d6rVy/vbeonp8MLCgrUsWNH75jCwsIG1b5ZjU72N910k7Kzs3XddddJkmbNmuU95nA4vL+JuN3ugAYIAEBAtPAT9C699FLt3bvXZ9+XX37p7YynpKQoMTFROTk56t+/vySppqZGGzdu1MKFC00E2lCjk/3q1av12GOPKS8vL6ABAABgRb/+9a81dOhQZWZmatKkSfr000+1YsUKrVixQlJ9oZyenq7MzEylpqYqNTVVmZmZatWqlaZMmRLQWBqd7A2j/lca5uoBAKGopd9nP2jQIK1bt04PPPCAHn74YaWkpCg7O1tTp071jpkzZ44qKys1c+ZMFRUVafDgwdqwYUNA77GX/JyzD/SCAQAAWkwQXoQzduxYjR079rTHHQ6HMjIylJGR0fS4GsGvZH/eeef9aMI/fvy4qYAAAEBg+ZXs58+fr/j4+OaKBQCAZtPSbfwziV/JfvLkyerQoUNzxQIAQPOx8fvsG/0EPebrAQAITX6vxgcAICTZuLJvdLL3eDzNGQcAAM2KOXsAAKzOxpW9X2+9AwAAoYfKHgBgDzau7En2AABbsPOcPW18AAAsjsoeAGAPtPEBALA22vgAAMCyqOwBAPZAGx8AAIuzcbKnjQ8AgMVR2QMAbMHx783M+aGKZA8AsAcbt/FJ9gAAW+DWOwAAYFlU9gAAe6CNDwCADYRwwjaDNj4AABZHZQ8AsAU7L9Aj2QMA7MHGc/a08QEAsDgqewCALdDGBwDA6mjjAwAAq6KyBwDYAm18AACszsZtfJI9AMAebJzsmbMHAMDiqOwBALbAnD0AAFZHGx8AAFgVlT0AwBYchiGH0fTy3My5wUayBwDYA218AABgVVT2AABbYDU+AABWRxsfAABYFZU9AMAWaOMDAGB1tPEBALC2k5W9ma2psrKy5HA4lJ6e7t1nGIYyMjKUlJSkmJgYDR8+XLt27TL/g54CyR4AgGa0detWrVixQhdeeKHP/kWLFmnx4sVaunSptm7dqsTERI0aNUqlpaUBj4FkDwCwByMAm6SSkhKfrbq6+rRfWVZWpqlTp+qZZ55R27Zt/xOKYSg7O1tz587VxIkT1bt3b61evVoVFRVau3ZtoH9ykj0AwD4C0cJPTk5WfHy8d8vKyjrt991+++26+uqrdeWVV/rsz8vLU0FBgdLS0rz7nE6nhg0bps2bNwf852aBHgAAfsjPz1dcXJz3s9PpPOW4F154Qdu3b9fWrVsbHCsoKJAkuVwun/0ul0sHDhwIYLT1SPYAAHswjPrNzPmS4uLifJL9qeTn5+uuu+7Shg0bFB0dfdpxDofje19hNNgXCLTxAQC20JKr8XNzc1VYWKgBAwYoIiJCERER2rhxo5588klFRER4K/qTFf5JhYWFDar9QCDZAwAQYCNHjtTnn3+unTt3ereBAwdq6tSp2rlzp7p166bExETl5OR4z6mpqdHGjRs1dOjQgMdDGx8AYA8t+FCd2NhY9e7d22df69at1a5dO+/+9PR0ZWZmKjU1VampqcrMzFSrVq00ZcoUE0GeGskeAGALDk/9Zub8QJozZ44qKys1c+ZMFRUVafDgwdqwYYNiY2MD+0Ui2QMA0CI++OADn88Oh0MZGRnKyMho9u8m2eNHXXfHv3TpVcVK7l6tmqow7d7WSs8u6KhDX51+hSlwJvl8S2v937IO+ufnrXT8X5Ga92yeho4p9h5f87tEffBqGx09HKnIKEPd+1TqxvuPqOdFFd4x9/53d/39k7N8rjvsmiL9v6cCf5sUmomNn41PssePunBIuf6yqr2+3NlK4RGGbrjviDL/9LVuGdZD1ZXhwQ4P+FFVFWHqdkGl0iYf1yM3pzQ4fk63Kt2+4JA6dqlRdVWY1q04Ww/8/Fyt3Lxbbdq5vePGTD2m6+/9z+ppZ3SA+7poVnZ+611QV+N/+OGHGjdunJKSkuRwOLR+/fpghoPTmDu1m3JeStCBL6P19e4Y/f7XneXqVKvUCyuDHRrQKIN+Uqob7ivQZVcVn/L4Tyae0EVXlKljlxp17VGlWzO+UUVpuPJ2x/iMc8YYSuhQ591ax5HsQ8rJ++zNbCEqqMm+vLxcffv21dKlS4MZBvzUOq6+0ik9QVUP66mtceiN59qpdZxb3c73/YX2/Vfa6mcX9NYtw3toxfwkVZRx9zJCQ1Db+GPGjNGYMWMaPb66utrnhQMlJSXNERZ+kKFbMw7ri7+11oG9MT8+HAgRW3LilPWrLqquDFOCq1ZZL+xT/Hda+CMmHldico0SOtRp/z+i9T9ZHfX17hg99uJXQYwa/rBzGz+k5uyzsrI0f/78YIdha7dnfqOUXpW6e0L3YIcCBFS/S8u0LGevSo5H6M3n22nBbV315F//qTbt6yRJV0097h3btWeVzulWrTt+2kP//HsMU1qhwsYL9EKqB/XAAw+ouLjYu+Xn5wc7JFuZ+eghDUkr0Zxrz9WxI1HBDgcIqOhWHp2TUqNeAyo0e3G+wiOkt/6UcNrx3ftUKiLSo2/yTv0SFOBMElKVvdPpPO3bhdCcDN2+4BsN/Wmx7r22u/6Vz/8DWJ9hSLXVp6+HDuyNVl1tmNq5alswKphBGx/4AXdkfqMR/1WkjBtTVFkWprZn1//jVl4arpqqkGoOwaYqy8N0+DsVeEF+lL76IkaxbeoUl+DW2idcGpJWrARXrUqOR+j11e117EikLh93QpJ0eH+U3nulrS4eWaK4BLcOfunUivnnqHvvCp0/qDxIPxX8FqC33oUikj1+1LgbvpUk/e4V34VIv0tPVs5Lp29zAmeKLz9rpTnX/medydMZ50iSRk06rlmP5evQPqce+b+uKjkeodi2bp3Xt0K/X/dPde1RJUmKiDS0c1Os1j97tqrKw9Q+qVaDR5Zo6uwChXNTCkJAUJN9WVmZ9u3b5/2cl5ennTt3KiEhQZ07dw5iZPiu0Ul9gx0CYErfoWV6+/DO0x5/6Nn9P3h+h3Nq9btX9v3gGJz5aOMHybZt2zRixAjv59mzZ0uSpk2bplWrVgUpKgCAJdl4NX5Qk/3w4cNlhPAcCAAAoYA5ewCALdDGBwDA6jxG/Wbm/BBFsgcA2ION5+y5SRoAAIujsgcA2IJDJufsAxZJyyPZAwDswcZP0KONDwCAxVHZAwBsgVvvAACwOlbjAwAAq6KyBwDYgsMw5DCxyM7MucFGsgcA2IPn35uZ80MUbXwAACyOyh4AYAu08QEAsDobr8Yn2QMA7IEn6AEAAKuisgcA2AJP0AMAwOpo4wMAAKuisgcA2ILDU7+ZOT9UkewBAPZAGx8AAFgVlT0AwB54qA4AANZm58fl0sYHAMDiqOwBAPZg4wV6JHsAgD0YMvdO+tDN9SR7AIA9MGcPAAAsi8oeAGAPhkzO2QcskhZHZQ8AsIeTC/TMbH7IysrSoEGDFBsbqw4dOmjChAnau3fv90IylJGRoaSkJMXExGj48OHatWtXIH9qSSR7AACaxcaNG3X77bdry5YtysnJUV1dndLS0lReXu4ds2jRIi1evFhLly7V1q1blZiYqFGjRqm0tDSgsdDGBwDYg0eSw+T5fnjrrbd8Pq9cuVIdOnRQbm6urrjiChmGoezsbM2dO1cTJ06UJK1evVoul0tr167VbbfdZiJYX1T2AABbOLka38wmSSUlJT5bdXV1o76/uLhYkpSQkCBJysvLU0FBgdLS0rxjnE6nhg0bps2bNwf0ZyfZAwDgh+TkZMXHx3u3rKysHz3HMAzNnj1bl112mXr37i1JKigokCS5XC6fsS6Xy3ssUGjjAwDsIUBP0MvPz1dcXJx3t9Pp/NFT77jjDv3973/Xpk2bGhxzOHznFgzDaLDPLJI9AMAeApTs4+LifJL9j7nzzjv12muv6cMPP1SnTp28+xMTEyXVV/gdO3b07i8sLGxQ7ZtFGx8AgGZgGIbuuOMOvfLKK3rvvfeUkpLiczwlJUWJiYnKycnx7qupqdHGjRs1dOjQgMZCZQ8AsIcWfhHO7bffrrVr1+rVV19VbGysdx4+Pj5eMTExcjgcSk9PV2ZmplJTU5WamqrMzEy1atVKU6ZMaXqcp0CyBwDYQwvferd8+XJJ0vDhw332r1y5UjfccIMkac6cOaqsrNTMmTNVVFSkwYMHa8OGDYqNjTURaEMkewCALbT0i3CMRox3OBzKyMhQRkZGE6NqHObsAQCwOCp7AIA9tPCc/ZmEZA8AsAePITlMJGxP6CZ72vgAAFgclT0AwB5o4wMAYHUmk71CN9nTxgcAwOKo7AEA9kAbHwAAi/MYMtWKZzU+AAA4U1HZAwDswfDUb2bOD1EkewCAPTBnDwCAxTFnDwAArIrKHgBgD7TxAQCwOEMmk33AImlxtPEBALA4KnsAgD3QxgcAwOI8Hkkm7pX3hO599rTxAQCwOCp7AIA90MYHAMDibJzsaeMDAGBxVPYAAHuw8eNySfYAAFswDI8ME2+uM3NusJHsAQD2YBjmqnPm7AEAwJmKyh4AYA+GyTn7EK7sSfYAAHvweCSHiXn3EJ6zp40PAIDFUdkDAOyBNj4AANZmeDwyTLTxQ/nWO9r4AABYHJU9AMAeaOMDAGBxHkNy2DPZ08YHAMDiqOwBAPZgGJLM3GcfupU9yR4AYAuGx5Bhoo1vkOwBADjDGR6Zq+y59Q4AAJyhqOwBALZAGx8AAKuzcRs/pJP9yd+y6lRr6jkJwJmspDR0/4EBfkxJWf3f75aoms3mijrVBi6YFhbSyb60tFSStElvBDkSoPm0PS/YEQDNr7S0VPHx8c1y7aioKCUmJmpTgflckZiYqKioqABE1bIcRghPQng8Hh0+fFixsbFyOBzBDscWSkpKlJycrPz8fMXFxQU7HCCg+Pvd8gzDUGlpqZKSkhQW1nxrxquqqlRTU2P6OlFRUYqOjg5ARC0rpCv7sLAwderUKdhh2FJcXBz/GMKy+Pvdspqrov+u6OjokEzSgcKtdwAAWBzJHgAAiyPZwy9Op1Pz5s2T0+kMdihAwPH3G1YV0gv0AADAj6OyBwDA4kj2AABYHMkeAACLI9kDAGBxJHs02rJly5SSkqLo6GgNGDBAH330UbBDAgLiww8/1Lhx45SUlCSHw6H169cHOyQgoEj2aJQXX3xR6enpmjt3rnbs2KHLL79cY8aM0cGDB4MdGmBaeXm5+vbtq6VLlwY7FKBZcOsdGmXw4MG66KKLtHz5cu++Xr16acKECcrKygpiZEBgORwOrVu3ThMmTAh2KEDAUNnjR9XU1Cg3N1dpaWk++9PS0rR58+YgRQUAaCySPX7UsWPH5Ha75XK5fPa7XC4VFBQEKSoAQGOR7NFo33+NsGEYvFoYAEIAyR4/qn379goPD29QxRcWFjao9gEAZx6SPX5UVFSUBgwYoJycHJ/9OTk5Gjp0aJCiAgA0VkSwA0BomD17tn75y19q4MCBGjJkiFasWKGDBw9qxowZwQ4NMK2srEz79u3zfs7Ly9POnTuVkJCgzp07BzEyIDC49Q6NtmzZMi1atEhHjhxR79699fjjj+uKK64IdliAaR988IFGjBjRYP+0adO0atWqlg8ICDCSPQAAFsecPQAAFkeyBwDA4kj2AABYHMkeAACLI9kDAGBxJHsAACyOZA8AgMWR7AEAsDiSPWBSRkaG+vXr5/18ww03aMKECS0ex/79++VwOLRz587Tjunatauys7Mbfc1Vq1apTZs2pmNzOBxav3696esAaBqSPSzphhtukMPhkMPhUGRkpLp166Z77rlH5eXlzf7dTzzxRKMfsdqYBA0AZvEiHFjWT3/6U61cuVK1tbX66KOPdPPNN6u8vFzLly9vMLa2tlaRkZEB+d74+PiAXAcAAoXKHpbldDqVmJio5ORkTZkyRVOnTvW2kk+23v/nf/5H3bp1k9PplGEYKi4u1q233qoOHTooLi5OP/nJT/TZZ5/5XPexxx6Ty+VSbGyspk+frqqqKp/j32/jezweLVy4UN27d5fT6VTnzp21YMECSVJKSookqX///nI4HBo+fLj3vJUrV6pXr16Kjo5Wz549tWzZMp/v+fTTT9W/f39FR0dr4MCB2rFjh99/RosXL1afPn3UunVrJScna+bMmSorK2swbv369TrvvPMUHR2tUaNGKT8/3+f4X/7yFw0YMEDR0dHq1q2b5s+fr7q6Or/jAdA8SPawjZiYGNXW1no/79u3Ty+99JL+/Oc/e9voV199tQoKCvTGG28oNzdXF110kUaOHKnjx49Lkl566SXNmzdPCxYs0LZt29SxY8cGSfj7HnjgAS1cuFAPPvigdu/erbVr18rlckmqT9iS9M477+jIkSN65ZVXJEnPPPOM5s6dqwULFmjPnj3KzMzUgw8+qNWrV0uSysvLNXbsWPXo0UO5ubnKyMjQPffc4/efSVhYmJ588kl98cUXWr16td577z3NmTPHZ0xFRYUWLFig1atX6+OPP1ZJSYkmT57sPf7222/rF7/4hWbNmqXdu3fr6aef1qpVq7y/0AA4AxiABU2bNs0YP3689/Pf/vY3o127dsakSZMMwzCMefPmGZGRkUZhYaF3zLvvvmvExcUZVVVVPtc699xzjaefftowDMMYMmSIMWPGDJ/jgwcPNvr27XvK7y4pKTGcTqfxzDPPnDLOvLw8Q5KxY8cOn/3JycnG2rVrffY98sgjxpAhQwzDMIynn37aSEhIMMrLy73Hly9ffsprfVeXLl2Mxx9//LTHX3rpJaNdu3bezytXrjQkGVu2bPHu27NnjyHJ+Nvf/mYYhmFcfvnlRmZmps911qxZY3Ts2NH7WZKxbt26034vgObFnD0s6/XXX9dZZ52luro61dbWavz48VqyZIn3eJcuXXT22Wd7P+fm5qqsrEzt2rXzuU5lZaW++uorSdKePXs0Y8YMn+NDhgzR+++/f8oY9uzZo+rqao0cObLRcR89elT5+fmaPn26brnlFu/+uro673qAPXv2qG/fvmrVqpVPHP56//33lZmZqd27d6ukpER1dXWqqqpSeXm5WrduLUmKiIjQwIEDvef07NlTbdq00Z49e3TxxRcrNzdXW7du9ank3W63qqqqVFFR4RMjgOAg2cOyRowYoeXLlysyMlJJSUkNFuCdTGYneTwedezYUR988EGDazX19rOYmBi/z/F4PJLqW/mDBw/2ORYeHi5JMgyjSfF814EDB3TVVVdpxowZeuSRR5SQkKBNmzZp+vTpPtMdUv2tc993cp/H49H8+fM1ceLEBmOio6NNxwnAPJI9LKt169bq3r17o8dfdNFFKigoUEREhLp27XrKMb169dKWLVt0/fXXe/dt2bLltNdMTU1VTEyM3n33Xd18880NjkdFRUmqr4RPcrlcOuecc/T1119r6tSpp7zu+eefrzVr1qiystL7C8UPxXEq27ZtU11dnX7/+98rLKx++c5LL73UYFxdXZ22bdumiy++WJK0d+9enThxQj179pRU/+e2d+9ev/6sAbQskj3wb1deeaWGDBmiCRMmaOHCherRo4cOHz6sN954QxMmTNDAgQN11113adq0aRo4cKAuu+wyPf/889q1a5e6det2ymtGR0frvvvu05w5cxQVFaVLL71UR48e1a5duzR9+nR16NBBMTExeuutt9SpUydFR0crPj5eGRkZmjVrluLi4jRmzBhVV1dr27ZtKioq0uzZszVlyhTNnTtX06dP129+8xvt379fv/vd7/z6ec8991zV1dVpyZIlGjdunD7++GM99dRTDcZFRkbqzjvv1JNPPqnIyEjdcccduuSSS7zJ/6GHHtLYsWOVnJysn/3sZwoLC9Pf//53ff7553r00Uf9/x8BIOBYjQ/8m8Ph0BtvvKErrrhCN910k8477zxNnjxZ+/fv966ev+666/TQQw/pvvvu04ABA3TgwAH96le/+sHrPvjgg7r77rv10EMPqVevXrruuutUWFgoqX4+/Mknn9TTTz+tpKQkjR8/XpJ08803649//KNWrVqlPn36aNiwYVq1apX3Vr2zzjpLf/nLX7R79271799fc+fO1cKFC/36efv166fFixdr4cKF6t27t55//nllZWU1GNeqVSvdd999mjJlioYMGaKYmBi98MIL3uOjR4/W66+/rpycHA0aNEiXXHKJFi9erC5duvgVD4Dm4zACMfkHAADOWFT2AABYHMkeAACLI9kDAGBxJHsAACyOZA8AgMWR7AEAsDiSPQAAFkeyBwDA4kj2AABYHMkeAACLI9kDAGBx/x8vTlx2CJMJ9gAAAABJRU5ErkJggg==",
            "text/plain": [
              "<Figure size 640x480 with 2 Axes>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "y_test_pred_nb = nb_gs.predict(X_test)\n",
        "print(classification_report(y_test, y_test_pred_nb))\n",
        "ConfusionMatrixDisplay.from_predictions(y_test, y_test_pred_nb);"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 40,
      "metadata": {},
      "outputs": [],
      "source": [
        "results_nb = pd.DataFrame({\"hist_text_classic\": X_test, \n",
        "                            \"true_label\": y_test,\n",
        "                            \"predicted_label\": y_test_pred_nb,\n",
        "                            \"result\": [\"Correct\" if true == pred else \"Incorrect\" for true, pred in zip(y_test, y_test_pred_nb)],\n",
        "                            \"n_words\": [len(text.split()) for text in X_test]\n",
        "                            })\n",
        "\n",
        "results_nb.true_label.replace({0: \"DC Comics\", 1: \"Marvel Comics\"}, inplace=True)\n",
        "results_nb.predicted_label.replace({0: \"DC Comics\", 1: \"Marvel Comics\"}, inplace=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 41,
      "metadata": {},
      "outputs": [],
      "source": [
        "results_nb.to_csv(\"data/error_nb.csv\",\n",
        "                  index=False)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### 3 Error analysis"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 42,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>hist_text_classic</th>\n",
              "      <th>true_label</th>\n",
              "      <th>predicted_label</th>\n",
              "      <th>result</th>\n",
              "      <th>n_words</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>1238</th>\n",
              "      <td>max save magda gas chamber execution ; member ...</td>\n",
              "      <td>Marvel Comics</td>\n",
              "      <td>DC Comics</td>\n",
              "      <td>Incorrect</td>\n",
              "      <td>269</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>494</th>\n",
              "      <td>vic stone son pair scientist decide use test s...</td>\n",
              "      <td>DC Comics</td>\n",
              "      <td>Marvel Comics</td>\n",
              "      <td>Incorrect</td>\n",
              "      <td>245</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1175</th>\n",
              "      <td>sam stern little slow - witte laborer complete...</td>\n",
              "      <td>Marvel Comics</td>\n",
              "      <td>DC Comics</td>\n",
              "      <td>Incorrect</td>\n",
              "      <td>135</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1584</th>\n",
              "      <td>vic sage , television investigative journalist...</td>\n",
              "      <td>DC Comics</td>\n",
              "      <td>Marvel Comics</td>\n",
              "      <td>Incorrect</td>\n",
              "      <td>318</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2071</th>\n",
              "      <td>mary macpherran bear prematurely suburb denver...</td>\n",
              "      <td>Marvel Comics</td>\n",
              "      <td>DC Comics</td>\n",
              "      <td>Incorrect</td>\n",
              "      <td>131</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>759</th>\n",
              "      <td>flash thompson bully - turn selfless - soldier...</td>\n",
              "      <td>Marvel Comics</td>\n",
              "      <td>DC Comics</td>\n",
              "      <td>Incorrect</td>\n",
              "      <td>166</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1481</th>\n",
              "      <td>dr . rudy jones ( die november 2016 ) research...</td>\n",
              "      <td>DC Comics</td>\n",
              "      <td>Marvel Comics</td>\n",
              "      <td>Incorrect</td>\n",
              "      <td>30</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>155</th>\n",
              "      <td>albert \" al \" rothstein ( die october 6 , 2015...</td>\n",
              "      <td>DC Comics</td>\n",
              "      <td>Marvel Comics</td>\n",
              "      <td>Incorrect</td>\n",
              "      <td>30</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2139</th>\n",
              "      <td>vili 2 brother odin ve , asgardian war frost g...</td>\n",
              "      <td>Marvel Comics</td>\n",
              "      <td>DC Comics</td>\n",
              "      <td>Incorrect</td>\n",
              "      <td>65</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>330</th>\n",
              "      <td>jaime assistance , start develop free independ...</td>\n",
              "      <td>DC Comics</td>\n",
              "      <td>Marvel Comics</td>\n",
              "      <td>Incorrect</td>\n",
              "      <td>194</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2114</th>\n",
              "      <td>speak jackson , come attack costume vigilante ...</td>\n",
              "      <td>Marvel Comics</td>\n",
              "      <td>DC Comics</td>\n",
              "      <td>Incorrect</td>\n",
              "      <td>210</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>39</th>\n",
              "      <td>ajax ( bear francis freeman ) human gain immun...</td>\n",
              "      <td>Marvel Comics</td>\n",
              "      <td>DC Comics</td>\n",
              "      <td>Incorrect</td>\n",
              "      <td>17</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>702</th>\n",
              "      <td>little know childhood youth fatale come dark b...</td>\n",
              "      <td>Marvel Comics</td>\n",
              "      <td>DC Comics</td>\n",
              "      <td>Incorrect</td>\n",
              "      <td>16</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>706</th>\n",
              "      <td>cannonball overpower feral , old friend lucia ...</td>\n",
              "      <td>Marvel Comics</td>\n",
              "      <td>DC Comics</td>\n",
              "      <td>Incorrect</td>\n",
              "      <td>30</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>56</th>\n",
              "      <td>have drink , rowdy guest penguin use gimmick u...</td>\n",
              "      <td>DC Comics</td>\n",
              "      <td>Marvel Comics</td>\n",
              "      <td>Incorrect</td>\n",
              "      <td>11</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2087</th>\n",
              "      <td>piper eventually chop hand trickster carry cor...</td>\n",
              "      <td>DC Comics</td>\n",
              "      <td>Marvel Comics</td>\n",
              "      <td>Incorrect</td>\n",
              "      <td>36</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1918</th>\n",
              "      <td>reassure crew lose earth remain matter , journ...</td>\n",
              "      <td>DC Comics</td>\n",
              "      <td>Marvel Comics</td>\n",
              "      <td>Incorrect</td>\n",
              "      <td>12</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1438</th>\n",
              "      <td>night thrasher parent kill young . start hate ...</td>\n",
              "      <td>Marvel Comics</td>\n",
              "      <td>DC Comics</td>\n",
              "      <td>Incorrect</td>\n",
              "      <td>72</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>32</th>\n",
              "      <td>mid - late 1973 , zero member team x , consist...</td>\n",
              "      <td>Marvel Comics</td>\n",
              "      <td>DC Comics</td>\n",
              "      <td>Incorrect</td>\n",
              "      <td>53</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1441</th>\n",
              "      <td>kurt wagner bear certain unusual physical char...</td>\n",
              "      <td>Marvel Comics</td>\n",
              "      <td>DC Comics</td>\n",
              "      <td>Incorrect</td>\n",
              "      <td>292</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>53</th>\n",
              "      <td>join father villain , kid amazo fight urge pro...</td>\n",
              "      <td>DC Comics</td>\n",
              "      <td>Marvel Comics</td>\n",
              "      <td>Incorrect</td>\n",
              "      <td>25</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>922</th>\n",
              "      <td>rex send role hourman android hourman return j...</td>\n",
              "      <td>DC Comics</td>\n",
              "      <td>Marvel Comics</td>\n",
              "      <td>Incorrect</td>\n",
              "      <td>71</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                      hist_text_classic     true_label  \\\n",
              "1238  max save magda gas chamber execution ; member ...  Marvel Comics   \n",
              "494   vic stone son pair scientist decide use test s...      DC Comics   \n",
              "1175  sam stern little slow - witte laborer complete...  Marvel Comics   \n",
              "1584  vic sage , television investigative journalist...      DC Comics   \n",
              "2071  mary macpherran bear prematurely suburb denver...  Marvel Comics   \n",
              "759   flash thompson bully - turn selfless - soldier...  Marvel Comics   \n",
              "1481  dr . rudy jones ( die november 2016 ) research...      DC Comics   \n",
              "155   albert \" al \" rothstein ( die october 6 , 2015...      DC Comics   \n",
              "2139  vili 2 brother odin ve , asgardian war frost g...  Marvel Comics   \n",
              "330   jaime assistance , start develop free independ...      DC Comics   \n",
              "2114  speak jackson , come attack costume vigilante ...  Marvel Comics   \n",
              "39    ajax ( bear francis freeman ) human gain immun...  Marvel Comics   \n",
              "702   little know childhood youth fatale come dark b...  Marvel Comics   \n",
              "706   cannonball overpower feral , old friend lucia ...  Marvel Comics   \n",
              "56    have drink , rowdy guest penguin use gimmick u...      DC Comics   \n",
              "2087  piper eventually chop hand trickster carry cor...      DC Comics   \n",
              "1918  reassure crew lose earth remain matter , journ...      DC Comics   \n",
              "1438  night thrasher parent kill young . start hate ...  Marvel Comics   \n",
              "32    mid - late 1973 , zero member team x , consist...  Marvel Comics   \n",
              "1441  kurt wagner bear certain unusual physical char...  Marvel Comics   \n",
              "53    join father villain , kid amazo fight urge pro...      DC Comics   \n",
              "922   rex send role hourman android hourman return j...      DC Comics   \n",
              "\n",
              "     predicted_label     result  n_words  \n",
              "1238       DC Comics  Incorrect      269  \n",
              "494    Marvel Comics  Incorrect      245  \n",
              "1175       DC Comics  Incorrect      135  \n",
              "1584   Marvel Comics  Incorrect      318  \n",
              "2071       DC Comics  Incorrect      131  \n",
              "759        DC Comics  Incorrect      166  \n",
              "1481   Marvel Comics  Incorrect       30  \n",
              "155    Marvel Comics  Incorrect       30  \n",
              "2139       DC Comics  Incorrect       65  \n",
              "330    Marvel Comics  Incorrect      194  \n",
              "2114       DC Comics  Incorrect      210  \n",
              "39         DC Comics  Incorrect       17  \n",
              "702        DC Comics  Incorrect       16  \n",
              "706        DC Comics  Incorrect       30  \n",
              "56     Marvel Comics  Incorrect       11  \n",
              "2087   Marvel Comics  Incorrect       36  \n",
              "1918   Marvel Comics  Incorrect       12  \n",
              "1438       DC Comics  Incorrect       72  \n",
              "32         DC Comics  Incorrect       53  \n",
              "1441       DC Comics  Incorrect      292  \n",
              "53     Marvel Comics  Incorrect       25  \n",
              "922    Marvel Comics  Incorrect       71  "
            ]
          },
          "execution_count": 42,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "results_logreg[results_logreg[\"result\"]==\"Incorrect\"]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 43,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>hist_text_classic</th>\n",
              "      <th>true_label</th>\n",
              "      <th>predicted_label</th>\n",
              "      <th>result</th>\n",
              "      <th>n_words</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>1238</th>\n",
              "      <td>max save magda gas chamber execution ; member ...</td>\n",
              "      <td>Marvel Comics</td>\n",
              "      <td>DC Comics</td>\n",
              "      <td>Incorrect</td>\n",
              "      <td>269</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>494</th>\n",
              "      <td>vic stone son pair scientist decide use test s...</td>\n",
              "      <td>DC Comics</td>\n",
              "      <td>Marvel Comics</td>\n",
              "      <td>Incorrect</td>\n",
              "      <td>245</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>302</th>\n",
              "      <td>channel positive energy , blackwulf able slice...</td>\n",
              "      <td>Marvel Comics</td>\n",
              "      <td>DC Comics</td>\n",
              "      <td>Incorrect</td>\n",
              "      <td>210</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1615</th>\n",
              "      <td>arsenal later come conflict vandal savage . sa...</td>\n",
              "      <td>DC Comics</td>\n",
              "      <td>Marvel Comics</td>\n",
              "      <td>Incorrect</td>\n",
              "      <td>281</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1951</th>\n",
              "      <td>decide stay future work legion , give flight r...</td>\n",
              "      <td>DC Comics</td>\n",
              "      <td>Marvel Comics</td>\n",
              "      <td>Incorrect</td>\n",
              "      <td>254</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1797</th>\n",
              "      <td>apparently shatterstar body merge russell , re...</td>\n",
              "      <td>Marvel Comics</td>\n",
              "      <td>DC Comics</td>\n",
              "      <td>Incorrect</td>\n",
              "      <td>216</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1150</th>\n",
              "      <td>korg kronan warrior reside sakaar force partic...</td>\n",
              "      <td>Marvel Comics</td>\n",
              "      <td>DC Comics</td>\n",
              "      <td>Incorrect</td>\n",
              "      <td>41</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2035</th>\n",
              "      <td>defense inject serum change lizard prevent osc...</td>\n",
              "      <td>Marvel Comics</td>\n",
              "      <td>DC Comics</td>\n",
              "      <td>Incorrect</td>\n",
              "      <td>42</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2071</th>\n",
              "      <td>mary macpherran bear prematurely suburb denver...</td>\n",
              "      <td>Marvel Comics</td>\n",
              "      <td>DC Comics</td>\n",
              "      <td>Incorrect</td>\n",
              "      <td>131</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>371</th>\n",
              "      <td>nathan summer ' birth carefully orchestrate ge...</td>\n",
              "      <td>Marvel Comics</td>\n",
              "      <td>DC Comics</td>\n",
              "      <td>Incorrect</td>\n",
              "      <td>309</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2139</th>\n",
              "      <td>vili 2 brother odin ve , asgardian war frost g...</td>\n",
              "      <td>Marvel Comics</td>\n",
              "      <td>DC Comics</td>\n",
              "      <td>Incorrect</td>\n",
              "      <td>65</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>39</th>\n",
              "      <td>ajax ( bear francis freeman ) human gain immun...</td>\n",
              "      <td>Marvel Comics</td>\n",
              "      <td>DC Comics</td>\n",
              "      <td>Incorrect</td>\n",
              "      <td>17</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1456</th>\n",
              "      <td>jean - paul beaubier parent kill car crash sho...</td>\n",
              "      <td>Marvel Comics</td>\n",
              "      <td>DC Comics</td>\n",
              "      <td>Incorrect</td>\n",
              "      <td>303</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>706</th>\n",
              "      <td>cannonball overpower feral , old friend lucia ...</td>\n",
              "      <td>Marvel Comics</td>\n",
              "      <td>DC Comics</td>\n",
              "      <td>Incorrect</td>\n",
              "      <td>30</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>167</th>\n",
              "      <td>prison serve father life sentence prison age 8...</td>\n",
              "      <td>DC Comics</td>\n",
              "      <td>Marvel Comics</td>\n",
              "      <td>Incorrect</td>\n",
              "      <td>57</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>490</th>\n",
              "      <td>doomsday eventually bring board space cruiser ...</td>\n",
              "      <td>DC Comics</td>\n",
              "      <td>Marvel Comics</td>\n",
              "      <td>Incorrect</td>\n",
              "      <td>275</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2124</th>\n",
              "      <td>angelo fortunato humiliate criminal father , o...</td>\n",
              "      <td>Marvel Comics</td>\n",
              "      <td>DC Comics</td>\n",
              "      <td>Incorrect</td>\n",
              "      <td>113</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>188</th>\n",
              "      <td>daughter gotham police commissioner james gord...</td>\n",
              "      <td>DC Comics</td>\n",
              "      <td>Marvel Comics</td>\n",
              "      <td>Incorrect</td>\n",
              "      <td>127</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1582</th>\n",
              "      <td>hippolyta sister , antiope amazon emerge soul ...</td>\n",
              "      <td>DC Comics</td>\n",
              "      <td>Marvel Comics</td>\n",
              "      <td>Incorrect</td>\n",
              "      <td>278</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                      hist_text_classic     true_label  \\\n",
              "1238  max save magda gas chamber execution ; member ...  Marvel Comics   \n",
              "494   vic stone son pair scientist decide use test s...      DC Comics   \n",
              "302   channel positive energy , blackwulf able slice...  Marvel Comics   \n",
              "1615  arsenal later come conflict vandal savage . sa...      DC Comics   \n",
              "1951  decide stay future work legion , give flight r...      DC Comics   \n",
              "1797  apparently shatterstar body merge russell , re...  Marvel Comics   \n",
              "1150  korg kronan warrior reside sakaar force partic...  Marvel Comics   \n",
              "2035  defense inject serum change lizard prevent osc...  Marvel Comics   \n",
              "2071  mary macpherran bear prematurely suburb denver...  Marvel Comics   \n",
              "371   nathan summer ' birth carefully orchestrate ge...  Marvel Comics   \n",
              "2139  vili 2 brother odin ve , asgardian war frost g...  Marvel Comics   \n",
              "39    ajax ( bear francis freeman ) human gain immun...  Marvel Comics   \n",
              "1456  jean - paul beaubier parent kill car crash sho...  Marvel Comics   \n",
              "706   cannonball overpower feral , old friend lucia ...  Marvel Comics   \n",
              "167   prison serve father life sentence prison age 8...      DC Comics   \n",
              "490   doomsday eventually bring board space cruiser ...      DC Comics   \n",
              "2124  angelo fortunato humiliate criminal father , o...  Marvel Comics   \n",
              "188   daughter gotham police commissioner james gord...      DC Comics   \n",
              "1582  hippolyta sister , antiope amazon emerge soul ...      DC Comics   \n",
              "\n",
              "     predicted_label     result  n_words  \n",
              "1238       DC Comics  Incorrect      269  \n",
              "494    Marvel Comics  Incorrect      245  \n",
              "302        DC Comics  Incorrect      210  \n",
              "1615   Marvel Comics  Incorrect      281  \n",
              "1951   Marvel Comics  Incorrect      254  \n",
              "1797       DC Comics  Incorrect      216  \n",
              "1150       DC Comics  Incorrect       41  \n",
              "2035       DC Comics  Incorrect       42  \n",
              "2071       DC Comics  Incorrect      131  \n",
              "371        DC Comics  Incorrect      309  \n",
              "2139       DC Comics  Incorrect       65  \n",
              "39         DC Comics  Incorrect       17  \n",
              "1456       DC Comics  Incorrect      303  \n",
              "706        DC Comics  Incorrect       30  \n",
              "167    Marvel Comics  Incorrect       57  \n",
              "490    Marvel Comics  Incorrect      275  \n",
              "2124       DC Comics  Incorrect      113  \n",
              "188    Marvel Comics  Incorrect      127  \n",
              "1582   Marvel Comics  Incorrect      278  "
            ]
          },
          "execution_count": 43,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "results_knn[results_knn[\"result\"]==\"Incorrect\"]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 44,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>hist_text_classic</th>\n",
              "      <th>true_label</th>\n",
              "      <th>predicted_label</th>\n",
              "      <th>result</th>\n",
              "      <th>n_words</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>2045</th>\n",
              "      <td>jonathan crane derange , criminally insane tee...</td>\n",
              "      <td>DC Comics</td>\n",
              "      <td>Marvel Comics</td>\n",
              "      <td>Incorrect</td>\n",
              "      <td>219</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1238</th>\n",
              "      <td>max save magda gas chamber execution ; member ...</td>\n",
              "      <td>Marvel Comics</td>\n",
              "      <td>DC Comics</td>\n",
              "      <td>Incorrect</td>\n",
              "      <td>269</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2040</th>\n",
              "      <td>montoya longstanding detective gcpd select que...</td>\n",
              "      <td>DC Comics</td>\n",
              "      <td>Marvel Comics</td>\n",
              "      <td>Incorrect</td>\n",
              "      <td>23</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>494</th>\n",
              "      <td>vic stone son pair scientist decide use test s...</td>\n",
              "      <td>DC Comics</td>\n",
              "      <td>Marvel Comics</td>\n",
              "      <td>Incorrect</td>\n",
              "      <td>245</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>302</th>\n",
              "      <td>channel positive energy , blackwulf able slice...</td>\n",
              "      <td>Marvel Comics</td>\n",
              "      <td>DC Comics</td>\n",
              "      <td>Incorrect</td>\n",
              "      <td>210</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1175</th>\n",
              "      <td>sam stern little slow - witte laborer complete...</td>\n",
              "      <td>Marvel Comics</td>\n",
              "      <td>DC Comics</td>\n",
              "      <td>Incorrect</td>\n",
              "      <td>135</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>174</th>\n",
              "      <td>roberto velasquez talented boxer , spar friend...</td>\n",
              "      <td>Marvel Comics</td>\n",
              "      <td>DC Comics</td>\n",
              "      <td>Incorrect</td>\n",
              "      <td>290</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2071</th>\n",
              "      <td>mary macpherran bear prematurely suburb denver...</td>\n",
              "      <td>Marvel Comics</td>\n",
              "      <td>DC Comics</td>\n",
              "      <td>Incorrect</td>\n",
              "      <td>131</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>759</th>\n",
              "      <td>flash thompson bully - turn selfless - soldier...</td>\n",
              "      <td>Marvel Comics</td>\n",
              "      <td>DC Comics</td>\n",
              "      <td>Incorrect</td>\n",
              "      <td>166</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2139</th>\n",
              "      <td>vili 2 brother odin ve , asgardian war frost g...</td>\n",
              "      <td>Marvel Comics</td>\n",
              "      <td>DC Comics</td>\n",
              "      <td>Incorrect</td>\n",
              "      <td>65</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>330</th>\n",
              "      <td>jaime assistance , start develop free independ...</td>\n",
              "      <td>DC Comics</td>\n",
              "      <td>Marvel Comics</td>\n",
              "      <td>Incorrect</td>\n",
              "      <td>194</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2114</th>\n",
              "      <td>speak jackson , come attack costume vigilante ...</td>\n",
              "      <td>Marvel Comics</td>\n",
              "      <td>DC Comics</td>\n",
              "      <td>Incorrect</td>\n",
              "      <td>210</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>39</th>\n",
              "      <td>ajax ( bear francis freeman ) human gain immun...</td>\n",
              "      <td>Marvel Comics</td>\n",
              "      <td>DC Comics</td>\n",
              "      <td>Incorrect</td>\n",
              "      <td>17</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>706</th>\n",
              "      <td>cannonball overpower feral , old friend lucia ...</td>\n",
              "      <td>Marvel Comics</td>\n",
              "      <td>DC Comics</td>\n",
              "      <td>Incorrect</td>\n",
              "      <td>30</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2087</th>\n",
              "      <td>piper eventually chop hand trickster carry cor...</td>\n",
              "      <td>DC Comics</td>\n",
              "      <td>Marvel Comics</td>\n",
              "      <td>Incorrect</td>\n",
              "      <td>36</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1918</th>\n",
              "      <td>reassure crew lose earth remain matter , journ...</td>\n",
              "      <td>DC Comics</td>\n",
              "      <td>Marvel Comics</td>\n",
              "      <td>Incorrect</td>\n",
              "      <td>12</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1438</th>\n",
              "      <td>night thrasher parent kill young . start hate ...</td>\n",
              "      <td>Marvel Comics</td>\n",
              "      <td>DC Comics</td>\n",
              "      <td>Incorrect</td>\n",
              "      <td>72</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>32</th>\n",
              "      <td>mid - late 1973 , zero member team x , consist...</td>\n",
              "      <td>Marvel Comics</td>\n",
              "      <td>DC Comics</td>\n",
              "      <td>Incorrect</td>\n",
              "      <td>53</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>53</th>\n",
              "      <td>join father villain , kid amazo fight urge pro...</td>\n",
              "      <td>DC Comics</td>\n",
              "      <td>Marvel Comics</td>\n",
              "      <td>Incorrect</td>\n",
              "      <td>25</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                      hist_text_classic     true_label  \\\n",
              "2045  jonathan crane derange , criminally insane tee...      DC Comics   \n",
              "1238  max save magda gas chamber execution ; member ...  Marvel Comics   \n",
              "2040  montoya longstanding detective gcpd select que...      DC Comics   \n",
              "494   vic stone son pair scientist decide use test s...      DC Comics   \n",
              "302   channel positive energy , blackwulf able slice...  Marvel Comics   \n",
              "1175  sam stern little slow - witte laborer complete...  Marvel Comics   \n",
              "174   roberto velasquez talented boxer , spar friend...  Marvel Comics   \n",
              "2071  mary macpherran bear prematurely suburb denver...  Marvel Comics   \n",
              "759   flash thompson bully - turn selfless - soldier...  Marvel Comics   \n",
              "2139  vili 2 brother odin ve , asgardian war frost g...  Marvel Comics   \n",
              "330   jaime assistance , start develop free independ...      DC Comics   \n",
              "2114  speak jackson , come attack costume vigilante ...  Marvel Comics   \n",
              "39    ajax ( bear francis freeman ) human gain immun...  Marvel Comics   \n",
              "706   cannonball overpower feral , old friend lucia ...  Marvel Comics   \n",
              "2087  piper eventually chop hand trickster carry cor...      DC Comics   \n",
              "1918  reassure crew lose earth remain matter , journ...      DC Comics   \n",
              "1438  night thrasher parent kill young . start hate ...  Marvel Comics   \n",
              "32    mid - late 1973 , zero member team x , consist...  Marvel Comics   \n",
              "53    join father villain , kid amazo fight urge pro...      DC Comics   \n",
              "\n",
              "     predicted_label     result  n_words  \n",
              "2045   Marvel Comics  Incorrect      219  \n",
              "1238       DC Comics  Incorrect      269  \n",
              "2040   Marvel Comics  Incorrect       23  \n",
              "494    Marvel Comics  Incorrect      245  \n",
              "302        DC Comics  Incorrect      210  \n",
              "1175       DC Comics  Incorrect      135  \n",
              "174        DC Comics  Incorrect      290  \n",
              "2071       DC Comics  Incorrect      131  \n",
              "759        DC Comics  Incorrect      166  \n",
              "2139       DC Comics  Incorrect       65  \n",
              "330    Marvel Comics  Incorrect      194  \n",
              "2114       DC Comics  Incorrect      210  \n",
              "39         DC Comics  Incorrect       17  \n",
              "706        DC Comics  Incorrect       30  \n",
              "2087   Marvel Comics  Incorrect       36  \n",
              "1918   Marvel Comics  Incorrect       12  \n",
              "1438       DC Comics  Incorrect       72  \n",
              "32         DC Comics  Incorrect       53  \n",
              "53     Marvel Comics  Incorrect       25  "
            ]
          },
          "execution_count": 44,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "results_svm[results_svm[\"result\"]==\"Incorrect\"]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 45,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>hist_text_classic</th>\n",
              "      <th>true_label</th>\n",
              "      <th>predicted_label</th>\n",
              "      <th>result</th>\n",
              "      <th>n_words</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>2040</th>\n",
              "      <td>montoya longstanding detective gcpd select que...</td>\n",
              "      <td>DC Comics</td>\n",
              "      <td>Marvel Comics</td>\n",
              "      <td>Incorrect</td>\n",
              "      <td>23</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>979</th>\n",
              "      <td>infinity man combine form forever people .</td>\n",
              "      <td>DC Comics</td>\n",
              "      <td>Marvel Comics</td>\n",
              "      <td>Incorrect</td>\n",
              "      <td>7</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>557</th>\n",
              "      <td>kindly one arrive dream allow destroy order sa...</td>\n",
              "      <td>DC Comics</td>\n",
              "      <td>Marvel Comics</td>\n",
              "      <td>Incorrect</td>\n",
              "      <td>14</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>703</th>\n",
              "      <td>record whereabouts mighty sorcerer know felix ...</td>\n",
              "      <td>DC Comics</td>\n",
              "      <td>Marvel Comics</td>\n",
              "      <td>Incorrect</td>\n",
              "      <td>18</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1166</th>\n",
              "      <td>lagoon boy originally underwater city atlantis...</td>\n",
              "      <td>DC Comics</td>\n",
              "      <td>Marvel Comics</td>\n",
              "      <td>Incorrect</td>\n",
              "      <td>12</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1481</th>\n",
              "      <td>dr . rudy jones ( die november 2016 ) research...</td>\n",
              "      <td>DC Comics</td>\n",
              "      <td>Marvel Comics</td>\n",
              "      <td>Incorrect</td>\n",
              "      <td>30</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1456</th>\n",
              "      <td>jean - paul beaubier parent kill car crash sho...</td>\n",
              "      <td>Marvel Comics</td>\n",
              "      <td>DC Comics</td>\n",
              "      <td>Incorrect</td>\n",
              "      <td>303</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>706</th>\n",
              "      <td>cannonball overpower feral , old friend lucia ...</td>\n",
              "      <td>Marvel Comics</td>\n",
              "      <td>DC Comics</td>\n",
              "      <td>Incorrect</td>\n",
              "      <td>30</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>637</th>\n",
              "      <td>crisis , dawn granger london mother . mother ,...</td>\n",
              "      <td>DC Comics</td>\n",
              "      <td>Marvel Comics</td>\n",
              "      <td>Incorrect</td>\n",
              "      <td>87</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>56</th>\n",
              "      <td>have drink , rowdy guest penguin use gimmick u...</td>\n",
              "      <td>DC Comics</td>\n",
              "      <td>Marvel Comics</td>\n",
              "      <td>Incorrect</td>\n",
              "      <td>11</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2087</th>\n",
              "      <td>piper eventually chop hand trickster carry cor...</td>\n",
              "      <td>DC Comics</td>\n",
              "      <td>Marvel Comics</td>\n",
              "      <td>Incorrect</td>\n",
              "      <td>36</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1918</th>\n",
              "      <td>reassure crew lose earth remain matter , journ...</td>\n",
              "      <td>DC Comics</td>\n",
              "      <td>Marvel Comics</td>\n",
              "      <td>Incorrect</td>\n",
              "      <td>12</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>94</th>\n",
              "      <td>garth origin similar mentor . amphibious human...</td>\n",
              "      <td>DC Comics</td>\n",
              "      <td>Marvel Comics</td>\n",
              "      <td>Incorrect</td>\n",
              "      <td>292</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>406</th>\n",
              "      <td>child , nelson gardner sickly asthmatic , year...</td>\n",
              "      <td>DC Comics</td>\n",
              "      <td>Marvel Comics</td>\n",
              "      <td>Incorrect</td>\n",
              "      <td>121</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1060</th>\n",
              "      <td>ability know people fear , reach john psyche f...</td>\n",
              "      <td>DC Comics</td>\n",
              "      <td>Marvel Comics</td>\n",
              "      <td>Incorrect</td>\n",
              "      <td>262</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1582</th>\n",
              "      <td>hippolyta sister , antiope amazon emerge soul ...</td>\n",
              "      <td>DC Comics</td>\n",
              "      <td>Marvel Comics</td>\n",
              "      <td>Incorrect</td>\n",
              "      <td>278</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                      hist_text_classic     true_label  \\\n",
              "2040  montoya longstanding detective gcpd select que...      DC Comics   \n",
              "979          infinity man combine form forever people .      DC Comics   \n",
              "557   kindly one arrive dream allow destroy order sa...      DC Comics   \n",
              "703   record whereabouts mighty sorcerer know felix ...      DC Comics   \n",
              "1166  lagoon boy originally underwater city atlantis...      DC Comics   \n",
              "1481  dr . rudy jones ( die november 2016 ) research...      DC Comics   \n",
              "1456  jean - paul beaubier parent kill car crash sho...  Marvel Comics   \n",
              "706   cannonball overpower feral , old friend lucia ...  Marvel Comics   \n",
              "637   crisis , dawn granger london mother . mother ,...      DC Comics   \n",
              "56    have drink , rowdy guest penguin use gimmick u...      DC Comics   \n",
              "2087  piper eventually chop hand trickster carry cor...      DC Comics   \n",
              "1918  reassure crew lose earth remain matter , journ...      DC Comics   \n",
              "94    garth origin similar mentor . amphibious human...      DC Comics   \n",
              "406   child , nelson gardner sickly asthmatic , year...      DC Comics   \n",
              "1060  ability know people fear , reach john psyche f...      DC Comics   \n",
              "1582  hippolyta sister , antiope amazon emerge soul ...      DC Comics   \n",
              "\n",
              "     predicted_label     result  n_words  \n",
              "2040   Marvel Comics  Incorrect       23  \n",
              "979    Marvel Comics  Incorrect        7  \n",
              "557    Marvel Comics  Incorrect       14  \n",
              "703    Marvel Comics  Incorrect       18  \n",
              "1166   Marvel Comics  Incorrect       12  \n",
              "1481   Marvel Comics  Incorrect       30  \n",
              "1456       DC Comics  Incorrect      303  \n",
              "706        DC Comics  Incorrect       30  \n",
              "637    Marvel Comics  Incorrect       87  \n",
              "56     Marvel Comics  Incorrect       11  \n",
              "2087   Marvel Comics  Incorrect       36  \n",
              "1918   Marvel Comics  Incorrect       12  \n",
              "94     Marvel Comics  Incorrect      292  \n",
              "406    Marvel Comics  Incorrect      121  \n",
              "1060   Marvel Comics  Incorrect      262  \n",
              "1582   Marvel Comics  Incorrect      278  "
            ]
          },
          "execution_count": 45,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "results_nb[results_nb[\"result\"]==\"Incorrect\"]"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "collapsed_sections": [
        "e__U2oo372zI"
      ],
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.10.8"
    },
    "orig_nbformat": 4,
    "vscode": {
      "interpreter": {
        "hash": "38cca0c38332a56087b24af0bc80247f4fced29cb4f7f437d91dc159adec9c4e"
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}