Tweet_Sentiment_Analysis.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "V4w8cgDLu9d9"
      },
      "source": [
        "## Helper Functions"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "iMPY4_zVwP2F",
        "outputId": "23ab7e0a-2416-4d6d-c05d-63c3ca87310d"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "--2022-10-01 14:46:12--  https://raw.githubusercontent.com/GSelvakumar/storage/main/Helpers/helper_functions.py\n",
            "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.108.133, 185.199.109.133, 185.199.110.133, ...\n",
            "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.108.133|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 10709 (10K) [text/plain]\n",
            "Saving to: ‘helper_functions.py’\n",
            "\n",
            "\rhelper_functions.py   0%[                    ]       0  --.-KB/s               \rhelper_functions.py 100%[===================>]  10.46K  --.-KB/s    in 0s      \n",
            "\n",
            "2022-10-01 14:46:12 (64.7 MB/s) - ‘helper_functions.py’ saved [10709/10709]\n",
            "\n"
          ]
        }
      ],
      "source": [
        "!wget https://raw.githubusercontent.com/GSelvakumar/storage/main/Helpers/helper_functions.py"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "xJK0r0jiwRFp"
      },
      "outputs": [],
      "source": [
        "from helper_functions import unzip_data, plot_loss_curves, create_tensorboard_callback, compare_historys"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "JPYgM3m0x2c2"
      },
      "outputs": [],
      "source": [
        "unzip_data(\"nlp_getting_started.zip\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_8wI_mrMx6ea"
      },
      "source": [
        "## Visualizing the Text Dataset"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "WvSHQuta0xyu"
      },
      "outputs": [],
      "source": [
        "import pandas as pd\n",
        "train_df = pd.read_csv(\"nlp_getting_started/train.csv\")\n",
        "test_df = pd.read_csv(\"nlp_getting_started/test.csv\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "dMug5GB70473",
        "outputId": "acd064af-9791-4e01-b3cc-922716638cf2"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-b9cf1298-4db8-4df4-bfdb-8a12670e5ed3\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>keyword</th>\n",
              "      <th>location</th>\n",
              "      <th>text</th>\n",
              "      <th>target</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Our Deeds are the Reason of this #earthquake M...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>4</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Forest fire near La Ronge Sask. Canada</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>5</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>All residents asked to 'shelter in place' are ...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>6</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>13,000 people receive #wildfires evacuation or...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>7</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Just got sent this photo from Ruby #Alaska as ...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-b9cf1298-4db8-4df4-bfdb-8a12670e5ed3')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-b9cf1298-4db8-4df4-bfdb-8a12670e5ed3 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-b9cf1298-4db8-4df4-bfdb-8a12670e5ed3');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "   id keyword location                                               text  \\\n",
              "0   1     NaN      NaN  Our Deeds are the Reason of this #earthquake M...   \n",
              "1   4     NaN      NaN             Forest fire near La Ronge Sask. Canada   \n",
              "2   5     NaN      NaN  All residents asked to 'shelter in place' are ...   \n",
              "3   6     NaN      NaN  13,000 people receive #wildfires evacuation or...   \n",
              "4   7     NaN      NaN  Just got sent this photo from Ruby #Alaska as ...   \n",
              "\n",
              "   target  \n",
              "0       1  \n",
              "1       1  \n",
              "2       1  \n",
              "3       1  \n",
              "4       1  "
            ]
          },
          "execution_count": 6,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "train_df.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "gVFKBC6R1IUj",
        "outputId": "d0e4d1ba-5123-4b17-ad7b-b8b010448ab9"
      },
      "outputs": [
        {
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'in response to trauma Children of Addicts develop a defensive self - one that decreases vulnerability. (3'"
            ]
          },
          "execution_count": 7,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# getting one data sample\n",
        "train_df[\"text\"][6845]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "oRhaKC-O12YA",
        "outputId": "09f2d272-70d6-4608-cdec-c7ad02184f25"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-448f3d8f-1293-47fd-988a-d766832c5055\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>keyword</th>\n",
              "      <th>location</th>\n",
              "      <th>text</th>\n",
              "      <th>target</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>2644</th>\n",
              "      <td>3796</td>\n",
              "      <td>destruction</td>\n",
              "      <td>NaN</td>\n",
              "      <td>So you have a new weapon that can cause un-ima...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2227</th>\n",
              "      <td>3185</td>\n",
              "      <td>deluge</td>\n",
              "      <td>NaN</td>\n",
              "      <td>The f$&amp;amp;@ing things I do for #GISHWHES Just...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5448</th>\n",
              "      <td>7769</td>\n",
              "      <td>police</td>\n",
              "      <td>UK</td>\n",
              "      <td>DT @georgegalloway: RT @Galloway4Mayor: ÛÏThe...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>132</th>\n",
              "      <td>191</td>\n",
              "      <td>aftershock</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Aftershock back to school kick off was great. ...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6845</th>\n",
              "      <td>9810</td>\n",
              "      <td>trauma</td>\n",
              "      <td>Montgomery County, MD</td>\n",
              "      <td>in response to trauma Children of Addicts deve...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-448f3d8f-1293-47fd-988a-d766832c5055')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-448f3d8f-1293-47fd-988a-d766832c5055 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-448f3d8f-1293-47fd-988a-d766832c5055');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "        id      keyword               location  \\\n",
              "2644  3796  destruction                    NaN   \n",
              "2227  3185       deluge                    NaN   \n",
              "5448  7769       police                     UK   \n",
              "132    191   aftershock                    NaN   \n",
              "6845  9810       trauma  Montgomery County, MD   \n",
              "\n",
              "                                                   text  target  \n",
              "2644  So you have a new weapon that can cause un-ima...       1  \n",
              "2227  The f$&amp;@ing things I do for #GISHWHES Just...       0  \n",
              "5448  DT @georgegalloway: RT @Galloway4Mayor: ÛÏThe...       1  \n",
              "132   Aftershock back to school kick off was great. ...       0  \n",
              "6845  in response to trauma Children of Addicts deve...       0  "
            ]
          },
          "execution_count": 8,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# Shuffling the data so that the supervised model won't learn any patterns to follow the prediction\n",
        "train_df_shuffled = train_df.sample(frac=1, random_state=42)\n",
        "train_df_shuffled.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6VokBe-_2M3x",
        "outputId": "22be5f69-0088-4220-9b9e-d71b295af2ae"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "0    4342\n",
              "1    3271\n",
              "Name: target, dtype: int64"
            ]
          },
          "execution_count": 9,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# Number of examples on the target class\n",
        "train_df.target.value_counts()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ypCVFJ0X3Erb",
        "outputId": "02ada601-5d1d-4b4d-8a60-b04c50fc75a2"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "(7613, 3263)"
            ]
          },
          "execution_count": 10,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "len(train_df), len(test_df)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "LnVOzkeH42L9",
        "outputId": "c1fb15b8-7e60-4fbe-ef18-4ca9824f5c2e"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Target Value: 0 (not real disaster)\n",
            "Text Value:\n",
            "Currently Blasting #Benediction - #SanelessTheory -on Metal Devastation Radio-  http://t.co/siGeeQ42cZ\n",
            "\n",
            "---\n",
            "\n",
            "Target Value: 0 (not real disaster)\n",
            "Text Value:\n",
            "sleeping with sirens vai vir pra sp\n",
            "\n",
            "---\n",
            "\n",
            "Target Value: 1 (real disaster)\n",
            "Text Value:\n",
            "The Latest: More Homes Razed by Northern California Wildfire - ABC News http://t.co/6AcSWzo7cw\n",
            "\n",
            "---\n",
            "\n",
            "Target Value: 0 (not real disaster)\n",
            "Text Value:\n",
            "*Jumps off of a cliff while drinking tea*\n",
            "\n",
            "This is how British people fall off cliffs.\n",
            "\n",
            "---\n",
            "\n",
            "Target Value: 1 (real disaster)\n",
            "Text Value:\n",
            "The Latest: More homes razed by Northern California wildfire - http://t.co/3tnuACIV3c http://t.co/SAkORGdqUL\n",
            "\n",
            "---\n",
            "\n"
          ]
        }
      ],
      "source": [
        "# Visualize the random samples\n",
        "import random\n",
        "random_index = random.randint(0, len(train_df)-5) # less the length so the rand won't exceed it\n",
        "for value in train_df_shuffled[[\"text\", \"target\"]][random_index:random_index+5].itertuples():\n",
        "  # itertuples are used to iterate over the dataframe rows and return them as tuples.\n",
        "  _, text, target = value\n",
        "  print(f\"Target Value: {target}\", \"(real disaster)\" if target > 0 else \"(not real disaster)\")\n",
        "  print(f\"Text Value:\\n{text}\\n\")\n",
        "  print(\"---\\n\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2Ja6afGutuJj"
      },
      "source": [
        "## Spliting Data into Train and Test Data"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "HXqjNY6zty88"
      },
      "outputs": [],
      "source": [
        "from sklearn.model_selection import train_test_split"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "5audtDkpuXaA"
      },
      "outputs": [],
      "source": [
        "train_sentences, test_sentences, train_labels, test_labels = train_test_split(train_df_shuffled['text'].to_numpy(),\n",
        "                                                                              train_df_shuffled['target'].to_numpy(),\n",
        "                                                                              test_size=0.1, # 10% of the train data to be test data\n",
        "                                                                              random_state=42)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WaAtsqzqvQnO",
        "outputId": "ad573cd9-352c-42b2-ee14-6b4fa9a49d91"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "(6851, 762, 6851, 762)"
            ]
          },
          "execution_count": 14,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "len(train_sentences), len(test_sentences), len(train_labels), len(test_labels)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eFgWTex2vg-M"
      },
      "source": [
        "## Converting Text into Numbers\n",
        "\n",
        "Before building the model need to convert the text into numbers using:\n",
        "\n",
        "1. Tokenization - directly map a token to a number\n",
        "2. Embedding - create a matrix of feature vector for each token(the size of the feature vector can be defined and this embedding can be learned)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VRxeO5cEi3l7"
      },
      "source": [
        "### Text Vectorization - Tokenization"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "RCCTtPe1jGCe"
      },
      "outputs": [],
      "source": [
        "import tensorflow as tf\n",
        "from tensorflow.keras.layers import TextVectorization"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "DgGMJl7QtHN9"
      },
      "outputs": [],
      "source": [
        "# Use the default TextVectorization parameters\n",
        "text_vectorizer = TextVectorization(max_tokens=None, # how many words in the vocabulary (automatically add <oov>)\n",
        "                                    standardize='lower_and_strip_punctuation',\n",
        "                                    split='whitespace',\n",
        "                                    ngrams=None, # create groups of n-words\n",
        "                                    output_mode=\"int\", # how to map tokens to numbers\n",
        "                                    output_sequence_length=None, # how long do you want your sequences to be\n",
        "                                    #pad_to_max_tokens=True # pad the feature axis which are less the max token axis, can be used only when max_tokens is mentioned\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "S1OyUtNG-e0T",
        "outputId": "c90e36f4-ce75-4fbc-8736-ab6fa5aa0deb"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "['@mogacola', '@zamtriossu', 'i', 'screamed', 'after', 'hitting', 'tweet']"
            ]
          },
          "execution_count": 17,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "train_sentences[0].split()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WueEMjRTAFQK",
        "outputId": "684ff865-b8ca-41ef-984a-4415e70cbb6e"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "7"
            ]
          },
          "execution_count": 18,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "len(train_sentences[0].split())"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ywDXtKogAIhU",
        "outputId": "049e15a6-f0fa-4d65-d19b-15e635a4eba9"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "102087"
            ]
          },
          "execution_count": 19,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# Find the average number of tokens (words) in the training tweets\n",
        "round(sum([len(i.split()) for i in train_sentences]))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "g_YCnpzia27K",
        "outputId": "1f8d854d-3b92-4d9f-fdd0-54fcc84f0a1e"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "15"
            ]
          },
          "execution_count": 20,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "round(sum([len(i.split()) for i in train_sentences]) / len(train_sentences))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "u0G2LDfMa_0C"
      },
      "outputs": [],
      "source": [
        "# Setup Text Vectorization Variables\n",
        "max_vocab_length = 10000 # max number of words to have in our vocabulary\n",
        "max_length = 15 # max length the sequences will be (e.g. how many words from a tweet does a model see?)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "e8jFxiZ6gy9_"
      },
      "outputs": [],
      "source": [
        "text_vectorizer = TextVectorization(max_tokens=max_vocab_length,\n",
        "                                    output_mode='int',\n",
        "                                    output_sequence_length=max_length)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "IucKmhmfhcPj"
      },
      "outputs": [],
      "source": [
        "# fit the text vectorizer to the training text\n",
        "\"\"\"\n",
        "Computes a vocabulary of string terms from tokens in a dataset. Calling adapt() on \n",
        "Text Vectorization layer is an alternative to passing in a precomputed vocabulary\n",
        "on construction via the vocabulary argument. A Text Vectorization layer should \n",
        "should always be either adapted over a dataset or supplied with a vocabulary.\n",
        "\"\"\"\n",
        "text_vectorizer.adapt(train_sentences)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lgn55fnwwQa1",
        "outputId": "2903aff8-4ed3-4c2f-dfa8-faeaed138334"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 15), dtype=int64, numpy=\n",
              "array([[264,   3, 232,   4,  13, 698,   0,   0,   0,   0,   0,   0,   0,\n",
              "          0,   0]])>"
            ]
          },
          "execution_count": 24,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# Sample a sentence and tokenize it\n",
        "sample_sentence = \"There's a flood in my street!\"\n",
        "text_vectorizer([sample_sentence])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "P-EyLCq6z916",
        "outputId": "22264b5a-9cfe-47e2-e3ea-b118bbde7627"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Original Text: \n",
            " .POTUS #StrategicPatience is a strategy for #Genocide; refugees; IDP Internally displaced people; horror; etc. https://t.co/rqWuoy1fm4 \n",
            "\n",
            " Vectorized Text: \n",
            " [[1065 2675    9    3 2279   10 1848  313 2933 1280  463   57  650 1213\n",
            "  3770]]\n"
          ]
        }
      ],
      "source": [
        "# Choose a random sentence from the training dataset and tokenize it\n",
        "random_sentence = random.choice(train_sentences)\n",
        "vectorize_sentence = text_vectorizer([random_sentence])\n",
        "print(f\"Original Text: \\n {random_sentence} \\n\\n Vectorized Text: \\n {vectorize_sentence}\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "i42frtoi070M"
      },
      "outputs": [],
      "source": [
        "# Get Unique words in the vocabulary\n",
        "words_in_vocab = text_vectorizer.get_vocabulary() # get all the unique words from the training data\n",
        "top_5_words = words_in_vocab[:5] # get the most common words\n",
        "bottom_5_words = words_in_vocab[-5:] # get the least common words"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HwdELMl525QR",
        "outputId": "328869a9-549f-46ef-fb9c-aba387298ebe"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Number of words in vocab: 10000\n",
            "5 most common words: ['', '[UNK]', 'the', 'a', 'in']\n",
            "5 least common words: ['pages', 'paeds', 'pads', 'padres', 'paddytomlinson1']\n"
          ]
        }
      ],
      "source": [
        "print(f\"Number of words in vocab: {len(words_in_vocab)}\")\n",
        "print(f\"5 most common words: {top_5_words}\")\n",
        "print(f\"5 least common words: {bottom_5_words}\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hUKgwNe43VyT"
      },
      "source": [
        "## Creating an Embedding Using the Embedding Layer\n",
        "\n",
        "Turns positive integers into dense vectors of fixed size\n",
        "\n",
        "* `input_dim` - the size of the vocabulary\n",
        "* `output_dim` - the size of the output embedding vector, for example, a value of 100 would mean each token gets represented by a vector 100 long\n",
        "* `input_length` - length of the sequences being passed to the embedding layer"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "29nH9c3tHA1j",
        "outputId": "488d7864-57ae-4f1f-ae30-1da7161a9524"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "<keras.layers.embeddings.Embedding at 0x7f8a400b5250>"
            ]
          },
          "execution_count": 28,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "from tensorflow.keras import layers\n",
        "\n",
        "embedding = layers.Embedding(input_dim=max_vocab_length, # set input shape\n",
        "                             output_dim=128, # output shape\n",
        "                             embeddings_initializer='uniform',\n",
        "                             input_length=max_length # how long is each input\n",
        "                             )\n",
        "\n",
        "embedding"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6tMssW_4Iobf",
        "outputId": "bae5ae61-580f-49c6-c34d-ad078a2e04e3"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Original Text:\n",
            " Good that the police are taking care of this and also have extra security  #HarryBeCareful \n",
            "\n",
            "Embedded Version: \n"
          ]
        },
        {
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 15, 128), dtype=float32, numpy=\n",
              "array([[[ 0.01984644,  0.00830482, -0.02854255, ...,  0.00801365,\n",
              "          0.00329381,  0.01413194],\n",
              "        [ 0.0418509 , -0.02610016,  0.02704556, ...,  0.01317421,\n",
              "          0.04718161,  0.02245113],\n",
              "        [-0.03159954, -0.03566319,  0.0407845 , ..., -0.03489129,\n",
              "          0.03081178,  0.04574866],\n",
              "        ...,\n",
              "        [ 0.00588577,  0.02289568, -0.02169272, ...,  0.01465872,\n",
              "          0.01658883,  0.04245586],\n",
              "        [ 0.04143627, -0.03601048,  0.04565748, ...,  0.04255717,\n",
              "         -0.03632572, -0.02699059],\n",
              "        [ 0.04307368,  0.00493417,  0.02824464, ..., -0.00372275,\n",
              "          0.0244881 , -0.0437237 ]]], dtype=float32)>"
            ]
          },
          "execution_count": 29,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# Get a Random sentence from the training set\n",
        "random_sentence = random.choice(train_sentences)\n",
        "\n",
        "print(f\"Original Text:\\n {random_sentence} \\n\\nEmbedded Version: \")\n",
        "\n",
        "# Embed the random sentence (turn positive integers into dense vectors of fixed size)\n",
        "sample_embed = embedding(text_vectorizer([random_sentence]))\n",
        "sample_embed"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FtELrmQeHom4"
      },
      "source": [
        "every single tweet are now in the form of 128 dense vectors. Always using the size as a divisible of 8 will increase the computation speed."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8g4sJC5rJTUq",
        "outputId": "cfb18c12-c970-4754-85d0-b42350d8f0bd"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "('Good that the police are taking care of this and also have extra security  #HarryBeCareful',\n",
              " <tf.Tensor: shape=(128,), dtype=float32, numpy=\n",
              " array([ 0.01984644,  0.00830482, -0.02854255, -0.0131613 ,  0.00325706,\n",
              "        -0.01029898, -0.01251417, -0.04096501, -0.01988413,  0.00140612,\n",
              "         0.0059981 ,  0.00981564,  0.00678983, -0.00233815,  0.00563979,\n",
              "        -0.03230121, -0.0147931 ,  0.00925279,  0.04607068, -0.01691993,\n",
              "        -0.01027177,  0.0496566 , -0.01788194,  0.04786152, -0.04479539,\n",
              "        -0.03010346,  0.01114365,  0.01989922, -0.03875159,  0.02081323,\n",
              "        -0.03444161,  0.01149971,  0.02774784, -0.00180935,  0.04884442,\n",
              "         0.02733567,  0.03970024,  0.03066006,  0.03174544,  0.00719982,\n",
              "        -0.00141595,  0.03227078, -0.0473914 , -0.04459813,  0.02357665,\n",
              "         0.03320308, -0.01177299, -0.03823646,  0.00553447,  0.01544705,\n",
              "         0.03169296, -0.0463669 ,  0.01536984, -0.04301878, -0.02385366,\n",
              "        -0.02187085, -0.04730109, -0.03271252, -0.03542906,  0.00037098,\n",
              "         0.01929698, -0.00593691, -0.02191129,  0.01224532,  0.02946725,\n",
              "         0.01937846, -0.04130434,  0.04633233, -0.04860544,  0.0098116 ,\n",
              "         0.01428062,  0.01530533, -0.03250907, -0.00474695, -0.00634193,\n",
              "        -0.03421745, -0.04984874, -0.00970024, -0.01710304, -0.02888839,\n",
              "         0.02891305,  0.0218257 , -0.0463082 , -0.00113047,  0.03461939,\n",
              "        -0.03550763,  0.04719131, -0.00289806, -0.03191346, -0.02736085,\n",
              "         0.00522421, -0.01843514, -0.01254319, -0.0380056 ,  0.00381025,\n",
              "         0.04548646, -0.03377936, -0.00559887, -0.04647366, -0.00854939,\n",
              "         0.02313909,  0.0434985 ,  0.03515447,  0.04450185,  0.03092709,\n",
              "         0.00751436, -0.03603584,  0.03577887,  0.01185121,  0.01549799,\n",
              "         0.0432312 ,  0.01258082,  0.00600643, -0.01110599, -0.01760417,\n",
              "         0.02448433, -0.00938492,  0.01187143, -0.04906317,  0.04511726,\n",
              "        -0.02934465,  0.00561317,  0.01583657,  0.04075951, -0.04727187,\n",
              "         0.00801365,  0.00329381,  0.01413194], dtype=float32)>,\n",
              " TensorShape([128]))"
            ]
          },
          "execution_count": 30,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# Checking out single token embedding\n",
        "random_sentence, sample_embed[0][0], sample_embed[0][0].shape"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "n4eWi479JgAr"
      },
      "source": [
        "## Modelling a Text Dataset\n",
        "\n",
        "- Model 0: Naive Bayes - sklearn(baseline)\n",
        "- Model 1: Feed-Forward Neural Network(dense model)\n",
        "- Model 2: LSTM Model (RNN)\n",
        "- Model 3: GRU Model (RNN)\n",
        "- Model 4: Bidirectional-LSTM Model (RNN)\n",
        "- Model 5: 1D Conventional Neural Network (CNN)\n",
        "- Model 6: Tensorflow Hub Pretrained Feature Extractor (using Transfer Learning for NLP)\n",
        "- Model 7: Same as Model 6 with 10% of training data"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4c6OkBu6MFIS"
      },
      "source": [
        "## Model 0: Getting a Baseline Model\n",
        "\n",
        "It's important to create a baseline model to attain a benchmark for future experiments/models to build upon. It's common practice to use non-DL algorithms as a baseline because of their speed and then later using DL to see if it can be improved upon them.\n",
        "\n",
        "Here the Baseline created is - SkLearn's Multinomial Naive Bayes using the TF-IDF formula to convert the words into numbers"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tDatsBODUq-u",
        "outputId": "c9ebe92b-2e4c-440f-c59c-c974b3982ba7"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "Pipeline(steps=[('tfidf', TfidfVectorizer()), ('clf', MultinomialNB())])"
            ]
          },
          "execution_count": 31,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "from sklearn.feature_extraction.text import TfidfVectorizer\n",
        "from sklearn.naive_bayes import MultinomialNB\n",
        "from sklearn.pipeline import Pipeline\n",
        "\n",
        "# Create Tokenization and Modelling pipeline\n",
        "model_0 = Pipeline([\n",
        "    (\"tfidf\", TfidfVectorizer()), # convert words into numbers using tfidf\n",
        "    (\"clf\", MultinomialNB()) # model the text, clf-classification\n",
        "])\n",
        "\n",
        "# Fit the Pipeline to Training Data\n",
        "model_0.fit(train_sentences, train_labels)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1L2CUpxCW44A",
        "outputId": "ad9f5261-9fea-4dfd-a7fb-62f99fc33c27"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "The Baseline Model achieves an accuracy of: 79.27%\n"
          ]
        }
      ],
      "source": [
        "# Evaluate the Baseline Model\n",
        "baseline_score = model_0.score(test_sentences, test_labels)\n",
        "print(f\"The Baseline Model achieves an accuracy of: {baseline_score*100:.2f}%\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "I32xU_HlYgTN",
        "outputId": "b2c02e31-fbb5-4fb8-87a5-f7034ba30065"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "array([1, 1, 1, 0, 0, 1, 1, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1])"
            ]
          },
          "execution_count": 33,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# make predictions\n",
        "baseline_preds = model_0.predict(test_sentences)\n",
        "baseline_preds[:20]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "4nLXHa8Yk9I8"
      },
      "outputs": [],
      "source": [
        "# Function to evaluate: accuracy, precision, recall, f1-score\n",
        "from sklearn.metrics import accuracy_score, precision_recall_fscore_support\n",
        "\n",
        "def calculate_results(y_true, y_pred):\n",
        "  \"\"\"\n",
        "  Calculate model accuracy, precision, recall and f1 score of a binary\n",
        "  classification model.\n",
        "  \"\"\"\n",
        "  # Calculate model accuracy\n",
        "  model_accuracy = accuracy_score(y_true, y_pred)*100\n",
        "\n",
        "  # Calculate model precision, recall and f1-score using weighted average\n",
        "  model_precision, model_recall, model_f1, _ = precision_recall_fscore_support(y_true, y_pred, average=\"weighted\")\n",
        "\n",
        "  model_results={\"accuracy\": model_accuracy,\n",
        "                 \"precision\": model_precision,\n",
        "                 \"recall\": model_recall,\n",
        "                 \"f1\": model_f1}\n",
        "\n",
        "  return model_results"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "iocHeLablID9",
        "outputId": "b3c159fa-2f91-4d3e-8559-4da6b220cd5b"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "{'accuracy': 79.26509186351706,\n",
              " 'precision': 0.8111390004213173,\n",
              " 'recall': 0.7926509186351706,\n",
              " 'f1': 0.7862189758049549}"
            ]
          },
          "execution_count": 35,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# Get Baseline results\n",
        "baseline_results = calculate_results(y_true=test_labels,\n",
        "                                     y_pred=baseline_preds)\n",
        "baseline_results"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "M1m6Fqrwo2ye"
      },
      "source": [
        "## Model 1: A Simple Dense Model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "xha83tYxjBEf"
      },
      "outputs": [],
      "source": [
        "# Create a tensorboard callback (need to create a new one for each model)\n",
        "from helper_functions import create_tensorboard_callback\n",
        "\n",
        "# Creating a directory to save tensorboard logs\n",
        "SAVE_DIR = \"model_logs\""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "45fTHFDFjKV7"
      },
      "outputs": [],
      "source": [
        "# Build Model with a Functional API\n",
        "from tensorflow.keras import layers\n",
        "inputs = layers.Input(shape=(1,), dtype=tf.string) # inputs are 1-dimensional strings\n",
        "x = text_vectorizer(inputs) # turn the input text into numbers\n",
        "x = embedding(x) # create a embedding of the numberized inputs\n",
        "x = layers.GlobalAveragePooling1D()(x) # lower the dimensionality of the embedding\n",
        "outputs = layers.Dense(1, activation=\"sigmoid\")(x) # Create the output layer, want binary outputs so using sigmoid activation function\n",
        "model_1 = tf.keras.Model(inputs, outputs, name=\"model_1_dense\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZvJQmvbklLQ3",
        "outputId": "ba1bac06-8a05-4c72-c3ed-ba7d77dbc0cc"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Model: \"model_1_dense\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " input_1 (InputLayer)        [(None, 1)]               0         \n",
            "                                                                 \n",
            " text_vectorization_1 (TextV  (None, 15)               0         \n",
            " ectorization)                                                   \n",
            "                                                                 \n",
            " embedding (Embedding)       (None, 15, 128)           1280000   \n",
            "                                                                 \n",
            " global_average_pooling1d (G  (None, 128)              0         \n",
            " lobalAveragePooling1D)                                          \n",
            "                                                                 \n",
            " dense (Dense)               (None, 1)                 129       \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 1,280,129\n",
            "Trainable params: 1,280,129\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n"
          ]
        }
      ],
      "source": [
        "model_1.summary()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "m4k-QvLDlNbE"
      },
      "outputs": [],
      "source": [
        "# Compile the model\n",
        "model_1.compile(loss=\"binary_crossentropy\",\n",
        "                optimizer=tf.keras.optimizers.Adam(),\n",
        "                metrics=[\"accuracy\"])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ipY0n_Gzmhdo",
        "outputId": "97e7fb37-7157-4f18-d787-972304fa8c64"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Saving TensorBoard log files to: model_logs/model_1_dense/20221001-144621\n",
            "Epoch 1/5\n",
            "215/215 [==============================] - 8s 9ms/step - loss: 0.6118 - accuracy: 0.6849 - val_loss: 0.5385 - val_accuracy: 0.7520\n",
            "Epoch 2/5\n",
            "215/215 [==============================] - 2s 8ms/step - loss: 0.4420 - accuracy: 0.8148 - val_loss: 0.4703 - val_accuracy: 0.7887\n",
            "Epoch 3/5\n",
            "215/215 [==============================] - 2s 8ms/step - loss: 0.3473 - accuracy: 0.8591 - val_loss: 0.4624 - val_accuracy: 0.7887\n",
            "Epoch 4/5\n",
            "215/215 [==============================] - 2s 8ms/step - loss: 0.2840 - accuracy: 0.8901 - val_loss: 0.4632 - val_accuracy: 0.7822\n",
            "Epoch 5/5\n",
            "215/215 [==============================] - 2s 8ms/step - loss: 0.2373 - accuracy: 0.9140 - val_loss: 0.4852 - val_accuracy: 0.7874\n"
          ]
        }
      ],
      "source": [
        "model_1_history = model_1.fit(x=train_sentences,\n",
        "                              y=train_labels,\n",
        "                              epochs=5,\n",
        "                              validation_data=(test_sentences, test_labels),\n",
        "                              callbacks=[create_tensorboard_callback(dir_name=SAVE_DIR,\n",
        "                                                                     experiment_name=\"model_1_dense\")])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6BT7jzqbqokc",
        "outputId": "8c042b37-85fa-4c34-b3b9-15168d55798a"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "24/24 [==============================] - 0s 5ms/step - loss: 0.4852 - accuracy: 0.7874\n"
          ]
        },
        {
          "data": {
            "text/plain": [
              "[0.4852340519428253, 0.787401556968689]"
            ]
          },
          "execution_count": 41,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# Check the results\n",
        "model_1.evaluate(test_sentences, test_labels)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ElCMaVarwwUc"
      },
      "outputs": [],
      "source": [
        "# make Predictions\n",
        "model_1_pred_probs = model_1.predict(test_sentences)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6gnkWceaxBwC",
        "outputId": "a6eb0e23-b535-4cff-a000-b4701c9c8351"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "(762, 1)"
            ]
          },
          "execution_count": 43,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "model_1_pred_probs.shape"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "oM8YgSBzyHoy",
        "outputId": "b2d58923-3fac-4231-9560-69b64c2acf7d"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Model: \"model_1_dense\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " input_1 (InputLayer)        [(None, 1)]               0         \n",
            "                                                                 \n",
            " text_vectorization_1 (TextV  (None, 15)               0         \n",
            " ectorization)                                                   \n",
            "                                                                 \n",
            " embedding (Embedding)       (None, 15, 128)           1280000   \n",
            "                                                                 \n",
            " global_average_pooling1d (G  (None, 128)              0         \n",
            " lobalAveragePooling1D)                                          \n",
            "                                                                 \n",
            " dense (Dense)               (None, 1)                 129       \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 1,280,129\n",
            "Trainable params: 1,280,129\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n"
          ]
        }
      ],
      "source": [
        "model_1.summary()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fG-KMORbxFVr",
        "outputId": "6bf11768-7292-4127-e324-4b4c80569763"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "array([0.3209695], dtype=float32)"
            ]
          },
          "execution_count": 45,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "model_1_pred_probs[0]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Bfzm1918yi3i"
      },
      "outputs": [],
      "source": [
        "model_1_preds = tf.squeeze(tf.round(model_1_pred_probs))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MfbyCOiO1LTN",
        "outputId": "2c876d68-0afe-4755-bfef-6b5f81bf7389"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(10,), dtype=float32, numpy=array([0., 1., 1., 0., 0., 1., 1., 1., 1., 0.], dtype=float32)>"
            ]
          },
          "execution_count": 47,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "model_1_preds[:10]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1sAkPzrs1O5i",
        "outputId": "6a715c02-a222-4e31-b3e5-1cd981359749"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "{'accuracy': 78.74015748031496,\n",
              " 'precision': 0.7932296029485675,\n",
              " 'recall': 0.7874015748031497,\n",
              " 'f1': 0.7841130596930417}"
            ]
          },
          "execution_count": 48,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# Calculate the Model Results\n",
        "model_1_results = calculate_results(y_true=test_labels,\n",
        "                                    y_pred=model_1_preds)\n",
        "model_1_results"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "RMRe-8C01mlt",
        "outputId": "f0fbf236-1812-4063-c116-9466e942c359"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "{'accuracy': 79.26509186351706,\n",
              " 'precision': 0.8111390004213173,\n",
              " 'recall': 0.7926509186351706,\n",
              " 'f1': 0.7862189758049549}"
            ]
          },
          "execution_count": 49,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "baseline_results"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7NgvCHsP2CWa",
        "outputId": "e89dbf67-fe5c-47d5-ae96-9d6d3678e3d7"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "array([False, False, False, False])"
            ]
          },
          "execution_count": 50,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# check whether the baseline is out performing the first DEEP model\n",
        "import numpy as np\n",
        "np.array(list(model_1_results.values())) > np.array(list(baseline_results.values()))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JsrLYCEZ3Odt"
      },
      "source": [
        "## Visualizing Learned Embeddings"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IFgLxwhb3jZ_",
        "outputId": "471c6d24-f9d2-46e5-c62f-a8d75ac49640"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "(10000, ['', '[UNK]', 'the', 'a', 'in', 'to', 'of', 'and', 'i', 'is'])"
            ]
          },
          "execution_count": 51,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# Get the vocabulary from the text vectorization layer\n",
        "words_in_vocab = text_vectorizer.get_vocabulary()\n",
        "len(words_in_vocab), words_in_vocab[:10]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WQ1T0-bS4LrF",
        "outputId": "9e2ea7e5-54e4-467a-8510-5b0bbd09c9a4"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Model: \"model_1_dense\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " input_1 (InputLayer)        [(None, 1)]               0         \n",
            "                                                                 \n",
            " text_vectorization_1 (TextV  (None, 15)               0         \n",
            " ectorization)                                                   \n",
            "                                                                 \n",
            " embedding (Embedding)       (None, 15, 128)           1280000   \n",
            "                                                                 \n",
            " global_average_pooling1d (G  (None, 128)              0         \n",
            " lobalAveragePooling1D)                                          \n",
            "                                                                 \n",
            " dense (Dense)               (None, 1)                 129       \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 1,280,129\n",
            "Trainable params: 1,280,129\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n"
          ]
        }
      ],
      "source": [
        "# Model 1 Summary\n",
        "model_1.summary()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QUSpaY5I4VhX",
        "outputId": "70ede5ff-ec18-4e98-bf2e-351be7fd3f49"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "(10000, 128)"
            ]
          },
          "execution_count": 53,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# Get the weight matrix of embedding layer\n",
        "\"\"\"\n",
        "These are the numerical representations of each token in the training data, \n",
        "which have been learned for 5 epochs.\n",
        "\"\"\"\n",
        "\n",
        "embed_weights=model_1.get_layer(\"embedding\").get_weights()[0]\n",
        "embed_weights.shape # same size as vocab size and embedding_dim (output_dim of the embedding layer)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "o1LWIGUs5LbD"
      },
      "source": [
        "Every token(10000) is embedded into a 128 shaped vector "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "6O3hv6AY6ETv"
      },
      "outputs": [],
      "source": [
        "# Create Embedding Files\n",
        "import io\n",
        "out_v = io.open('vectors.tsv', 'w', encoding='utf-8')\n",
        "out_m = io.open('metadata.tsv', 'w', encoding='utf-8')\n",
        "\n",
        "for index, word in enumerate(words_in_vocab):\n",
        "  if index == 0:\n",
        "    continue # skip 0, it's padding\n",
        "  vec = embed_weights[index]\n",
        "  out_v.write('\\t'.join([str(x) for x in vec]) + \"\\n\")\n",
        "  out_m.write(word + \"\\n\")\n",
        "out_v.close()\n",
        "out_m.close()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "YsqzvddCYEsx"
      },
      "outputs": [],
      "source": [
        "# # Download files from colab to upload to projector\n",
        "# try:\n",
        "#   from google.colab import files\n",
        "#   files.download('vectors.tsv')\n",
        "#   files.download('metadata.tsv')\n",
        "# except Exception:\n",
        "#   pass"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FxyU64KvaSoC"
      },
      "source": [
        "## Recurrent Neural Network\n",
        "\n",
        "RNN is useful for Sequence Data. The Premise of a recurrent neural network is to use the representation of a previous input to aid the representation of a later input."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "baUfQVxOBq7F"
      },
      "source": [
        "## Model 2: LSTM(Long Short Term Memory)\n",
        "\n",
        "Structure of RNN\n",
        "\n",
        "``` \n",
        "Input (Text) => Tokenize => Embedding =>  Layers (RNN/Dense) => Output (Label Probability)\n",
        "```"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "A6WtkmJxBv4H"
      },
      "outputs": [],
      "source": [
        "# Create an LSTM Model\n",
        "from tensorflow.keras import layers\n",
        "inputs = layers.Input(shape=(1,), dtype=\"string\")\n",
        "x = text_vectorizer(inputs)\n",
        "x = embedding(x)\n",
        "x = layers.LSTM(64)(x)\n",
        "outputs = layers.Dense(1, activation=\"sigmoid\")(x)\n",
        "model_2 = tf.keras.Model(inputs, outputs, name=\"model_2_LSTM\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2h6AYpqWY31P",
        "outputId": "6a9b1751-3869-4087-ff23-1a1b3992f161"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Model: \"model_2_LSTM\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " input_2 (InputLayer)        [(None, 1)]               0         \n",
            "                                                                 \n",
            " text_vectorization_1 (TextV  (None, 15)               0         \n",
            " ectorization)                                                   \n",
            "                                                                 \n",
            " embedding (Embedding)       (None, 15, 128)           1280000   \n",
            "                                                                 \n",
            " lstm (LSTM)                 (None, 64)                49408     \n",
            "                                                                 \n",
            " dense_1 (Dense)             (None, 1)                 65        \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 1,329,473\n",
            "Trainable params: 1,329,473\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n"
          ]
        }
      ],
      "source": [
        "# Get Model_2 summary\n",
        "model_2.summary()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "WkNY8LbrY7jl"
      },
      "outputs": [],
      "source": [
        "# Compile the model\n",
        "model_2.compile(loss=\"binary_crossentropy\",\n",
        "                optimizer=tf.keras.optimizers.Adam(),\n",
        "                metrics=[\"accuracy\"])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nCKNRnqBZymC",
        "outputId": "9ba38033-655e-4a84-e558-b97b131c4d60"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Saving TensorBoard log files to: model_logs/model_2_LSTM/20221001-144641\n",
            "Epoch 1/5\n",
            "215/215 [==============================] - 6s 9ms/step - loss: 0.2286 - accuracy: 0.9206 - val_loss: 0.5425 - val_accuracy: 0.7822\n",
            "Epoch 2/5\n",
            "215/215 [==============================] - 1s 6ms/step - loss: 0.1562 - accuracy: 0.9423 - val_loss: 0.5995 - val_accuracy: 0.7769\n",
            "Epoch 3/5\n",
            "215/215 [==============================] - 1s 6ms/step - loss: 0.1270 - accuracy: 0.9502 - val_loss: 0.7013 - val_accuracy: 0.7848\n",
            "Epoch 4/5\n",
            "215/215 [==============================] - 1s 6ms/step - loss: 0.1062 - accuracy: 0.9600 - val_loss: 0.7790 - val_accuracy: 0.7808\n",
            "Epoch 5/5\n",
            "215/215 [==============================] - 1s 6ms/step - loss: 0.0868 - accuracy: 0.9670 - val_loss: 1.0326 - val_accuracy: 0.7743\n"
          ]
        }
      ],
      "source": [
        "# Fit the model\n",
        "model_2_history = model_2.fit(train_sentences,\n",
        "                              train_labels,\n",
        "                              epochs=5,\n",
        "                              validation_data=(test_sentences, test_labels),\n",
        "                              callbacks=[create_tensorboard_callback(SAVE_DIR, \"model_2_LSTM\")])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8--lm-NcajSP",
        "outputId": "87717dee-c2f1-484c-a4cb-8fa4ee11f041"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "array([[2.1888032e-03],\n",
              "       [6.0524374e-01],\n",
              "       [9.9980730e-01],\n",
              "       [2.2326315e-02],\n",
              "       [4.7604076e-04],\n",
              "       [9.9937612e-01],\n",
              "       [8.5028791e-01],\n",
              "       [9.9989057e-01],\n",
              "       [9.9980205e-01],\n",
              "       [4.0506905e-01]], dtype=float32)"
            ]
          },
          "execution_count": 60,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# Make predictions with LSTM Model\n",
        "model_2_pred_probs = model_2.predict(test_sentences)\n",
        "model_2_pred_probs[:10]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "r7uWsttXa8c_",
        "outputId": "6c60cdc4-2d95-4a10-9d52-5c7746bdd9df"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(10,), dtype=float32, numpy=array([0., 1., 1., 0., 0., 1., 1., 1., 1., 0.], dtype=float32)>"
            ]
          },
          "execution_count": 61,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# Convert model 2 pred probs to labels\n",
        "model_2_preds = tf.squeeze(tf.round(model_2_pred_probs))\n",
        "model_2_preds[:10]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "eY8kcxRubYeL",
        "outputId": "5382d9a9-25eb-4e65-9664-3a077ab68032"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "{'accuracy': 77.42782152230971,\n",
              " 'precision': 0.7799224267091094,\n",
              " 'recall': 0.7742782152230971,\n",
              " 'f1': 0.7706229446239603}"
            ]
          },
          "execution_count": 62,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# Calculate model 2 results\n",
        "model_2_results = calculate_results(test_labels, model_2_preds)\n",
        "model_2_results"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-R0kbcnibqSH"
      },
      "source": [
        "## Model 3: GRU\n",
        "\n",
        "Another popular and effective RNN component is the GRU(Gated Recurrent Unit). The GRU cell has similar features to an LSTM cell but has less parameters."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "_2_woiTniyBn"
      },
      "outputs": [],
      "source": [
        "# Build RNN using the GRU Cell\n",
        "from tensorflow.keras import layers\n",
        "\n",
        "inputs = layers.Input(shape=(1,), dtype=tf.string)\n",
        "x = text_vectorizer(inputs)\n",
        "x = embedding(x)\n",
        "x = layers.GRU(64)(x)\n",
        "#x = layers.GRU(64, return_sequences=True)(x) # if the recurrent layers stacked, then the return_sequences must be True, the 64 refers the hidden layers.\n",
        "# x = layers.LSTM(64, return_sequences=True)(x)\n",
        "# x = layers.GRU(64)(x)\n",
        "# x = layers.Dense(64, activation=\"relu\")(x)\n",
        "# x = layers.GlobalAveragePooling1D()(x)\n",
        "outputs = layers.Dense(1, activation=\"sigmoid\")(x)\n",
        "model_3 = tf.keras.Model(inputs, outputs, name=\"model_3_GRU\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cgiuBqrMz6FK"
      },
      "source": [
        "without the global average pooling 1D layer the output returns the predictions for every token, but what need is the prediction of each sequence of data."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "t_bAkAgGybtW",
        "outputId": "ad0a1938-1fa1-4a76-a728-c57e1f16cb99"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Model: \"model_3_GRU\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " input_3 (InputLayer)        [(None, 1)]               0         \n",
            "                                                                 \n",
            " text_vectorization_1 (TextV  (None, 15)               0         \n",
            " ectorization)                                                   \n",
            "                                                                 \n",
            " embedding (Embedding)       (None, 15, 128)           1280000   \n",
            "                                                                 \n",
            " gru (GRU)                   (None, 64)                37248     \n",
            "                                                                 \n",
            " dense_2 (Dense)             (None, 1)                 65        \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 1,317,313\n",
            "Trainable params: 1,317,313\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n"
          ]
        }
      ],
      "source": [
        "model_3.summary()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "pnAd86nByrKI"
      },
      "outputs": [],
      "source": [
        "# compile the model\n",
        "model_3.compile(loss=\"binary_crossentropy\",\n",
        "                optimizer=tf.keras.optimizers.Adam(),\n",
        "                metrics=[\"accuracy\"])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dc40G4CG2swq",
        "outputId": "a54d3e13-54c0-450a-b997-451d0f155ab5"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Saving TensorBoard log files to: model_logs/model_3_GRU/20221001-144654\n",
            "Epoch 1/5\n",
            "215/215 [==============================] - 3s 8ms/step - loss: 0.1614 - accuracy: 0.9378 - val_loss: 0.7339 - val_accuracy: 0.7743\n",
            "Epoch 2/5\n",
            "215/215 [==============================] - 1s 6ms/step - loss: 0.0799 - accuracy: 0.9688 - val_loss: 0.8325 - val_accuracy: 0.7743\n",
            "Epoch 3/5\n",
            "215/215 [==============================] - 1s 6ms/step - loss: 0.0682 - accuracy: 0.9733 - val_loss: 1.1380 - val_accuracy: 0.7795\n",
            "Epoch 4/5\n",
            "215/215 [==============================] - 1s 6ms/step - loss: 0.0578 - accuracy: 0.9759 - val_loss: 1.1193 - val_accuracy: 0.7730\n",
            "Epoch 5/5\n",
            "215/215 [==============================] - 1s 6ms/step - loss: 0.0488 - accuracy: 0.9785 - val_loss: 1.1913 - val_accuracy: 0.7730\n"
          ]
        }
      ],
      "source": [
        "# Fit the model\n",
        "model_3_history = model_3.fit(train_sentences,\n",
        "                              train_labels,\n",
        "                              epochs=5,\n",
        "                              validation_data=(test_sentences, test_labels),\n",
        "                              callbacks=[create_tensorboard_callback(SAVE_DIR, \"model_3_GRU\")])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Kg6-l3di29rL",
        "outputId": "e4cb2967-c506-4187-a544-509edc9a281a"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "array([[3.1173374e-03],\n",
              "       [7.4438709e-01],\n",
              "       [9.9987614e-01],\n",
              "       [9.1521583e-02],\n",
              "       [1.1745404e-04],\n",
              "       [9.9978775e-01],\n",
              "       [9.7995752e-01],\n",
              "       [9.9996018e-01],\n",
              "       [9.9991822e-01],\n",
              "       [5.0216836e-01]], dtype=float32)"
            ]
          },
          "execution_count": 67,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# Prediction with GRU Model\n",
        "model_3_pred_probs = model_3.predict(test_sentences)\n",
        "model_3_pred_probs[:10]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "yECu38RJ3RHA",
        "outputId": "101db4c9-152a-4988-e9d0-b9bbc2bf3b42"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(10,), dtype=float32, numpy=array([0., 1., 1., 0., 0., 1., 1., 1., 1., 1.], dtype=float32)>"
            ]
          },
          "execution_count": 68,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# Convert model 3 pred probs to labels\n",
        "model_3_preds = tf.squeeze(tf.round(model_3_pred_probs))\n",
        "model_3_preds[:10]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KEW28ndF3qjH",
        "outputId": "8dda0be7-6703-45d9-b8f9-3312ff017ea5"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "{'accuracy': 77.29658792650919,\n",
              " 'precision': 0.7740517401498704,\n",
              " 'recall': 0.7729658792650919,\n",
              " 'f1': 0.7712160418848196}"
            ]
          },
          "execution_count": 69,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# Calculate model 3 results\n",
        "model_3_results = calculate_results(y_true=test_labels,\n",
        "                                    y_pred=model_3_preds)\n",
        "model_3_results"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zLvW63Ox3-g5"
      },
      "source": [
        "## Model 4: Bidirectional RNN\n",
        "\n",
        "RNN usually travel from left to right, but Bidirectional RNN travels on Both sides"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "8JGEQHheBN9C"
      },
      "outputs": [],
      "source": [
        "from tensorflow.keras import layers\n",
        "\n",
        "inputs = layers.Input(shape=(1,), dtype=tf.string)\n",
        "x = text_vectorizer(inputs)\n",
        "x = embedding(x)\n",
        "# x = layers.Bidirectional(layers.LSTM(64, return_sequences=True))(x)\n",
        "x = layers.Bidirectional(layers.GRU(64))(x) # bidirectional so 64 + 64 = 128\n",
        "outputs = layers.Dense(1, activation=\"sigmoid\")(x)\n",
        "model_4 = tf.keras.Model(inputs, outputs, name=\"model_4_bidirectional\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QumuSln-KtBv",
        "outputId": "3eb6c4a8-525b-45e5-e0df-0d43653e7724"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Model: \"model_4_bidirectional\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " input_4 (InputLayer)        [(None, 1)]               0         \n",
            "                                                                 \n",
            " text_vectorization_1 (TextV  (None, 15)               0         \n",
            " ectorization)                                                   \n",
            "                                                                 \n",
            " embedding (Embedding)       (None, 15, 128)           1280000   \n",
            "                                                                 \n",
            " bidirectional (Bidirectiona  (None, 128)              74496     \n",
            " l)                                                              \n",
            "                                                                 \n",
            " dense_3 (Dense)             (None, 1)                 129       \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 1,354,625\n",
            "Trainable params: 1,354,625\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n"
          ]
        }
      ],
      "source": [
        "model_4.summary()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "b5QKUv_gKvcI"
      },
      "outputs": [],
      "source": [
        "# Compile the model\n",
        "model_4.compile(loss=\"binary_crossentropy\",\n",
        "                optimizer=tf.keras.optimizers.Adam(),\n",
        "                metrics=[\"accuracy\"])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0i1xvJ-cNNLk",
        "outputId": "d2d582fc-7bd7-4cdb-b752-83ff8ae69200"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Saving TensorBoard log files to: model_logs/model_4_bidirectional/20221001-144704\n",
            "Epoch 1/5\n",
            "215/215 [==============================] - 6s 11ms/step - loss: 0.1211 - accuracy: 0.9568 - val_loss: 0.8303 - val_accuracy: 0.7703\n",
            "Epoch 2/5\n",
            "215/215 [==============================] - 2s 8ms/step - loss: 0.0589 - accuracy: 0.9762 - val_loss: 1.0657 - val_accuracy: 0.7756\n",
            "Epoch 3/5\n",
            "215/215 [==============================] - 2s 8ms/step - loss: 0.0499 - accuracy: 0.9769 - val_loss: 1.0906 - val_accuracy: 0.7677\n",
            "Epoch 4/5\n",
            "215/215 [==============================] - 2s 8ms/step - loss: 0.0445 - accuracy: 0.9796 - val_loss: 1.3066 - val_accuracy: 0.7795\n",
            "Epoch 5/5\n",
            "215/215 [==============================] - 2s 9ms/step - loss: 0.0432 - accuracy: 0.9801 - val_loss: 1.3673 - val_accuracy: 0.7730\n"
          ]
        }
      ],
      "source": [
        "# Fit the model\n",
        "model_4_history = model_4.fit(train_sentences,\n",
        "                              train_labels,\n",
        "                              epochs=5,\n",
        "                              validation_data=(test_sentences, test_labels),\n",
        "                              callbacks=[create_tensorboard_callback(SAVE_DIR, \"model_4_bidirectional\")])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IxCy8hFGNru8",
        "outputId": "4eb6b953-5a52-4423-c074-49dbf5d075d1"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "array([[4.9363112e-04],\n",
              "       [6.5757275e-01],\n",
              "       [9.9988055e-01],\n",
              "       [2.3434408e-01],\n",
              "       [2.2506198e-05],\n",
              "       [9.9990547e-01],\n",
              "       [9.5602793e-01],\n",
              "       [9.9998486e-01],\n",
              "       [9.9993181e-01],\n",
              "       [9.9881601e-01]], dtype=float32)"
            ]
          },
          "execution_count": 74,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# Make predictions with bidirectional model\n",
        "model_4_pred_probs = model_4.predict(test_sentences)\n",
        "model_4_pred_probs[:10]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ffvTkO02N8x2",
        "outputId": "d429ae34-ec83-47a3-ea32-14f95745f651"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(10,), dtype=float32, numpy=array([0., 1., 1., 0., 0., 1., 1., 1., 1., 1.], dtype=float32)>"
            ]
          },
          "execution_count": 75,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# Convert probs to labels\n",
        "model_4_preds = tf.squeeze(tf.round(model_4_pred_probs))\n",
        "model_4_preds[:10]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "q1scokrUOKRT",
        "outputId": "0d7d03d6-b558-4711-8b64-401d6e8acb28"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "{'accuracy': 77.29658792650919,\n",
              " 'precision': 0.77365375135567,\n",
              " 'recall': 0.7729658792650919,\n",
              " 'f1': 0.77144803030122}"
            ]
          },
          "execution_count": 76,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# Calculate the results of the bidirectional model\n",
        "model_4_results = calculate_results(y_true=test_labels,\n",
        "                                    y_pred=model_4_preds)\n",
        "model_4_results"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uQiRyHdmOX_P"
      },
      "source": [
        "## Convolutional Neural Network for Text (and other types of sequences)\n",
        "\n",
        "The Typical structure of a **Conv1D** for sequences(text)\n",
        "\n",
        "```\n",
        "Inputs(Text) => Tokenization => Embedding => Layer(s) (typically Conv1D + Pooling) => Outputs (class probabilities)\n",
        "```"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hBzF9MsXY3cP"
      },
      "source": [
        "### Model 5: Conv1D"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "T9hljaqEaVT9"
      },
      "outputs": [],
      "source": [
        "# Testing the embedding layer, Conv1D layer and max pooling\n",
        "from tensorflow.keras import layers\n",
        "\n",
        "embedding_test = embedding(text_vectorizer([\"this is a test sentence\"]))\n",
        "conv_1d = layers.Conv1D(filters=32,\n",
        "                        kernel_size=5, # this is ngram of 5 means looks 5 words at a time\n",
        "                        activation=\"relu\",\n",
        "                        padding=\"same\") # default is valid, the output is smaller than the input shape, same means the output is same as input shape.\n",
        "conv_1d_output = conv_1d(embedding_test)\n",
        "max_pool = layers.GlobalMaxPool1D()\n",
        "max_pool_output = max_pool(conv_1d_output) # get the features with the highest value"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jOiMTijXdVhm",
        "outputId": "1deb730e-500c-4402-a334-438729a12b2c"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "(TensorShape([1, 15, 128]), TensorShape([1, 15, 32]), TensorShape([1, 32]))"
            ]
          },
          "execution_count": 78,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "embedding_test.shape, conv_1d_output.shape, max_pool_output.shape"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vZOY-4LWdbWS",
        "outputId": "ca937e8b-98f1-4f5b-abdb-667eb9de4a32"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Model: \"model_5_Conv1D\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " input_5 (InputLayer)        [(None, 1)]               0         \n",
            "                                                                 \n",
            " text_vectorization_1 (TextV  (None, 15)               0         \n",
            " ectorization)                                                   \n",
            "                                                                 \n",
            " embedding (Embedding)       (None, 15, 128)           1280000   \n",
            "                                                                 \n",
            " conv1d_1 (Conv1D)           (None, 11, 64)            41024     \n",
            "                                                                 \n",
            " global_max_pooling1d_1 (Glo  (None, 64)               0         \n",
            " balMaxPooling1D)                                                \n",
            "                                                                 \n",
            " dense_4 (Dense)             (None, 1)                 65        \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 1,321,089\n",
            "Trainable params: 1,321,089\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n"
          ]
        }
      ],
      "source": [
        "# Create 1-dimensional Convolutional layer to model sequences\n",
        "from tensorflow.keras import layers\n",
        "inputs = layers.Input(shape=(1,), dtype=tf.string)\n",
        "x = text_vectorizer(inputs)\n",
        "x = embedding(x)\n",
        "x = layers.Conv1D(filters=64, kernel_size=5, strides=1, activation=\"relu\", padding=\"valid\")(x)\n",
        "x = layers.GlobalMaxPool1D()(x)\n",
        "outputs = layers.Dense(1, activation=\"sigmoid\")(x)\n",
        "model_5 = tf.keras.Model(inputs, outputs, name=\"model_5_Conv1D\")\n",
        "\n",
        "# Compile the Model\n",
        "model_5.compile(loss=\"binary_crossentropy\",\n",
        "                optimizer=tf.keras.optimizers.Adam(),\n",
        "                metrics=[\"accuracy\"])\n",
        "\n",
        "# Model summary\n",
        "model_5.summary()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9PWxRsEKm1vZ",
        "outputId": "0dd37989-e236-4ebe-eb54-c04e787d1c54"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Saving TensorBoard log files to: model_logs/Conv1D/20221001-144723\n",
            "Epoch 1/5\n",
            "215/215 [==============================] - 3s 6ms/step - loss: 0.1228 - accuracy: 0.9587 - val_loss: 0.8996 - val_accuracy: 0.7782\n",
            "Epoch 2/5\n",
            "215/215 [==============================] - 1s 5ms/step - loss: 0.0737 - accuracy: 0.9736 - val_loss: 1.0327 - val_accuracy: 0.7743\n",
            "Epoch 3/5\n",
            "215/215 [==============================] - 1s 5ms/step - loss: 0.0607 - accuracy: 0.9755 - val_loss: 1.1426 - val_accuracy: 0.7664\n",
            "Epoch 4/5\n",
            "215/215 [==============================] - 1s 5ms/step - loss: 0.0541 - accuracy: 0.9780 - val_loss: 1.1775 - val_accuracy: 0.7664\n",
            "Epoch 5/5\n",
            "215/215 [==============================] - 1s 5ms/step - loss: 0.0503 - accuracy: 0.9801 - val_loss: 1.2310 - val_accuracy: 0.7638\n"
          ]
        }
      ],
      "source": [
        "# Fit the model\n",
        "model_5_history = model_5.fit(train_sentences,\n",
        "                              train_labels,\n",
        "                              epochs=5,\n",
        "                              validation_data=(test_sentences, test_labels),\n",
        "                              callbacks=[create_tensorboard_callback(SAVE_DIR, \"Conv1D\")])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "sODN-8V7nS1D",
        "outputId": "a0e5180c-e7c3-419e-9142-c61421c1b261"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "array([[3.2880846e-02],\n",
              "       [8.8012457e-01],\n",
              "       [9.9996030e-01],\n",
              "       [4.2408999e-02],\n",
              "       [7.5127036e-07],\n",
              "       [9.9764627e-01],\n",
              "       [9.4113952e-01],\n",
              "       [9.9999654e-01],\n",
              "       [9.9999964e-01],\n",
              "       [8.0233634e-01]], dtype=float32)"
            ]
          },
          "execution_count": 81,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# make predictions \n",
        "model_5_pred_probs = model_5.predict(test_sentences)\n",
        "model_5_pred_probs[:10]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UZuHOST5pQvw",
        "outputId": "28f267b2-80e8-4e0e-ed8f-b990b3105e2d"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(10,), dtype=float32, numpy=array([0., 1., 1., 0., 0., 1., 1., 1., 1., 1.], dtype=float32)>"
            ]
          },
          "execution_count": 82,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# Convert model_5_pred_probs to labels\n",
        "model_5_preds = tf.squeeze(tf.round(model_5_pred_probs))\n",
        "model_5_preds[:10]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VndbHj03phaY",
        "outputId": "290f3afc-c594-496d-8699-bf8b4896e373"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "{'accuracy': 76.37795275590551,\n",
              " 'precision': 0.7647678424871814,\n",
              " 'recall': 0.7637795275590551,\n",
              " 'f1': 0.761896339990086}"
            ]
          },
          "execution_count": 83,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# Evaluate Model 5 Predictions\n",
        "model_5_results = calculate_results(y_true=test_labels,\n",
        "                                    y_pred=model_5_preds)\n",
        "model_5_results"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zwbqgysUh9t8"
      },
      "source": [
        "## Model 6: Tensorflow Hub Pretrained Sentence Encoder"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "oYGe5iEJj7lj",
        "outputId": "fac0a1e9-b3b3-46b2-fc8f-32d689167587"
      },
      "outputs": [
        {
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "\"There's a flood in my street!\""
            ]
          },
          "execution_count": 84,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "sample_sentence"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "RohqKcIbpvkc",
        "outputId": "8e20e729-dfd3-4ce7-b991-1da9e3694c93"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tf.Tensor(\n",
            "[-0.01157028  0.0248591   0.02878048 -0.012715    0.03971538  0.0882776\n",
            "  0.02680984  0.05589836 -0.0106873  -0.00597291  0.00639323 -0.01819518\n",
            "  0.00030813  0.09105888  0.05874644 -0.03180628  0.01512474 -0.05162929\n",
            "  0.00991367 -0.06865347 -0.04209306  0.02678981  0.03011006  0.00321069\n",
            " -0.00337973 -0.04787357  0.0226672  -0.00985925 -0.04063613 -0.01292092\n",
            " -0.04666384  0.05630299 -0.03949255  0.00517686  0.02495829 -0.0701444\n",
            "  0.02871508  0.04947684 -0.00633979 -0.08960192  0.02807118 -0.00808364\n",
            " -0.01360602  0.0599865  -0.10361787 -0.05195374  0.00232954 -0.02332531\n",
            " -0.03758105  0.03327728], shape=(50,), dtype=float32)\n"
          ]
        }
      ],
      "source": [
        "import tensorflow_hub as hub\n",
        "\n",
        "embed = hub.load(\"https://tfhub.dev/google/universal-sentence-encoder/4\")\n",
        "embed_samples = embed([sample_sentence, \"When you can the universal sentence encoder on a sentence, it turns into numbers.\"])\n",
        "print(embed_samples[0][:50])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "yKGMwSNDkFma",
        "outputId": "d58fde61-59d2-41a2-920f-de202a681d6b"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "TensorShape([512])"
            ]
          },
          "execution_count": 86,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "embed_samples[0].shape"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "F2USJBk2n1EF"
      },
      "source": [
        "In The USE (Universal Sentence Encoder) whatever the input is it will return 512 outputs."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "S8R8-g7wnSKt"
      },
      "outputs": [],
      "source": [
        "# Create a keras layer using the USE Pretrained layer from tensorflow hub\n",
        "sentence_encoder_layer = hub.KerasLayer(\"https://tfhub.dev/google/universal-sentence-encoder/4\",\n",
        "                                        input_shape=[],\n",
        "                                        dtype=tf.string,\n",
        "                                        trainable=False,\n",
        "                                        name=\"USE\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WO_YaBgcn0F1",
        "outputId": "c37cad2e-cd82-4560-d193-381e1d3c814f"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Model: \"model_6_USE\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " USE (KerasLayer)            (None, 512)               256797824 \n",
            "                                                                 \n",
            " dense_5 (Dense)             (None, 64)                32832     \n",
            "                                                                 \n",
            " dense_6 (Dense)             (None, 1)                 65        \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 256,830,721\n",
            "Trainable params: 32,897\n",
            "Non-trainable params: 256,797,824\n",
            "_________________________________________________________________\n"
          ]
        }
      ],
      "source": [
        "# Create Model using the Sequential API\n",
        "model_6 = tf.keras.Sequential([\n",
        "    sentence_encoder_layer,\n",
        "    layers.Dense(64, activation=\"relu\"),\n",
        "    layers.Dense(1, activation=\"sigmoid\")\n",
        "], name=\"model_6_USE\")\n",
        "\n",
        "# Compile\n",
        "model_6.compile(loss=\"binary_crossentropy\",\n",
        "                optimizer=tf.keras.optimizers.Adam(),\n",
        "                metrics=[\"accuracy\"])\n",
        "\n",
        "# summary\n",
        "model_6.summary()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "BqYlJrmrpo3x",
        "outputId": "8690fc61-abd8-40ed-eb2f-d8b9d2ad28f5"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Saving TensorBoard log files to: model_logs/tf_hub_sentence_encoder/20221001-144754\n",
            "Epoch 1/5\n",
            "215/215 [==============================] - 5s 14ms/step - loss: 0.5028 - accuracy: 0.7865 - val_loss: 0.4464 - val_accuracy: 0.8031\n",
            "Epoch 2/5\n",
            "215/215 [==============================] - 3s 12ms/step - loss: 0.4143 - accuracy: 0.8155 - val_loss: 0.4420 - val_accuracy: 0.8058\n",
            "Epoch 3/5\n",
            "215/215 [==============================] - 4s 19ms/step - loss: 0.4020 - accuracy: 0.8209 - val_loss: 0.4337 - val_accuracy: 0.8123\n",
            "Epoch 4/5\n",
            "215/215 [==============================] - 3s 12ms/step - loss: 0.3938 - accuracy: 0.8257 - val_loss: 0.4335 - val_accuracy: 0.8123\n",
            "Epoch 5/5\n",
            "215/215 [==============================] - 3s 12ms/step - loss: 0.3875 - accuracy: 0.8292 - val_loss: 0.4274 - val_accuracy: 0.8176\n"
          ]
        }
      ],
      "source": [
        "# Train a classifier on top of the USE pretrained embedding\n",
        "model_6_history = model_6.fit(\n",
        "    train_sentences,\n",
        "    train_labels,\n",
        "    epochs=5,\n",
        "    validation_data=(test_sentences, test_labels),\n",
        "    callbacks=[create_tensorboard_callback(SAVE_DIR, \"tf_hub_sentence_encoder\")]\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_K8LIv1xqYl-",
        "outputId": "2c45f973-fe33-4535-db22-4c7e758041ea"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "array([[0.17687963],\n",
              "       [0.78243446],\n",
              "       [0.9885632 ],\n",
              "       [0.18210103],\n",
              "       [0.7412013 ],\n",
              "       [0.7048023 ],\n",
              "       [0.9802706 ],\n",
              "       [0.9793012 ],\n",
              "       [0.94333357],\n",
              "       [0.09389146]], dtype=float32)"
            ]
          },
          "execution_count": 90,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# make predictions\n",
        "model_6_pred_probs = model_6.predict(test_sentences)\n",
        "model_6_pred_probs[:10]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "DZ1MbUlaqr6l",
        "outputId": "1c9c5bff-093b-455a-bfec-89012ad93439"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(10,), dtype=float32, numpy=array([0., 1., 1., 0., 1., 1., 1., 1., 1., 0.], dtype=float32)>"
            ]
          },
          "execution_count": 91,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# squeeze to labels\n",
        "model_6_preds = tf.squeeze(tf.round(model_6_pred_probs))\n",
        "model_6_preds[:10]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-leFbO5dq8gs",
        "outputId": "1147a57e-2db2-4dc5-b8fd-05332f7ca249"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "{'accuracy': 81.75853018372703,\n",
              " 'precision': 0.8190970972170162,\n",
              " 'recall': 0.8175853018372703,\n",
              " 'f1': 0.816365758450113}"
            ]
          },
          "execution_count": 92,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# Calculate Model 6 peformance metrics\n",
        "model_6_results = calculate_results(y_true=test_labels,\n",
        "                                    y_pred=model_6_preds)\n",
        "model_6_results"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4AF4O-VUrNBV",
        "outputId": "3e18e5ed-b058-4580-d2cc-75a8a46cacd4"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "{'accuracy': 79.26509186351706,\n",
              " 'precision': 0.8111390004213173,\n",
              " 'recall': 0.7926509186351706,\n",
              " 'f1': 0.7862189758049549}"
            ]
          },
          "execution_count": 93,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "baseline_results"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CsAsc2M8rSRv"
      },
      "source": [
        "## Model 7: TF Hub Pretrained USE but with 10% Training Data"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "itGX7CVjLARs"
      },
      "source": [
        "making data splits like this leads to data leakage. This data splits leak data from validation/test to training data set."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "KyjF55Bi9fFk"
      },
      "outputs": [],
      "source": [
        "# Create subsets of 10% of the training data\n",
        "# train_10_percent = train_df_shuffled[[\"text\", \"target\"]].sample(frac=0.1, random_state=42)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "enbzTqeCALdn"
      },
      "outputs": [],
      "source": [
        "# train_sentences_10_percent = train_10_percent['text'].to_list()\n",
        "# train_labels_10_percent = train_10_percent['target'].to_list()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Bio8fhs_AxjR"
      },
      "outputs": [],
      "source": [
        "# len(train_sentences_10_percent), len(train_labels_10_percent)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "4XtsZaO0L34_"
      },
      "outputs": [],
      "source": [
        "train_10_percent_split = int(0.1 * len(train_sentences))\n",
        "train_sentences_10_percent = train_sentences[:train_10_percent_split]\n",
        "train_labels_10_percent = train_labels[:train_10_percent_split]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "knEVbApsO75C",
        "outputId": "1dbafb9c-1f03-42ed-d14d-2d855721bc68"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "0    406\n",
              "1    279\n",
              "dtype: int64"
            ]
          },
          "execution_count": 98,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "pd.Series(np.array(train_labels_10_percent)).value_counts()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "heBVveDgD1nO",
        "outputId": "eaccb91b-c42f-4fde-f798-4842c3a5135d"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Model: \"model_7_USE\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " USE (KerasLayer)            (None, 512)               256797824 \n",
            "                                                                 \n",
            " dense_7 (Dense)             (None, 64)                32832     \n",
            "                                                                 \n",
            " dense_8 (Dense)             (None, 1)                 65        \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 256,830,721\n",
            "Trainable params: 32,897\n",
            "Non-trainable params: 256,797,824\n",
            "_________________________________________________________________\n"
          ]
        }
      ],
      "source": [
        "# Build a model same as model_6 : Just cloning the layers not the weights\n",
        "# model_7 = tf.keras.models.clone_model(model_6)\n",
        "model_7 = tf.keras.Sequential([\n",
        "    sentence_encoder_layer,\n",
        "    layers.Dense(64, activation=\"relu\"),\n",
        "    layers.Dense(1, activation=\"sigmoid\")\n",
        "], name=\"model_7_USE\")\n",
        "\n",
        "# Compile the model\n",
        "model_7.compile(loss=\"binary_crossentropy\",\n",
        "                optimizer=tf.keras.optimizers.Adam(),\n",
        "                metrics=['accuracy'])\n",
        "\n",
        "# summary\n",
        "model_7.summary()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JPufDJM73sj5",
        "outputId": "a2714132-639c-4d62-8d4f-752f251ba3c8"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Saving TensorBoard log files to: model_logs/tf_hub_sentence_encoder_10_percent/20221001-144812\n",
            "Epoch 1/5\n",
            "22/22 [==============================] - 2s 42ms/step - loss: 0.6745 - accuracy: 0.6161 - val_loss: 0.6586 - val_accuracy: 0.6509\n",
            "Epoch 2/5\n",
            "22/22 [==============================] - 0s 22ms/step - loss: 0.6102 - accuracy: 0.7854 - val_loss: 0.6040 - val_accuracy: 0.7572\n",
            "Epoch 3/5\n",
            "22/22 [==============================] - 1s 26ms/step - loss: 0.5370 - accuracy: 0.8204 - val_loss: 0.5453 - val_accuracy: 0.7756\n",
            "Epoch 4/5\n",
            "22/22 [==============================] - 1s 26ms/step - loss: 0.4727 - accuracy: 0.8219 - val_loss: 0.5091 - val_accuracy: 0.7769\n",
            "Epoch 5/5\n",
            "22/22 [==============================] - 1s 26ms/step - loss: 0.4268 - accuracy: 0.8365 - val_loss: 0.4923 - val_accuracy: 0.7808\n"
          ]
        }
      ],
      "source": [
        "model_7_history = model_7.fit(train_sentences_10_percent,\n",
        "                              train_labels_10_percent,\n",
        "                              epochs=5,\n",
        "                              validation_data=(test_sentences, test_labels),\n",
        "                              callbacks=[create_tensorboard_callback(SAVE_DIR,\n",
        "                                                                     \"tf_hub_sentence_encoder_10_percent\")]\n",
        "                              )"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7sMmoBdABgI9",
        "outputId": "2aa351cc-d5d6-430c-f572-34853bce8475"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "array([[0.22676557],\n",
              "       [0.58281285],\n",
              "       [0.90893376],\n",
              "       [0.38268232],\n",
              "       [0.54316425],\n",
              "       [0.68539625],\n",
              "       [0.876606  ],\n",
              "       [0.8053596 ],\n",
              "       [0.84605384],\n",
              "       [0.1584504 ]], dtype=float32)"
            ]
          },
          "execution_count": 101,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# make predictions\n",
        "model_7_pred_probs = model_7.predict(test_sentences)\n",
        "model_7_pred_probs[:10]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "O7RGtVJ5Bm40",
        "outputId": "507fc0f3-56d6-45d7-e6da-3123198d814b"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(10,), dtype=float32, numpy=array([0., 1., 1., 0., 1., 1., 1., 1., 1., 0.], dtype=float32)>"
            ]
          },
          "execution_count": 102,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# turn probs to labels\n",
        "model_7_preds = tf.squeeze(tf.round(model_7_pred_probs))\n",
        "model_7_preds[:10]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pO2p51j_B5Vn",
        "outputId": "0f36f164-c487-4c90-bcce-7de96046754c"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "{'accuracy': 78.08398950131233,\n",
              " 'precision': 0.7818764826324955,\n",
              " 'recall': 0.7808398950131233,\n",
              " 'f1': 0.7792643495521726}"
            ]
          },
          "execution_count": 103,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# Evaluate model 7 results\n",
        "model_7_results = calculate_results(y_true=test_labels,\n",
        "                                    y_pred=model_7_preds)\n",
        "model_7_results"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1qoJp4wWCGFO"
      },
      "source": [
        "## Comparing the performance of each of the models"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 300
        },
        "id": "AZSLkv0HeFdx",
        "outputId": "97f9be0a-47ce-4cea-c930-2b899919ce74"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-1cecf468-9287-48da-b01e-edbef2dd461e\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>accuracy</th>\n",
              "      <th>precision</th>\n",
              "      <th>recall</th>\n",
              "      <th>f1</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0_baseline</th>\n",
              "      <td>79.265092</td>\n",
              "      <td>0.811139</td>\n",
              "      <td>0.792651</td>\n",
              "      <td>0.786219</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1_simple_dense</th>\n",
              "      <td>78.740157</td>\n",
              "      <td>0.793230</td>\n",
              "      <td>0.787402</td>\n",
              "      <td>0.784113</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2_lstm</th>\n",
              "      <td>77.427822</td>\n",
              "      <td>0.779922</td>\n",
              "      <td>0.774278</td>\n",
              "      <td>0.770623</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3_gru</th>\n",
              "      <td>77.296588</td>\n",
              "      <td>0.774052</td>\n",
              "      <td>0.772966</td>\n",
              "      <td>0.771216</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4_bidirectional</th>\n",
              "      <td>77.296588</td>\n",
              "      <td>0.773654</td>\n",
              "      <td>0.772966</td>\n",
              "      <td>0.771448</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5_conv1d</th>\n",
              "      <td>76.377953</td>\n",
              "      <td>0.764768</td>\n",
              "      <td>0.763780</td>\n",
              "      <td>0.761896</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6_tf_hub_use_encoder</th>\n",
              "      <td>81.758530</td>\n",
              "      <td>0.819097</td>\n",
              "      <td>0.817585</td>\n",
              "      <td>0.816366</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7_tf_hub_use_encoder_10_percent</th>\n",
              "      <td>78.083990</td>\n",
              "      <td>0.781876</td>\n",
              "      <td>0.780840</td>\n",
              "      <td>0.779264</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-1cecf468-9287-48da-b01e-edbef2dd461e')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-1cecf468-9287-48da-b01e-edbef2dd461e button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-1cecf468-9287-48da-b01e-edbef2dd461e');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "                                  accuracy  precision    recall        f1\n",
              "0_baseline                       79.265092   0.811139  0.792651  0.786219\n",
              "1_simple_dense                   78.740157   0.793230  0.787402  0.784113\n",
              "2_lstm                           77.427822   0.779922  0.774278  0.770623\n",
              "3_gru                            77.296588   0.774052  0.772966  0.771216\n",
              "4_bidirectional                  77.296588   0.773654  0.772966  0.771448\n",
              "5_conv1d                         76.377953   0.764768  0.763780  0.761896\n",
              "6_tf_hub_use_encoder             81.758530   0.819097  0.817585  0.816366\n",
              "7_tf_hub_use_encoder_10_percent  78.083990   0.781876  0.780840  0.779264"
            ]
          },
          "execution_count": 104,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "all_model_results = pd.DataFrame({\"0_baseline\": baseline_results,\n",
        "                                  \"1_simple_dense\": model_1_results,\n",
        "                                  \"2_lstm\": model_2_results,\n",
        "                                  \"3_gru\": model_3_results,\n",
        "                                  \"4_bidirectional\": model_4_results,\n",
        "                                  \"5_conv1d\": model_5_results,\n",
        "                                  \"6_tf_hub_use_encoder\": model_6_results,\n",
        "                                  \"7_tf_hub_use_encoder_10_percent\": model_7_results})\n",
        "all_model_results = all_model_results.transpose()\n",
        "all_model_results"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 300
        },
        "id": "2BsrvNPSet8v",
        "outputId": "1cd001ec-1df3-49a0-d55c-7b6294e27f05"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-72ac77b7-ee58-4ec1-9441-a85a91f8c27c\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>accuracy</th>\n",
              "      <th>precision</th>\n",
              "      <th>recall</th>\n",
              "      <th>f1</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0_baseline</th>\n",
              "      <td>0.792651</td>\n",
              "      <td>0.811139</td>\n",
              "      <td>0.792651</td>\n",
              "      <td>0.786219</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1_simple_dense</th>\n",
              "      <td>0.787402</td>\n",
              "      <td>0.793230</td>\n",
              "      <td>0.787402</td>\n",
              "      <td>0.784113</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2_lstm</th>\n",
              "      <td>0.774278</td>\n",
              "      <td>0.779922</td>\n",
              "      <td>0.774278</td>\n",
              "      <td>0.770623</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3_gru</th>\n",
              "      <td>0.772966</td>\n",
              "      <td>0.774052</td>\n",
              "      <td>0.772966</td>\n",
              "      <td>0.771216</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4_bidirectional</th>\n",
              "      <td>0.772966</td>\n",
              "      <td>0.773654</td>\n",
              "      <td>0.772966</td>\n",
              "      <td>0.771448</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5_conv1d</th>\n",
              "      <td>0.763780</td>\n",
              "      <td>0.764768</td>\n",
              "      <td>0.763780</td>\n",
              "      <td>0.761896</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6_tf_hub_use_encoder</th>\n",
              "      <td>0.817585</td>\n",
              "      <td>0.819097</td>\n",
              "      <td>0.817585</td>\n",
              "      <td>0.816366</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7_tf_hub_use_encoder_10_percent</th>\n",
              "      <td>0.780840</td>\n",
              "      <td>0.781876</td>\n",
              "      <td>0.780840</td>\n",
              "      <td>0.779264</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-72ac77b7-ee58-4ec1-9441-a85a91f8c27c')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-72ac77b7-ee58-4ec1-9441-a85a91f8c27c button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-72ac77b7-ee58-4ec1-9441-a85a91f8c27c');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "                                 accuracy  precision    recall        f1\n",
              "0_baseline                       0.792651   0.811139  0.792651  0.786219\n",
              "1_simple_dense                   0.787402   0.793230  0.787402  0.784113\n",
              "2_lstm                           0.774278   0.779922  0.774278  0.770623\n",
              "3_gru                            0.772966   0.774052  0.772966  0.771216\n",
              "4_bidirectional                  0.772966   0.773654  0.772966  0.771448\n",
              "5_conv1d                         0.763780   0.764768  0.763780  0.761896\n",
              "6_tf_hub_use_encoder             0.817585   0.819097  0.817585  0.816366\n",
              "7_tf_hub_use_encoder_10_percent  0.780840   0.781876  0.780840  0.779264"
            ]
          },
          "execution_count": 105,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "all_model_results['accuracy'] = all_model_results['accuracy']/100\n",
        "all_model_results"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 590
        },
        "id": "RuK6kMZmgoW_",
        "outputId": "8d9cc60f-c19f-4913-865d-827f85110d3f"
      },
      "outputs": [
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAqkAAAI9CAYAAAAZ0eGSAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzde5iVdb3+8fseDiICHnAkFBBUBEZFUSTzWB5Kt4qW7sQ0rV2xrVDTTnZQyyzTTIvytzdmpnmIzEzxSFYKu9QENZSjIhKCgiMiqIgw8Pn98Twji2FgBh3m+cLzfl0XF+s5zJrPrAvW3Ot7dEQIAAAASElV0QUAAAAADRFSAQAAkBxCKgAAAJJDSAUAAEByCKkAAABITtuivvH2228fvXv3LurbAwAANNsTTzzxakRUF11HmRQWUnv37q2JEycW9e0BAACazfa/i66hbOjuBwAAQHIIqQAAAEgOIRUAAADJKWxMKgAAwKbsiSee2KFt27bXSdpTNPxtqFWSJtfV1X1+v/32e6WxGwipAAAA70Hbtm2v+8AHPjCgurp6UVVVVRRdz6Zk1apVrq2trZk/f/51koY2dg+pHwAA4L3Zs7q6egkBdcNVVVVFdXX1YmWt0I3f04r1AAAAbE6qCKjvXf7arTOLElIBAACQHMakAgAAtIDeF9y7X0s+3+wfH/tESz7fpoaWVAAAAKzXihUrWv17ElIBAAA2YUceeeSue+yxx4DddtttjyuvvHJ7Sbr99tu71NTUDOjXr1/Nhz70od0lafHixVUnn3xy7913371m9913r7nhhhu2kaSOHTsOqn+u3/zmN9uedNJJvSXppJNO6v2pT32q18CBA/t/8Ytf7PHQQw913GefffoPGDCgZtCgQf0nTZq0hSTV1dVp+PDhPfr27bvH7rvvXvPDH/5whzFjxnQ+8sgjd61/3j/96U9djjrqqF21AejuBwAA2ITdcssts7t167byzTff9KBBg2pOOeWU10eMGNH74Ycfnt6/f//lCxYsaCNJF1xwQfcuXbqsfPbZZ6dKUm1tbZumnvvll19u/+STT05v27atXnvttaoJEyZMb9eune68887O3/jGN3qMHTv2+Z/+9KfVc+bMaT916tQp7dq104IFC9pUV1evPPfcc3u99NJLbXfccce666+/vutnP/vZVzfk5yKkAgAAbMIuv/zybvfee+82kjR//vx2I0eOrB4yZMgb/fv3Xy5J3bp1WylJ48eP7zJ69OhZ9V9XXV29sqnn/sQnPrGobdssLr722mttTjnllD6zZ8/uYDtWrFhhSfrb3/7W5ayzzqpt166dKr/fJz/5yYW/+tWvtvvyl7+88Mknn+x0xx13vLAhPxchFQAAYBN1zz33dB43blzniRMnTu/cufOqIUOG9Bs0aNDSGTNmdGjuc9h+9/Hbb7/tymudOnVaVf/4m9/85k6HHXbYGw8++ODzM2bMaH/44Yf3W9/zfvGLX1x47LHH7tahQ4c4/vjjF9WH2OZiTCoAAMAm6vXXX2+z9dZbr+zcufOqp556qsOkSZO2WrZsWdXjjz/eefr06e0lqb67/7DDDlty9dVX71D/tfXd/V27dl3x5JNPdli5cqXuuuuubdf1vZYsWdKmR48eyyVp1KhR29efP+KII5aMGjVq+/rJVfXfr3fv3iu6deu24qc//Wn34cOHb1BXv0RLKgAAQIsoYsmok046afG1115bvcsuu+yxyy67LNt7773f2mGHHepGjhw5++Mf//huq1atUteuXVc88sgjz1122WUvf/azn+3Vt2/fPaqqquLb3/72S2eeeebr3//+9+edcMIJu2233XZ1e++999K33nqr0UbMb37zm/M///nP97n88st3POqoo16vP3/eeefVPvvss1v0799/j7Zt28aZZ55Z++1vf7tWkoYNG7bwmmuuabvvvvsu29CfzRHFbJQwePDgmDhxYiHfGwCAVvW9rZu4vrh16sB7ZvuJiBhceW7SpEmz99577w1uISyTM844o9egQYOWnnfeeY2+TpMmTdp+77337t3YNVpSAQB4H3pfcG+T98xuYnTgXjfu1eRzPHPmM80tCUjCHnvsMWDLLbdcNWrUqBffy9cTUgEA2ARM6z+gyXsGTJ/WCpUAzTNlypT39Q+SiVMAAABIzubfktrUOCCJsUAAAACJoSUVAAAAyWlWSLV9tO0ZtmfavqCR671sP2T7KdtP2/6Pli8VAAAAZdFkd7/tNpKukXSUpLmSJtgeExFTK277rqTbIuJ/bNdIuk9S741QLwAAQJq+t/V+Lft8i1t93VVJGj9+fMfrr7++6w033NDorPzZs2e3O+uss3o+8MADsxq73lKaMyZ1iKSZETFLkmyPlnSCpMqQGpK65I+3lvRSSxYJAACA96aurk5t2zZ/GtKhhx669NBDD126ruu9e/desbEDqtS87v6dJFUm6bn5uUrfk3S67bnKWlHPbuyJbA+3PdH2xNra2vdQLgAAAOrNmDGjfZ8+ffYYOnRon1122WWPo48+epc33nijaqeddtrri1/84k41NTUDrr/++m3vuOOOLvvss0//mpqaAcccc8wuixcvrpKkcePGdRw0aFD/fv361ey1114DFi1aVHXPPfd0/shHPrKbJN17772d+vfvX9O/f/+aAQMG1CxatKhqxowZ7fv27buHJC1dutQnn3xy7913371mwIABNXfffXdnSRo5cmTXj370o7secsghfXfeeec9zzrrrB4b+rO11MSpUyXdEBE9JP2HpJtsr/XcEXFtRAyOiMHV1dUt9K0BAADKa/bs2R1GjBjxyqxZs6Z07tx51U9+8pNqSeratWvd1KlTpx1//PFv/OhHP+o+fvz4Z6dOnTpt3333XfqDH/yg27Jly3zaaaft+rOf/WzOjBkzpo4bN25Gp06dVlU+909/+tMPjBw58t/Tp0+f+thjj01veP3yyy/fwbaeffbZqbfeeuus4cOH9166dKklaerUqR3vvPPOWdOmTZsyZsyYbWfOnNluQ36u5oTUeZJ6Vhz3yM9V+pyk2yQpIh6V1EHS9htSCAAAADbcBz7wgeUf/ehH35KkT3/60wsfeeSRTpJ0xhlnLJKkhx9+eKvnn3++w5AhQ/r379+/ZvTo0V3nzJnT/umnn+6www47rDjssMOWStJ22223ql27NXPkAQcc8ObXvva1npdeeukOr776apuG1x955JFOn/70pxdK0qBBg5btuOOOy5955pkOknTwwQcv6dq168qOHTvGbrvttuz555/fYkN+ruYMUJggqa/tPsrC6TBJn2pwzxxJR0i6wfYAZSG1Vfrzm9qOrqmt6KSSbkfH+rEAAGwWbDd63Llz51WSFBE6+OCDl9x9990vVN73+OOPb9nUc//oRz+af+KJJy6+6667tj7kkEP633vvvc917NhxVVNfJ0nt27eP+sdt2rSJFStWeH33N9RkS2pE1EkaIWmspGnKZvFPsX2J7aH5bV+V9AXbkyT9TtJnIiIaf0YAAAC0lJdffrn9X/7yl60k6ZZbbtnuwAMPfLPy+oc//OG3Jk6c2Gny5MlbSNKSJUuqnn766S0GDhy47JVXXmk3bty4jpK0aNGiqhUrVqzx3FOmTNliyJAhb//whz+cP3DgwLcmT568RvPfQQcd9ObNN9+8nSQ9/fTTW7z88svtBw4cuKwlfq5mTfWKiPuUTYiqPHdRxeOpkg5qiYLw/jXVuiy1TAvzZte6DADA+1HQklG9e/de9otf/GKH4cOHd+zbt++yr33ta7XXXXfdDvXXd9xxx7pRo0bNHjZs2C7Lly+3JF188cXzBg4c+M4tt9zy/DnnnNNr2bJlVR06dFg1fvz4Zyuf+4orrtjhkUce6WI7+vXr9/bJJ5+8eM6cOe/2+X/jG9945Ywzzth59913r2nTpo1GjRo1e8stt2yRhsrNf1vUFjKt/4D1Xh8wfVorVQIAALBa27Ztddddd63RlT9v3rw1WpKGDh36xtChQ9cKK4cddtjSSZMmTa88d9xxx71x3HHHvSFJN95441prpfbr12/5c889N0WSOnbsGLfffvvshvecc845CyUtrD9+6KGHZm7YT0VIxfvQVHCXCO8AAOC9IaQCAIAW1/TE5oZzsNe2V59eTd5T9qFnla2amxtCKlCpqVUPWPEAAJJCr97mq6UW8wcAAABaDC2pKI2WWPWANXXXdQ8tzACAlkVIBUqODTEa11rj6W67rG691+mmBFBWhFSghTE+qnG8LgA2d3vduNd+Lfl8z5z5TCHrro4cObLrxIkTt/rtb3875/zzz9+xU6dOKy+55JIFrV0HY1IBAAA2A6tWrdLKlSuLLqPF0JIKAGg5jGEGWtWMGTPaf+xjH9t90KBBbz7zzDNbnXDCCa+NHTt2m+XLl/vYY499/eqrr35Jkn75y192HTlyZDfbGjBgwNt33nnnC7feeuvWP/7xj7uvWLGiatttt637/e9/P6tnz57rH4PUigipAIBmYctlIE1z5szZ4te//vULixcvfu0Pf/jDtk8//fS0iNCRRx652/3339+purq67sorr+z+6KOPTu/evXvdggUL2kjSUUcd9eawYcOmV1VV6aqrrtr+kksu+cCvfvWruUX/PPUIqQCApDB+Gdgw3bt3X37EEUe8NXz48B7jx4/vUlNTUyNJS5curZo+fXqHJ598sur4449f1L179zpJ6tat20pJeuGFF9qfeOKJPWpra9stX768qmfPnu8U+XM0xJhUAACATVjHjh1XSVJE6Ctf+crL06dPnzp9+vSpc+bMmXzeeee9uq6vGzFiRK8vfelLrzz77LNTf/nLX/77nXfeSSoXJlUMAAAA3ptjjjlmyU033bT94sWLqyTphRdeaDdv3ry2H/vYx5bcfffd286fP7+NJNV397/xxhttevXqtUKSbrjhhq7FVd44uvsBAABaQFFLRtX7xCc+sWTKlCkd9t9///5S1sJ6yy23vDB48OBlX/3qV18+5JBD+ldVVcWee+659I9//OPs73znOy+deuqpu2699dZ1Bx988Btz5szZosj6GyKkAgAAbKL69eu3/LnnnptSf3zhhRe+cuGFF77S8L6zzz574dlnn72w8tzpp5/++umnn/56w3vPOeechZIWStJVV1310kYou1no7gcAAEByCKkAAABIDiEVAAAAySGkAgAAIDmEVAAAACSHkAoAAIDksAQVAABAC5jWf8B+Lfl8A6ZPa3Ld1UsvvXSH66+/vrpv377LFixY0G7q1KkdL7jggnmXXHLJgpaspQiEVAAAgE3Ur3/96+q//OUvz3bo0CFmzpzZ/vbbb9+26JpaCt39AAAAm6BPfepTvebOnbvFMccc0/e6667b7rDDDlvarl27KLqulkJLKgAAwCbo1ltvnTNu3Litx40b92z37t3riq6npdGSCgAAgOQQUgEAAJAcQioAAACSw5hUAACAFtCcJaM2ljlz5rTdf//9a9566602tmPUqFHdpk2bNnm77bZbVVRN7xchFQAAYBM1b968Z+ofL1iw4Okia2lpdPcDAAAgOYRUAAAAJIeQCgAA8N6sWrVqlYsuYlOVv3brHDNLSAUAAHhvJtfW1m5NUN1wq1atcm1t7daSJq/rnmZNnLJ9tKSfS2oj6bqI+HGD61dL+kh+2FHSDhGxzXuqGgAAYBNQV1f3+fnz5183f/78PUXD34ZaJWlyXV3d59d1Q5Mh1XYbSddIOkrSXEkTbI+JiKn190TEeRX3ny1p0PupGgAAIHX77bffK5KGFl3H5qo5qX+IpJkRMSsilksaLemE9dx/qqTftURxAAAAKKfmhNSdJL1YcTw3P7cW2ztL6iPpb+u4Ptz2RNsTa2trN7RWAAAAlERLj58YJun2iFjZ2MWIuDYiBkfE4Orq6hb+1gAAANhcNCekzpPUs+K4R36uMcNEVz8AAADep+aE1AmS+truY7u9siA6puFNtvtL2lbSoy1bIgAAAMqmyZAaEXWSRkgaK2mapNsiYortS2xXzmgbJml0RMTGKRUAAABl0ax1UiPiPkn3NTh3UYPj77VcWQAAACgzFp4FAABAcgipAAAASA4hFQAAAMkhpAIAACA5hFQAAAAkh5AKAACA5BBSAQAAkBxCKgAAAJJDSAUAAEByCKkAAABIDiEVAAAAySGkAgAAIDmEVAAAACSHkAoAAIDkEFIBAACQHEIqAAAAkkNIBQAAQHIIqQAAAEgOIRUAAADJIaQCAAAgOYRUAAAAJIeQCgAAgOQQUgEAAJAcQioAAACSQ0gFAABAcgipAAAASA4hFQAAAMkhpAIAACA5hFQAAAAkh5AKAACA5BBSAQAAkBxCKgAAAJJDSAUAAEByCKkAAABIDiEVAAAAyWlWSLV9tO0ZtmfavmAd93zS9lTbU2zf2rJlAgAAoEzaNnWD7TaSrpF0lKS5kibYHhMRUyvu6SvpW5IOiohFtnfYWAUDAABg89ecltQhkmZGxKyIWC5ptKQTGtzzBUnXRMQiSYqIV1q2TAAAAJRJc0LqTpJerDiem5+rtLuk3W3/w/Zjto9u7IlsD7c90fbE2tra91YxAAAANnstNXGqraS+kj4s6VRJv7K9TcObIuLaiBgcEYOrq6tb6FsDAABgc9OckDpPUs+K4x75uUpzJY2JiBUR8YKkZ5WFVgAAAGCDNSekTpDU13Yf2+0lDZM0psE9dyprRZXt7ZV1/89qwToBAABQIk2G1IiokzRC0lhJ0yTdFhFTbF9ie2h+21hJC21PlfSQpK9HxMKNVTQAAAA2b00uQSVJEXGfpPsanLuo4nFIOj//AwAAALwv7DgFAACA5BBSAQAAkBxCKgAAAJJDSAUAAEByCKkAAABIDiEVAAAAySGkAgAAIDmEVAAAACSHkAoAAIDkEFIBAACQHEIqAAAAkkNIBQAAQHIIqQAAAEgOIRUAAADJIaQCAAAgOYRUAAAAJIeQCgAAgOQQUgEAAJAcQioAAACSQ0gFAABAcgipAAAASA4hFQAAAMkhpAIAACA5hFQAAAAkh5AKAACA5BBSAQAAkBxCKgAAAJJDSAUAAEByCKkAAABIDiEVAAAAySGkAgAAIDmEVAAAACSHkAoAAIDkEFIBAACQHEIqAAAAkkNIBQAAQHKaFVJtH217hu2Zti9o5PpnbNfa/lf+5/MtXyoAAADKom1TN9huI+kaSUdJmitpgu0xETG1wa2/j4gRG6FGAAAAlExzWlKHSJoZEbMiYrmk0ZJO2LhlAQAAoMyaE1J3kvRixfHc/FxDJ9l+2vbttns29kS2h9ueaHtibW3teygXAAAAZdBSE6fultQ7IgZKelDSjY3dFBHXRsTgiBhcXV3dQt8aAAAAm5vmhNR5kipbRnvk594VEQsj4p388DpJ+7VMeQAAACij5oTUCZL62u5ju72kYZLGVN5gu3vF4VBJ01quRAAAAJRNk7P7I6LO9ghJYyW1kXR9REyxfYmkiRExRtI5todKqpP0mqTPbMSaAQAAsJlrMqRKUkTcJ+m+Bucuqnj8LUnfatnSAAAAUFbsOAUAAIDkEFIBAACQHEIqAAAAkkNIBQAAQHIIqQAAAEgOIRUAAADJIaQCAAAgOYRUAAAAJIeQCgAAgOQQUgEAAJAcQioAAACSQ0gFAABAcgipAAAASA4hFQAAAMkhpAIAACA5hFQAAAAkh5AKAACA5BBSAQAAkBxCKgAAAJJDSAUAAEByCKkAAABIDiEVAAAAySGkAgAAIDmEVAAAACSHkAoAAIDkEFIBAACQHEIqAAAAkkNIBQAAQHIIqQAAAEgOIRUAAADJIaQCAAAgOYRUAAAAJIeQCgAAgOQQUgEAAJAcQioAAACS06yQavto2zNsz7R9wXruO8l22B7cciUCAACgbJoMqbbbSLpG0jGSaiSdarumkfs6SzpX0j9bukgAAACUS3NaUodImhkRsyJiuaTRkk5o5L4fSLpc0rIWrA8AAAAl1JyQupOkFyuO5+bn3mV7X0k9I+Le9T2R7eG2J9qeWFtbu8HFAgAAoBze98Qp21WSrpL01abujYhrI2JwRAyurq5+v98aAAAAm6nmhNR5knpWHPfIz9XrLGlPSQ/bni3pAEljmDwFAACA96o5IXWCpL62+9huL2mYpDH1FyNicURsHxG9I6K3pMckDY2IiRulYgAAAGz2mgypEVEnaYSksZKmSbotIqbYvsT20I1dIAAAAMqnbXNuioj7JN3X4NxF67j3w++/LAAAAJQZO04BAAAgOYRUAAAAJIeQCgAAgOQQUgEAAJAcQioAAACSQ0gFAABAcgipAAAASA4hFQAAAMkhpAIAACA5hFQAAAAkh5AKAACA5BBSAQAAkBxCKgAAAJJDSAUAAEByCKkAAABIDiEVAAAAySGkAgAAIDmEVAAAACSHkAoAAIDkEFIBAACQHEIqAAAAkkNIBQAAQHIIqQAAAEgOIRUAAADJIaQCAAAgOYRUAAAAJIeQCgAAgOQQUgEAAJAcQioAAACSQ0gFAABAcgipAAAASA4hFQAAAMkhpAIAACA5hFQAAAAkh5AKAACA5DQrpNo+2vYM2zNtX9DI9bNsP2P7X7b/brum5UsFAABAWTQZUm23kXSNpGMk1Ug6tZEQemtE7BUR+0i6QtJVLV4pAAAASqM5LalDJM2MiFkRsVzSaEknVN4QEUsqDreSFC1XIgAAAMqmbTPu2UnSixXHcyV9sOFNtr8s6XxJ7SUd3tgT2R4uabgk9erVa0NrBQAAQEm02MSpiLgmInaV9E1J313HPddGxOCIGFxdXd1S3xoAAACbmeaE1HmSelYc98jPrctoSSe+n6IAAABQbs0JqRMk9bXdx3Z7ScMkjam8wXbfisNjJT3XciUCAACgbJockxoRdbZHSBorqY2k6yNiiu1LJE2MiDGSRtg+UtIKSYsknbkxiwYAAMDmrTkTpxQR90m6r8G5iyoen9vCdQEAAKDE2HEKAAAAySGkAgAAIDmEVAAAACSHkAoAAIDkEFIBAACQHEIqAAAAkkNIBQAAQHIIqQAAAEgOIRUAAADJIaQCAAAgOYRUAAAAJIeQCgAAgOQQUgEAAJAcQioAAACSQ0gFAABAcgipAAAASA4hFQAAAMkhpAIAACA5hFQAAAAkh5AKAACA5BBSAQAAkBxCKgAAAJJDSAUAAEByCKkAAABIDiEVAAAAySGkAgAAIDmEVAAAACSHkAoAAIDkEFIBAACQHEIqAAAAkkNIBQAAQHIIqQAAAEgOIRUAAADJIaQCAAAgOYRUAAAAJKdZIdX20bZn2J5p+4JGrp9ve6rtp23/1fbOLV8qAAAAyqLJkGq7jaRrJB0jqUbSqbZrGtz2lKTBETFQ0u2SrmjpQgEAAFAezWlJHSJpZkTMiojlkkZLOqHyhoh4KCKW5oePSerRsmUCAACgTJoTUneS9GLF8dz83Lp8TtL9jV2wPdz2RNsTa2trm18lAAAASqVFJ07ZPl3SYEk/aex6RFwbEYMjYnB1dXVLfmsAAABsRto24555knpWHPfIz63B9pGSviPpsIh4p2XKAwAAQBk1pyV1gqS+tvvYbi9pmKQxlTfYHiRplKShEfFKy5cJAACAMmkypEZEnaQRksZKmibptoiYYvsS20Pz234iqZOkP9j+l+0x63g6AAAAoEnN6e5XRNwn6b4G5y6qeHxkC9cFAACAEmPHKQAAACSHkAoAAIDkEFIBAACQHEIqAAAAkkNIBQAAQHIIqQAAAEgOIRUAAADJIaQCAAAgOYRUAAAAJIeQCgAAgOQQUgEAAJAcQioAAACSQ0gFAABAcgipAAAASA4hFQAAAMkhpAIAACA5hFQAAAAkh5AKAACA5BBSAQAAkBxCKgAAAJJDSAUAAEByCKkAAABIDiEVAAAAySGkAgAAIDmEVAAAACSHkAoAAIDkEFIBAACQHEIqAAAAkkNIBQAAQHIIqQAAAEgOIRUAAADJIaQCAAAgOYRUAAAAJIeQCgAAgOQQUgEAAJCcZoVU20fbnmF7pu0LGrl+qO0nbdfZPrnlywQAAECZNBlSbbeRdI2kYyTVSDrVdk2D2+ZI+oykW1u6QAAAAJRP22bcM0TSzIiYJUm2R0s6QdLU+hsiYnZ+bdVGqBEAAAAl05zu/p0kvVhxPDc/BwAAAGwUrTpxyvZw2xNtT6ytrW3Nbw0AAIBNSHNC6jxJPSuOe+TnNlhEXBsRgyNicHV19Xt5CgAAAJRAc0LqBEl9bfex3V7SMEljNm5ZAAAAKLMmQ2pE1EkaIWmspGmSbouIKbYvsT1Ukmzvb3uupP+UNMr2lI1ZNAAAADZvzZndr4i4T9J9Dc5dVPF4grJhAAAAAMD7xo5TAAAASA4hFQAAAMkhpAIAACA5hFQAAAAkh5AKAACA5BBSAQAAkBxCKgAAAJJDSAUAAEByCKkAAABIDiEVAAAAySGkAgAAIDmEVAAAACSHkAoAAIDkEFIBAACQHEIqAAAAkkNIBQAAQHIIqQAAAEgOIRUAAADJIaQCAAAgOYRUAAAAJIeQCgAAgOQQUgEAAJAcQioAAACSQ0gFAABAcgipAAAASA4hFQAAAMkhpAIAACA5hFQAAAAkh5AKAACA5BBSAQAAkBxCKgAAAJJDSAUAAEByCKkAAABIDiEVAAAAySGkAgAAIDmEVAAAACSnWSHV9tG2Z9ieafuCRq5vYfv3+fV/2u7d0oUCAACgPJoMqbbbSLpG0jGSaiSdarumwW2fk7QoInaTdLWky1u6UAAAAJRHc1pSh0iaGRGzImK5pNGSTmhwzwmSbswf3y7pCNtuuTIBAABQJo6I9d9gnyzp6Ij4fH78aUkfjIgRFfdMzu+Zmx8/n9/zaoPnGi5peH7YT9KMlvpB3qftJb3a5F3lw+uyNl6TxvG6NI7XpXG8LmvjNWlcSq/LzhFRXXQRZdK2Nb9ZRFwr6drW/J7NYXtiRAwuuo7U8LqsjdekcbwujeN1aRyvy9p4TRrH61JuzenunyepZ8Vxj/xco/fYbitpa0kLW6JAAAAAlE9zQuoESX1t97HdXtIwSWMa3DNG0pn545Ml/S2aGkcAAAAArEOT3f0RUWd7hKSxkif/k3MAACAASURBVNpIuj4ipti+RNLEiBgj6deSbrI9U9JryoLspiS5IQiJ4HVZG69J43hdGsfr0jhel7XxmjSO16XEmpw4BQAAALQ2dpwCAABAcgipAAAASA4hFQAAAMkhpAIAUADbVbYPLLoOIFWlnThlu6Okr0rqFRFfsN1XUr+IuKfg0pJgu2NELC26jpTY3lbZesDvrooREU8WV1GxbB/a2PmIGN/ataTA9ifWdz0i7mitWrDpsP1URAwquo6U2L4pIj7d1Dls/lp1x6nE/EbSE5I+lB/Pk/QHSaUOqfmn+uskdZLUy/bekv47Ir5UbGXFsv0DSZ+R9Lyk+k92IenwompKwNcrHneQNETZ/6myvibHr+daSCplSLX9hlb/n1lLRHRpxXJS9FfbJ0m6g/XF37VH5YHtNpL2K6gWFKjMLakTI2Jw5adY25MiYu+iayuS7X8q25BhTMXrMjki9iy2smLZniFpr4hYXnQtqbLdU9LPIuKkomtBevIPei9LukmSJZ0mqXtEXFRoYQXLQ/xWklZKelvZaxNlDO+2vyXp25K2lFTfk2dJyyVdGxHfKqo2FKPMLanLbW+p/BO+7V0lvVNsSWmIiBdtV55aWVQtCZksaRtJrxRdSMLmShpQdBEpsH2sstagDvXnIuKS4ipKwtAGjQD/Y3uSpFKH1IjoXHQNqYiIyyRdZvsyAimkcofUiyU9IKmn7VskHaSsO7fsXsy7/MN2O0nnSppWcE0puEzSU7Ynq+LDTEQMLa6kYtn+hVZ341ZJ2kdSacfo1rP9v5I6SvqIsqEzJ0t6vNCi0vCW7dMkjVb27+ZUSW8VW1LxnLUInCapT0T8IO+R6B4Rpf03ExHfsr2TpJ215hyAUo53L7PSdvdLku2ukg5Q1p3wWES8WnBJhbO9vaSfSzpS2evyZ0nnRsTCQgsrmO0pkkZJekbSqvrzETGusKIKZvvMisM6SbMj4h9F1ZMK209HxMCKvztJuj8iDim6tiLZ7q3sveUgZSH1H5K+EhGzi6uqeLb/R9l7yuERMSCfoPnniNi/4NIKY/vHyrZXn6rVPXlR5kaBsipzS6qUdcUtUvY61Ngu/Se1PKifVnQdCVoaESOLLiIV+USGj0YE/1bW9nb+91LbO0paKKl7gfUkIQ+jJxRdR4I+GBH72n5KkiJike32RRdVsI8rW22HIXglV9qQavtySadImqLVLWMhqdQh1fYVki5V9ov2AUkDJZ0XETcXWljx/s/2ZZLGaM3u/lJ2b0fESts7227PZLK13GN7G0k/UTb8IZR1+6MB2xcxVlcr8g999fMjqlXRW1NSsyS1E/NESq+03f35bO2BfFJbk+1/RcQ+tj8u6ThJ50saz6oHfqiR0xERZV1uSbZ/q2yi1BhVjC2MiKsKKyoxtreQ1CEiFhddS4psz4mIXkXXUaR8nO4pkvaVdKOyMczfjYg/FFpYgWz/UdLekv6qNRsFzimsKBSitC2p4pPautT/mzhW0h8iYnGDmf5l9bmImFV5wvYuRRWTiOfzP1WSmKFcIZ982Fv5/6d8KNFvCy2qILaXrOuSsqWGSi0ibrH9hKQjlL0mJ0ZE2Serjsn/oOTK3JLKJ7VG5APWT1TW3T9E2bJL90TEBwstrGC2n4yIfRuceyIiWGAaa7B9k6RdJf1La076KOV7i+05kvaPiAWNXHsxInoWUFbhbG+3vusR8Vpr1ZKifInIXhExo+haUJwyt6TySa0REXFBPi51cT7u8C2VeLKD7f7K1rvcusG2l11UsQZmGdm+W2vvJLRY0kRJoyJiWetXlYTBkmrYPehdv1W2lNBaIVXSra1cS0qeUPb/x5J6KZvEa2UNA3Mk9SmutGLZPl7SlZLaS+pjex9JlzC7v3xK25KKdWvYVSmpzF2VJyhrWR6qNT/UvCFpdEQ8UkhhCbD9c0nVkn6XnzpF0hJlv3i7lHWfbdt/kHRORLxcdC1In+1fSfpTRNyXHx+jrMv/v4utrDj58IfDJT3MzoflVrqWVNu3RcQnbT+jRvaTjoiBBZSVjHV1VSprDSmdiLhL0l22PxQRjxZdT2IObLCW4922J0TE/vm6smW1vaSpth8XGz+8K295/52kuyKi9Iv4VzggIr5QfxAR9+e9WWW2opH5EGVf8aCUShdSle2gJGUz17E2uiob9/E8eLE012qdbPeKiDmSZLuXpE75tTIvS/W9ogtI1JXKWtsvsz1B2c5T95R4WEi9l2x/V1L9e8lpkl4qsJ4UTLH9KUltbPeVdI6k0vZalRnd/VgDXZWNY2mutdn+D0n/q2yGv5WNofuSpIclfSEiflZcdcWy3U1SfSvz4xHxSpH1pCRfE/RwSV+QdHREdCm4pELlE6gulnRofmq8pO+XeeKU7Y6SviPpo/mpsZIu5QNN+ZQupNp+Q6u7+ev7EuoHrwdvmH5I2R7sdFVWsD0lIvawfZ2k2yPiAduTyhxSpXfXAe2fH86o/CVi+6iIeLCYyopj+5PKFvJ/WNn7yiGSvh4RtxdZVwryGdvHa/W6oPdExNnFVpUG252V/Q56s+hagFSULqRi/Wwf1tj5Mu9RL7E013vR2LJdZWB7kqSj6ltP8x2E/sIHGt+m7P/OA5J+L2lcRJR+nKHtvZSN+a9fkupVSWdGxOTiqiqW7Qcl/WdEvJ4fb6tsourHiq0Mra2MY1LfZftgSX0j4je2t5fUOSJeKLquIkXEONs7K3td/pJ3u7Qpuq6isTTXe1LWXSCqGnTvL1S24UHZ/VrSqRGxssk7y2WUpPMj4iFJsv1hSddKOrDIogq2fX1AlaSIWGR7hyILQjFKG1JtX6xsklA/Sb9Rth7bzZIOKrKuotn+gqThyj7V7yppJ2XjDo8osq6iNFgbtf5c5eEdrVfNJqes3TQP2B6rNZfmuq/AepIQEWNtH2i7t1jertJW9QFVkiLiYdtbFVlQAlY1mJS5s8r7flJqpQ2pkj4uaZCkJyUpIl7KxwSV3ZeVdcn9U5Ii4rmSf4I9fj3XQoRUNBARX7d9klZ/4L02Iv5UZE0pYHm7dZpl+0JJN+XHpyvbtrvMvi3p77bHafW47uHFloQilDmkLo+IsB2SxCfXd70TEcvrWwttt1WJP8FGxGebc5/tMyPixo1dTypsD1E2yWOC7RpJR0uaXr8geW52IcUlICL+KOmPRdeRGJa3a9x/Sfq+sg+8Ien/8nOlZLtK0tbKJtYdkJ/+SkS8WlxVKEppJ07Z/pqkvpKOknSZsjeFWyPiF4UWVrB83OXrks6QdLayJYWmRsR3Ci0scWWaJJQPlTlG2YfcByV9UNJDyv4vjY2IHxZYXmFs/z0iDm6wgojEyiGSWN4OzWd7YkQMLroOFK+0IVXKlshRtg6blf1yLd1yOQ3ln2I/p4rXRdJ1tH6sn+2n6rfv29zlu7XtI2kLSfMl9YiIJfnyQv8s+65taBzL2zWOmexry1dTeVXZKhDv7k5W5rVjy6q03f159/7fIuJB2/0k9bPdLiJWFF1bkfIlYX6V/0HzlSnE1+UztJfafj4ilkhSRLxtmyWF7Jsi4tNNnSuh7xVdQKKYyb62U/K/v1xxLiTtUkAtKFBpQ6qyXT0OyT+1PiBporL/GKcVWlVB8taxdQYtWseaVKbllpbb7hgRSyXtV3/S9tZif21J2qPyIB/Xvd867i2NfHk7duJaGzPZG4iIPkXXgDSUOaQ6Ipba/pyk/4mIK2z/q+iiCnRc/nf9J9fKmaalfsO03V/ZUlz/rNwNxvbREfFAfviPQoorxqER8Y70bst7vXaSziympOLZ/payWclb2l5Sf1rScmXrXpZaIztx/cI2O3Fl238yk71Cvj73+ZJ6RcRw230l9YuIewouDa2stGNSbT+lbFLQ1ZI+FxFTbD8TEXsVXFqhGhtbWaZJQQ3ZPkdZcJ+mbDzduRFxV36ttK8L1s32ZRHxraLrSA07ca1bvplM/Uz2x8o+k9327yU9IemMiNgzD62PRMQ+BZeGVlbmXVDOlfQtSX/KA+ouymYol51tH1RxcKDK/e/kC5L2i4gTJX1Y0oW2z82vlamLH833eD70QZJkexvbJxZZUCLYiWvdtpD0mqQlkmpsH1pwPUXbNSKukLRCkvKhRbzfllBpu/sjYryycan1x7MknVNcRcn4nKTrK37Jvq4Sr9mn7Bfrm5IUEbPzLQtvz8eN8aaJxlxcuXh/RLyeL9t1Z4E1paCxnbjuL7CeJNi+XNlrMUWrx3SHKn4/ldDyfLWQ+nXMd1XFihAoj9KG1Lyr6RvKJjl0qD8fEYcXVlQCIuIJSXvXh9SIWFx5vWyL1ktaYHufiPiXJEXEm7aPk3S9pFIPDcE6NdY6WNr32nr5TlyfkHRwfoqduDInKhtvSQhb7WJlE5p72r5F2e5tnym0IhSizGNS/6xsDbavSTpL2YSP2oj4ZqGFJa5s4zBt91C25NL8Rq4dFBFlmjCFZrB9vbIeiGvyU1+WtF1EfKawohJgu4+klyNiWX68paRuETG70MIKZvt+ZeukvtnkzSViu6uycboW43RLq8wh9YmI2M/20/XLK9meEBH7N/W1ZVamReuB9yJfg/lCSUcq6658UNIPI+Kt9X7hZs72REkHRsTy/Li9pH+U/T3X9h8l7S3pr1pzk4NSDz+raHUPSX+n1b2cytwFVb9o/8u2j5X0kqTtCqxnU1HOTzVAM+Vh9ALbW5U9mDbQtj6gSlJELM+DatmNyf8gZ/v/SdpNq8cv/7ftIyPiy+v5MmyGyhxSL83HXX5V0i8kdZF0XrElbRKYLASsR74ixnWSOknqZXtvSf8dEV8qtrLC1doeGhFjJMn2Ccq2viy1iLgxH/rQKyJmFF1PIg6XNKB+O27bNyqbWIaSKW1IrVgUeLGkjxRZyyaGMZjA+l0t6WPKW8ciYhJLCknKxv7fYvuX+fFcSWXfKla2j5d0paT2kvrY3kfSJRExtNjKCjVTUi9J/86Pe+bnUDKlXaPO9i6277b9qu1XbN+Vr5Vaara72f51PphftmvyXbkkSRExorjqgE1DRLzY4NTKQgpJSEQ8HxEHSKqRVBMRB0bE8/XXbZd1t7LvSRqibLKd8pVEyv67qLOkabYftv2QpKmSutgeY5uhESVS2pZUSbcqm3378fx4mLLxLx8srKI03CDpN8q26pOkZ5WtgvDrogoCNjEv5l3+Ybudso1DphVcUzLWM4v9XEllWt6u3oqIWGyvMZJq1bpuLomLii4AaShzSO0YETdVHN9s++uFVZOO7SPitnwfckVEne3StwIBG+AsST+XtJOkeZL+rGwZKqxfWce7T7H9KUlt8j3qz5H0SME1FSoixq3vuu1HI+JDrVUPilO6kGq7fgb//bYvkDRa2Yz1UyTdV1hh6XgrX5+ufsD6AcrG7QJogu02kn4eEacVXcsmqKwrh5ytrOfqHWU9fGMlXVpoRenr0PQt2ByUbp1U2y8oezNs7FN7RESpxwLZ3lfZagd7SposqVrSyRHxdKGFAZsI23+XdHjlcktoGmswN872LyLi7KLrSEnZNpUps9K1pEZEn+bcZ/uoiHhwY9eTmoh40vZhkvopC/IzImJFE18GYLVZkv6RT/B4d53UiLiquJLSYftgZROFJkfEnysusXJI4w4qugCgKKULqRvgcmU7xZRCvrtHY3a3rYi4o1ULAjZdz+d/qpTNUi41249HxJD88ReUjc/9k6SLbe8bET+WWDkEG6Ss45dLh5C6bmX7T3D8eq6FJEIq0AwR8f2ia0hMu4rHwyUdFRG1tq+U9JikHxdTFlJlu5uyiYeSNC8iFjS4pfTr65YFIXXdSjVYNyI+W3QNwKbM9s8i4iu271Yj7x8lXpy9yva2ylqWHRG1UrZ9rO26YkvbJJSmwSTfyOB/JW2tbGUMSeph+3VJX4qIJyUpIiYXVCJaGSEVa8hn9l8s6WBlv2j/rmz3k4WFFgakr35JuysLrSI9W0t6QlnYCtvdI+Jl251UogDWFNsdI2JpI5d+3urFFOcGZVsI/7PyZL7KzG8k7V1EUShO6Wb3S5Lt/pJOUEV3gqQxETGt4p47ImJd4zQ3W7YflDRe0s35qdMkfTgijiyuKgCbG9sdJXWLiBeKrqVI+cYP10nqFBG9bO+tLKh9qeDSWp3t5yKi7zquzYyI3Vq7JhSrdCHV9jclnapsfdS5+ekeynacGl0/iL+sbE+OiD0bnHsmIvYqqiZgU2D7Ga1nmFBEDGzFcrCJsP1PSScraygZlJ9b6324DGyPlLSrpN9Kqt9auKekMyS9wOS68iljd//nJO3RcFkl21dJmiIG8f/Z9jBJt+XHJytbXBrA+h2X/12/u1R99//pKtkYd2yYiHixwbaopdzlLyLOsX2M1u7pvCYi2GynhMrYkjpd0sci4t8Nzu8s6c8R0a+YytJg+w1JW2n13tFVWr3WY0REl0IKAzYRjS1Kz+LjWBfbt0u6StIvJX1Q0rmSBkfEsEILAxJQxpbUr0j6q+3ntLo7oZek3SSVvishIkq/riPwPtn2QRHxj/zgQGUf9oDGnKVsctROyloN/6zVrfHI2b42IoYXXQdaV+laUiXJdpWyHU8quxMmREQpu1gasj1QUm9VfIhhMX+geWzvJ+l6ZbPaLWmRpP+qXz4HQONsb7euS5ImRUSP1qwHxStlSMW62b5e0kBl43Pru/wjIv6ruKqATY/trSUpIhYXXQvSZfsKSZdKelvSA8ref8+LiJvX+4WbIdsrJf1bay5NFvnxThHRvpDCUBhCKtZge2pE1BRdB7CpsX16RNxs+/zGrkfEVa1dE9Jn+18RsY/tjyubfHe+pPERUbo1QfNheEdExJxGrr0YET0LKAsFYpwUGnrUNiEV2HBb5X93XscfoDH1w6qOlfSHkre8/0zStuu4dkVrFoI00JKKNdg+TNIYSfMlvaN8lxjWeASAlmf7x5JOVNbdP0TSNpLuiYgPFlpYwmwfFREPFl0HNj5CKtZge6ay7qZntHpMqhou2QWgcbZ3UTZb+wBl4+keVTbGcFahhSFZ+YShxRGxMt+Jq0tEzC+6rlSxpFt5lHEJKqxfbUSMKboIYBN2q6RrJH08Px4m6XfK1sAE1mD7jIrHlZd+2/rVbDLc9C3YHBBS0dBTtm+VdLey7n5JLEEFbICOEXFTxfHNtr9eWDVI3f4VjztIOkLSkyKkrg9dwCVBSEVDWyoLpx+tOBeSCKnAelSs8Xi/7QskjVb2f+cUSWzpiEZFxNmVx7a3UfZvByg9xqQCQAuw/YJWr+nYUETELq1cEjZBtttJmlzWLbrzzXYOiIhH1nPPHRHxiVYsCwUhpEKSZPsbEXGF7V+oka6UiDingLKAzQ4zk1HJ9t1a/Z5bJalG0m0RcUFxVRXL9lMRMajoOlA8uvtRb1r+98RCqwA2f5dLIqSi3pUVj+sk/Tsi5hZVTCL+avskSXcELWmlRksq1invdukUEUuKrgXYXNBKhA1h+9GI+FDRdbQm228o2xxjpbL1Y+vX6+5SaGFodew4hTXYvtV2F9tbSZosaSozk4EWRcsANkSHogtobRHROSKqIqJdRHTJjwmoJURIRUM1ecvpiZLul9RH0qeLLQkASqt0H2qcOd32hflxT9tDiq4LrY+Qioba5bNLT5Q0JiJWqIRvkkBLsN3YWpezW7sOYBPz/yR9SNKn8uM3lW2QgZJh4hQaGqXsl+gkSeNt7yyJMalAE2w33KnNkj6Sr3upiBia/83SOdgQZdxd6YMRsa/tpyQpIhbZbl90UWh9hFSsISJGShpZf2x7jqSPVByfGRE3FlEbkLgekqZKuk6r10sdLOmnRRaF9Nn+gKQhyv7dTIiI+RWXyzjcaoXtNsp78WxXS1pVbEkoAt39WK/I1FWcOrewYoC0DZb0hKTvSFocEQ9LejsixkXEuEIrQ7Jsf17S45I+IelkSY/Z/q/66xExuajaCjRS0p8k7WD7h5L+LulHxZaEIrAEFTYIy+cA62e7h6SrJS2QNDQiehVcEhJme4akAyNiYX7cVdIjZd1xqp7t/pKOUNYj8deImNbEl2AzRHc/NhSfaoD1yBdi/0/bx4rx3GjaQklvVBy/kZ8rHdvbVRy+Iul3ldci4rXWrwpFIqRiQ5VxED+wwSLiXkn3Fl0H0mT7/PzhTEn/tH2XskaAEyQ9XVhhxXpCq8dz95K0KH+8jaQ5ypZERIkwJhVNsv3ZisN/FFYIAGw+Oud/npd0p1b3Ut0l6YWiiipSRPSJiF0k/UXS8RGxfUR0lXScpD8XWx2KwJhUNMn2HMbVAQBag+1nImKvps5h80d3PyRJttfVvWRJ3VqzFgAoC9sPqZGx/hFxeAHlpOIl29+VdHN+fJqklwqsBwUhpKJeN0kfUzYGqJIlPdL65QBAKXyt4nEHSSdJqlvHvWVxqqSLlS1DJUnj83MoGUIq6t0jqVNE/KvhBdsPt345ALD5i4gnGpz6h+3HCykmEfks/nNtd84O482ia0IxGJMKAEBBGiy7VCVpP0kjy7xOqu29JP1WUv1r86qkM0u6sUGp0ZIKAEBxKpddqlM2s/9zhVZUvFGSzo+IhyTJ9oclXSvpwCKLQusjpAIAUJCIYO3PtW1VH1AlKSIetr1VkQWhGIRUAAAKZPtASb1V8Ts5In5bWEHFm2X7Qkk35cenS5pVYD0oCGNSAQAoiO2bJO0q6V+SVuanIyLOKa6qYtneVtL3JR2sbCjE/0n6fkQ0XH0GmzlCKgAABbE9TVJN8MsYWAvbogIAUJzJkj5QdBEpsf2g7W0qjre1PbbImlAMxqQCANDKbN+trCu7s6Sp+dqo79Rfj4ihRdWWgO0j4vX6g4hYZHuHIgtCMQipAAC0viuLLiBhq2z3iog5kmR7ZzWydSw2f4RUAABaWUSMa859th+NiA9t7HoS8x1Jf7c9Ttn6sYdIGl5sSSgCE6cAAEiU7aciYlDRdbQ229tLOiA/fCwiXi2yHhSDllQAANJV1pakLSS9piyn1NhWRIwvuCa0MkIqAABIhu3LJZ0iaYqkVfnpkERILRlCKgAArcz2FhHxTtN3yhu9mPScKOn/t3d/IXvWdRzH35/FdBu4DUsYaJH9UZCgNm3hIoISLWoFzpM021odGNiEqBM7KerEyANJOqy0gyJxoOvAAqmDdGmoM5Z/Iq3QA8N2MMWlOPt28NyPe3zadCe7vr9n9/sFN/d1Xb/fDR+ek+fL79914Un+fXQa85xUSZKmtx9ef+PUm7l2giyjeRpY3R1C/RxJlSRpemckuRrYluTK5Y1VtXf2fXDyZP2OAAeS3Msbz46d21fFziuLVEmSpncdcA2wEdi+rK2AvZMnGsfds4/mnEdQSZLUJMn1VXXrsmcnu171tJVkLfCuqnqyO4v6uCZVkqQ+u4/zbP/kKQaSZDtwALhndv+hJI6sziGn+yVJmliSTcC5wNokmzm2i389sK4t2Bi+A2wFfg9QVQeSvKczkHpYpEqSNL0rgF3AecDNHCtSXwBubMo0iler6nDyhtO3/nuizjp9WaRKkjSxqroNuC3Jjqq680T9kuyc9Z0nf5mdfPC2JO8H9gD3N2dSAzdOSZI0qCQPV9WW7hxTSrIO+DZw+ezRb4DvV9XLfanUwSJVkqRBJXmkqjZ35xhJkh9V1de7c+jUc3e/JEnjciTp/320O4CmYZEqSdK48tZdpNOTRaokSRNL8pEk62fXa5N8N8m+JDcl2bCk631NEaV2FqmSJE3vJyy8ox7gFmADcNPs2U8XO1XV9dNHG56jy3PCI6gkSZreqqo6Oru+ZMkO/j8kOdAVaiRJ1lXVkeM03TJ5GLVwJFWSpOkdTPLl2fWjSS4BSHIB8GpfrH5JtiV5DHhidv/BJD9ebK+qn3Vl07Q8gkqSpInN1p3eAnwM+DewBXhm9tlTVY82xmuV5AHgKuDuxeO3khysqg/0JtPUnO6XJGliVXUY2DXbPHU+C/+Pn62qf/UmG0NVPbPstaivdWVRH4tUSZKaVNULwNyOmp7AM0m2AZVkNXAD8HhzJjVwul+SJA0jyTtYWApxGQs7+X8L3FBVh1qDaXIWqZIkSRqOu/slSdIwkvwgyfokq5Pcm+T5JF/szqXpWaRKkqSRXD5bq/tZ4B/A+4BvtSZSC4tUSZI0ksVN3Z8B7pidhKA55O5+SZI0kl8neQL4D/C1JOcALzdnUgM3TkmSpKEkORs4XFWvJVkHrK+q57pzaVqOpEqSpGEk+dKS66VNt0+fRp0sUiVJ0kg+vOR6DfBJ4GEsUueO0/2SJGlYSTYCv6yqT3Vn0bTc3S9Jkkb2EnB+dwhNz+l+SZI0jCT7gMVp3lXARcCv+hKpi9P9kiRpGEk+vuT2KPDPqnq2K4/6WKRKkqQVI8n+qrq0O4dOPdekSpKklWRNdwBNwyJVkiStJE4BzwmLVEmSJA3HIlWSJK0keesuOh14BJUkSRpKkk3AVham9v9UVc8tab62J5Wm5kiqJEkaRpKvAg8CVwJXAX9MsnuxvaoOdmXTtDyCSpIkDSPJk8C2qjo0u387cH9VXdibTFNzJFWSJI3kEPDikvsXZ880Z1yTKkmS2iX5xuzyb8ADSe5iYU3q54E/twVTG4tUSZI0grNm30/NPovuasiiAbgmVZIkScNxJFWSJA0jye84zlulquoTDXHUyCJVkiSN5JtLrtcAO4CjTVnUyOl+SZI0tCQPVtXW7hyaliOpkiRpGEnOXnK7CrgY2NAUR40sUiVJ0kgeYmFNaliY5v878JXWRGrhdL8kSZKG40iqJEkaSpJtwLtZUqdU1e1tgdTCIlWSJA0jyc+B9wIHgNdmjwuwSJ0zTvdLkqRhJHkcuKgsUObequ4AkiRJSxwENnWHUD+n+yVJUrsk+1iY1j8LeCzJg8Ari+1V9bmubOphkSpJkkbww+4AGotrUiVJ0oqRZH9VXdqdQ6eea1IlSdJKOg7mYQAAAapJREFUsqY7gKZhkSpJklYSp4DnhEWqJEmShmORKkmS2iU582S7ntIgGoZFqiRJGsF+eP2NU2/m2gmyaAAeQSVJkkZwRpKrgW1JrlzeWFV7Z98HJ0+mFhapkiRpBNcB1wAbge3L2grYO3kitfKcVEmSNIwk11fVrcuenVlVr5zoNzo9uSZVkiSNZPdxnu2fPIXaOd0vSZLaJdkEnAusTbKZY7v41wPr2oKpjUWqJEkawRXALuA84GaOFakvADc2ZVIj16RKkqRhJNlRVXe+SfvOqrptykzqYZEqSZJWjCQPV9WW7hw69dw4JUmSVhLfODUnLFIlSdJK4hTwnLBIlSRJK4kjqXPCIlWSJLVLsifJO0+i632nPIyG4MYpSZLULslh4CXgKeAXwB1V9XxvKnVyJFWSJI3gaRbOSP0ecDHwWJJ7kuxMclZvNHVwJFWSJLVbfrRUktXAp4EvAJdV1Tlt4dTCIlWSJLVL8khVbT5B27qqOjJ1JvWySJUkSe2SXFBVf+3OoXFYpEqSJGk4bpySJEnScCxSJUmSNByLVEmSJA3HIlWSJEnD+R/NbrRXbD/WcgAAAABJRU5ErkJggg==",
            "text/plain": [
              "<Figure size 720x504 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          },
          "output_type": "display_data"
        }
      ],
      "source": [
        "# Plot and compare all of the model results\n",
        "all_model_results.plot(kind=\"bar\", figsize=(10, 7)).legend(bbox_to_anchor=(1.0, 1.0));"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 608
        },
        "id": "8nIlG0u-g9pE",
        "outputId": "97cead1b-ddfd-4351-c8e7-5a51a5ee096a"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "<matplotlib.axes._subplots.AxesSubplot at 0x7f890a16d750>"
            ]
          },
          "execution_count": 107,
          "metadata": {},
          "output_type": "execute_result"
        },
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlMAAAI9CAYAAAAev/3CAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3debyndV3//8eTTUTBJSbtxyJopE3ujqhoZS6JqeBWQmqaJvkrhLL8fjHLhernkllqVJJpiimhUg6GoplLKiqDorJEjbgAtoyoaGqy+Pr9cV2H+XA4M+fMXOec93Xmetxvt3M7n2thzsvLszw/7zVVhSRJknbObq0LkCRJWssMU5IkSQMYpiRJkgYwTEmSJA1gmJIkSRpgj1ZfeP/9969DDjmk1ZeXJElasvPPP/9rVbVuoWvNwtQhhxzCpk2bWn15SZKkJUvy5W1ds5tPkiRpAMOUJEnSAIYpSZKkAQxTkiRJAximJEmSBjBMSZIkDWCYkiRJGsAwJUmSNIBhSpIkaQDDlCRJ0gCGKUmSpAEMU5IkSQMYpiRJkgYwTEmSJA1gmJIkSRrAMCVJkjTAHq0LGOqQk/6xdQkAfOllj2pdgiRJasCWKUmSpAEMU5IkSQMYpiRJkgYwTEmSJA2wpDCV5MgklybZnOSkBa4fnOSDST6T5HNJfm75S5UkSRqfRcNUkt2BU4BHAuuBY5Osn3fb7wJnVNW9gGOAP1/uQiVJksZoKS1ThwObq+qyqroGOB04et49BezXv74V8NXlK1GSJGm8lhKmDgAunzm+oj8368XAU5JcAZwNPGehfyjJcUk2Jdm0ZcuWnShXkiRpXJZrAPqxwN9U1YHAzwGnJbnJv11Vp1bVhqrasG7dumX60pIkSe0sJUxdCRw0c3xgf27WM4EzAKrqXGBvYP/lKFCSJGnMlhKmzgMOS3Jokr3oBphvnHfPV4CHAiT5cbowZT+eJEna5S0apqrqOuB44BzgErpZexclOTnJUf1tvwU8K8lngbcBT6+qWqmiJUmSxmJJGx1X1dl0A8tnz71w5vXFwAOXtzRJkqTxW1KY0tpyyEn/2LqEG3zpZY9qXYIkSSvKMKXJMGRKklaCe/NJkiQNYMuUNHG22EnSMLZMSZIkDWCYkiRJGsBuPklawFi6P8fU9TmWZwLjei6SYUqSpAEMmbKbT5IkaQDDlCRJ0gCGKUmSpAEcMyVJkpbdlMaS2TIlSZI0gGFKkiRpAMOUJEnSAIYpSZKkAQxTkiRJAximJEmSBjBMSZIkDWCYkiRJGsAwJUmSNIBhSpIkaQDDlCRJ0gCGKUmSpAEMU5IkSQMYpiRJkgYwTEmSJA1gmJIkSRrAMCVJkjSAYUqSJGkAw5QkSdIAhilJkqQBDFOSJEkDGKYkSZIGMExJkiQNsKQwleTIJJcm2ZzkpAWu/0mSC/qPf0vyzeUvVZIkaXz2WOyGJLsDpwAPB64Azkuysaounrunqn5z5v7nAPdagVolSZJGZyktU4cDm6vqsqq6BjgdOHo79x8LvG05ipMkSRq7pYSpA4DLZ46v6M/dRJI7AIcC/7yN68cl2ZRk05YtW3a0VkmSpNFZ7gHoxwDvqKrrF7pYVadW1Yaq2rBu3bpl/tKSJEmrbylh6krgoJnjA/tzCzkGu/gkSdKELCVMnQccluTQJHvRBaaN829KchfgNsC5y1uiJEnSeC0apqrqOuB44BzgEuCMqrooyclJjpq59Rjg9KqqlSlVkiRpfBZdGgGgqs4Gzp537oXzjl+8fGVJkiStDa6ALkmSNIBhSpIkaQDDlCRJ0gCGKUmSpAEMU5IkSQMYpiRJkgYwTEmSJA1gmJIkSRrAMCVJkjSAYUqSJGkAw5QkSdIAhilJkqQBDFOSJEkDGKYkSZIGMExJkiQNYJiSJEkawDAlSZI0gGFKkiRpAMOUJEnSAIYpSZKkAQxTkiRJAximJEmSBjBMSZIkDWCYkiRJGsAwJUmSNIBhSpIkaQDDlCRJ0gCGKUmSpAEMU5IkSQMYpiRJkgYwTEmSJA1gmJIkSRrAMCVJkjSAYUqSJGkAw5QkSdIASwpTSY5McmmSzUlO2sY9v5Dk4iQXJXnr8pYpSZI0TnssdkOS3YFTgIcDVwDnJdlYVRfP3HMY8HzggVX1jSQ/vFIFS5IkjclSWqYOBzZX1WVVdQ1wOnD0vHueBZxSVd8AqKr/Xt4yJUmSxmkpYeoA4PKZ4yv6c7N+DPixJB9L8okkRy70DyU5LsmmJJu2bNmycxVLkiSNyHINQN8DOAx4MHAs8FdJbj3/pqo6tao2VNWGdevWLdOXliRJamcpYepK4KCZ4wP7c7OuADZW1bVV9UXg3+jClSRJ0i5tKWHqPOCwJIcm2Qs4Btg4755/oGuVIsn+dN1+ly1jnZIkSaO0aJiqquuA44FzgEuAM6rqoiQnJzmqv+0c4KokFwMfBJ5XVVetVNGSJEljsejSCABVdTZw9rxzL5x5XcBz+w9JkqTJcAV0SZKkAQxTkiRJAximJEmSBjBMSZIkDWCYkiRJGsAwJUmSNIBhSpIkaQDDlCRJ0gCGKUmSpAEMU5IkSQMYpiRJkgYwTEmSJA1gmJIkSRrAMCVJkjSAYUqSJGkAw5QkSdIAhilJkqQBDFOSJEkDGKYkSZIGMExJkiQNYJiSJEkawDAlSZI0gGFKkiRpAMOUJEnSAIYpSZKkAQxTkiRJAximJEmSBjBMSZIkDWCYkiRJGsAwJUmSNIBhSpIkaQDDlCRJ0gCGKUmSpAEMU5IkSQMYpiRJkgYwTEmSJA2wpDCV5MgklybZnOSkBa4/PcmWJBf0H7+y/KVKkiSNzx6L3ZBkd+AU4OHAFcB5STZW1cXzbv27qjp+BWqUJEkaraW0TB0ObK6qy6rqGuB04OiVLUuSJGltWEqYOgC4fOb4iv7cfE9I8rkk70hy0EL/UJLjkmxKsmnLli07Ua4kSdK4LNcA9LOAQ6rq7sD7gTctdFNVnVpVG6pqw7p165bpS0uSJLWzlDB1JTDb0nRgf+4GVXVVVX2/P3w9cJ/lKU+SJGnclhKmzgMOS3Jokr2AY4CNszck+ZGZw6OAS5avREmSpPFadDZfVV2X5HjgHGB34A1VdVGSk4FNVbUROCHJUcB1wNeBp69gzZIkSaOxaJgCqKqzgbPnnXvhzOvnA89f3tIkSZLGzxXQJUmSBjBMSZIkDWCYkiRJGsAwJUmSNIBhSpIkaQDDlCRJ0gCGKUmSpAEMU5IkSQMYpiRJkgYwTEmSJA1gmJIkSRrAMCVJkjSAYUqSJGkAw5QkSdIAhilJkqQBDFOSJEkDGKYkSZIGMExJkiQNYJiSJEkawDAlSZI0gGFKkiRpAMOUJEnSAIYpSZKkAQxTkiRJAximJEmSBjBMSZIkDWCYkiRJGsAwJUmSNIBhSpIkaQDDlCRJ0gCGKUmSpAEMU5IkSQMYpiRJkgYwTEmSJA1gmJIkSRpgSWEqyZFJLk2yOclJ27nvCUkqyYblK1GSJGm8Fg1TSXYHTgEeCawHjk2yfoH79gVOBD653EVKkiSN1VJapg4HNlfVZVV1DXA6cPQC9/0+8HLgf5exPkmSpFFbSpg6ALh85viK/twNktwbOKiq/nF7/1CS45JsSrJpy5YtO1ysJEnS2AwegJ5kN+BVwG8tdm9VnVpVG6pqw7p164Z+aUmSpOaWEqauBA6aOT6wPzdnX+CuwIeSfAm4P7DRQeiSJGkKlhKmzgMOS3Jokr2AY4CNcxer6uqq2r+qDqmqQ4BPAEdV1aYVqViSJGlEFg1TVXUdcDxwDnAJcEZVXZTk5CRHrXSBkiRJY7bHUm6qqrOBs+ede+E27n3w8LIkSZLWBldAlyRJGsAwJUmSNIBhSpIkaQDDlCRJ0gCGKUmSpAEMU5IkSQMYpiRJkgYwTEmSJA1gmJIkSRrAMCVJkjSAYUqSJGkAw5QkSdIAhilJkqQBDFOSJEkDGKYkSZIGMExJkiQNYJiSJEkawDAlSZI0gGFKkiRpAMOUJEnSAIYpSZKkAQxTkiRJAximJEmSBjBMSZIkDWCYkiRJGsAwJUmSNIBhSpIkaQDDlCRJ0gCGKUmSpAEMU5IkSQMYpiRJkgYwTEmSJA1gmJIkSRrAMCVJkjSAYUqSJGkAw5QkSdIASwpTSY5McmmSzUlOWuD6s5N8PskFST6aZP3ylypJkjQ+i4apJLsDpwCPBNYDxy4Qlt5aVXerqnsCrwBeteyVSpIkjdBSWqYOBzZX1WVVdQ1wOnD07A1V9a2Zw1sAtXwlSpIkjdceS7jnAODymeMrgPvNvynJrwPPBfYCHrLQP5TkOOA4gIMPPnhHa5UkSRqdZRuAXlWnVNWdgP8L/O427jm1qjZU1YZ169Yt15eWJElqZilh6krgoJnjA/tz23I68NghRUmSJK0VSwlT5wGHJTk0yV7AMcDG2RuSHDZz+Cjg35evREmSpPFadMxUVV2X5HjgHGB34A1VdVGSk4FNVbUROD7Jw4BrgW8AT1vJoiVJksZiKQPQqaqzgbPnnXvhzOsTl7kuSZKkNcEV0CVJkgYwTEmSJA1gmJIkSRrAMCVJkjSAYUqSJGkAw5QkSdIAhilJkqQBDFOSJEkDGKYkSZIGMExJkiQNYJiSJEkawDAlSZI0gGFKkiRpAMOUJEnSAIYpSZKkAQxTkiRJAximJEmSBjBMSZIkDWCYkiRJGsAwJUmSNIBhSpIkaQDDlCRJ0gCGKUmSpAEMU5IkSQMYpiRJkgYwTEmSJA1gmJIkSRrAMCVJkjSAYUqSJGkAw5QkSdIAhilJkqQBDFOSJEkDGKYkSZIGMExJkiQNYJiSJEkaYElhKsmRSS5NsjnJSQtcf26Si5N8LskHktxh+UuVJEkan0XDVJLdgVOARwLrgWOTrJ9322eADVV1d+AdwCuWu1BJkqQxWkrL1OHA5qq6rKquAU4Hjp69oao+WFXf7Q8/ARy4vGVKkiSN01LC1AHA5TPHV/TntuWZwHsWupDkuCSbkmzasmXL0quUJEkaqWUdgJ7kKcAG4I8Wul5Vp1bVhqrasG7duuX80pIkSU3ssYR7rgQOmjk+sD93I0keBrwA+Omq+v7ylCdJkjRuS2mZOg84LMmhSfYCjgE2zt6Q5F7A64Cjquq/l79MSZKkcVo0TFXVdcDxwDnAJcAZVXVRkpOTHNXf9kfALYG3J7kgycZt/HOSJEm7lKV081FVZwNnzzv3wpnXD1vmuiRJktYEV0CXJEkawDAlSZI0gGFKkiRpAMOUJEnSAIYpSZKkAQxTkiRJAximJEmSBjBMSZIkDWCYkiRJGsAwJUmSNIBhSpIkaQDDlCRJ0gCGKUmSpAEMU5IkSQMYpiRJkgYwTEmSJA1gmJIkSRrAMCVJkjSAYUqSJGkAw5QkSdIAhilJkqQBDFOSJEkDGKYkSZIGMExJkiQNYJiSJEkawDAlSZI0gGFKkiRpAMOUJEnSAIYpSZKkAQxTkiRJAximJEmSBjBMSZIkDWCYkiRJGsAwJUmSNIBhSpIkaYAlhakkRya5NMnmJCctcP2nknw6yXVJnrj8ZUqSJI3TomEqye7AKcAjgfXAsUnWz7vtK8DTgbcud4GSJEljtscS7jkc2FxVlwEkOR04Grh47oaq+lJ/7QcrUKMkSdJoLaWb7wDg8pnjK/pzOyzJcUk2Jdm0ZcuWnfknJEmSRmVVB6BX1alVtaGqNqxbt241v7QkSdKKWEqYuhI4aOb4wP6cJEnS5C0lTJ0HHJbk0CR7AccAG1e2LEmSpLVh0TBVVdcBxwPnAJcAZ1TVRUlOTnIUQJL7JrkC+HngdUkuWsmiJUmSxmIps/moqrOBs+ede+HM6/Pouv8kSZImxRXQJUmSBjBMSZIkDWCYkiRJGsAwJUmSNIBhSpIkaQDDlCRJ0gCGKUmSpAEMU5IkSQMYpiRJkgYwTEmSJA1gmJIkSRrAMCVJkjSAYUqSJGkAw5QkSdIAhilJkqQBDFOSJEkDGKYkSZIGMExJkiQNYJiSJEkawDAlSZI0gGFKkiRpAMOUJEnSAIYpSZKkAQxTkiRJAximJEmSBjBMSZIkDWCYkiRJGsAwJUmSNIBhSpIkaQDDlCRJ0gCGKUmSpAEMU5IkSQMYpiRJkgYwTEmSJA1gmJIkSRrAMCVJkjTAksJUkiOTXJpkc5KTFrh+syR/11//ZJJDlrtQSZKkMVo0TCXZHTgFeCSwHjg2yfp5tz0T+EZV/SjwJ8DLl7tQSZKkMVpKy9ThwOaquqyqrgFOB46ed8/RwJv61+8AHpoky1emJEnSOKWqtn9D8kTgyKr6lf74qcD9qur4mXsu7O+5oj/+Qn/P1+b9W8cBx/WHdwYuXa7/IQPtD3xt0bumx+dyUz6ThflcFuZzWZjP5aZ8Jgsb03O5Q1WtW+jCHqtZRVWdCpy6ml9zKZJsqqoNresYG5/LTflMFuZzWZjPZWE+l5vymSxsrTyXpXTzXQkcNHN8YH9uwXuS7AHcCrhqOQqUJEkas6WEqfOAw5IcmmQv4Bhg47x7NgJP618/EfjnWqz/UJIkaRewaDdfVV2X5HjgHGB34A1VdVGSk4FNVbUR+GvgtCSbga/TBa61ZHRdjyPhc7kpn8nCfC4L87kszOdyUz6Tha2J57LoAHRJkiRtmyugS5IkDWCYkiRJGsAwJUmSNMAkw1SS3ZIc0boOSZK09k12AHqSz1TVvVrXMTZJ9gF+Czi4qp6V5DDgzlX17saljUKSfarqu63rGIMkp1XVUxc7NxVJHr+961V15mrVMkZJfmqh81X1kdWuZWyS3IZurcYbZthX1afbVaQdtaoroI/MB5I8ATjTNbFu5I3A+cAD+uMrgbcDkw5TfUvm64FbAgcnuQfwq1X1a20ra+onZg/6TdHv06iWMXjMdq4VMOkwBTxv5vXedPu+ng88pE0545Dk94GnA1+g+z6h/zy555Lk22x9BjdRVfutYjk7ZMotU98GbgFcD3wPCFBj/j9rNcwt3T/bcpfks1V1j9a1tZTkk3QL0m6ceS4XVtVd21a2+pI8H/gd4ObAXCtdgGuAU6vq+a1q09qR5CDgT6vqCa1raSnJpcDdquqa1rWMRR8w/wM4je53y5OBH6mqFzYtbDsm2zJVVfu2rmGkrklyc/p3B0nuBHy/bUnjUFWXJ5k9dX2rWlqqqpcCL03yUoPTwpI8iq7lbu+5c1V1cruKRukK4MdbFzECFwK3Bv67dSEjctS8N/B/keSzgGFqbNL9VXwycGhV/X7/LulHqupTjUtr7UXAe4GDkvwt8EC6Juipu7zv6qskewInApc0rqmpqnp+kgOAO3DjsR6THgOT5C+BfYCfoesafiIw9d8rJHktW7twdgPuCTguCF4KfCbJhcy8ca2qo9qV1Nx3kjwZOJ3ue+ZY4DttS9q+KXfz/QXwA+AhVfXj/QDA91XVfRuX1lySHwLuT9e8+omq+lrjkppLsj/wauBhdM/lfcCJVTXZDb2TvIxu66iL2dpKVxP/I0CSz1XV3Wc+3xJ4T1X9ZOvaWkrytJnD64AvVdXHWtUzFkkuAl4HfJ7ubxIAVfXhZkU1luQQut+3D6QLUx8DfqOqvtSuqu2bbMsUcL+quneSzwBU1Tf6jZzVdU18g+77Y32Sybc29IHyya3rGJnH0c30tBv4xr7Xf/5ukv8HuAr4kYb1NNdPTvjZqvJn6Ka+W1WvaV3EmPSh6ejWdeyIKYepa/sf8LmxQeuYeVcwVUleDjwJuIitz6OASYepJK8A/oDuD+V7gbsDv1lVb2laWFuXAXvimLr53p3k1sAf0XVjFV1332RV1fVJ7pBkLwda38S/JHkpsJEbd/PZBTojyQvHPO5wyt18T6YLDfcG3kQ3ruF3q+rtTQtrrJ9ZcndbG24syQVVdc8kjwMeDTwX+MiUZzkmeSdwD+AD3PiPwAnNihqZJDcD9q6qq1vX0lqSN9MNON/IzPiXqnpVs6JGIMkHFzhdVTW5pRG2J8lXqurg1nVsy2Rbpqrqb5OcDzyUbgzMY6tq0gOKe7Y2LGzuZ+VRwNur6up5M/umaGP/oXn6yQqH0H/f9F3lb25aVHtf6D92A5xNvdUzq+qy2RNJ7tiqmJaSfGtbl+iWYhmtybVMJbnt9q5X1ddXq5YxsrVhYf1g68fSdfMdTjeV+d1Vdb+mhTXWL6NxcFVd2rqWsUhyGnAn4AJuPDB/0j9DWliST1fVveedO7+qJrcAbpKvAPetqv9a4NrlVXVQg7KWZIotU+fTjWEIcDDdQOvQ/XH8CnBou9JGwdaGBVTVSf24qav78R/fYY0NkFxuSR4DvBLYCzg0yT2Bk6c+mw/YAKx3Z4UbS3IWN13d+mpgE/C6qvrf1a+qnSR3oVuL7FbztiLaj5n1ySbmzXRLrdwkTAFvXeVadsjkWqbmJPkr4O+r6uz++JF0XX2/2rYyjdX8rhtg0l03fTf5Q4APTX1V+FlJ3g6cUFX/0bqWMUnyamAd8Lb+1JOAb9EFrP2mtqdjkqPpWruP4sZvYL8NnF5VH29SmHbKFFum5ty/qp41d1BV7+lbHiYpyRlV9QtJPs8CeyNV1d0blDUa2+q6oXsnNVXXLjB2bPIzYoH9gYuTfAoXYZx1xLx1/M5Kcl5V3bdfa2lSqupdwLuSPKCqzm1dz5j0rZhvA95VVaNerHPOlMPUV5P8LjA3tf3JwFcb1tPaif3nRzetYrzsurmpi5L8IrB7ksOAEwDfTcOLWxcwUrdMcnBVfQUgycF0G4dDt6/jVD2uD5Muu7LVK+laLl+a5Dy6ldDfPeau4Cl3892WbuuUn+pPfQR4ydQHoGthdt3cVJJ9gBcAP9ufOgf4gzH/wlstSW4HzLXCfKqqJr/vWpKfA/6SbkZf6Man/hrwIeBZVfWn7aprx2VXtq1fC/IhwLOAI6tqv8YlbdNkw9ScJPvSzbT5n9a1tJTk22zt3pvrt5kbqF9j/iZeDf1aMPek22PNrhttU5JfoFuw80N0Pz8/CTyvqt7Rsq4x6Nfdukt/eOls8E7y8Kp6f5vK2klyUVX9RJLXA++oqvcm+ezUw1Q/U/gxbF0P8t1V9Zy2VW3bZMNUkrvRjXeZWyrha8DTqurCdlVprJL89ELnJ75/1vuBn6+qb/bHt6EbOPuItpW11e9u//C51qh+d4V/mvofx8UstETAFLjsyk0lOYPuWbwX+Dvgw1U16vGYUx4z9TrguVX1QYAkDwZOBY5oWdQYJHkQcFhVvbHf4Hffqvpi67paqqoPJ7kD3XP5p76La/fWdTW2/1yQghv2t/zhlgWNxG7zuvWuoluoUts3yVVwXXZlQX8NHFtV1y9650hMOUzdYi5IAVTVh5LcomVBY5DkRXSDre8MvJFuDaG30O3ePVlJngUcR9eSeSfgALrxHw9tWVdjP5g3oPgOLDATdILem+QcbrwEwNkN61krJvW9M29tqblzs4dnrl4141JV5yQ5IskhrJGlaKYcpi5L8nvAaf3xU+i2Upm6xwH3otuglar6aj+ubOp+na7Z+ZMAVfXvtsLwO8BHk3yYrWODjmtbUntV9bwkT2DrG5BTq+rvW9akUXrMdq4VEw5Ta3EpmimHqWcAL6H7hi3gX/pzU3dNVVWSArC17gbfr6pr5t45JtmDib2TnpVkN+BWdAND79+f/o2q+lq7qsajqt4JvLN1HWOR5HC6iSznJVkPHAn869yiyb0vNSmukar65aXcl+RpVfWmla5nZNbcUjSTHYCuhSX5beAw4OHAS+kC5lur6rVNC2usH9PwTeCXgOfQTem+uKpe0LSwhpJsqqoNresYiyQfraoHzZsZCxOfEdsPHXgk3Zv39wP3Az5I9zvmnKr6w4bljd4UB+avxaVoJhumnIm0bUkeTrd2UOh+2U1uuvJ8fUvMM5l5LsDr19I7p+XWz0L6Gt1smxtWKXatNs3qd1W4J3Az4D+BA6vqW/3U909OfXeFxST5zNx2TVOxFpeimXI3nzORFtB36/1zVb0/yZ2BOyfZs6qubV1bS/203L/qP9R5Uv/512fOFXDHBrWMRpLT5u8zt9C5Cbmun5X13SRfqKpvAVTV95KMerr7SEzxDduLWxewo6YcppyJtLCPAD/Zt9S9l25H9yfRbbczOdvaq3DOlN9VV9WhrWsYqZ+YPejH192nUS1jcE2Sfarqu8w8hyS3wr0cl2JyS0b0S9GsqV0EphymXoAzkRaSqvpukmcCf1FVr0hyQeuiGprbq3Cu9WV29uekw3e/1tZzgYOr6rh+f747V9W7G5fWRJLn081wvHmSb82dptt37tRmhbX3U1X1fbihhXfOnsDT2pQ0DknuQrfMyidnd+FIcmRVvbc//FiT4hpaYBeB1yYZ9S4Ckx0zBdAvSDk3E+kTzkTq+ufpBlf/CfDMqrooyeer6m6NS2tqoXELUxwYOivJ3wHnA79UVXftw9XHq+qejUtrKslLq+r5revQuCU5ge5N2iV044NOrKp39dem/rtlze0iMPVVeW8GfB34FrA+yU8tcv8UnAg8H/j7PkjdkW7mzdQlyQNnDo7An587VdUrgGsB+m6cyXVJLOBTfRcWAEluneSxLQvSKD0LuE9VPRZ4MPB7SU7sr03952jN7SIw2W6+JC+nGwt0EVv77YtuzNBkVdVHmHkGVXUZcEK7ikbjmcAbZv5IfhPXJbumn5E1tybZnZiZeTNhL5pdpLOqvtkvD/APDWvS+Ow217VXVV/qtzR7Rz9+d+phaqFdBN7TsJ5FTTZM0W0seee5vnx1+ubU/0M3iHbvufNV9ZBmRY1AVZ0P3GMuTFXV1bPXJ7qw3ovoJikclORv6Vb8fnrTisZhoXfQU/5dq4X9V5J7VtUFAFX1P0keDbwBmPSwin4XgccDD+pPjX4XgcmOmUryHrp1pv5n0ZsnJMn76NYN+m3g2XQDRLdU1f9tWtjITXWMQ5Ifoht3GBx3CECSN9C1XJ7Sn/p14LZV9fRmRWl0khxIt2zEfy5w7YFVNbmB53OSHAr8R1X9b398c+B2VfWlpoVtx5TD1DuBewAf4MaLgk26SyvJ+VV1nySfm5v2n+S8qrrvYv/tlE1xYT24YbPWB9F19X107O8eV0O/Vn20rmsAABN9SURBVNvvAQ+jey7vB/6wqr6z3f9QEtDtrgAcUVXX9Md7AR8b89+hKTc9b+w/dGNzi3P+R5JHAV8FbtuwnrVicu9Kkvw58KNsHdfwq0keVlW/vp3/bJfXh6aTktzCACXtlD3mghRAvy/qXi0LWsxkw1RVvalvOjy4qi5tXc+I/EE/Lui3gNcC+wG/2bakNWGKA0YfAvz43JY6Sd5EN6Fj0vqZnq8HbgkcnOQewK9W1a+1rUxaM7YkOaqqNgIkOZpu66rRmmyYSvIY4JXAXsChSe4JnDzmvX9Ww8yCi1cDP9OyljVmiuMbNgMHA1/ujw/qz03dnwCPoG/5rqrPuuyKtEOeDfxtkj/rj68ARr0d06jXbVhhLwYOpxsoSj+jYtJ7igEkuWOSs5J8Lcl/J3lXv9bUpCW5XZK/7icukGR9v0o8AFV1fLvqmtkXuCTJh/qNSS8G9kuyMcmku9Cr6vJ5p65vUoi0BlXVF6rq/sB6YH1VHVFVX5i7nmR0K+dPtmUKuLaqrk5u1DvjPlHwVrpZSI/rj4+hGxNzv2YVjcPfAG+k24YI4N/oZj3+dauCRuCFrQsYqcv7rr5KsifdQriXNK5JWnO2M9v+RGBUS9FMOUxdlOQXgd37PcVOAD7euKYx2KeqTps5fkuS5zWrZjz2r6oz+v3XqKrrkky6taGqPry960nOraoHrFY9I/Js4NV0e65dCbyPrXs7ShpudGNUpxymnkPXyvB9utaYc4A/aFpRQ0nmZuy9J8lJwOl0M9SeBJzdrLDx+E6/ptLcYOv7040r07btvfgtu5YkuwOvrqont65F2oWNbvb0ZNeZWkyS11bVc1rXsVqSfJHuG3ShxF9VNelxU0nuTTe78a7AhcA64IlV9bmmhY3YhBcy/SjwkNmp3ZKWzxjX9Ztyy9RiHrj4LbuOqjp0KfcleXhVvX+l6xmbqvp0kp8G7kwXOC+tqmsX+c80TZcBH+sH4d+wzlRVvapdSdLak+RBdBPFLqyq981cGt3sacOUdtTL6VZ0noR+he+F/FgSqurMVS1obRnduIZV8oX+Yze6GY+SliDJp6rq8P71s+jGGv498KIk966ql8E4Z08bprSjpvYH8jHbuVbAZMNUktvRDbIGuLKq/mveLaNeF2alVNVLWtcgrVF7zrw+Dnh4VW1J8krgE8DL2pS1OMPUtk0tNCzVpAbZVdUvt65hbPoFbv8SuBXdbDWAA5N8E/i1qvo0QFVd2KjEJpL8aVX9RpKzWODnZOoLAktLsFuS29C16qaqtkC3RVOS69qWtn2TD1NJ9qmq7y5w6dWrXoxGq5/J9yJmNvWlWzH/qqaFtfE3dNujfHL2ZD/D8Y10G4hP0dySIq9sWoW0dt0KOJ+uMaOS/EhV/UeSWzLyBo7Jzuab3T+rqtw/C0hyF+BoZrpugI1VdcnMPWdW1bbGEe2ykrwf+Ajwlv7Uk4EHV9XD2lXVRpJ/r6rDtnFtc1X96GrXJGnXlWQf4HZV9cXWtWzLlMPUJ4En0oWFe/XnLqyqu7atrI0k/xc4lm59qSv60wfSrYB++tzAv6la6Hsjyeer6m6tamolyWuAOwFvBua2TTkI+CXgi2McHLoaknye7XSDV9XdV7EcSato0t18VXX5vO1kpryi9TOBn5g/3T/Jq4CLGPHAv1XyviTHAGf0x0+kW+h1cqrqhCSP5KatmKdU1ZQXeH10/3lutfO5br+nMLGxhtLUTLll6h3Aq4A/o9t37kRgQ1Ud07SwRpL8K/CIqvryvPN3AN5XVXduU9k4JPk2cAu27t+4G1vXEKqq2q9JYRqdhRYUnOoCptJUTLllyv2zbuw3gA8k+Xe2dt0cDPwoMMlum1lV5XpBS5Dk1Ko6rnUdjSXJA6vqY/3BEXThW9IuarItU7qpJLvRrTY723VzXlVNufvzBknuDhzCzJuQKS7aObOP400uAZ+tqgNXs56xSXIf4A10M5MCfAN4xtySEZJ2PZMNU0leQbex8feA9wJ3B36zqt6y3f9Qk5TkDXTfIxextauvquoZ7apqI8n1wJe58VTluX0dD6iqvZoUNjJJbgVQVW6ILe3iphymLqiqeyZ5HN3A0ecCH6mqqa6Ro+1IcnFVrW9dxxj0XcEPraqvLHDt8qo6qEFZzSV5SlW9JclzF7ru3nzSrmvK/fhzXTWPAt7uu0ct4twkhqnOnwK32ca1V6xmISNzi/7zvtv4kLSLmnLL1MuAx9J18x0O3Bp4d1Xdr2lhGqUkPw1sBP4T+D79Cr2uHbRtSR5eVZPZFFvSdE02TMENA2mvrqrr+xVW96uq/2xdl8YnyWa6ruDPs3XMFPOXktBWU10OIMkd6WYK359uLNm5dOMxL2tamKQVM9mlEZL80szr2UtvXv1qtAZsqaqNrYtYY0a9l9YKeitwCvC4/vgY4G1069lJ2gVNNkwB9515vTfwUODTGKa0sM8keStwFl03HzDNpRF2wFSbvfepqtNmjt+S5HnNqpG04iYbpqrqObPHSW5Nty+dtJCb04Won505V4BhSsCN1t96T5KT6H6fFPAkYMrb7Ei7vEmPmZqVZE/gwqlvmyItRb/A6/2r6uPbuefMqnr8KpbVVJIvsnW9rfmqqu64yiVJWiWTDVNJzmJrN8RuwHrgjKo6qV1VGpsk/6eqXpHktSzQbVVVJzQoaxQW2oNOi3OWo7TrmWw3H/DKmdfXAV+uqitaFaPRuqT/vKlpFeP0gSRPAM6sqb4r2zkvBwxT0i5ksi1Ti0lyblU9oHUdGp++i+uWVfWt1rW0lOTbdAtVXk+3Xtvc2lv7NS1s5GzRk3Y9U14BfTF7ty5A45HkrUn2S3IL4ELg4qnP0Kqqfatqt6ras6r2648NUovzHay0izFMbZu/8DRrfd8S9VjgPcChwFPbltRWOk9J8nv98UFJDm9dlyStNsOUtDR79jM+HwtsrKprMXD/OfAA4Bf74/+hW6xSvSQLrVv3pdWuQ9LKmvIA9MVMdfVmLex1dH8EPwt8JMkdgEmPmQLuV1X3TvIZgKr6RpK9WhfVSpL5K+QH+Jl+DTuq6qj+82SWi5CmYtJhKsnt6TY5LuC8efvyTboLRzdWVa8BXjN3nOQrwM/MHD+tqt7UoraGrk2yO30LXZJ1zOxbOEEHAhcDr2frelMbgD9uWZSklTfZbr4kvwJ8Cng88ETgE0meMXe9qi5sVZvGrzrXzZw6sVkx7bwG+Hvgh5P8IfBR4P9rW1JTG4DzgRfQbaD+IeB7VfXhqvpw08okrajJLo2Q5FLgiKq6qj/+IeDjroCunTHV6e5J7kK3r2WAD1TVJYv8J7u8JAcCfwL8F3BUVR3cuCRJK2zK3XxXAd+eOf52f07aGZN5VzKzBx3AfwNvm71WVV9f/arGo1/89+eTPArH1UmTMLkwleS5/cvNwCeTvIvuD+HRwOeaFaa1bkoTFs5n65igg4Fv9K9vDXyFbtmIyauqfwT+sXUdklbeFMdM7dt/fAH4B7a2KLwL+GKrorT2JPnlmcOPNStklVXVof2mvf8EPKaq9q+qHwIeDbyvbXWStPomO2ZKGirJV6Y8HibJ56vqboudk6Rd3eS6+eYk+SALjHOpqoc0KEcjlWRbXb8BbreatYzQV5P8LvCW/vjJwFcb1iNJTUw2TAG/PfN6b+AJwHXbuFfTdTvgEXTjgmYF+PjqlzMqxwIvolseAeAj/TlJmpTJhqmqOn/eqY8l+VSTYjRm7wZuWVUXzL+Q5EOrX8549LP2Tkyyb3dY/9O6JklqYbJjpuZN794NuA/wGteZkpYmyd2ANwNzP0tfA57mgreSpmayLVPceHr3dXQz+Z7ZtCJpbXkd8Nyq+iBAkgcDpwJHtCxKklbbZMNUVbkWjjTMLeaCFEBVfSjJLVoWJEktTDZMASQ5AjiEmedQVW9uVpC0tlyW5PeA0/rjpwCXNaxHkpqY8pip04A7ARcA1/enq6pOaFeVtHYkuQ3wEuBBdF3m/wK8pKrmz3yUpF3alMPUJcD6muoDkCRJy2KK28nMuRC4fesipLUqyfuT3Hrm+DZJzmlZkyS1MLkxU0nOouuS2Be4uF9b6vtz16vqqFa1SWvM/lX1zbmDqvpGkh9uWZAktTC5MAW8snUB0i7iB0kOrqqvACS5Awts0SRJu7rJhamq+vBS7ktyblU9YKXrkdawFwAfTfJhuvXafhI4rm1JkrT6JjsAfTFJPlNV92pdhzRmSfYH7t8ffqKqvtayHklqYXItUzvAlCkt7mbA1+l+l6xPQlV9pHFNkrSqDFOSdkqSlwNPAi4CftCfLsAwJWlSJhemktysqr6/+J1kxYuR1rbHAnde4s+TJO2yprjO1Llwwwro2/PUVahFWssuA/ZsXYQktTa5lilgryS/CByR5PHzL1bVmf3nC1e9Mmlt+S5wQZIPcOO12tySSdKkTDFMPRt4MnBr4DHzrhVw5qpXJK1NG/sPSZq0yS6NkOT4qvqzeeeWOp5KEpDk5sDBVXVp61okqZUpjpma84wFzp276lVIa1SSxwAXAO/tj++ZxJYqSZMzuW6+JLcHDgBunuRebJ21tx+wT7PCpLXnxcDhwIcAquqCJHdsWZAktTC5MAU8Ang6cCDwx2wNU98CfqdRTdJadG1VXZ3caBWRH2zrZknaVU0uTFXVm4A3JXlCVb1zW/cleVp/r6SFXdTPjN09yWHACcDHG9ckSatusgPQF5Pk01V179Z1SGOVZB+6zY5/tj91DvAHVfW/7aqSpNVnmNoGNzqWhkny2qp6Tus6JGmlTXk232JMmdIwD2xdgCStBsPUtrk3nyRJWtTkwlSS+yXZr3998yQvSXJWkpcnudXMrR9rVKIkSVpDJhemgDfQ7SkG8GrgVsDL+3NvnLupqo5f/dKkXYqtu5ImYXJLIwC7VdV1/esNMzP2PprkglZFSWtVkn2q6rsLXHr1qhcjSQ1MsWXqwiS/3L/+bJINAEl+DLi2XVnS2pLkiCQXA//aH98jyZ/PXa+qv2lVmyStpsktjdCPi3o18JPA14B7A5f3HydU1WcblietGUk+CTwR2Di3jEiSC6vqrm0rk6TVNbluvqq6Gnh6Pwj9ULpncEVV/VfbyqS1p6oun7edzPWtapGkViYXpuZU1bcAW6GknXd5kiOASrIncCJwSeOaJGnVTa6bT9LySLI/XZf5w+hm7r0POLGqrmpamCStMsOUJEnSAFOczSdpGSR5RZL9kuyZ5ANJtiR5Suu6JGm1GaYk7ayf7ccePhr4EvCjwPOaViRJDRimJO2suQksjwLe3s+UlaTJmexsPkmDvTvJvwLfA/7fJOuA/21ckyStOgegS9ppSW4LXF1V1yfZB9ivqv6zdV2StJpsmZK0U5L80szr2UtvXv1qJKkdw5SknXXfmdd7Aw8FPo1hStLE2M0naVkkuTVwelUd2boWSVpNzuaTtFy+Q7ffpSRNit18knZKkrOAuabt3YD1wBntKpKkNuzmk7RTkvz0zOF1wJer6opW9UhSK4YpSSsiyblV9YDWdUjSSnPMlKSVsnfrAiRpNRimJK0Um70lTYJhSpIkaQDDlKSVksVvkaS1z6URJO20JLcHDqfr0jtv3r58T21TlSStLlumJO2UJL8CfAp4PPBE4BNJnjF3vaoubFWbJK0ml0aQtFOSXAocUVVX9cc/BHy8qu7ctjJJWl22TEnaWVcB3545/nZ/TpImxTFTknZIkuf2LzcDn0zyLroxU0cDn2tWmCQ1YpiStKP27T9/of+Y864GtUhSc46ZkiRJGsCWKUk7JckHWWCV86p6SINyJKkZw5SknfXbM6/3Bp4AXNeoFklqxm4+Scsmyaeq6vDWdUjSarJlStJOSXLbmcPdgPsAt2pUjiQ1Y5iStLPOpxszFbruvS8Cz2xakSQ1YDefJEnSALZMSdppSY4ADmHmd0lVvblZQZLUgGFK0k5JchpwJ+AC4Pr+dAGGKUmTYjefpJ2S5BJgfflLRNLEudGxpJ11IXD71kVIUmt280naIUnOouvO2xe4OMmngO/PXa+qo1rVJkktGKYk7ahXti5AksbEMVOSVkSSc6vqAa3rkKSV5pgpSStl79YFSNJqMExJWik2e0uaBMOUJEnSAIYpSTskyc2WeuuKFiJJI2GYkrSjzoUbVkDfnqeuQi2S1JxLI0jaUXsl+UXgiCSPn3+xqs7sP1+46pVJUgOGKUk76tnAk4FbA4+Zd62AM1e9IklqyHWmJO2UJMdX1Z/NO3ezqvr+tv4bSdoVOWZK0s56xgLnzl31KiSpMbv5JO2QJLcHDgBunuRebJ21tx+wT7PCJKkRw5SkHfUI4OnAgcAfszVMfQv4nUY1SVIzjpmStFOSPKGq3rmd60+rqjetZk2S1IJhStKKSPLpqrp36zokaaU5AF3SSnEFdEmTYJiStFJs9pY0CYYpSSvFlilJk2CYkrRDkpyQ5KAl3PqxFS9GkkbAAeiSdkiSq4HvAF8A3ga8vaq2tK1KktqxZUrSjrqMbo2p3wfuA1yc5L1JnpZk37alSdLqs2VK0g6Zv+RBkj2BRwLHAg+rqnXNipOkBgxTknZIks9U1b22cW2fqvruatckSS0ZpiTtkCQ/VlX/1roOSRoLw5QkSdIADkCXJEkawDAlSZI0gGFKkiRpAMOUJEnSAP8/bnsPzk/YAs4AAAAASUVORK5CYII=",
            "text/plain": [
              "<Figure size 720x504 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          },
          "output_type": "display_data"
        }
      ],
      "source": [
        "# sort model results by f1-score\n",
        "all_model_results.sort_values('f1', ascending=False)['f1'].plot(kind='bar', figsize=(10, 7))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LxvOyH5uR5g5"
      },
      "source": [
        "## Tensorboard Logs"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "OQ5dVBqNR9rU"
      },
      "outputs": [],
      "source": [
        "# !tensorboard dev upload --logdir ./model_logs/ \\\n",
        "#   --name \"NLP Modelling Experiments\" \\\n",
        "#   --description \"Comparing multiple different types of model architecture on the Kaggle Tweets text classification dataset\" \\\n",
        "#   --one_shot"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zTKcpzjrU9Y-"
      },
      "source": [
        "https://tensorboard.dev/experiment/HMB5GQ5MSKeS0nw6ZSoGsw/#scalars - tensorboard link"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "JTe-pg27U_IR"
      },
      "outputs": [],
      "source": [
        "# See Previous experiments\n",
        "# !tensorboard dev list"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "JZPEfme5bCXS"
      },
      "outputs": [],
      "source": [
        "#!tensorboard dev delete --experiment_id <experiment_id>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "y8jBVyhgowV7"
      },
      "source": [
        "## Saving and Loading the Best performing Model\n",
        "\n",
        "Two formats to save the model in tensorflow\n",
        "\n",
        "1. The HDF5 Model\n",
        "2. The SavedModel Format"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "jT81uZeFozPm"
      },
      "outputs": [],
      "source": [
        "model_6.save(\"model_6.h5\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "d2hzmq02r5jE"
      },
      "outputs": [],
      "source": [
        "import tensorflow_hub as hub\n",
        "loaded_model_6 = tf.keras.models.load_model(\"model_6.h5\", \n",
        "                                            custom_objects={\"KerasLayer\": hub.KerasLayer})"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fhIN1u1bsn3X",
        "outputId": "a4f706c3-c244-4eb2-b779-e8b123498475"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "24/24 [==============================] - 1s 11ms/step - loss: 0.4274 - accuracy: 0.8176\n"
          ]
        },
        {
          "data": {
            "text/plain": [
              "[0.4274255037307739, 0.817585289478302]"
            ]
          },
          "execution_count": 113,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "loaded_model_6.evaluate(test_sentences, test_labels)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "RuRU_AfXtY5F",
        "outputId": "36bf84f2-7ebe-47ad-e8f1-e84183ee234b"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:absl:Function `_wrapped_model` contains input name(s) USE_input with unsupported characters which will be renamed to use_input in the SavedModel.\n"
          ]
        }
      ],
      "source": [
        "model_6.save(\"model_6_SavedModel_format\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "NRHHBvMGuZyY"
      },
      "outputs": [],
      "source": [
        "loaded_model_model_6_savedModel_format = tf.keras.models.load_model(\"model_6_SavedModel_format\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zDXOi4gSu1gF",
        "outputId": "6e5fa57f-a38d-4a90-bcbe-9f071a3337c3"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "24/24 [==============================] - 1s 10ms/step - loss: 0.4274 - accuracy: 0.8176\n"
          ]
        },
        {
          "data": {
            "text/plain": [
              "[0.4274255037307739, 0.817585289478302]"
            ]
          },
          "execution_count": 116,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "loaded_model_model_6_savedModel_format.evaluate(test_sentences, test_labels)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "x-fFzXmMu6Ka"
      },
      "source": [
        "## Finding the Most Wrong Predictions"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2X4Ycx0_wRbS",
        "outputId": "63ac01c8-8840-4c94-b0b9-98964bb467d2"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "--2022-10-01 14:48:43--  https://storage.googleapis.com/ztm_tf_course/08_model_6_USE_feature_extractor.zip\n",
            "Resolving storage.googleapis.com (storage.googleapis.com)... 172.217.203.128, 172.253.123.128, 142.251.107.128, ...\n",
            "Connecting to storage.googleapis.com (storage.googleapis.com)|172.217.203.128|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 960779165 (916M) [application/zip]\n",
            "Saving to: ‘08_model_6_USE_feature_extractor.zip’\n",
            "\n",
            "08_model_6_USE_feat 100%[===================>] 916.27M  49.5MB/s    in 21s     \n",
            "\n",
            "2022-10-01 14:49:04 (44.0 MB/s) - ‘08_model_6_USE_feature_extractor.zip’ saved [960779165/960779165]\n",
            "\n",
            "Archive:  08_model_6_USE_feature_extractor.zip\n",
            "   creating: 08_model_6_USE_feature_extractor/\n",
            "   creating: 08_model_6_USE_feature_extractor/assets/\n",
            "   creating: 08_model_6_USE_feature_extractor/variables/\n",
            "  inflating: 08_model_6_USE_feature_extractor/variables/variables.data-00000-of-00001  \n",
            "  inflating: 08_model_6_USE_feature_extractor/variables/variables.index  \n",
            "  inflating: 08_model_6_USE_feature_extractor/saved_model.pb  \n"
          ]
        }
      ],
      "source": [
        "!wget https://storage.googleapis.com/ztm_tf_course/08_model_6_USE_feature_extractor.zip\n",
        "!unzip 08_model_6_USE_feature_extractor.zip"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "y9Xk2UpsyPCd",
        "outputId": "23077c74-7019-4dba-85c8-3a3594eafebc"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:SavedModel saved prior to TF 2.5 detected when loading Keras model. Please ensure that you are saving the model with model.save() or tf.keras.models.save_model(), *NOT* tf.saved_model.save(). To confirm, there should be a file named \"keras_metadata.pb\" in the SavedModel directory.\n"
          ]
        }
      ],
      "source": [
        "model_6_pretrained = tf.keras.models.load_model(\"08_model_6_USE_feature_extractor\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FbaPS8OqyxQt",
        "outputId": "6d5bbcd2-b1a2-405c-8717-24790315c806"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "24/24 [==============================] - 1s 10ms/step - loss: 0.4272 - accuracy: 0.8163\n"
          ]
        },
        {
          "data": {
            "text/plain": [
              "[0.42723122239112854, 0.8162729740142822]"
            ]
          },
          "execution_count": 119,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "model_6_pretrained.evaluate(test_sentences, test_labels)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "01KoHVVkzUWT",
        "outputId": "7b8721e1-53cf-49a5-9fba-987f80892bcd"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(10,), dtype=float32, numpy=array([0., 1., 1., 0., 1., 1., 1., 1., 1., 0.], dtype=float32)>"
            ]
          },
          "execution_count": 120,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# Make Predictions with the loaded model from GS\n",
        "model_6_pretrained_pred_probs = model_6_pretrained.predict(test_sentences)\n",
        "model_6_pretrained_preds = tf.squeeze(tf.round(model_6_pretrained_pred_probs))\n",
        "model_6_pretrained_preds[:10]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "D_KLVL4Q0yph",
        "outputId": "55223e8f-87c2-4ce0-9c66-13ec14131518"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-989604fb-b3d1-4bb6-9d9c-a5ac8f1ecaf2\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>target</th>\n",
              "      <th>pred</th>\n",
              "      <th>pred_prob</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>DFR EP016 Monthly Meltdown - On Dnbheaven 2015...</td>\n",
              "      <td>0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.159757</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>FedEx no longer to transport bioterror germs i...</td>\n",
              "      <td>0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.747162</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Gunmen kill four in El Salvador bus attack: Su...</td>\n",
              "      <td>1</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.988749</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>@camilacabello97 Internally and externally scr...</td>\n",
              "      <td>1</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.196229</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Radiation emergency #preparedness starts with ...</td>\n",
              "      <td>1</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.707808</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-989604fb-b3d1-4bb6-9d9c-a5ac8f1ecaf2')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-989604fb-b3d1-4bb6-9d9c-a5ac8f1ecaf2 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-989604fb-b3d1-4bb6-9d9c-a5ac8f1ecaf2');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "                                                text  target  pred  pred_prob\n",
              "0  DFR EP016 Monthly Meltdown - On Dnbheaven 2015...       0   0.0   0.159757\n",
              "1  FedEx no longer to transport bioterror germs i...       0   1.0   0.747162\n",
              "2  Gunmen kill four in El Salvador bus attack: Su...       1   1.0   0.988749\n",
              "3  @camilacabello97 Internally and externally scr...       1   0.0   0.196229\n",
              "4  Radiation emergency #preparedness starts with ...       1   1.0   0.707808"
            ]
          },
          "execution_count": 121,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "val_df = pd.DataFrame({\"text\": test_sentences,\n",
        "                       \"target\": test_labels,\n",
        "                       \"pred\": model_6_pretrained_preds,\n",
        "                       \"pred_prob\": tf.squeeze(model_6_pretrained_pred_probs)})\n",
        "val_df.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "oof7ePzr1l_d",
        "outputId": "8489a976-3caa-4735-b9f0-4e283e3f583d"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-a075388f-a1c2-43c3-87ae-4b5b6f15790c\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>target</th>\n",
              "      <th>pred</th>\n",
              "      <th>pred_prob</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>31</th>\n",
              "      <td>? High Skies - Burning Buildings ? http://t.co...</td>\n",
              "      <td>0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.910196</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>759</th>\n",
              "      <td>FedEx will no longer transport bioterror patho...</td>\n",
              "      <td>0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.876982</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>628</th>\n",
              "      <td>@noah_anyname That's where the concentration c...</td>\n",
              "      <td>0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.852300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>209</th>\n",
              "      <td>Ashes 2015: AustraliaÛªs collapse at Trent Br...</td>\n",
              "      <td>0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.835454</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>251</th>\n",
              "      <td>@AshGhebranious civil rights continued in the ...</td>\n",
              "      <td>0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.827213</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-a075388f-a1c2-43c3-87ae-4b5b6f15790c')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-a075388f-a1c2-43c3-87ae-4b5b6f15790c button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-a075388f-a1c2-43c3-87ae-4b5b6f15790c');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "                                                  text  target  pred  \\\n",
              "31   ? High Skies - Burning Buildings ? http://t.co...       0   1.0   \n",
              "759  FedEx will no longer transport bioterror patho...       0   1.0   \n",
              "628  @noah_anyname That's where the concentration c...       0   1.0   \n",
              "209  Ashes 2015: AustraliaÛªs collapse at Trent Br...       0   1.0   \n",
              "251  @AshGhebranious civil rights continued in the ...       0   1.0   \n",
              "\n",
              "     pred_prob  \n",
              "31    0.910196  \n",
              "759   0.876982  \n",
              "628   0.852300  \n",
              "209   0.835454  \n",
              "251   0.827213  "
            ]
          },
          "execution_count": 122,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "most_wrong = val_df[val_df['pred'] != val_df['target']].sort_values(\"pred_prob\", ascending=False)\n",
        "most_wrong.head() # False Positives"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "TleIAKCt3zER",
        "outputId": "a4f71297-52d7-4e1a-8fbe-08563152d310"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-dc2434f5-5a94-4f25-a7f9-ad151645931c\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>target</th>\n",
              "      <th>pred</th>\n",
              "      <th>pred_prob</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>411</th>\n",
              "      <td>@SoonerMagic_ I mean I'm a fan but I don't nee...</td>\n",
              "      <td>1</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.043918</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>233</th>\n",
              "      <td>I get to smoke my shit in peace</td>\n",
              "      <td>1</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.042087</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>38</th>\n",
              "      <td>Why are you deluged with low self-image? Take ...</td>\n",
              "      <td>1</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.038998</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>244</th>\n",
              "      <td>Reddit Will Now QuarantineÛ_ http://t.co/pkUA...</td>\n",
              "      <td>1</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.038949</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>23</th>\n",
              "      <td>Ron &amp;amp; Fez - Dave's High School Crush https...</td>\n",
              "      <td>1</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.037186</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-dc2434f5-5a94-4f25-a7f9-ad151645931c')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-dc2434f5-5a94-4f25-a7f9-ad151645931c button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-dc2434f5-5a94-4f25-a7f9-ad151645931c');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "                                                  text  target  pred  \\\n",
              "411  @SoonerMagic_ I mean I'm a fan but I don't nee...       1   0.0   \n",
              "233                    I get to smoke my shit in peace       1   0.0   \n",
              "38   Why are you deluged with low self-image? Take ...       1   0.0   \n",
              "244  Reddit Will Now QuarantineÛ_ http://t.co/pkUA...       1   0.0   \n",
              "23   Ron &amp; Fez - Dave's High School Crush https...       1   0.0   \n",
              "\n",
              "     pred_prob  \n",
              "411   0.043918  \n",
              "233   0.042087  \n",
              "38    0.038998  \n",
              "244   0.038949  \n",
              "23    0.037186  "
            ]
          },
          "execution_count": 123,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "most_wrong.tail() # False negatives"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gQGFASO86in-",
        "outputId": "5493d81b-f24f-4e96-a79e-6d6aa2a2d5dd"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Target: 0, Pred: 1.0, Prob: 0.9101957678794861\n",
            "Text:\n",
            "? High Skies - Burning Buildings ? http://t.co/uVq41i3Kx2 #nowplaying\n",
            "\n",
            "---\n",
            "\n",
            "Target: 0, Pred: 1.0, Prob: 0.8769820928573608\n",
            "Text:\n",
            "FedEx will no longer transport bioterror pathogens in wake of anthrax lab mishaps http://t.co/lHpgxc4b8J\n",
            "\n",
            "---\n",
            "\n",
            "Target: 0, Pred: 1.0, Prob: 0.8523001074790955\n",
            "Text:\n",
            "@noah_anyname That's where the concentration camps and mass murder come in. \n",
            " \n",
            "EVERY. FUCKING. TIME.\n",
            "\n",
            "---\n",
            "\n",
            "Target: 0, Pred: 1.0, Prob: 0.8354544043540955\n",
            "Text:\n",
            "Ashes 2015: AustraliaÛªs collapse at Trent Bridge among worst in history: England bundled out Australia for 60 ... http://t.co/t5TrhjUAU0\n",
            "\n",
            "---\n",
            "\n",
            "Target: 0, Pred: 1.0, Prob: 0.8272132277488708\n",
            "Text:\n",
            "@AshGhebranious civil rights continued in the 60s. And what about trans-generational trauma? if anything we should listen to the Americans.\n",
            "\n",
            "---\n",
            "\n",
            "Target: 0, Pred: 1.0, Prob: 0.8148158192634583\n",
            "Text:\n",
            "@SonofLiberty357 all illuminated by the brightly burning buildings all around the town!\n",
            "\n",
            "---\n",
            "\n",
            "Target: 0, Pred: 1.0, Prob: 0.8108396530151367\n",
            "Text:\n",
            "[55436] 1950 LIONEL TRAINS SMOKE LOCOMOTIVES WITH MAGNE-TRACTION INSTRUCTIONS http://t.co/xEZBs3sq0y http://t.co/C2x0QoKGlY\n",
            "\n",
            "---\n",
            "\n",
            "Target: 0, Pred: 1.0, Prob: 0.80312180519104\n",
            "Text:\n",
            "@madonnamking RSPCA site multiple 7 story high rise buildings next to low density character residential in an area that floods\n",
            "\n",
            "---\n",
            "\n",
            "Target: 0, Pred: 1.0, Prob: 0.7669008374214172\n",
            "Text:\n",
            "@freefromwolves GodsLove &amp; #thankU brother Danny for RT of NEW VIDEO http://t.co/cybKsXHF7d The Coming Apocalyptic US Earthquake &amp; Tsunami\n",
            "\n",
            "---\n",
            "\n",
            "Target: 0, Pred: 1.0, Prob: 0.7666252851486206\n",
            "Text:\n",
            "Air Group is here to the rescue! We have 24/7 Emergency Service! Learn more about it here - http://t.co/9lyx7zMtHE http://t.co/5PbC96rTMJ\n",
            "\n",
            "---\n",
            "\n"
          ]
        }
      ],
      "source": [
        "# Check the False positives (model predicted 1 when should've been 0)\n",
        "for row in most_wrong[:10].itertuples():\n",
        "  _, text, target, pred, pred_prob = row\n",
        "  print(f\"Target: {target}, Pred: {pred}, Prob: {pred_prob}\")\n",
        "  print(f\"Text:\\n{text}\\n\")\n",
        "  print(\"---\\n\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "87IlWuG99poq",
        "outputId": "e79d1290-79b0-42b9-b531-1c394861ac5b"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Target: 1, Pred: 0.0, Prob: 0.06730346381664276\n",
            "Text:\n",
            "@DavidVonderhaar At least you were sincere ??\n",
            "\n",
            "---\n",
            "\n",
            "Target: 1, Pred: 0.0, Prob: 0.05507579818367958\n",
            "Text:\n",
            "@willienelson We need help! Horses will die!Please RT &amp; sign petition!Take a stand &amp; be a voice for them! #gilbert23 https://t.co/e8dl1lNCVu\n",
            "\n",
            "---\n",
            "\n",
            "Target: 1, Pred: 0.0, Prob: 0.05460337549448013\n",
            "Text:\n",
            "Lucas Duda is Ghost Rider. Not the Nic Cage version but an actual 'engulfed in flames' badass. #Mets\n",
            "\n",
            "---\n",
            "\n",
            "Target: 1, Pred: 0.0, Prob: 0.054597001522779465\n",
            "Text:\n",
            "going to redo my nails and watch behind the scenes of desolation of smaug ayyy\n",
            "\n",
            "---\n",
            "\n",
            "Target: 1, Pred: 0.0, Prob: 0.04963727295398712\n",
            "Text:\n",
            "You can never escape me. Bullets don't harm me. Nothing harms me. But I know pain. I know pain. Sometimes I share it. With someone like you.\n",
            "\n",
            "---\n",
            "\n",
            "Target: 1, Pred: 0.0, Prob: 0.043918490409851074\n",
            "Text:\n",
            "@SoonerMagic_ I mean I'm a fan but I don't need a girl sounding off like a damn siren\n",
            "\n",
            "---\n",
            "\n",
            "Target: 1, Pred: 0.0, Prob: 0.042086850851774216\n",
            "Text:\n",
            "I get to smoke my shit in peace\n",
            "\n",
            "---\n",
            "\n",
            "Target: 1, Pred: 0.0, Prob: 0.038997918367385864\n",
            "Text:\n",
            "Why are you deluged with low self-image? Take the quiz: http://t.co/XsPqdOrIqj http://t.co/CQYvFR4UCy\n",
            "\n",
            "---\n",
            "\n",
            "Target: 1, Pred: 0.0, Prob: 0.038949452340602875\n",
            "Text:\n",
            "Reddit Will Now QuarantineÛ_ http://t.co/pkUAMXw6pm #onlinecommunities #reddit #amageddon #freespeech #Business http://t.co/PAWvNJ4sAP\n",
            "\n",
            "---\n",
            "\n",
            "Target: 1, Pred: 0.0, Prob: 0.03718579187989235\n",
            "Text:\n",
            "Ron &amp; Fez - Dave's High School Crush https://t.co/aN3W16c8F6 via @YouTube\n",
            "\n",
            "---\n",
            "\n"
          ]
        }
      ],
      "source": [
        "# Check the False positives (model predicted 1 when should've been 0)\n",
        "for row in most_wrong[-10:].itertuples():\n",
        "  _, text, target, pred, pred_prob = row\n",
        "  print(f\"Target: {target}, Pred: {pred}, Prob: {pred_prob}\")\n",
        "  print(f\"Text:\\n{text}\\n\")\n",
        "  print(\"---\\n\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "x32-6KXRBLzN"
      },
      "source": [
        "## Making Predictions on a Test Dataset"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5saxl9RzBOis",
        "outputId": "8281d7c9-9062-449c-e536-96802c8b87a3"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "['Just happened a terrible car crash',\n",
              " 'Heard about #earthquake is different cities, stay safe everyone.',\n",
              " 'there is a forest fire at spot pond, geese are fleeing across the street, I cannot save them all',\n",
              " 'Apocalypse lighting. #Spokane #wildfires',\n",
              " 'Typhoon Soudelor kills 28 in China and Taiwan',\n",
              " \"We're shaking...It's an earthquake\",\n",
              " \"They'd probably still show more life than Arsenal did yesterday, eh? EH?\",\n",
              " 'Hey! How are you?',\n",
              " 'What a nice hat?',\n",
              " 'Fuck off!']"
            ]
          },
          "execution_count": 126,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# make predictions on test data and visualize them\n",
        "test_sentences = test_df['text'].to_list()\n",
        "test_sentences[:10]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "OwbVuGFiBhKb",
        "outputId": "92d7e87c-9e3e-499c-fabd-82f9bf4e7466"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Pred: 1, Prob: 0.9500387907028198\n",
            "Text:\n",
            "VIDEO: 'We're picking up bodies from water': Rescuers are searching for hundreds of migrants in the Mediterran... http://t.co/xJm6Snk1G0\n",
            "\n",
            "------\n",
            "\n",
            "Pred: 0, Prob: 0.42041850090026855\n",
            "Text:\n",
            "Finnish Nuclear Plant to Move Ahead After Financing Secured http://t.co/S9Jhcf3lD7 @JukkaOksaharju @ollirehn @juhasipila\n",
            "\n",
            "------\n",
            "\n",
            "Pred: 0, Prob: 0.09595008194446564\n",
            "Text:\n",
            "WHAT AN INCREDIBLE CHARACTER MY HEART IS BROKEN THAT HE IS ACTUALLY DEAD!! #RIPROSS WE WILL MISS YOU!  https://t.co/LgqmdnaVDf\n",
            "\n",
            "------\n",
            "\n",
            "Pred: 0, Prob: 0.017011698335409164\n",
            "Text:\n",
            "Just want someone to smoke a blunt &amp; talk about life with ??\n",
            "\n",
            "------\n",
            "\n",
            "Pred: 1, Prob: 0.9833021759986877\n",
            "Text:\n",
            "Sinjar Massacre Yazidis Blast Lack of Action Over Hostages http://t.co/fdU8aCnC2W #denver #billings #rapidcity #seattle #cheyenne #lasvegas\n",
            "\n",
            "------\n",
            "\n",
            "Pred: 1, Prob: 0.9168365597724915\n",
            "Text:\n",
            "Train derailed at Smithsonian Metro. Sidewalks outside L'Enfant mobbed to get on buses @wmata #nightmarecommute\n",
            "\n",
            "------\n",
            "\n",
            "Pred: 0, Prob: 0.18547047674655914\n",
            "Text:\n",
            "@edsheeran tf is innit\n",
            "\n",
            "------\n",
            "\n",
            "Pred: 0, Prob: 0.18304049968719482\n",
            "Text:\n",
            "@mishavelgos @MattBacal8 @ComplexMag this is so accurate I can't even speak haha... Comparing hazard to harden is so true\n",
            "\n",
            "------\n",
            "\n",
            "Pred: 0, Prob: 0.07691879570484161\n",
            "Text:\n",
            "I liked a @YouTube video from @teamtwiistz http://t.co/OCurjyDRcn FUTURISTIC HOUSE (Built To Survive The Apocalypse!) - Minecraft\n",
            "\n",
            "------\n",
            "\n",
            "Pred: 1, Prob: 0.9053698182106018\n",
            "Text:\n",
            "SH 29 Kaimai Ranges - Eastbound. Hazardous driving conditions. Due To A Truck Breakdown... http://t.co/0cuCMZCmaO via @AA_Traffic\n",
            "\n",
            "------\n",
            "\n"
          ]
        }
      ],
      "source": [
        "test_samples = random.sample(test_sentences, 10)\n",
        "for test_sample in test_samples:\n",
        "  pred_prob = tf.squeeze(model_6_pretrained.predict([test_sample]))# model expects a list as input\n",
        "  pred = tf.round(pred_prob)\n",
        "  print(f\"Pred: {int(pred)}, Prob: {pred_prob}\")\n",
        "  print(f\"Text:\\n{test_sample}\\n\")\n",
        "  print(\"------\\n\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0DFbGjA9DP8L"
      },
      "source": [
        "## The Speed/Score Tradeoff"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "LG4_1iWpFFnj"
      },
      "outputs": [],
      "source": [
        "# Function to measure the time taken for prediction\n",
        "import time\n",
        "\n",
        "def pred_timer(model, samples):\n",
        "  \"\"\"\n",
        "  Time how long a model takes to make predictions on samples.\n",
        "  \"\"\"\n",
        "  start_time = time.perf_counter() # get start time\n",
        "  model.predict(samples) # make predictions\n",
        "  end_time = time.perf_counter() # get finish time\n",
        "  total_time = end_time - start_time # calculate how long predictions took to make\n",
        "  time_per_pred = total_time / len(samples)\n",
        "  return total_time, time_per_pred"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YdHQtVVtDS9U",
        "outputId": "c97aab5d-54aa-4470-ab30-d78ae9c8bb2e"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "(0.9216799079999873, 0.00028246396199815734)"
            ]
          },
          "execution_count": 130,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# calculate TF hub sentence encoder time per pred\n",
        "model_6_total_pred_time, model_6_time_per_pred = pred_timer(model=model_6_pretrained,\n",
        "                                                            samples=test_sentences)\n",
        "model_6_total_pred_time, model_6_time_per_pred"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YSjHH0dYEapA",
        "outputId": "29f494b7-767c-48c6-80ea-410487cc4737"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "(0.1033885920001012, 3.168513392586614e-05)"
            ]
          },
          "execution_count": 131,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# Calculate the Baseline Model time per pred\n",
        "baseline_total_pred_time, baseline_time_per_pred = pred_timer(model_0, test_sentences)\n",
        "baseline_total_pred_time, baseline_time_per_pred"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "r23C4gZcFUeW",
        "outputId": "ff89cc0a-d722-4277-e638-7e60828d898a"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "{'accuracy': 81.62729658792651,\n",
              " 'precision': 0.818446310697231,\n",
              " 'recall': 0.8162729658792651,\n",
              " 'f1': 0.8148082644367335}"
            ]
          },
          "execution_count": 133,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "model_6_pretrained_results = calculate_results(y_true=test_labels,\n",
        "                                               y_pred=model_6_pretrained_preds)\n",
        "model_6_pretrained_results "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 458
        },
        "id": "c7wRnRqaFtbT",
        "outputId": "483d0812-25b2-4b08-a4a4-beaac47741c0"
      },
      "outputs": [
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAm4AAAG5CAYAAAA3e7gZAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dfbxVZZ338c9PQDHLVKS5U1CwDOXhyMMRRStRMywbtUYN0ia1MsfMuZ2JSSctc/Ieze5xRsPUupXGUnwqI7VkUhy1fDqEoqgoKgnoGBKoECjg7/5jr3PcHM8TwubsBZ/367VeZ+1rXde1rrXW2fJ1PZwVmYkkSZLq3xbdPQBJkiR1jcFNkiSpJAxukiRJJWFwkyRJKgmDmyRJUkkY3CRJkkrC4CZJ6yAi/jkiftzd46h3ETE2IhZUfZ4dEWPfQT8fiYg5G3RwUokZ3KQ6FBHzImJFRCyrmnYqll0REXMi4s2IOL6bh7pJax0+ADLz/2Tml7prTGWVmUMy867O6kVERsQHq9rdk5mDajo4qUQMblL9+uvMfHfV9EJR/ghwCvCHbhwbABHRc3Ncd9lsiH0VET02xFgkrR+Dm1QymTkpM+8AVnZWNyJ6R8RPI2JxRCyNiIci4q+KZTtExFUR8UJELImIm6vafTki5kbEnyNiavPZvmJZRsRXI+Jp4Omi7FMR8XCxjt9HREM74/lhRHy/VdkvI+IfivmdIuKmiFgUEc9FxGlV9c6JiBuL7XkVOD4iRkdEU0S8GhEvRcS/FXXfdqasOIv5sWK+zXat6m8D/BrYqfqsZzGOnxZ1BhT744SImF/sx5MjYu+ImFXsjx+06vfEiHiiqHt7ROzazr5q7vuk4hi9GBFfr1q+RUScERHPFMf3+ojYoVXbL0bE88CdbfQ/NiIWFJd+Xy72z7FVyycXx+u2iFgOHNjJ8dm6aLMkIh4H9u5g//co1vtMRLwWETMion9E3F1Uf6TY359tfSwjYs+IuKvYt7Mj4vBWY54UEbcW/T4QER9oa/9KpZWZTk5OdTYB84CPdVLnXuD4Tup8BfgV8C6gBzAK2LZYditwHbA90As4oCg/CHgZGAlsBVwC3F3VZwL/BewAbA2MAP4E7FOs4wvF+LdqYzwfBeYDUXzeHlgB7ETlfyRnAN8CtgR2A54FxhV1zwFWAUcWdbcG7gM+Xyx/N7BvMT8WWNDePm2vXRvjbaufc4CfFvMDiv1xGdAb+DiVQH0z8D5g52LfNO/bI4C5wJ5AT+As4PftrLu572uBbYBhwKKqbfh74H6gX3GcLgeubdX2P4u2W7ezbauBfyvaHwAsBwYVyycDrwD7F/v7XZ0cn/OBe4rfi/7AY9X7rtX+nwg8CgwCAtgL6FP1+/XBto4Bld/TucA/F2M4CHit1ZgXA6OL/fszYEp3f5+dnDbk5Bk3qX7dXJxVWFp9NmwdrQL6UPmHcE1mzsjMVyPi/cAngJMzc0lmrsrM/y7aHAtcmZl/yMzXgTOBMRExoKrff83MP2fmCuAk4PLMfKBYx0+A14F92xjPPVT+Yf5I8fko4L6sXAbeG+ibmedm5huZ+SzwI2B8Vfv7MvPmzHyzWPcq4IMRsWNmLsvM+9dhv7yTdu35l8xcmZnTqISfazPzT5m5sNjmEUW9k6nsuycyczXwf4Dh7Z11K3wnM5dn5qPAVcCEqr6+mZkLiuN0DnBUrH1Z9Jyi7YoO+j87M18vjv+twDFVy36Zmb/LzDepBMeOjs8xwHnF78V84OIO1vkl4KzMnJMVj2Tm4g7qN9uXStA+vxjDncAtVfsE4BeZ+WCxf38GDO9Cv1JpGNyk+nVkZm5XTEd2pUGs/TDDLsDVwO3AlOJy2/cioheVMyJ/zswlbXSzE/DH5g+ZuYzKWYydq+rMr5rfFfjHqpC5tOh/J1rJzASm8NY/tJ+j8o9rcz87ternn4G/ame9AF8EPgQ8GZXLwJ9qb99soHbtealqfkUbn99dzO8K/EfV9v2Zyhmn6n3bWvU2/5G39uuuwC+q+noCWEPH+6u1JZm5vJ3+W7fv7Pjs1MZY29MfeKaTsbVlJ2B+ESSr11O9//6nav4vvLXvpU2CN/dKm5DMbOsfqe8A3ynOmN0GzCl+7hAR22Xm0lb1X6DyjzTQcq9XH2Bh9aqq5udTOdNyXheHeS0wLSLOp3J59dNV/TyXmbt30DbX+pD5NDAhIrYAPgPcGBF9qJz1elfVNvQA+nbWrlWIedv6NoDmffWzTmu+pT/wZDG/C5Xj09zXiZn5u9YNqs6Odjb+7SNim6rt3oXKJc5mrY9zR8fnxWKss6v6as984AOt1tUVLwD9I2KLqvC2C/DUOvYjlZZn3KSSiYgtI6I3lTM1vaLyAEKb3+WIODAihhXB5VUqlwjfzMwXqdx4f2lEbB8RvSLio0Wza4ETImJ4RGxF5XLeA5k5r50h/Qg4OSL2iYptIuKwiHhPW5UzcyaVe+h+DNxeFRwfBF6LiG8UN7r3iIihEbF3W/0U23dcRPQt/hFv7udNKv+Q9y7G0YvKvWRbdaFday8BfSLive2NYR1dBpwZEUOKcbw3Io7upM3ZEfGuos0JVO5LbO7rvObLrBHRNyKOeAdj+k7xO/UR4FPADe3U6+z4XF9s2/YR0Q/4Wgfr/DHwLxGxe/E701AEbqjs893aafcAlbNo/1T8zo4F/prKWVxps2Bwk8pnGpXLb/sBVxTzH22n7v8CbqQS2p4A/pvK5VOAz1MJck9SuYH+fwNk5m+Bs4GbqJxF+QBr32e2lsxsAr4M/ABYQuXm8eM72YZrgI8VP5v7WUMlOAwHnuOtcNdRaDoUmB0Ry4D/AMZn5orMfIXKn0z5MZUzhcuBBZ21a2PbnqQSZJ8tLg++7fLvusjMXwAXULl0/SqVM06f6KTZf1PZp3cA3y/uo6MY91QqZy9fo/Kgwj7rOKT/oXLMXqByyfrkYpvbGntnx+c7VC5bPkfld/TqNrpp9m9Ugt40Kr+b/4/KwyZQuVfvJ8X+rr7fjsx8g0pQ+0Sx/kuBv21vzNKmqPnJLklSHSkudz4H9CputN/Q/Y+l8nRsvw3dt6Ta8YybJElSSRjcJEmSSsJLpZIkSSXhGTdJkqSS2Cz+jtuOO+6YAwYM6O5hSJIkdWrGjBkvZ2bftpZtFsFtwIABNDU1dfcwJEmSOhUR7b55xEulkiRJJWFwkyRJKgmDmyRJUklsFve4tWXVqlUsWLCAlStXdvdQtJnr3bs3/fr1o1evXt09FElSndtsg9uCBQt4z3vew4ABA4iI7h6ONlOZyeLFi1mwYAEDBw7s7uFIkurcZnupdOXKlfTp08fQpm4VEfTp08czv5KkLtlsgxtgaFNd8PdQktRVm3VwkyRJKhODWzeaN28eQ4cOrUnfd911F5/61KcAmDp1Kueff35N1iNJkjaezfbhhM3J4YcfzuGHH97dw5AkSeuppmfcIuLQiJgTEXMj4ow2lu8SEdMjYmZEzIqITxblfYryZRHxg1Zt7ir6fLiY3lfLbWh288yF7H/+nQw841b2P/9Obp65cIP0u3r1ao499lj23HNPjjrqKP7yl79w7rnnsvfeezN06FBOOukkMhOAiy++mMGDB9PQ0MD48eMBWL58OSeeeCKjR49mxIgR/PKXv3zbOiZPnsypp54KwPHHH89pp53Gfvvtx2677caNN97YUu/CCy9k7733pqGhgW9/+9sbZPskSdKGU7PgFhE9gEnAJ4DBwISIGNyq2lnA9Zk5AhgPXFqUrwTOBr7eTvfHZubwYvrThh/92m6euZAzf/4oC5euIIGFS1dw5s8f3SDhbc6cOZxyyik88cQTbLvttlx66aWceuqpPPTQQzz22GOsWLGCW265BYDzzz+fmTNnMmvWLC677DIAzjvvPA466CAefPBBpk+fzsSJE1m+fHmH63zxxRe59957ueWWWzjjjEqenjZtGk8//TQPPvggDz/8MDNmzODuu+9e7+2TJEkbTi3PuI0G5mbms5n5BjAFOKJVnQS2LebfC7wAkJnLM/NeKgGu2114+xxWrFqzVtmKVWu48PY56913//792X///QE47rjjuPfee5k+fTr77LMPw4YN484772T27NkANDQ0cOyxx/LTn/6Unj0rV7mnTZvG+eefz/Dhwxk7diwrV67k+eef73CdRx55JFtssQWDBw/mpZdeauln2rRpjBgxgpEjR/Lkk0/y9NNPr/f2SZKkDaeW97jtDMyv+rwA2KdVnXOAaRHxNWAb4GNd7PuqiFgD3AR8N5uvJVaJiJOAkwB22WWXdRt5Ky8sXbFO5eui9Z+CiAhOOeUUmpqa6N+/P+ecc07L3/i69dZbufvuu/nVr37Feeedx6OPPkpmctNNNzFo0KC1+mkOZG3ZaqutWuabd11mcuaZZ/KVr3xlvbdJkqRNyqzr4Y5z4ZUF8N5+cPC3oOGYbhlKdz9VOgGYnJn9gE8CV0dEZ2M6NjOHAR8pps+3VSkzr8jMxsxs7Nu373oNcqfttl6n8nXx/PPPc9999wFwzTXX8OEPfxiAHXfckWXLlrXcg/bmm28yf/58DjzwQC644AJeeeUVli1bxrhx47jkkktaAtjMmTPf0TjGjRvHlVdeybJlywBYuHAhf/pTza9CS5JU32ZdD786DV6ZD2Tl569Oq5R3g1oGt4VA/6rP/Yqyal8ErgfIzPuA3sCOHXWamQuLn68B11C5JFtTE8cNYutePdYq27pXDyaOG9ROi64bNGgQkyZNYs8992TJkiX83d/9HV/+8pcZOnQo48aNY++99wZgzZo1HHfccQwbNowRI0Zw2mmnsd1223H22WezatUqGhoaGDJkCGefffY7GsfHP/5xPve5zzFmzBiGDRvGUUcdxWuvvbbe2ydJUqndcS6sanWFbdWKSnk3iDauMm6YjiN6Ak8BB1MJbA8Bn8vM2VV1fg1cl5mTI2JP4A5g5+ZLnxFxPNCYmadW9bldZr4cEb2Aa4HfZuZlHY2lsbExm5qa1ip74okn2HPPPbu8PTfPXMiFt8/hhaUr2Gm7rZk4bhBHjti5y+2ljqzr76MkaSM5Zzsqt+S3FnDO0pqsMiJmZGZjW8tqdo9bZq6OiFOB24EewJWZOTsizgWaMnMq8I/AjyLidCp75fiq0DaPyoMLW0bEkcDHgT8CtxehrQfwW+BHtdqGakeO2NmgJknS5ua9/YrLpG2Ud4Oa/gHezLwNuK1V2beq5h8H9m+n7YB2uh21ocYnSZLUoYO/Vbmnrfpyaa+tK+XdoLsfTpAkSapfDcfAX18M7+0PROXnX1/cbU+V+sorSZKkjjQc021BrTXPuEmSJJWEwU2SJKkkDG6SJEklYXDrJkuXLuXSSy9t+Txx4kSGDBnCxIkT26x//PHHt7xFoasGDBjAyy+/vF7jXFf//u//zl/+8peNus7udNddd/GpT32qu4chSdpMGNy6atb1cNHQyh/iu2joer/qonVwu+KKK5g1axYXXnjh+o60W21uwW1drV69uruHIEkqMYNbV9TgPWVnnHEGzzzzDMOHD+eQQw5h2bJljBo1iuuuu67dNnfffTf77bcfu+22W8vZt9ZnfE499VQmT57c8vl73/sew4YNY/To0cydO7fdvm+44QaGDh3KXnvtxUc/+lGg8pqtiRMnsvfee9PQ0MDll1/ess6xY8dy1FFHsccee3DssceSmVx88cW88MILHHjggRx44IEATJs2jTFjxjBy5EiOPvrolnehDhgwgG9/+9uMHDmSYcOG8eSTTwKwbNkyTjjhBIYNG0ZDQwM33XRTh/20ZcaMGRxwwAGMGjWKcePG8eKLLwIwduxYvvGNbzB69Gg+9KEPcc8997Rs59e//nWGDh1KQ0MDl1xyCQB33HEHI0aMYNiwYZx44om8/vrrAPzmN79hjz32YOTIkfz85z9vWe/y5cs58cQTGT16NCNGjOCXv/wlAJMnT+bwww/noIMO4uCDD2533JIkdSozN/lp1KhR2drjjz/+trJ2/duQzG9v+/bp34Z0vY9WnnvuuRwy5K3222yzTYf1v/CFL+RRRx2Va9asydmzZ+cHPvCBzMycPn16HnbYYS31vvrVr+ZVV12VmZm77rprfve7383MzJ/85Cdr1Wtt6NChuWDBgszMXLJkSWZmXn755fkv//IvmZm5cuXKHDVqVD777LM5ffr03HbbbXP+/Pm5Zs2a3HffffOee+5pWeeiRYsyM3PRokX5kY98JJctW5aZmeeff35+5zvfaal38cUXZ2bmpEmT8otf/GJmZv7TP/1T/v3f/33LuP785z932E9rb7zxRo4ZMyb/9Kc/ZWbmlClT8oQTTsjMzAMOOCD/4R/+ITMzb7311jz44IMzM/PSSy/Nv/mbv8lVq1ZlZubixYtzxYoV2a9fv5wzZ05mZn7+85/Piy66qKX8qaeeyjfffDOPPvrolv165pln5tVXX92yD3ffffdctmxZXnXVVbnzzjvn4sWL293/6/T7KEnapFF5w1Sbmca/49YVryxYt/IaOfLII9liiy0YPHgwL730UpfaTJgwoeXn6aef3m69/fffn+OPP55jjjmGz3zmM0DlLNesWbNazu698sorPP3002y55ZaMHj2afv0qr/sYPnw48+bN48Mf/vBafd5///08/vjj7L9/5eUYb7zxBmPGjGlZ3ryeUaNGtZy5+u1vf8uUKVNa6my//fbccsstHfZTbc6cOTz22GMccsghQOVs2vvf//421zlv3ryWdZ588sn07Fn5Ouywww488sgjDBw4kA996EMAfOELX2DSpEmMHTuWgQMHsvvuuwNw3HHHccUVV7Tsr6lTp/L9738fgJUrV/L8888DcMghh7DDDju0u/8lSeoKg1tX1Ml7yrbaaquW+Uogh549e/Lmm2+2lK9cuXKtNhHR5nxrl112GQ888AC33noro0aNYsaMGWQml1xyCePGjVur7l133bXWWHr06NHmvVuZySGHHMK1117b4fa0176r/bSuO2TIEO677771Wuc7kZncdNNNDBo0aK3yBx54gG222WaDrkuStHnyHreuOPhblfeSVVvP95S95z3v4bXXXlvPgcGuu+7K448/zuuvv87SpUu544471lrefM/cdddd1+5ZKoBnnnmGffbZh3PPPZe+ffsyf/58xo0bxw9/+ENWrVoFwFNPPcXy5cs7HE/1du2777787ne/a7m3bvny5Tz11FMdtj/kkEOYNGlSy+clS5asUz+DBg1i0aJFLcFt1apVzJ49u9N1Xn755S1B7s9//jODBg1i3rx5Leu8+uqrOeCAA9hjjz2YN28ezzzzDMBaYXLcuHFccsklLaF65syZHa5XkqR1ZXDrihq8p6xPnz7sv//+DB06tN0/AdIV/fv355hjjmHo0KEcc8wxjBgxYq3lS5YsoaGhgf/4j//goosuarefiRMnMmzYMIYOHcp+++3HXnvtxZe+9CUGDx7MyJEjGTp0KF/5ylc6PUt10kknceihh3LggQfSt29fJk+ezIQJE2hoaGDMmDEtDyG056yzzmLJkiUtD0pMnz59nfrZcsstufHGG/nGN77BXnvtxfDhw/n973/f4Tq/9KUvscsuu9DQ0MBee+3FNddcQ+/evbnqqqs4+uijGTZsGFtssQUnn3wyvXv35oorruCwww5j5MiRvO9972vp5+yzz2bVqlU0NDQwZMgQzj777A7XK0nSuormswObssbGxmxqalqr7IknnmDPPffsphFJa/P3UZLULCJmZGZjW8s84yZJklQSPpxQZ8477zxuuOGGtcqOPvpovvnNb5ai/43p05/+NM8999xaZRdccMHbHqaQJGlTsVlfKt1jjz06fNJS2hgykyeffNJLpZIkwEulberduzeLFy9mcwiuql+ZyeLFi+ndu3d3D0WSVAKb7aXSfv36sWDBAhYtWtTdQ9Fmrnfv3i1/zFiSpI5stsGtV69eDBw4sLuHIUmS1GWb7aVSSZKksjG4SZIklYTBTZIkqSQMbpIkSSVhcJMkSSoJg5skSVJJGNwkSZJKwuAmSZJUEgY3SZKkkjC4SZIklYTBTZIkqSQMbpIkSSVhcJMkSSoJg5skSVJJGNwkSZJKwuAmSZJUEgY3SZKkkjC4SZIklYTBTZIkqSQMbpIkSSVhcJMkSSoJg5skSVJJGNwkSZJKwuAmSZJUEgY3SZKkkjC4SZIklYTBTZIkqSQMbpIkSSVhcJMkSSoJg5skSVJJGNwkSZJKwuAmSZJUEjUNbhFxaETMiYi5EXFGG8t3iYjpETEzImZFxCeL8j5F+bKI+EGrNqMi4tGiz4sjImq5DZIkSfWiZsEtInoAk4BPAIOBCRExuFW1s4DrM3MEMB64tChfCZwNfL2Nrn8IfBnYvZgO3fCjlyRJqj+1POM2Gpibmc9m5hvAFOCIVnUS2LaYfy/wAkBmLs/Me6kEuBYR8X5g28y8PzMT+E/gyBpugyRJUt2oZXDbGZhf9XlBUVbtHOC4iFgA3AZ8rQt9LuikTwAi4qSIaIqIpkWLFq3LuCVJkupSdz+cMAGYnJn9gE8CV0fEBhlTZl6RmY2Z2di3b98N0aUkSVK3qmVwWwj0r/rcryir9kXgeoDMvA/oDezYSZ/9OulTkiRpk1TL4PYQsHtEDIyILak8fDC1VZ3ngYMBImJPKsGt3euamfki8GpE7Fs8Tfq3wC9rMXhJkqR607NWHWfm6og4Fbgd6AFcmZmzI+JcoCkzpwL/CPwoIk6n8qDC8cVDB0TEPCoPLmwZEUcCH8/Mx4FTgMnA1sCvi0mSJGmTF0VO2qQ1NjZmU1NTdw9DkiSpUxExIzMb21rW3Q8nSJIkqYsMbpIkSSVhcJMkSSoJg5skSVJJGNwkSZJKwuAmSZJUEgY3SZKkkjC4SZIklYTBTZIkqSQMbpIkSSVhcJMkSSoJg5skSVJJGNwkSZJKwuAmSZJUEgY3SZKkkjC4SZIklYTBTZIkqSQMbpIkSSVhcJMkSSoJg5skSVJJGNwkSZJKwuAmSZJUEgY3SZKkkjC4SZIklYTBTZIkqSQMbpIkSSVhcJMkSSoJg5skSVJJGNwkSZJKwuAmSZJUEgY3SZKkkjC4SZIklYTBTZIkqSQMbpIkSSVhcJMkSSoJg5skSVJJGNwkSZJKwuAmSZJUEgY3SZKkkjC4SZIklYTBTZIkqSQMbpIkSSVhcJMkSSoJg5skSVJJGNwkSZJKwuAmSZJUEgY3SZKkkjC4SZIklYTBTZIkqSQMbpIkSSVhcJMkSSqJmga3iDg0IuZExNyIOKON5btExPSImBkRsyLik1XLzizazYmIcVXl8yLi0Yh4OCKaajl+SZKketKzVh1HRA9gEnAIsAB4KCKmZubjVdXOAq7PzB9GxGDgNmBAMT8eGALsBPw2Ij6UmWuKdgdm5su1GrskSVI9quUZt9HA3Mx8NjPfAKYAR7Sqk8C2xfx7gReK+SOAKZn5emY+B8wt+pMkSdps1TK47QzMr/q8oCirdg5wXEQsoHK27WtdaJvAtIiYEREntbfyiDgpIpoiomnRokXvfCskSZLqRHc/nDABmJyZ/YBPAldHRGdj+nBmjgQ+AXw1Ij7aVqXMvCIzGzOzsW/fvht21JIkSd2glsFtIdC/6nO/oqzaF4HrATLzPqA3sGNHbTOz+eefgF/gJVRJkrSZqGVwewjYPSIGRsSWVB42mNqqzvPAwQARsSeV4LaoqDc+IraKiIHA7sCDEbFNRLynqL8N8HHgsRpugyRJUt2o2VOlmbk6Ik4Fbgd6AFdm5uyIOBdoysypwD8CP4qI06ncu3Z8ZiYwOyKuBx4HVgNfzcw1EfFXwC8ionns12Tmb2q1DZIkSfUkKjlp09bY2JhNTf7JN0mSVP8iYkZmNra1rLsfTpAkSVIXGdwkSZJKwuAmSZJUEgY3SZKkkjC4SZIklYTBTZIkqSQMbpIkSSVhcJMkSSoJg5skSVJJGNwkSZJKwuAmSZJUEgY3SZKkkjC4SZIklYTBTZIkqSQMbpIkSSVhcJMkSSoJg5skSVJJGNwkSZJKwuAmSZJUEgY3SZKkkjC4SZIklYTBTZIkqSQMbpIkSSVhcJMkSSoJg5skSVJJGNwkSZJKwuAmSZJUEgY3SZKkkjC4SZIklYTBTZIkqSQMbpIkSSXRpeAWER+KiDsi4rHic0NEnFXboUmSJKlaV8+4/Qg4E1gFkJmzgPG1GpQkSZLerqvB7V2Z+WCrstUbejCSJElqX1eD28sR8QEgASLiKODFmo1KkiRJb9Ozi/W+ClwB7BERC4HngGNrNipJkiS9TafBLSJ6AKdk5sciYhtgi8x8rfZDkyRJUrVOg1tmromIDxfzy2s/JEmSJLWlq5dKZ0bEVOAGoCW8ZebPazIqSZIkvU1Xg1tvYDFwUFVZAgY3SZKkjaRLwS0zT6j1QCRJktSxrr45oV9E/CIi/lRMN0VEv1oPTpIkSW/p6t9xuwqYCuxUTL8qyiRJkrSRdDW49c3MqzJzdTFNBvrWcFySJElqpavBbXFEHBcRPYrpOCoPK0iSJGkj6WpwOxE4BvgfKq+6OgrwgQVJkqSNqKtPlf4ROLzGY5EkSVIHuvpU6U8iYruqz9tHxJW1G5YkSZJa6+ql0obMXNr8ITOXACNqMyRJkiS1pavBbYuI2L75Q0TsQNffuiBJkqQNoKvh6/8C90XEDUBQeTjhvJqNSpIkSW/T1YcT/jMimnjrXaWfyczHazcsSZIktdbVhxM+ADyTmT8AHgM+Vv2wQgftDo2IORExNyLOaGP5LhExPSJmRsSsiPhk1bIzi3ZzImJcV/uUJEnaVHX1HrebgDUR8UHgcqA/cE1HDSKiBzAJ+AQwGJgQEYNbVTsLuD4zRwDjgUuLtoOLz0OAQ4FLm//4bxf6lCRJ2iR1Nbi9mZmrgc8AP8jMicD7O2kzGpibmc9m5hvAFOCIVnUS2LaYfy/wQjF/BDAlM1/PzOeAuUV/XelTkiRpk9TV4LYqIiYAfwvcUpT16qTNzsD8qs8LirJq5wDHRcQC4Dbga5207UqfAETESRHRFBFNixYt6mSokiRJ9a+rwe0EYAxwXmY+FxEDgas3wPonAE5OdcEAABJsSURBVJMzsx/wSeDqiOjqmDqUmVdkZmNmNvbt23dDdClJktStuvpU6ePAaQARMTIz/wBc0EmzhVTuhWvWryir9kUq97CRmfdFRG9gx07adtanJEnSJumdnN36cRfrPQTsHhEDI2JLKg8bTG1V53ngYICI2BPoDSwq6o2PiK2Ks3u7Aw92sU9JkqRN0jt5+0F0pVJmro6IU4HbgR7AlZk5OyLOBZoycyrwj8CPIuJ0Kg8qHJ+ZCcyOiOuBx4HVwFczcw1AW32+g22QJEkqnajkpHVoEHFkZt5co/HURGNjYzY1NXX3MCRJkjoVETMys7GtZet8qbQ5tEXEHus7MEmSJHXd+jzBOW2DjUKSJEmd6vAet4i4uL1FQKevvJIkSdKG09nDCSdQeYDg9TaWTdjww5EkSVJ7OgtuDwGPZebvWy+IiHNqMiJJkiS1qbPgdhSwsq0FmTlwww9HkiRJ7ens4YR3Z+ZfNspIJEmS1KHOglvL32uLiJtqPBZJkiR1oLPgVv2WhN1qORBJkiR1rLPglu3MS5IkaSPr7OGEvSLiVSpn3rYu5ik+Z2ZuW9PRSZIkqUWHwS0ze2ysgUiSJKlj6/PKK0mSJG1EBjdJkqSSMLhJkiSVhMFNkiSpJAxukiRJJWFwkyRJKgmDmyRJUkkY3CRJkkrC4CZJklQSBjdJkqSSMLhJkiSVhMFNkiSpJAxukiRJJWFwkyRJKgmDmyRJUkkY3CRJkkrC4CZJklQSBjdJkqSSMLhJkiSVhMFNkiSpJAxukiRJJWFwkyRJKgmDmyRJUkkY3CRJkkrC4CZJklQSBjdJkqSSMLhJkiSVhMFNkiSpJAxukiRJJWFwkyRJKgmDmyRJUkkY3CRJkkrC4CZJklQSBjdJkqSSMLhJkiSVhMFNkiSpJAxukiRJJWFwkyRJKgmDmyRJUknUNLhFxKERMSci5kbEGW0svygiHi6mpyJiadWyCyLisWL6bFX55Ih4rqrd8FpugyRJUr3oWauOI6IHMAk4BFgAPBQRUzPz8eY6mXl6Vf2vASOK+cOAkcBwYCvgroj4dWa+WlSfmJk31mrskiRJ9aiWZ9xGA3Mz89nMfAOYAhzRQf0JwLXF/GDg7sxcnZnLgVnAoTUcqyRJUt2rZXDbGZhf9XlBUfY2EbErMBC4syh6BDg0It4VETsCBwL9q5qcFxGzikutW7XT50kR0RQRTYsWLVrfbZEkSep29fJwwnjgxsxcA5CZ04DbgN9TOQt3H7CmqHsmsAewN7AD8I22OszMKzKzMTMb+/btW+PhS5Ik1V4tg9tC1j5L1q8oa8t43rpMCkBmnpeZwzPzECCAp4ryF7PideAqKpdkJUmSNnm1DG4PAbtHxMCI2JJKOJvaulJE7AFsT+WsWnNZj4joU8w3AA3AtOLz+4ufARwJPFbDbZAkSaobNXuqNDNXR8SpwO1AD+DKzJwdEecCTZnZHOLGA1MyM6ua9wLuqWQzXgWOy8zVxbKfRURfKmfhHgZOrtU2SJIk1ZNYOy9tmhobG7Opqam7hyFJktSpiJiRmY1tLauXhxMkSZLUCYObJElSSRjcJEmSSsLgJkmSVBIGN0mSpJIwuEmSJJWEwU2SJKkkDG6SJEklYXCTJEkqCYObJElSSRjcJEmSSsLgJkmSVBIGN0mSpJIwuEmSJJWEwU2SJKkkDG6SJEklYXCTJEkqCYObJElSSRjcJEmSSsLgJkmSVBIGN0mSpJIwuEmSJJWEwU2SJKkkDG6SJEklYXCTJEkqCYObJElSSRjcJEmSSsLgJkmSVBIGN0mSpJIwuEmSJJWEwU2SJKkkDG6SJEklYXCTJEkqCYObJElSSRjcJEmSSsLgJkmSVBIGN0mSpJIwuEmSJJWEwU2SJKkkDG6SJEklYXCTJEkqCYObJElSSRjcJEmSSsLgJkmSVBIGN0mSpJIwuEmSJJWEwU2SJKkkDG6SJEklYXCTJEkqCYObJElSSRjcJEmSSsLgJkmSVBI1DW4RcWhEzImIuRFxRhvLL4qIh4vpqYhYWrXsgoh4rJg+W1U+MCIeKPq8LiK2rOU2SJIk1YuaBbeI6AFMAj4BDAYmRMTg6jqZeXpmDs/M4cAlwM+LtocBI4HhwD7A1yNi26LZBcBFmflBYAnwxVptgyRJUj2p5Rm30cDczHw2M98ApgBHdFB/AnBtMT8YuDszV2fmcmAWcGhEBHAQcGNR7yfAkTUZvSRJUp2pZXDbGZhf9XlBUfY2EbErMBC4syh6hEpQe1dE7AgcCPQH+gBLM3N1F/o8KSKaIqJp0aJF670xkiRJ3a1eHk4YD9yYmWsAMnMacBvweypn4e4D1qxLh5l5RWY2ZmZj3759N/R4JUmSNrpaBreFVM6SNetXlLVlPG9dJgUgM88r7n87BAjgKWAxsF1E9OxCn5IkSZuUWga3h4Ddi6dAt6QSzqa2rhQRewDbUzmr1lzWIyL6FPMNQAMwLTMTmA4cVVT9AvDLGm6DJElS3ejZeZV3JjNXR8SpwO1AD+DKzJwdEecCTZnZHOLGA1OKUNasF3BP5VkEXgWOq7qv7RvAlIj4LjAT+H+12gZJkqR6EmvnpU1TY2NjNjU1dfcwJEmSOhURMzKzsa1l9fJwgiRJkjphcJMkSSoJg5skSVJJGNwkSZJKwuAmSZJUEgY3SZKkkjC4SZIklYTBTZIkqSQMbpIkSSVhcJMkSSoJg5skSVJJGNwkSZJKwuAmSZJUEgY3SZKkkjC4SZIklYTBTZIkqSQMbpIkSSVhcJMkSSoJg5skSVJJGNwkSZJKwuAmSZJUEgY3SZKkkjC4SZIklYTBTZIkqSQMbpIkSSVhcJMkSSoJg5skSVJJGNwkSZJKwuAmSZJUEgY3SZKkkjC4SZIklYTBTZIkqSQMbpIkSSVhcJMkSSoJg5skSVJJGNwkSZJKwuAmSZJUEj27ewBld/PMhVx4+xxeWLqCnbbbmonjBnHkiJ27e1iSJGkTZHBbDzfPXMiZP3+UFavWALBw6QrO/PmjAIY3SZK0wXmpdD1cePucltDWbMWqNVx4+5xuGpEkSdqUGdzWwwtLV6xTuSRJ0vowuK2Hnbbbep3KJUmS1ofBbT1MHDeIrXv1WKts6149mDhuUDeNSJIkbcp8OGE9ND+A4FOlkiRpYzC4racjR+xsUJMkSRuFl0olSZJKwuAmSZJUEgY3SZKkkjC4SZIklYTBTZIkqSQMbpIkSSVR0+AWEYdGxJyImBsRZ7Sx/KKIeLiYnoqIpVXLvhcRsyPiiYi4OCKiKL+r6LO53ftquQ2SJEn1omZ/xy0iegCTgEOABcBDETE1Mx9vrpOZp1fV/xowopjfD9gfaCgW3wscANxVfD42M5tqNXZJkqR6VMszbqOBuZn5bGa+AUwBjuig/gTg2mI+gd7AlsBWQC/gpRqOVZIkqe7VMrjtDMyv+rygKHubiNgVGAjcCZCZ9wHTgReL6fbMfKKqyVXFZdKzmy+httHnSRHRFBFNixYtWv+tkSRJ6mb18nDCeODGzFwDEBEfBPYE+lEJewdFxEeKusdm5jDgI8X0+bY6zMwrMrMxMxv79u1b8w2QJEmqtVoGt4VA/6rP/YqytoznrcukAJ8G7s/MZZm5DPg1MAYgMxcWP18DrqFySVaSJGmTV8uXzD8E7B4RA6kEtvHA51pXiog9gO2B+6qKnwe+HBH/CgSVBxP+PSJ6Attl5ssR0Qv4FPDbzgYyY8aMlyPij+u7QSW0I/Bydw9Ca/GY1CePS/3xmNQfj8nGs2t7C2oW3DJzdUScCtwO9ACuzMzZEXEu0JSZU4uq44EpmZlVzW8EDgIepfKgwm8y81cRsQ1wexHaelAJbT/qwlg2y2ulEdGUmY3dPQ69xWNSnzwu9cdjUn88JvUh1s5L2pT4Jas/HpP65HGpPx6T+uMxqQ/18nCCJEmSOmFw27Rd0d0D0Nt4TOqTx6X+eEzqj8ekDnipVJIkqSQ84yZJklQSBjdJkqSSMLjVmYg4NCLmRMTciDijjeVbRcR1xfIHImJA1bIzi/I5ETGusz4jYmDRx9yizy2L8uMjYlHxWrGHI+JLtd3q+reRj8upRVlGxI5V5RERFxfLZkXEyNptcf2rk2MyNiJeqfqufKt2W1z/NvIx+VlR/lhEXFn8mSi/J63UyTHxe7IhZaZTnUxU/jbdM8BuwJbAI8DgVnVOAS4r5scD1xXzg4v6W1F57+szRX/t9glcD4wv5i8D/q6YPx74QXfvj3qZuuG4jAAGAPOAHavW8UkqbxEJYF/gge7eNx4TxgK3dPf+qIepG47JJ4vvQlB5887fVZX7PamvY+L3ZANOnnGrL6OBuZn5bGa+AUwBjmhV5wjgJ8X8jcDBERFF+ZTMfD0znwPmFv212WfR5qCiD4o+j6zhtpXZRjsuAJk5MzPntTGOI4D/zIr7ge0i4v0bdEvLo16Oid6ysY/JbcV3IYEHqbxWsXkdfk8q6uWYaAMyuNWXnYH5VZ8XFGVt1snM1cArQJ8O2rZX3gdYWvTR1rr+prjMcGNEVL9zdnO0MY/L+o5jc1EvxwRgTEQ8EhG/jogh67IRm5huOSbF5bjPA79Zh3FsLurlmIDfkw3G4Ka2/AoYkJkNwH/x1v+NSVrbH4BdM3Mv4BLg5m4ez+boUuDuzLynuweiFq2Pid+TDcjgVl8WAtVnt/oVZW3WiYiewHuBxR20ba98MZVLCD1blZOZizPz9aL8x8Co9dqq8tuYx2V9x7G5qItjkpmvZuayYv42oFf1wwubmY1+TCLi20Bf4B/WcRybi7o4Jn5PNrDuvsnO6a0J6Ak8S+VG0OabPoe0qvNV1r6R9Ppifghr30j6LJWbSNvtE7iBtR9OOKWYf3/V+j4N3N/d+2ZzOi5Vfc5j7RvhD2Ptm64f7O594zHhf/HWHzIfDTzf/Hlzm7rhv19fAn4PbN1qHX5P6u+Y+D3ZkMe1uwfg1OqAVJ7KeYrKUzvfLMrOBQ4v5ntTCVxzqdz8uVtV228W7eYAn+ioz6J8t6KPuUWfWxXl/wrMLr6Q04E9unu/dPe0kY/LaVTuG1kNvAD8uCgPYFJR/1Ggsbv3i8eEU6u+K/cD+3X3ftmMjsnqouzhYvpWUe73pP6Oid+TDTj5yitJkqSS8B43SZKkkjC4SZIklYTBTZIkqSQMbpIkSSVhcJMkSSoJg5ukbhMRfSLi4WL6n4hYWMwvi4hLa7C+c6rW8VhEHF4nfQ2IiMeK+caIuLiT+v/c6vPv3+m6JZWLfw5EUl2IiHOAZZn5/Y2xjojYE7gHeF9mvtmFtj0yc8269BURPfOt9wF31PcA4JbMHNrF7ViWme/uSl1JmxbPuEmqOxExNiJuKebPiYifRMQ9EfHHiPhMRHwvIh6NiN8UL7QmIkZFxH9HxIyIuD0i3t/ROjLzCSp/MHTHiPh4RNwXEX+IiBsi4t1Fn/Mi4oKI+ANwdBf7uisi/j0imoC/b29cRfkjEfEIlb9e39a2vzsiriq2dVZE/E1EnA9sXZzp+1lRb1nxMyLiwuIM4KMR8dmqPu+KiBsj4smI+FlExDs5NpK6l8FNUhl8ADgIOBz4KTA9M4cBK4DDivB2CXBUZo4CrgTO66jDiNgHeBNI4CzgY5k5Emhi7XdfLs7MkZk5pQt9LSqKtszMRuDiDsZ1FfC1rLx4uz1nA69k5rDMbADuzMwzgBWZOTwzj21V/zPAcGAv4GPAhVUBdgTwv4HBVN6asn8H65VUp3p2XkWSut2vM3NVRDxK5X2JvynKHwUGAIOAocB/FSeSegAvttPX6RFxHPAa8FlgHyph5ndF2y2B+6rqX9fBuNbqKzOz6KO5TZvjiojtgO0y8+6i3tXAJ9ro/2NU3h8JQGYu6WAsAB8Gri0u6b4UEf8N7A28SuWdnQsAIuJhKvvt3k76k1RnDG6SyuB1gMx8MyJW5Vs3575J5b9jAczOzDFd6Oui6vvoIuKvgf/KzAnt1F/e1b7aaNPmuIrgtrG9XjW/Bv/7L5WSl0olbQrmAH0jYgxARPSKiCFdbHs/sH9EfLBou01EfKiW48rMpcDSiPhwUa/1Jc9m/8Xa979tX8yuar63r5V7gM9GRI+I6At8lMqLwyVtIgxukkovM98AjgIuKG72fxjYr4ttFwHHA9dGxCwql0n32AjjOgGYVFy2bO9Bge8C2xcPGzwCHFiUXwHMan44ocovgFnAI8CdwD9l5v9siG2RVB/8cyCSJEkl4Rk3SZKkkjC4SZIklYTBTZIkqSQMbpIkSSVhcJMkSSoJg5skSVJJGNwkSZJK4v8DkZdiLqe2f28AAAAASUVORK5CYII=",
            "text/plain": [
              "<Figure size 720x504 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          },
          "output_type": "display_data"
        }
      ],
      "source": [
        "import matplotlib.pyplot as plt\n",
        "\n",
        "plt.figure(figsize=(10, 7))\n",
        "plt.scatter(baseline_time_per_pred, baseline_results['f1'], label=\"baseline\")\n",
        "plt.scatter(model_6_time_per_pred, model_6_pretrained_results['f1'], label=\"tf_hub_sentence_encoder\")\n",
        "plt.legend()\n",
        "plt.title(\"F1-score versus time per prediction\")\n",
        "plt.xlabel(\"Time Per Prediction\")\n",
        "plt.ylabel(\"F1-score\");"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "GopIrZ-yGXzT"
      },
      "outputs": [],
      "source": []
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}