notebook.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 69,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: transformers in c:\\users\\tim\\anaconda3\\lib\\site-packages (4.16.1)\n",
            "Requirement already satisfied: filelock in c:\\users\\tim\\anaconda3\\lib\\site-packages (from transformers) (3.0.12)\n",
            "Requirement already satisfied: pyyaml>=5.1 in c:\\users\\tim\\anaconda3\\lib\\site-packages (from transformers) (5.4.1)\n",
            "Requirement already satisfied: requests in c:\\users\\tim\\anaconda3\\lib\\site-packages (from transformers) (2.25.1)\n",
            "Note: you may need to restart the kernel to use updated packages.\n",
            "Requirement already satisfied: regex!=2019.12.17 in c:\\users\\tim\\anaconda3\\lib\\site-packages (from transformers) (2021.4.4)\n",
            "Requirement already satisfied: tokenizers!=0.11.3,>=0.10.1 in c:\\users\\tim\\anaconda3\\lib\\site-packages (from transformers) (0.11.4)\n",
            "Requirement already satisfied: packaging>=20.0 in c:\\users\\tim\\anaconda3\\lib\\site-packages (from transformers) (20.9)\n",
            "Requirement already satisfied: numpy>=1.17 in c:\\users\\tim\\anaconda3\\lib\\site-packages (from transformers) (1.20.1)\n",
            "Requirement already satisfied: huggingface-hub<1.0,>=0.1.0 in c:\\users\\tim\\anaconda3\\lib\\site-packages (from transformers) (0.4.0)\n",
            "Requirement already satisfied: sacremoses in c:\\users\\tim\\anaconda3\\lib\\site-packages (from transformers) (0.0.47)\n",
            "Requirement already satisfied: tqdm>=4.27 in c:\\users\\tim\\anaconda3\\lib\\site-packages (from transformers) (4.59.0)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in c:\\users\\tim\\anaconda3\\lib\\site-packages (from huggingface-hub<1.0,>=0.1.0->transformers) (3.7.4.3)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in c:\\users\\tim\\anaconda3\\lib\\site-packages (from packaging>=20.0->transformers) (2.4.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in c:\\users\\tim\\anaconda3\\lib\\site-packages (from requests->transformers) (2020.12.5)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in c:\\users\\tim\\anaconda3\\lib\\site-packages (from requests->transformers) (1.26.4)\n",
            "Requirement already satisfied: chardet<5,>=3.0.2 in c:\\users\\tim\\anaconda3\\lib\\site-packages (from requests->transformers) (4.0.0)\n",
            "Requirement already satisfied: idna<3,>=2.5 in c:\\users\\tim\\anaconda3\\lib\\site-packages (from requests->transformers) (2.10)\n",
            "Requirement already satisfied: joblib in c:\\users\\tim\\anaconda3\\lib\\site-packages (from sacremoses->transformers) (1.0.1)\n",
            "Requirement already satisfied: click in c:\\users\\tim\\anaconda3\\lib\\site-packages (from sacremoses->transformers) (7.1.2)\n",
            "Requirement already satisfied: six in c:\\users\\tim\\anaconda3\\lib\\site-packages (from sacremoses->transformers) (1.15.0)\n"
          ]
        }
      ],
      "source": [
        "pip install transformers"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "execution": {
          "iopub.execute_input": "2022-02-09T18:18:21.343718Z",
          "iopub.status.busy": "2022-02-09T18:18:21.343339Z",
          "iopub.status.idle": "2022-02-09T18:18:21.351870Z",
          "shell.execute_reply": "2022-02-09T18:18:21.351104Z",
          "shell.execute_reply.started": "2022-02-09T18:18:21.343677Z"
        },
        "id": "04QpofV2wG-o",
        "trusted": true
      },
      "outputs": [],
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "import random\n",
        "import pandas as pd\n",
        "\n",
        "import os\n",
        "import json\n",
        "\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "from torch.utils.data import Dataset, DataLoader\n",
        "\n",
        "from transformers import AutoTokenizer, AutoModelForQuestionAnswering\n",
        "import transformers\n",
        "\n",
        "from tqdm import tqdm\n",
        "import re"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "# Load tokenizer."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {},
      "outputs": [],
      "source": [
        "use_type_ids = True # Disables type_ids if the model does not support that feature.\n",
        "model_name = \"bert-base-uncased\"\n",
        "tokenizer = AutoTokenizer.from_pretrained(model_name)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "# Dataset."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {},
      "outputs": [],
      "source": [
        "squad_train_file_name = \"train-v2.0.json\"\n",
        "squad_dev_file_name = \"dev-v2.0.json\"\n",
        "\n",
        "triviaQA_train_file_name = \"triviaqa_train.json\"\n",
        "triviaQA_dev_file_name = \"triviaqa_dev.json\""
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Convert TriviaQA to Squad form."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 73,
      "metadata": {},
      "outputs": [],
      "source": [
        "# Download the converter, the dataset, util functions from the official dataset repo.\n",
        "# The scripts require about 15-20 munites to transform the files to the appropriate SQUAD form.\n",
        "\n",
        "# This procedure can be disabled manually. Re-enable if needed.\n",
        "convertion_needed = False\n",
        "\n",
        "if convertion_needed:\n",
        "    !wget https://nlp.cs.washington.edu/triviaqa/data/triviaqa-rc.tar.gz\n",
        "    !tar -xf triviaqa-rc.tar.gz\n",
        "    !git clone https://github.com/amazon-research/qa-dataset-converter\n",
        "    !git clone https://github.com/mandarjoshi90/triviaqa\n",
        "    !cp qa-dataset-converter/triviaqa/triviaqa_to_squad.py triviaqa/\n",
        "    !python triviaqa/triviaqa_to_squad.py --triviaqa_file qa/wikipedia-train.json --data_dir evidence/wikipedia/ --output_file triviaqa_train.json\n",
        "    !python triviaqa/triviaqa_to_squad.py --triviaqa_file qa/wikipedia-dev.json --data_dir evidence/wikipedia/ --output_file triviaqa_dev.json"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Json format"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {},
      "outputs": [],
      "source": [
        "def PrintJSONFormat(data, identetation, keyName, isList = False):\n",
        "    # Prints the squad dataset format.\n",
        "    left_string = keyName + \" ->\"\n",
        "    ident_string = \"    \" * identetation\n",
        "\n",
        "    if isList:\n",
        "        left_string = left_string + \" (list)\"\n",
        "\n",
        "    if isinstance(data, list) and data:\n",
        "        # Skip list of items (not important for the format).\n",
        "        PrintJSONFormat(data[0], identetation, keyName, True)\n",
        "    elif isinstance(data, dict):\n",
        "        keys = list(data.keys())\n",
        "        print(ident_string + left_string, keys)\n",
        "        for key in keys: # Iterate each key.\n",
        "            PrintJSONFormat(data[key], identetation + 1, key)\n",
        "    else:\n",
        "        print(ident_string + keyName + \":\", type(data).__name__)\n",
        "\n",
        "def PrintFileFormat(file_name):\n",
        "    file = open(file_name)\n",
        "    json_obj = json.load(file)\n",
        "    PrintJSONFormat(json_obj, 0, \"Root\")\n",
        "    file.close()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 75,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Root -> ['version', 'data']\n",
            "    version: str\n",
            "    data -> (list) ['title', 'paragraphs']\n",
            "        title: str\n",
            "        paragraphs -> (list) ['qas', 'context']\n",
            "            qas -> (list) ['question', 'id', 'answers', 'is_impossible']\n",
            "                question: str\n",
            "                id: str\n",
            "                answers -> (list) ['text', 'answer_start']\n",
            "                    text: str\n",
            "                    answer_start: int\n",
            "                is_impossible: bool\n",
            "            context: str\n"
          ]
        }
      ],
      "source": [
        "# The following function is helpfull for understanding the dataset's format.\n",
        "PrintFileFormat(squad_train_file_name)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Number of answered / non answered questions."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "File name: train-v2.0.json\n",
            "Answered questions: 86821\n",
            "Not answered questions: 43498\n",
            "Total questions: 130319\n",
            " \n",
            "File name: dev-v2.0.json\n",
            "Answered questions: 20302\n",
            "Not answered questions: 5945\n",
            "Total questions: 26247\n",
            "- - -\n",
            "File name: triviaqa_train.json\n",
            "Answered questions: 77112\n",
            "Not answered questions: 33535\n",
            "Total questions: 110647\n",
            " \n",
            "File name: triviaqa_dev.json\n",
            "Answered questions: 9842\n",
            "Not answered questions: 4387\n",
            "Total questions: 14229\n"
          ]
        }
      ],
      "source": [
        "def PrintStats(file_name):\n",
        "    # Number of answered / not answered questions on the set.\n",
        "    file = open(file_name)\n",
        "    data_json = json.load(file)\n",
        "    file.close()\n",
        "\n",
        "    qas_count = 0\n",
        "    no_answers = 0\n",
        "    for topics in data_json['data']:\n",
        "        for par in topics['paragraphs']:\n",
        "            for sample in par['qas']:\n",
        "                n = len(sample['answers'])\n",
        "                qas_count += n\n",
        "                if n == 0:\n",
        "                    no_answers += 1\n",
        "\n",
        "    print(\"File name:\", file_name)\n",
        "    print(\"Answered questions:\", qas_count)\n",
        "    print(\"Not answered questions:\", no_answers)\n",
        "    print(\"Total questions:\", no_answers + qas_count)\n",
        "\n",
        "PrintStats(squad_train_file_name)\n",
        "print(\" \")\n",
        "PrintStats(squad_dev_file_name)\n",
        "\n",
        "print(\"- - -\")\n",
        "\n",
        "PrintStats(triviaQA_train_file_name)\n",
        "print(\" \")\n",
        "PrintStats(triviaQA_dev_file_name)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Make pairs, calculate end idx."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {},
      "outputs": [],
      "source": [
        "def CalculateEndIndex(sample):\n",
        "    # NOTE: we could extract multiple answers idxs. (The dataset contains 1 per answer)\n",
        "    context = sample['context']\n",
        "    answer = sample['answer']\n",
        "    answer_start_idx = sample['answer_start']\n",
        "\n",
        "    answer_len = len(answer)\n",
        "    answer_end_idx = answer_start_idx + answer_len\n",
        "\n",
        "    '''\n",
        "    answer_from_idxs = context[answer_start_idx:answer_end_idx]\n",
        "    if answer != answer_from_idxs:\n",
        "        raise Exception(\"Invalid sentence idx\", \"Ans1:\", answer, \"Ans2\", answer_from_idxs)\n",
        "    '''\n",
        "\n",
        "    return answer_end_idx\n",
        "\n",
        "def MakePairs(json_obj):\n",
        "    samples = []\n",
        "\n",
        "    for topics in json_obj['data']:\n",
        "        for paragraph in topics['paragraphs']:\n",
        "            context = paragraph['context']\n",
        "            for qas in paragraph['qas']:\n",
        "                question = qas['question']\n",
        "                # Edge case, plausible answers.\n",
        "                keys = qas.keys()\n",
        "                if \"plausible_answers\" in keys:\n",
        "                    answers = []\n",
        "                    #answers = qas[\"plausible_answers\"]\n",
        "                else:\n",
        "                    answers = qas['answers']\n",
        "\n",
        "                if not answers:\n",
        "                    # Does not contain answers because the question is not\n",
        "                    # able to be answered from the context. Provide a space\n",
        "                    # string as the answer. (We consider empty string the true answer)\n",
        "                    empty_sample = {\n",
        "                        \"text\": \" \",\n",
        "                        \"answer_start\": context.find(\" \")\n",
        "                    }\n",
        "                    answers.append(empty_sample)\n",
        "\n",
        "                # Extract info into pairs.\n",
        "                for answer in answers:\n",
        "                    sample = {\n",
        "                        \"id\": qas['id'],\n",
        "                        \"context\": context,\n",
        "                        \"question\": question,\n",
        "                        \"answer\": answer['text'],\n",
        "                        \"answer_start\": answer['answer_start']\n",
        "                    }\n",
        "                    sample['answer_end'] = CalculateEndIndex(sample)\n",
        "                    samples.append(sample)\n",
        "\n",
        "    # Return a list of dictionaries.  \n",
        "    return samples"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Dataset class."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {},
      "outputs": [],
      "source": [
        "class SQUAD(Dataset):\n",
        "    def __init__(self, file_name):\n",
        "        file = open(file_name)\n",
        "        data_json = json.load(file)\n",
        "        file.close()\n",
        "\n",
        "        # Split json into list of pairs.\n",
        "        self.pairs = MakePairs(data_json)\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.pairs)\n",
        "\n",
        "    def __getitem__(self, idx):\n",
        "        pair = self.pairs[idx]\n",
        "        return pair\n",
        "\n",
        "class CustomQuestions(Dataset):\n",
        "    id_c = 0\n",
        "\n",
        "    def __init__(self, contexts, questions):\n",
        "        samples = []\n",
        "        for context, question in zip(contexts, questions):\n",
        "            sample = {\n",
        "                \"id\": CustomQuestions.id_c,\n",
        "                \"context\": context,\n",
        "                \"question\": question,\n",
        "                }\n",
        "            samples.append(sample)\n",
        "            CustomQuestions.id_c += 1\n",
        "        \n",
        "        self.pairs = samples\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.pairs)\n",
        "\n",
        "    def __getitem__(self, idx):\n",
        "        pair = self.pairs[idx]\n",
        "        return pair"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Datasets creation."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {},
      "outputs": [],
      "source": [
        "squad_training_dataset = SQUAD(squad_train_file_name)\n",
        "squad_dev_dataset = SQUAD(squad_dev_file_name)\n",
        "\n",
        "# Same SQUAD class is used since the dataset is converted to the squad format.\n",
        "triviaQA_training_dataset = SQUAD(triviaQA_train_file_name)\n",
        "triviaQA_dev_dataset = SQUAD(triviaQA_dev_file_name)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Histogram."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "def GetLens(data):\n",
        "    data_loader = DataLoader(dataset=data, batch_size=1024)\n",
        "\n",
        "    data_lengths = []\n",
        "    for batch in data_loader:\n",
        "        contexts = batch['context']\n",
        "        questions = batch['question']\n",
        "        tokenized_cq = tokenizer(list(contexts), list(questions))\n",
        "        batch_input_ids = tokenized_cq['input_ids']\n",
        "\n",
        "        for sample in batch_input_ids:\n",
        "            sample_len = len(sample)\n",
        "            data_lengths.append(sample_len)\n",
        "\n",
        "    return data_lengths"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 48,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYQAAAD4CAYAAADsKpHdAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAARgklEQVR4nO3dbYidZ53H8e9vE619oNpupyUmcRPZoLYFqR260YKIcWmWiukLC3GpzUqXQIlaRXAT3+ibQBbE1cK2EKo2XbtmQy00KHUtUVkWuq3TVqhpzDaYbjI2NuM+1CLYmvrfF+dKPZmcSTLnTGfOzHw/MJz7/O+HuebKw2+u6344qSokSfqTuW6AJGk4GAiSJMBAkCQ1BoIkCTAQJEnN0rluQL8uu+yyWrVq1Vw3Q5LmlSeeeOLXVTXSa928DYRVq1YxNjY2182QpHklyX9Ntc4pI0kSYCBIkhoDQZIEGAiSpMZAkCQBBoIkqTEQJEmAgSBJagwESRIwj+9UXghWbf3ea8vP7bhxDlsiSY4QJEmNgSBJAgwESVJjIEiSAANBktQYCJIkwECQJDUGgiQJMBAkSc1ZAyHJN5IcT/KzrtqlSR5J8mx7vaRr3bYkh5IcTHJDV/3aJE+3dXcmSaufl+RfWv2xJKtm+GeUJJ2Dcxkh3Ausn1TbCuyrqjXAvvaeJFcCG4Gr2j53JVnS9rkb2AysaV8nj3kb8L9V9efAPwB/3+8PI0nq31kDoar+DfifSeUNwK62vAu4qau+u6perqrDwCHguiTLgIur6tGqKuC+SfucPNYDwLqTowdJ0uzp9xzCFVV1DKC9Xt7qy4GjXduNt9rytjy5fso+VXUCeBH40z7bJUnq00yfVO71m32doX6mfU4/eLI5yViSsYmJiT6bKEnqpd9AeKFNA9Fej7f6OLCya7sVwPOtvqJH/ZR9kiwF3szpU1QAVNXOqhqtqtGRkZE+my5J6qXfQNgLbGrLm4CHuuob25VDq+mcPH68TSu9lGRtOz9w66R9Th7ro8AP23kGSdIsOusH5CT5NvAB4LIk48AXgR3AniS3AUeAmwGqan+SPcAzwAlgS1W92g51O50rls4HHm5fAF8H/inJITojg40z8pNJkqblrIFQVR+bYtW6KbbfDmzvUR8Dru5R/x0tUCRJc8eP0BxyfsympNnioyskSYCBIElqDARJEmAgSJIaA0GSBBgIkqTGQJAkAQaCJKnxxrQh4Q1okuaaIwRJEmAgSJIaA0GSBHgOYSh1n0+QpNniCEGSBBgIkqTGQJAkAQaCJKkxECRJgIEgSWoMBEkSYCBIkhpvTJtl3nQmaVg5QpAkAQaCJKkxECRJgIEgSWoMBEkSYCBIkhoDQZIEDBgIST6bZH+SnyX5dpI3Jbk0ySNJnm2vl3Rtvy3JoSQHk9zQVb82ydNt3Z1JMki7JEnT13cgJFkOfBoYraqrgSXARmArsK+q1gD72nuSXNnWXwWsB+5KsqQd7m5gM7Cmfa3vt12SpP4MOmW0FDg/yVLgAuB5YAOwq63fBdzUljcAu6vq5ao6DBwCrkuyDLi4qh6tqgLu69pHkjRL+g6Eqvol8GXgCHAMeLGqfgBcUVXH2jbHgMvbLsuBo12HGG+15W15cv00STYnGUsyNjEx0W/TJUk9DDJldAmd3/pXA28FLkxyy5l26VGrM9RPL1btrKrRqhodGRmZbpMlSWcwyJTRh4DDVTVRVb8HHgTeB7zQpoFor8fb9uPAyq79V9CZYhpvy5PrkqRZNEggHAHWJrmgXRW0DjgA7AU2tW02AQ+15b3AxiTnJVlN5+Tx421a6aUka9txbu3aR5I0S/p+/HVVPZbkAeBJ4ATwFLATuAjYk+Q2OqFxc9t+f5I9wDNt+y1V9Wo73O3AvcD5wMPtS5I0iwb6PISq+iLwxUnll+mMFnptvx3Y3qM+Blw9SFskSYPxTmVJEmAgSJIaA0GSBBgIkqTGQJAkAQaCJKkZ6LJTza5VW7/32vJzO26cw5ZIWogcIUiSAANBktQYCJIkwECQJDUGgiQJMBAkSY2BIEkCDARJUuONaQuAN6xJmgmOECRJgIEgSWqcMpqnuqeJJGkmOEKQJAEGgiSpMRAkSYCBIElqDARJEmAgSJIaA0GSBBgIkqTGQJAkAQaCJKkxECRJwICBkOQtSR5I8vMkB5K8N8mlSR5J8mx7vaRr+21JDiU5mOSGrvq1SZ5u6+5MkkHaJUmavkFHCF8Dvl9V7wTeDRwAtgL7qmoNsK+9J8mVwEbgKmA9cFeSJe04dwObgTXta/2A7ZIkTVPfTztNcjHwfuBvAKrqFeCVJBuAD7TNdgE/Bv4O2ADsrqqXgcNJDgHXJXkOuLiqHm3HvQ+4CXi437YNG59MKmk+GGSE8HZgAvhmkqeS3JPkQuCKqjoG0F4vb9svB4527T/easvb8uT6aZJsTjKWZGxiYmKApkuSJhskEJYC7wHurqprgN/Spoem0Ou8QJ2hfnqxamdVjVbV6MjIyHTbK0k6g0ECYRwYr6rH2vsH6ATEC0mWAbTX413br+zafwXwfKuv6FGXJM2ivgOhqn4FHE3yjlZaBzwD7AU2tdom4KG2vBfYmOS8JKvpnDx+vE0rvZRkbbu66NaufSRJs2TQj9D8FHB/kjcCvwA+QSdk9iS5DTgC3AxQVfuT7KETGieALVX1ajvO7cC9wPl0TiYvmBPKkjRfDBQIVfVTYLTHqnVTbL8d2N6jPgZcPUhbJEmD8U5lSRJgIEiSGgNBkgQYCJKkxkCQJAEGgiSpMRAkScDgN6ZpyHQ/WfW5HTfOYUskzTeOECRJgIEgSWoMBEkSYCBIkhoDQZIEGAiSpMZAkCQBBoIkqTEQJEmAgSBJagwESRJgIEiSGgNBkgQYCJKkxkCQJAEGgiSp8QNyFjA/LEfSdDhCkCQBjhAWDUcLks7GEYIkCTAQJEnNwIGQZEmSp5J8t72/NMkjSZ5tr5d0bbstyaEkB5Pc0FW/NsnTbd2dSTJouyRJ0zMTI4Q7gANd77cC+6pqDbCvvSfJlcBG4CpgPXBXkiVtn7uBzcCa9rV+BtolSZqGgQIhyQrgRuCervIGYFdb3gXc1FXfXVUvV9Vh4BBwXZJlwMVV9WhVFXBf1z6SpFky6Ajhq8DngT901a6oqmMA7fXyVl8OHO3abrzVlrflyXVJ0izqOxCSfBg4XlVPnOsuPWp1hnqv77k5yViSsYmJiXP8tpKkczHICOF64CNJngN2Ax9M8i3ghTYNRHs93rYfB1Z27b8CeL7VV/Son6aqdlbVaFWNjoyMDNB0SdJkfQdCVW2rqhVVtYrOyeIfVtUtwF5gU9tsE/BQW94LbExyXpLVdE4eP96mlV5KsrZdXXRr1z6SpFnyetypvAPYk+Q24AhwM0BV7U+yB3gGOAFsqapX2z63A/cC5wMPt695rfvOYEmaD2YkEKrqx8CP2/J/A+um2G47sL1HfQy4eibaIknqj3cqS5IAA0GS1BgIkiTAQJAkNX4ewiI01RVQfk6CtLg5QpAkAQaCJKkxECRJgIEgSWoMBEkSYCBIkhoDQZIEGAiSpMZAkCQBBoIkqTEQJEmAgSBJagwESRJgIEiSGgNBkgQYCJKkxkCQJAEGgiSp8SM09Zruj9b04zSlxccRgiQJMBAkSY1TRnPouTf99WvLq373z3PYEkkyEHQOPLcgLQ5OGUmSAEcImkL3qEDS4tD3CCHJyiQ/SnIgyf4kd7T6pUkeSfJse72ka59tSQ4lOZjkhq76tUmebuvuTJLBfixJ0nQNMmV0AvhcVb0LWAtsSXIlsBXYV1VrgH3tPW3dRuAqYD1wV5Il7Vh3A5uBNe1r/QDtkiT1oe9AqKpjVfVkW34JOAAsBzYAu9pmu4Cb2vIGYHdVvVxVh4FDwHVJlgEXV9WjVVXAfV37SJJmyYycQ0iyCrgGeAy4oqqOQSc0klzeNlsO/EfXbuOt9vu2PLne6/tspjOS4G1ve9tMNH1GDTLvfvISVC8/lTRXBr7KKMlFwHeAz1TVb860aY9anaF+erFqZ1WNVtXoyMjI9BsrSZrSQCOEJG+gEwb3V9WDrfxCkmVtdLAMON7q48DKrt1XAM+3+ooe9aHldfmSFqJBrjIK8HXgQFV9pWvVXmBTW94EPNRV35jkvCSr6Zw8frxNL72UZG075q1d+0iSZskgI4TrgY8DTyf5aat9AdgB7ElyG3AEuBmgqvYn2QM8Q+cKpS1V9Wrb73bgXuB84OH2NS+8ntfrD+OjLRwdSQtX34FQVf9O7/l/gHVT7LMd2N6jPgZc3W9bFpLuEJCk2eSjKyRJgIEgSWp8ltE5mMnzBE4JSRpWBoL6NjkoPckszW9OGUmSAEcI88ZUU03DcjkqeEmqNN8ZCPPcMN6rIGl+cspIkgQ4QpiSnxgmabExELoYApIWM6eMJEmAI4QFyRPNkvrhCEGSBDhCWFCG6bEY53I+xnsVpOHiCEGSBDhCmBVz+Zv7fLjDWdJwcIQgSQIMBElSYyBIkgDPIbxuhumKn2Hl01Gl4bLoA2GxPq7Cm9ckTbboA0HDwdGCNPcMBA3daMFwkOaGgaBTnAyHcwmGYQsSSYMxENSTN7RJi4+BoKHmM5Gk2eN9CJIkwBGCpqnf+ys83yANPwNBM2Iub8Sb6qokr1aSpsdAmEHenTz3pjrnYDhIZzc0gZBkPfA1YAlwT1XteL2+V793J/f6D9/pj+mbavpoJqaVzuUY0/3zN0C0WKSq5roNJFkC/Cfwl8A48BPgY1X1zFT7jI6O1tjYWF/fbzr/Ifhb//CY7r0R/R5jUAaIhlmSJ6pqtNe6YRkhXAccqqpfACTZDWwApgyEmeB/9vPLTPx5zcro5Eun1k8er597OyafE5myD7704jm3cypTTqt96c0z+n2GxkL9uQYwLCOEjwLrq+pv2/uPA39RVZ+ctN1mYHN7+w7gYFu+DPj1LDV3PrJ/zs4+Ojv76OzmQx/9WVWN9FoxLCOE9KidllRVtRPYedrOydhUQyDZP+fCPjo7++js5nsfDcuNaePAyq73K4Dn56gtkrQoDUsg/ARYk2R1kjcCG4G9c9wmSVpUhmLKqKpOJPkk8K90Ljv9RlXtn8YhTptG0insn7Ozj87OPjq7ed1HQ3FSWZI094ZlykiSNMcMBEkSMM8DIcn6JAeTHEqyda7bM1eSrEzyoyQHkuxPckerX5rkkSTPttdLuvbZ1vrtYJIb5q71syfJkiRPJflue2//dEnyliQPJPl5+7v0XvvoVEk+2/6N/SzJt5O8aSH10bwNhPa4i38E/gq4EvhYkivntlVz5gTwuap6F7AW2NL6Yiuwr6rWAPvae9q6jcBVwHrgrtafC90dwIGu9/bPqb4GfL+q3gm8m05f2UdNkuXAp4HRqrqazgUwG1lAfTRvA4Gux11U1SvAycddLDpVdayqnmzLL9H5h7ycTn/sapvtAm5qyxuA3VX1clUdBg7R6c8FK8kK4Ebgnq6y/dMkuRh4P/B1gKp6par+D/tosqXA+UmWAhfQuV9qwfTRfA6E5cDRrvfjrbaoJVkFXAM8BlxRVcegExrA5W2zxdh3XwU+D/yhq2b//NHbgQngm21a7Z4kF2Ifvaaqfgl8GTgCHANerKofsID6aD4Hwjk97mIxSXIR8B3gM1X1mzNt2qO2YPsuyYeB41X1xLnu0qO2YPunWQq8B7i7qq4Bfkub+pjCouujdm5gA7AaeCtwYZJbzrRLj9pQ99F8DgQfd9ElyRvohMH9VfVgK7+QZFlbvww43uqLre+uBz6S5Dk6U4sfTPIt7J9u48B4VT3W3j9AJyDsoz/6EHC4qiaq6vfAg8D7WEB9NJ8DwcddNElCZ+73QFV9pWvVXmBTW94EPNRV35jkvCSrgTXA47PV3tlWVduqakVVraLz9+SHVXUL9s9rqupXwNEk72ildXQeP28f/dERYG2SC9q/uXV0ztctmD4aikdX9GMGHnexkFwPfBx4OslPW+0LwA5gT5Lb6PxlvhmgqvYn2UPnH/wJYEtVvTrrrZ579s+pPgXc337B+gXwCTq/NNpHQFU9luQB4Ek6P/NTdB5VcRELpI98dIUkCZjfU0aSpBlkIEiSAANBktQYCJIkwECQJDUGgiQJMBAkSc3/A6fIqJcuK2aDAAAAAElFTkSuQmCC",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          },
          "output_type": "display_data"
        }
      ],
      "source": [
        "plt.hist(GetLens(squad_training_dataset), bins=100);\n",
        "plt.hist(GetLens(squad_dev_dataset), bins=100);"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 55,
      "metadata": {},
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Token indices sequence length is longer than the specified maximum sequence length for this model (892 > 512). Running this sequence through the model will result in indexing errors\n"
          ]
        },
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYQAAAD4CAYAAADsKpHdAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAWEklEQVR4nO3dfYxdd53f8fenyeJldzELyYBcO3QMGKQQtQaP0lQURJXdxQSWhC2wTquNq41kiIIKopVIilRQpUjJbtmoUUVQ2ERJEORhCVEsQbpkYQuqFAiTYPJIlgkxm8GuMwsIUgFpHb794/4mHI/vPN07M/fGeb+koznzPQ/zvWfs+dzzO+fem6pCkqR/NOoGJEnjwUCQJAEGgiSpMRAkSYCBIElqTh51A4M69dRTa3JyctRtSNJzyr333vsPVTXRb9lzNhAmJyeZnp4edRuS9JyS5AeLLXPISJIEGAiSpMZAkCQBBoIkqTEQJEmAgSBJagwESRJgIEiSGgNBkgQ8h1+pLG2UyUu++Oz8wcvfPsJOpPXlGYIkCTAQJEmNgSBJAgwESVJjIEiSAANBktQYCJIkwECQJDXLBkKS65I8meTBTu2WJAfadDDJgVafTPKLzrJPdbbZleSBJDNJrkqSVt/U9jeT5JtJJtf+YUqSlrOSM4Trgd3dQlX9cVXtrKqdwG3AFzqLH5tfVlXv79SvBvYBO9o0v88LgZ9U1auBK4ErBnkgkqThLBsIVfV14Mf9lrVn+e8FblpqH0m2AJur6u6qKuBG4Ly2+Fzghjb/eeDs+bMHSdLGGfYawpuAI1X1vU5te5JvJ/lakje12lZgtrPObKvNL3sCoKqOAj8FTun3w5LsSzKdZHpubm7I1iVJXcMGwvkce3ZwGHhFVb0e+DDwuSSbgX7P+Kt9XWrZscWqa6pqqqqmJiYmhmhbkrTQwO92muRk4I+AXfO1qnoaeLrN35vkMeA19M4ItnU23wYcavOzwGnAbNvni1lkiEqStH6GOUP4PeC7VfXsUFCSiSQntflX0rt4/P2qOgw8leSsdn3gAuCOttl+YG+bfzfw1XadQZK0gVZy2+lNwN3Aa5PMJrmwLdrD8ReT3wzcn+Q79C4Qv7+q5p/tXwT8JTADPAbc2erXAqckmaE3zHTJEI9HkjSgZYeMqur8Rer/rk/tNnq3ofZbfxo4o0/9l8B7lutDkrS+fKWyJAkwECRJjZ+pLPXR/Rxl6fnCMwRJEmAgSJIaA0GSBBgIkqTGQJAkAQaCJKnxtlNpFbq3ox68/O0j7ERae54hSJIAA0GS1BgIkiTAQJAkNQaCJAkwECRJjYEgSQIMBElSYyBIkoAVBEKS65I8meTBTu3jSX6Y5ECbzuksuzTJTJJHk7y1U9+V5IG27KokafVNSW5p9W8mmVzjxyhJWoGVnCFcD+zuU7+yqna26UsASU4H9gCva9t8MslJbf2rgX3AjjbN7/NC4CdV9WrgSuCKAR+LJGkIywZCVX0d+PEK93cucHNVPV1VjwMzwJlJtgCbq+ruqirgRuC8zjY3tPnPA2fPnz1IkjbOMNcQPpDk/jak9JJW2wo80VlnttW2tvmF9WO2qaqjwE+BU/r9wCT7kkwnmZ6bmxuidUnSQoMGwtXAq4CdwGHgE63e75l9LVFfapvji1XXVNVUVU1NTEysqmFJ0tIGCoSqOlJVz1TVr4BPA2e2RbPAaZ1VtwGHWn1bn/ox2yQ5GXgxKx+ikiStkYECoV0TmPcuYP4OpP3Annbn0HZ6F4/vqarDwFNJzmrXBy4A7uhss7fNvxv4arvOIEnaQMt+QE6Sm4C3AKcmmQU+BrwlyU56QzsHgfcBVNVDSW4FHgaOAhdX1TNtVxfRu2PphcCdbQK4FvhMkhl6ZwZ71uBxSZJWadlAqKrz+5SvXWL9y4DL+tSngTP61H8JvGe5PiRJ68tXKkuSAANBktQYCJIkwECQJDUGgiQJMBAkSY2BIEkCDARJUmMgSJIAA0GS1BgIkiTAQJAkNQaCJAkwECRJjYEgSQIMBElSYyBIkgADQZLUGAiSJGAFgZDkuiRPJnmwU/vzJN9Ncn+S25P8bqtPJvlFkgNt+lRnm11JHkgyk+SqJGn1TUluafVvJplc+4cpSVrOSs4Qrgd2L6jdBZxRVf8U+Dvg0s6yx6pqZ5ve36lfDewDdrRpfp8XAj+pqlcDVwJXrPpRSJKGtmwgVNXXgR8vqH25qo62b78BbFtqH0m2AJur6u6qKuBG4Ly2+Fzghjb/eeDs+bMHSdLGWYtrCH8K3Nn5fnuSbyf5WpI3tdpWYLazzmyrzS97AqCFzE+BU/r9oCT7kkwnmZ6bm1uD1iVJ84YKhCQfBY4Cn22lw8Arqur1wIeBzyXZDPR7xl/zu1li2bHFqmuqaqqqpiYmJoZpXZK0wMmDbphkL/AO4Ow2DERVPQ083ebvTfIY8Bp6ZwTdYaVtwKE2PwucBswmORl4MQuGqCRJ62+gM4Qku4GPAO+sqp936hNJTmrzr6R38fj7VXUYeCrJWe36wAXAHW2z/cDeNv9u4KvzASNJ2jjLniEkuQl4C3BqklngY/TuKtoE3NWu/36j3VH0ZuC/JDkKPAO8v6rmn+1fRO+OpRfSu+Ywf93hWuAzSWbonRnsWZNHJq2zyUu++Oz8wcvfPsJOpLWxbCBU1fl9ytcusu5twG2LLJsGzuhT/yXwnuX6kCStL1+pLEkCDARJUmMgSJIAA0GS1BgIkiTAQJAkNQaCJAkwECRJjYEgSQIMBElSYyBIkgADQZLUGAiSJMBAkCQ1BoIkCTAQJEmNgSBJAgwESVKzks9Uvg54B/BkVZ3Rai8FbgEmgYPAe6vqJ23ZpcCF9D5T+d9X1V+3+i5+/ZnKXwI+WFWVZBNwI7AL+BHwx1V1cM0eobRC3c9Ilp6PVnKGcD2we0HtEuArVbUD+Er7niSnA3uA17VtPpnkpLbN1cA+YEeb5vd5IfCTqno1cCVwxaAPRpI0uGUDoaq+Dvx4Qflc4IY2fwNwXqd+c1U9XVWPAzPAmUm2AJur6u6qKnpnBOf12dfngbOTZLCHI0ka1KDXEF5eVYcB2teXtfpW4InOerOttrXNL6wfs01VHQV+CpwyYF+SpAGt9UXlfs/sa4n6Utscv/NkX5LpJNNzc3MDtihJ6mfQQDjShoFoX59s9VngtM5624BDrb6tT/2YbZKcDLyY44eoAKiqa6pqqqqmJiYmBmxdktTPoIGwH9jb5vcCd3Tqe5JsSrKd3sXje9qw0lNJzmrXBy5YsM38vt4NfLVdZ5AkbaCV3HZ6E/AW4NQks8DHgMuBW5NcCPw98B6Aqnooya3Aw8BR4OKqeqbt6iJ+fdvpnW0CuBb4TJIZemcGe9bkkUmSVmXZQKiq8xdZdPYi618GXNanPg2c0af+S1qgSJJGx1cqS5IAA0GS1BgIkiTAQJAkNQaCJAkwECRJjYEgSQIMBElSYyBIkgADQZLUGAiSJMBAkCQ1BoIkCVjBu51KWt7kJV98dv7g5W8fYSfS4DxDkCQBBoIkqTEQJEmAgSBJagwESRIwRCAkeW2SA53pZ0k+lOTjSX7YqZ/T2ebSJDNJHk3y1k59V5IH2rKrkmTYByZJWp2BA6GqHq2qnVW1E9gF/By4vS2+cn5ZVX0JIMnpwB7gdcBu4JNJTmrrXw3sA3a0afegfUmSBrNWQ0ZnA49V1Q+WWOdc4OaqerqqHgdmgDOTbAE2V9XdVVXAjcB5a9SXJGmF1ioQ9gA3db7/QJL7k1yX5CWtthV4orPObKttbfML68dJsi/JdJLpubm5NWpdkgRrEAhJXgC8E/irVroaeBWwEzgMfGJ+1T6b1xL144tV11TVVFVNTUxMDNO2JGmBtThDeBtwX1UdAaiqI1X1TFX9Cvg0cGZbbxY4rbPdNuBQq2/rU5ckbaC1CITz6QwXtWsC894FPNjm9wN7kmxKsp3exeN7quow8FSSs9rdRRcAd6xBX5KkVRjqze2S/Bbw+8D7OuU/S7KT3rDPwfllVfVQkluBh4GjwMVV9Uzb5iLgeuCFwJ1tkiRtoKECoap+DpyyoPYnS6x/GXBZn/o0cMYwvUiShuMrlSVJgIEgSWoMBEkSYCBIkhoDQZIEGAiSpMZAkCQBBoIkqTEQJEmAgSBJagwESRJgIEiSGgNBkgQYCJKkxkCQJAEGgiSpMRAkSYCBIElqDARJEjBkICQ5mOSBJAeSTLfaS5PcleR77etLOutfmmQmyaNJ3tqp72r7mUlyVZIM05ckafXW4gzhX1XVzqqaat9fAnylqnYAX2nfk+R0YA/wOmA38MkkJ7Vtrgb2ATvatHsN+pIkrcLJ67DPc4G3tPkbgP8JfKTVb66qp4HHk8wAZyY5CGyuqrsBktwInAfcuQ69Setu8pIvHvP9wcvfPqJOpNUZ9gyhgC8nuTfJvlZ7eVUdBmhfX9bqW4EnOtvOttrWNr+wfpwk+5JMJ5mem5sbsnVJUtewZwhvrKpDSV4G3JXku0us2++6QC1RP75YdQ1wDcDU1FTfdSRJgxnqDKGqDrWvTwK3A2cCR5JsAWhfn2yrzwKndTbfBhxq9W196pKkDTRwICT57SQvmp8H/gB4ENgP7G2r7QXuaPP7gT1JNiXZTu/i8T1tWOmpJGe1u4su6GwjSdogwwwZvRy4vd0hejLwuar6H0m+Bdya5ELg74H3AFTVQ0luBR4GjgIXV9UzbV8XAdcDL6R3MdkLypK0wQYOhKr6PvDP+tR/BJy9yDaXAZf1qU8DZwzaiyRpeL5SWZIEGAiSpMZAkCQBBoIkqTEQJEmAgSBJagwESRJgIEiSGgNBkgQYCJKkxkCQJAHr84lp0nPGwk83k57PDARpnXVDx4/T1DhzyEiSBBgIkqTGQJAkAQaCJKkxECRJgIEgSWoGDoQkpyX52ySPJHkoyQdb/eNJfpjkQJvO6WxzaZKZJI8meWunvivJA23ZVUky3MOSJK3WMK9DOAr8h6q6L8mLgHuT3NWWXVlV/7W7cpLTgT3A64B/DPxNktdU1TPA1cA+4BvAl4DdwJ1D9CZJWqWBzxCq6nBV3dfmnwIeAbYuscm5wM1V9XRVPQ7MAGcm2QJsrqq7q6qAG4HzBu1LkjSYNbmGkGQSeD3wzVb6QJL7k1yX5CWtthV4orPZbKttbfML6/1+zr4k00mm5+bm1qJ1SVIzdCAk+R3gNuBDVfUzesM/rwJ2AoeBT8yv2mfzWqJ+fLHqmqqaqqqpiYmJYVuXJHUMFQhJfoNeGHy2qr4AUFVHquqZqvoV8GngzLb6LHBaZ/NtwKFW39anLknaQMPcZRTgWuCRqvqLTn1LZ7V3AQ+2+f3AniSbkmwHdgD3VNVh4KkkZ7V9XgDcMWhfkqTBDHOX0RuBPwEeSHKg1f4TcH6SnfSGfQ4C7wOoqoeS3Ao8TO8OpYvbHUYAFwHXAy+kd3eRdxhpbBz8zX9zXG3yl58bQSfS+ho4EKrqf9F//P9LS2xzGXBZn/o0cMagvUiShufnIUgbyM9G0DgzEKQ++g0TSSc638tIkgQYCJKkxiEjaQDdISXvONKJwjMESRLgGYI0Mt5xpHHjGYIkCTAQJEmNQ0ZSM+hrD7zArBOFgSCNAa8naBw4ZCRJAjxD0PNQ99m4pF8zEKQ15PUEPZc5ZCRJAjxDkMaOF5g1KgaCtE4cPtJzjYGg5zU/90D6NQNBy1rpXTnDDG8sNkyy2uGTcb2DaNCzBYePtJFSVaPuAYAku4H/BpwE/GVVXb7U+lNTUzU9Pb0hvW2ElfxB1Nob5RnCMMNIhoMGleTeqprqu2wcAiHJScDfAb8PzALfAs6vqocX22YcA8E/3s8d4zxUtNqgMBy0GksFwrgMGZ0JzFTV9wGS3AycCywaCOthI/+gL/YHqd8fg8WGG1ayj3H+w6f+VvNvA1b/79YA0WLGJRC2Ak90vp8F/vnClZLsA/a1b/9PkkdX8TNOBf5h4A7XWPqXT4V3HNfjseu+Y7l9rHCdgYzVMVzEuPc4RH/vWH6VFcgVy65yAh/DDTPOPf6TxRaMSyD0+7t13FhWVV0DXDPQD0imFztNGhfj3uO49wfj3+O49wfj3+O49wfPjR77GZdXKs8Cp3W+3wYcGlEvkvS8NC6B8C1gR5LtSV4A7AH2j7gnSXpeGYsho6o6muQDwF/Tu+30uqp6aI1/zEBDTRts3Hsc9/5g/Hsc9/5g/Hsc9/7gudHjccbitlNJ0uiNy5CRJGnEDARJEvA8CIQku5M8mmQmySUj7OO0JH+b5JEkDyX5YKt/PMkPkxxo0zmdbS5tfT+a5K0b0OPBJA+0PqZb7aVJ7kryvfb1JSPs77Wd43Qgyc+SfGiUxzDJdUmeTPJgp7bqY5ZkVzv2M0muSrJmLyFZpMc/T/LdJPcnuT3J77b6ZJJfdI7lp9a7x0X6W/XvdATH8JZOfweTHGj1DT+Ga6aqTtiJ3gXqx4BXAi8AvgOcPqJetgBvaPMvovdWHacDHwf+Y5/1T2/9bgK2t8dx0jr3eBA4dUHtz4BL2vwlwBWj6q/P7/Z/03uRzciOIfBm4A3Ag8McM+Ae4F/Qe03OncDb1rnHPwBObvNXdHqc7K63YD/r0uMi/a36d7rRx3DB8k8A/3lUx3CtphP9DOHZt8Soqv8LzL8lxoarqsNVdV+bfwp4hN4rtBdzLnBzVT1dVY8DM/Qez0Y7F7ihzd8AnNepj7K/s4HHquoHS6yz7j1W1deBH/f5uSs+Zkm2AJur6u7q/dW4sbPNuvRYVV+uqqPt22/Qe+3Potazx0WO4WLG5hjOa8/y3wvctNQ+1rvHtXCiB0K/t8RY6o/whkgyCbwe+GYrfaCdul/XGV4YRe8FfDnJvem9TQjAy6vqMPRCDXjZCPvr2sOx/wHH5RjC6o/Z1ja/0X3O+1N6z1bnbU/y7SRfS/KmVhtFj6v5nY7yGL4JOFJV3+vUxuUYrsqJHggrekuMjZTkd4DbgA9V1c+Aq4FXATuBw/ROPWE0vb+xqt4AvA24OMmbl1h3ZMc2vRcvvhP4q1Yap2O4lMX6GeWx/ChwFPhsKx0GXlFVrwc+DHwuyeYR9Lja3+kof9fnc+yTk3E5hqt2ogfCWL0lRpLfoBcGn62qLwBU1ZGqeqaqfgV8ml8PaWx471V1qH19Eri99XKknerOn/I+Oar+Ot4G3FdVR1q/Y3MMm9Ues1mOHbLZkD6T7KX3jnn/tg1h0IZiftTm76U3Rv+aje5xgN/pqI7hycAfAbfM18blGA7iRA+EsXlLjDbOeC3wSFX9Rae+pbPau4D5uxj2A3uSbEqyHdhB74LUevX320leND9P76Ljg62PvW21vcAdo+hvgWOekY3LMexY1TFrw0pPJTmr/Tu5oLPNukjvA6k+Aryzqn7eqU+k9/kkJHll6/H7G93jan+noziGze8B362qZ4eCxuUYDmTUV7XXewLOoXdHz2PAR0fYx7+kd3p4P3CgTecAnwEeaPX9wJbONh9tfT/KOt+NQO9OrO+06aH5YwWcAnwF+F77+tJR9Nf5mb8F/Ah4cac2smNIL5gOA/+P3jPACwc5ZsAUvT96jwH/nfYuAuvY4wy9sfj5f4ufauv+6/b7/w5wH/CH693jIv2t+ne60cew1a8H3r9g3Q0/hms1+dYVkiTgxB8ykiStkIEgSQIMBElSYyBIkgADQZLUGAiSJMBAkCQ1/x9hiS2DQZfPYAAAAABJRU5ErkJggg==",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          },
          "output_type": "display_data"
        }
      ],
      "source": [
        "plt.hist(GetLens(triviaQA_training_dataset), bins=100);\n",
        "plt.hist(GetLens(triviaQA_dev_dataset), bins=100);"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 20,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "triviaqa_train.json 69 %\n",
            "triviaqa_dev.json 70 %\n"
          ]
        }
      ],
      "source": [
        "def AnswersNotAffectedByTruncationPercentage(data):\n",
        "    cut = 0\n",
        "    ok = 0\n",
        "    data_loader = DataLoader(dataset=data, batch_size=1024)\n",
        "\n",
        "    for batch in data_loader:\n",
        "        ends = batch['answer_end']\n",
        "\n",
        "        for sample in ends:\n",
        "            if sample <= 512:\n",
        "                ok += 1\n",
        "            else:\n",
        "                cut += 1\n",
        "\n",
        "    return ok / (cut + ok)\n",
        "\n",
        "# Percentage of answers not affected by truncation.\n",
        "print(triviaQA_train_file_name, int(AnswersNotAffectedByTruncationPercentage(triviaQA_training_dataset) * 100), \"%\")\n",
        "print(triviaQA_dev_file_name, int(AnswersNotAffectedByTruncationPercentage(triviaQA_dev_dataset) * 100), \"%\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "# Utility functions."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Get answer."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 49,
      "metadata": {},
      "outputs": [],
      "source": [
        "def GetStartEnd(starts, ends):\n",
        "    # Maximize propabilities while keeping end > start.\n",
        "    # Following brute force method.\n",
        "\n",
        "    sums = []\n",
        "    start_i = 0\n",
        "    for start in starts:\n",
        "        end_i = 0\n",
        "        for end in ends:\n",
        "            sum = start + end\n",
        "\n",
        "            # Skip invalid pairs.\n",
        "            if end >= start:\n",
        "                element = (sum, start_i, end_i)\n",
        "                sums.append(element)\n",
        "\n",
        "            end_i += 1\n",
        "        start_i += 1\n",
        "\n",
        "    max_element = max(sums, key=lambda s: s[0])\n",
        "    return max_element[1], max_element[2]\n",
        "\n",
        "\n",
        "def GetAnswer(input_ids, starts, ends):\n",
        "    # NOTE: way too slow.\n",
        "    #start, end = GetStartEnd(starts, ends)\n",
        "\n",
        "    start = torch.argmax(starts)\n",
        "    end = torch.argmax(ends)\n",
        "    tokenized_context = tokenizer.convert_ids_to_tokens(input_ids)\n",
        "\n",
        "    # Join list of words.\n",
        "    answer = \" \".join(tokenized_context[start:end+1])\n",
        "    # Concat subwords.\n",
        "    answer = re.sub(r\"( ##|##)\", \"\", answer)\n",
        "    return answer"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Save / Checkpoint."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 57,
      "metadata": {},
      "outputs": [],
      "source": [
        "def GetSaveName(epoch, prefix=\"\"):\n",
        "    save_name = prefix + \"model_epoch\" + str(epoch + 1)\n",
        "    return \"./saved_models/\" + save_name\n",
        "\n",
        "\n",
        "def LoadSavedModel(max_epochs, prefix=\"\"):\n",
        "    # Loop through the epochs in reverse order. \n",
        "    # Trying to find the most trained checkpoint.\n",
        "    for epoch in reversed(range(max_epochs)):\n",
        "        save_name = GetSaveName(epoch, prefix)\n",
        "        if os.path.isdir(save_name):\n",
        "            finetuned_model = AutoModelForQuestionAnswering.from_pretrained(save_name)\n",
        "            loaded_epoch = epoch + 1\n",
        "\n",
        "            print(f\"Loaded finetuned model trained with {loaded_epoch} epochs\")\n",
        "            return finetuned_model, loaded_epoch\n",
        "\n",
        "    raise Exception(\"Model not found.\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Subset."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 51,
      "metadata": {},
      "outputs": [],
      "source": [
        "#Since the dataset is too big we can take a subset for faster training. (Ended up not being used)\n",
        "def GetSubset(full_dataset, percentage):\n",
        "    full_size = len(set)\n",
        "    new_size = int(percentage * full_size)\n",
        "    discard_size = full_size - new_size\n",
        "    new_dataset, _ = torch.utils.data.random_split(full_dataset, [new_size, discard_size])\n",
        "    return new_dataset"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Training."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 52,
      "metadata": {},
      "outputs": [],
      "source": [
        "def BatchTokenization(contexts, questions, max_length):\n",
        "    tokens = tokenizer(\n",
        "        list(contexts),\n",
        "        list(questions),\n",
        "        max_length=max_length,\n",
        "        truncation=True,\n",
        "        padding=True,\n",
        "        return_tensors='pt'\n",
        "    )\n",
        "\n",
        "    return tokens\n",
        "\n",
        "\n",
        "def UpdateStartEnd(tokenized_cq, answers_start, answers_end):\n",
        "    new_answers_start = []\n",
        "    new_answers_end = []\n",
        "\n",
        "    # Update start / end indices to match real sentences.\n",
        "    for i, (start, end) in enumerate(zip(answers_start, answers_end)):\n",
        "        new_start = tokenized_cq.char_to_token(i, start)\n",
        "        if new_start is None:\n",
        "            new_start = tokenizer.model_max_length\n",
        "        new_answers_start.append(new_start)\n",
        "\n",
        "        new_end = tokenized_cq.char_to_token(i, end)\n",
        "        curr_count = 1\n",
        "        while new_end is None:\n",
        "            new_end = tokenized_cq.char_to_token(i, end-curr_count)\n",
        "            curr_count += 1\n",
        "        new_answers_end.append(new_end)\n",
        "\n",
        "    return new_answers_start, new_answers_end\n",
        "\n",
        "\n",
        "def ModelOutput(model, max_length, contexts, questions, answers_start=None, answers_end=None):\n",
        "    tokenized_cq = BatchTokenization(contexts, questions, max_length)\n",
        "    batch_input_ids = tokenized_cq['input_ids'].to(device)\n",
        "    batch_attention_mask = tokenized_cq['attention_mask'].to(device)\n",
        "\n",
        "    # Only needed on training data.\n",
        "    if answers_start != None and answers_end != None:\n",
        "        new_answers_start, new_answers_end = UpdateStartEnd(tokenized_cq, answers_start, answers_end)\n",
        "        answers_start = torch.tensor(new_answers_start).to(device)\n",
        "        answers_end = torch.tensor(new_answers_end).to(device)\n",
        "    \n",
        "    # Forward.\n",
        "    # Some model do not use token type ids. Disabled by global variable.\n",
        "    if use_type_ids:\n",
        "        batch_token_type_ids = tokenized_cq['token_type_ids'].to(device)\n",
        "        output = model(\n",
        "            input_ids=batch_input_ids,\n",
        "            attention_mask=batch_attention_mask,\n",
        "            token_type_ids=batch_token_type_ids,\n",
        "            start_positions=answers_start,\n",
        "            end_positions=answers_end\n",
        "        )\n",
        "    else:\n",
        "        output = model(\n",
        "            input_ids=batch_input_ids,\n",
        "            attention_mask=batch_attention_mask,\n",
        "            start_positions=answers_start,\n",
        "            end_positions=answers_end\n",
        "        )\n",
        "\n",
        "    return output, batch_input_ids\n",
        "\n",
        "\n",
        "def SetSeed(seed):\n",
        "    torch.manual_seed(seed)\n",
        "    np.random.seed(seed)\n",
        "    random.seed(seed)\n",
        "\n",
        "\n",
        "def TrainModel(model, args):\n",
        "    # Main function arguments through a dictionary.\n",
        "    starting_epoch = args['starting_epoch']\n",
        "    n_epochs = args['n_epochs']\n",
        "    lr = args['lr']\n",
        "    max_length = args['max_length']\n",
        "    training_data_loader = args['training_data_loader']\n",
        "    name_prefix = args['name_prefix']\n",
        "\n",
        "    optimizer = torch.optim.Adam(model.parameters(), lr=lr)\n",
        "\n",
        "    # Training loop.\n",
        "    for epoch in range(n_epochs - starting_epoch):\n",
        "        # tqdm provides a progress bar.\n",
        "        for batch in tqdm(training_data_loader):\n",
        "            contexts = batch['context']\n",
        "            questions = batch['question']\n",
        "            answers_start = batch['answer_start'].to(device)\n",
        "            answers_end = batch['answer_end'].to(device)\n",
        "\n",
        "            output, _ = ModelOutput(model, max_length, contexts, questions, answers_start, answers_end)\n",
        "\n",
        "            # Backward.\n",
        "            optimizer.zero_grad()\n",
        "            output.loss.backward()\n",
        "            optimizer.step()\n",
        "\n",
        "        # Checkpoint each epoch.\n",
        "        model.save_pretrained(GetSaveName(starting_epoch + epoch, name_prefix))\n",
        "\n",
        "\n",
        "def TryModel(args):\n",
        "    batch_s = args['batch_s']\n",
        "    training_set = args['training_set']\n",
        "    load_saved_model = args['load_saved_model']\n",
        "    max_epochs = args['n_epochs']\n",
        "    name_prefix = args['name_prefix']\n",
        "\n",
        "    # Set static seed to prevent random results from batch shufling.\n",
        "    SetSeed(153)\n",
        "\n",
        "    # Load previous checkpoint (if enabled).\n",
        "    # If max_epochs > saved_epoch then training is continued from the last checkpoint.\n",
        "    starting_epoch = 0\n",
        "    if load_saved_model:\n",
        "        model, starting_epoch = LoadSavedModel(max_epochs, name_prefix)\n",
        "    else:\n",
        "        model = AutoModelForQuestionAnswering.from_pretrained(model_name) \n",
        "    \n",
        "    args['starting_epoch'] = starting_epoch\n",
        "    curr_model = model.to(device)\n",
        "\n",
        "    training_data_loader = DataLoader(dataset=training_set,\n",
        "                                      batch_size=batch_s,\n",
        "                                      shuffle=True,\n",
        "                                      # num_workers=2\n",
        "                                      )\n",
        "    args['training_data_loader'] = training_data_loader\n",
        "\n",
        "    # Using pretrained bert model.\n",
        "    TrainModel(curr_model, args)\n",
        "\n",
        "    return curr_model"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Predictions."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {},
      "outputs": [],
      "source": [
        "def CalculatePredictions(model, data, batch_s, max_length=None):\n",
        "    # Create a dictionary with format <question_id:answer> for each question-context pair.\n",
        "    with torch.no_grad():\n",
        "        predictions = dict()\n",
        "\n",
        "        model = model.to(device)\n",
        "        data_loader = DataLoader(dataset=data, batch_size=batch_s)\n",
        "        for batch in tqdm(data_loader):\n",
        "            contexts = batch['context']\n",
        "            questions = batch['question']\n",
        "            qas_id = batch['id']\n",
        "            \n",
        "            output, inputs_ids = ModelOutput(model, max_length, contexts, questions)\n",
        "            batch_start_logits = output['start_logits']\n",
        "            batch_end_logits = output['end_logits']\n",
        "\n",
        "            sample_zip = zip(batch_start_logits, batch_end_logits, inputs_ids, questions, qas_id, contexts)\n",
        "            for start_logits, end_logits, input_ids, question, qas_id, context in sample_zip:\n",
        "                answer = GetAnswer(input_ids, start_logits, end_logits)\n",
        "                predictions[qas_id] = answer\n",
        "                \n",
        "                #print(question)\n",
        "                #print(answer)\n",
        "                #print(\"---\")\n",
        "\n",
        "        return predictions\n",
        "\n",
        "\n",
        "def GetPredictionsName(prefix):\n",
        "    return f\"./{prefix}predictions.json\"\n",
        "\n",
        "\n",
        "def CreatePredictionsJSON(predictions, prefix):\n",
        "    pred_file_name = GetPredictionsName(prefix)\n",
        "    with open(pred_file_name, 'w') as file:\n",
        "        json.dump(predictions, file)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "# Setup cuda / cpu device"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 54,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Using Cuda\n"
          ]
        }
      ],
      "source": [
        "# If cuda is available GPU will be used for\n",
        "# faster training. Restricted usage in colab.\n",
        "run_gpu = True\n",
        "if torch.cuda.is_available and run_gpu:\n",
        "    print(\"Using Cuda\")\n",
        "    device = torch.device(\"cuda\")\n",
        "else:\n",
        "    print(\"Using the CPU\")\n",
        "    device = torch.device(\"cpu\")\n",
        "\n",
        "transformers.logging.set_verbosity_error()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "# Squad2."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Fine tuning."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 60,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded finetuned model trained with 2 epochs\n"
          ]
        }
      ],
      "source": [
        "squad_prefix = \"squad_\"\n",
        "triviaQA_prefix = \"triviaQA_\"\n",
        "\n",
        "TryModel({\n",
        "    'name_prefix': squad_prefix,\n",
        "    'load_saved_model': False,\n",
        "    'training_set': squad_training_dataset,\n",
        "    'n_epochs': 2,\n",
        "    'batch_s': 24, # Make this smaller if vram problems occure.\n",
        "    'lr': 3e-5, \n",
        "    'max_length': 384 # None for max\n",
        "});"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Create predictions file."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "squad_finetuned_model, _ = LoadSavedModel(2, squad_prefix)\n",
        "\n",
        "predictions = CalculatePredictions(squad_finetuned_model, squad_dev_dataset, 8)\n",
        "CreatePredictionsJSON(predictions, squad_prefix + squad_prefix)\n",
        "\n",
        "predictions = CalculatePredictions(squad_finetuned_model, triviaQA_dev_dataset, 8)\n",
        "CreatePredictionsJSON(predictions, squad_prefix + triviaQA_prefix)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Custom questions / contexts."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 66,
      "metadata": {},
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 1/1 [00:00<00:00,  2.30it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Context: The modern Olympic Games are leading international multi-sport events in which thousands of athletes from around the world participate in various competitions. The Summer and Winter Games are each held in a different city every four years, offset from each other by two. Inspired by the ancient Greek Olympic Games, Pierre de Coubertin founded the International Olympic Committee (IOC) in 1894, leading to the first modern Games in 1896.\n",
            "\n",
            "Question: who invented the olympic games?\n",
            "Answer: pierre de coubertin\n",
            "- - - -\n",
            "Context: SQuAD2.0 combines the 100,000 questions in SQuAD1.1 with over 50,000 unanswerable questions written adversarially by crowdworkers to look similar to answerable ones. To do well on SQuAD2.0, systems must not only answer questions when possible, but also determine when no answer is supported by the paragraph and abstain from answering.\n",
            "\n",
            "Question: How many questions does squad 1 contain?\n",
            "Answer: 100 , 000\n",
            "- - - -\n",
            "Context: Elon Reeve Musk FRS is an entrepreneur and business magnate. He is the founder, CEO, and Chief Engineer at SpaceX; early-stage investor, CEO, and Product Architect of Tesla, Inc.; founder of The Boring Company; and co-founder of Neuralink and OpenAI.\n",
            "\n",
            "Question: Who is elon musk?\n",
            "Answer: an entrepreneur and business magnate .\n",
            "- - - -\n",
            "Context: I don’t trust stairs because they’re always up to somethin\n",
            "\n",
            "Question: Why dont i trust stairs?\n",
            "Answer: they ’ re always up to somethin\n",
            "- - - -\n",
            "Context: Tetris is a puzzle video game created by Soviet software engineer Alexey Pajitnov in 1984. It has been published by several companies for multiple platforms, most prominently during a dispute over the appropriation of the rights in the late 1980s.\n",
            "\n",
            "Question: When was tetris released?\n",
            "Answer: 1984 .\n",
            "- - - -\n",
            "Context: In competitive games, the piece colors are allocated to players by the organizers; in informal games, the colors are usually decided randomly, for example by a coin toss, or by one player concealing a white pawn in one hand and a black pawn in the other, and having the opponent choose. White moves first, after which players alternate turns, moving one piece per turn (except for castling, when two pieces are moved). A piece is moved to either an unoccupied square or one occupied by an opponent's piece, which is captured and removed from play. With the sole exception of en passant, all pieces capture by moving to the square that the opponent's piece occupies. Moving is compulsory; a player may not skip a turn, even when having to move is detrimental.\n",
            "\n",
            "Question: Who plays first in chess?\n",
            "Answer: white\n",
            "- - - -\n",
            "Context: George's dog is barking outside the house while George's wife is at the supermarket.\n",
            "\n",
            "Question: What is george's dog doing?\n",
            "Answer: barking\n",
            "- - - -\n",
            "Context: Alex likes Susan\n",
            "\n",
            "Question: What is the wheather like?\n",
            "Answer: \n",
            "- - - -\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\n"
          ]
        }
      ],
      "source": [
        "contexts = [] \n",
        "questions = []\n",
        "\n",
        "contexts.append('The modern Olympic Games are leading international multi-sport events in which thousands of athletes from around the world participate in various competitions. The Summer and Winter Games are each held in a different city every four years, offset from each other by two. Inspired by the ancient Greek Olympic Games, Pierre de Coubertin founded the International Olympic Committee (IOC) in 1894, leading to the first modern Games in 1896.')\n",
        "questions.append('who invented the olympic games?')\n",
        "\n",
        "contexts.append('SQuAD2.0 combines the 100,000 questions in SQuAD1.1 with over 50,000 unanswerable questions written adversarially by crowdworkers to look similar to answerable ones. To do well on SQuAD2.0, systems must not only answer questions when possible, but also determine when no answer is supported by the paragraph and abstain from answering.')\n",
        "questions.append('How many questions does squad 1 contain?')\n",
        "\n",
        "contexts.append('Elon Reeve Musk FRS is an entrepreneur and business magnate. He is the founder, CEO, and Chief Engineer at SpaceX; early-stage investor, CEO, and Product Architect of Tesla, Inc.; founder of The Boring Company; and co-founder of Neuralink and OpenAI.')\n",
        "questions.append('Who is elon musk?')\n",
        "\n",
        "contexts.append('I don’t trust stairs because they’re always up to somethin')\n",
        "questions.append('Why dont i trust stairs?')\n",
        "\n",
        "contexts.append('Tetris is a puzzle video game created by Soviet software engineer Alexey Pajitnov in 1984. It has been published by several companies for multiple platforms, most prominently during a dispute over the appropriation of the rights in the late 1980s.')\n",
        "questions.append('When was tetris released?')\n",
        "\n",
        "contexts.append('In competitive games, the piece colors are allocated to players by the organizers; in informal games, the colors are usually decided randomly, for example by a coin toss, or by one player concealing a white pawn in one hand and a black pawn in the other, and having the opponent choose. White moves first, after which players alternate turns, moving one piece per turn (except for castling, when two pieces are moved). A piece is moved to either an unoccupied square or one occupied by an opponent\\'s piece, which is captured and removed from play. With the sole exception of en passant, all pieces capture by moving to the square that the opponent\\'s piece occupies. Moving is compulsory; a player may not skip a turn, even when having to move is detrimental.')\n",
        "questions.append('Who plays first in chess?')\n",
        "\n",
        "contexts.append('George\\'s dog is barking outside the house while George\\'s wife is at the supermarket.')\n",
        "questions.append(\"What is george\\'s dog doing?\")\n",
        "\n",
        "contexts.append('Alex likes Susan')\n",
        "questions.append(\"What is the wheather like?\")\n",
        "\n",
        "custom_set = CustomQuestions(contexts, questions)\n",
        "custom_predictions = CalculatePredictions(squad_finetuned_model, custom_set, 8)\n",
        "\n",
        "for i, q_id in enumerate(custom_predictions):\n",
        "    print(\"Context:\", custom_set[i]['context'])\n",
        "    print(\"\")\n",
        "    print(\"Question:\", custom_set[i]['question'])\n",
        "    print(\"Answer:\", custom_predictions[q_id])\n",
        "    print(\"- - - -\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "# TriviaQA"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Fine tuning."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "TryModel({\n",
        "    'name_prefix': triviaQA_prefix,\n",
        "    'load_saved_model': False,\n",
        "    'training_set': triviaQA_training_dataset,\n",
        "    'n_epochs': 2,\n",
        "    'batch_s': 24, # Make this smaller if vram problems occure.\n",
        "    'lr': 3e-5, \n",
        "    'max_length': 384 # None for max\n",
        "});"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Create predictions file."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "triviaQA_finetuned_model, _ = LoadSavedModel(2, triviaQA_prefix)\n",
        "\n",
        "predictions = CalculatePredictions(triviaQA_finetuned_model, triviaQA_dev_dataset, 8)\n",
        "CreatePredictionsJSON(predictions, triviaQA_prefix + triviaQA_prefix)\n",
        "\n",
        "predictions = CalculatePredictions(triviaQA_finetuned_model, squad_dev_dataset, 8)\n",
        "CreatePredictionsJSON(predictions, triviaQA_prefix + squad_prefix)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "# Predictions evaluation.\n",
        "The following convention corresponds to the tuples <model, dataset>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Squad2-Squad2"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 33,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/plain": [
              "['{',\n",
              " '  \"exact\": 64.51612903225806,',\n",
              " '  \"f1\": 69.4742652718149,',\n",
              " '  \"total\": 11873,',\n",
              " '  \"HasAns_exact\": 61.11673414304993,',\n",
              " '  \"HasAns_f1\": 71.04722529896368,',\n",
              " '  \"HasAns_total\": 5928,',\n",
              " '  \"NoAns_exact\": 67.90580319596299,',\n",
              " '  \"NoAns_f1\": 67.90580319596299,',\n",
              " '  \"NoAns_total\": 5945',\n",
              " '}']"
            ]
          },
          "execution_count": 33,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "predictions_name = GetPredictionsName(squad_prefix + squad_prefix)\n",
        "squad_squad_accuracy = !python ./evaluate-v2.0.py $squad_dev_file_name $predictions_name\n",
        "squad_squad_accuracy"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Squad2-TriviaQA"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 37,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/plain": [
              "['{',\n",
              " '  \"exact\": 36.43966547192353,',\n",
              " '  \"f1\": 41.75575469008161,',\n",
              " '  \"total\": 14229,',\n",
              " '  \"HasAns_exact\": 25.16764885185938,',\n",
              " '  \"HasAns_f1\": 32.853346218773474,',\n",
              " '  \"HasAns_total\": 9842,',\n",
              " '  \"NoAns_exact\": 61.72783223159335,',\n",
              " '  \"NoAns_f1\": 61.72783223159335,',\n",
              " '  \"NoAns_total\": 4387',\n",
              " '}']"
            ]
          },
          "execution_count": 37,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "predictions_name = GetPredictionsName(squad_prefix + triviaQA_prefix)\n",
        "squad_trivia_accuracy = !python ./evaluate-v2.0.py $triviaQA_dev_file_name $predictions_name\n",
        "squad_trivia_accuracy"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## TriviaQA-TriviaQA"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 36,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/plain": [
              "['{',\n",
              " '  \"exact\": 52.75845105067116,',\n",
              " '  \"f1\": 56.193190058036436,',\n",
              " '  \"total\": 14229,',\n",
              " '  \"HasAns_exact\": 43.131477342003656,',\n",
              " '  \"HasAns_f1\": 48.097226309266205,',\n",
              " '  \"HasAns_total\": 9842,',\n",
              " '  \"NoAns_exact\": 74.35605197173467,',\n",
              " '  \"NoAns_f1\": 74.35605197173467,',\n",
              " '  \"NoAns_total\": 4387',\n",
              " '}']"
            ]
          },
          "execution_count": 36,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "predictions_name = GetPredictionsName(triviaQA_prefix + triviaQA_prefix)\n",
        "trivia_trivia_accuracy = !python ./evaluate-v2.0.py $triviaQA_dev_file_name $predictions_name\n",
        "trivia_trivia_accuracy"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## TriviaQA-Squad2"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 29,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/plain": [
              "['{',\n",
              " '  \"exact\": 30.447233218226227,',\n",
              " '  \"f1\": 34.253412183549884,',\n",
              " '  \"total\": 11873,',\n",
              " '  \"HasAns_exact\": 27.10863697705803,',\n",
              " '  \"HasAns_f1\": 34.73191006330798,',\n",
              " '  \"HasAns_total\": 5928,',\n",
              " '  \"NoAns_exact\": 33.77628259041211,',\n",
              " '  \"NoAns_f1\": 33.77628259041211,',\n",
              " '  \"NoAns_total\": 5945',\n",
              " '}']"
            ]
          },
          "execution_count": 29,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "predictions_name = GetPredictionsName(triviaQA_prefix + squad_prefix)\n",
        "trivia_squad_accuracy = !python ./evaluate-v2.0.py $squad_dev_file_name $predictions_name\n",
        "trivia_squad_accuracy"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Final table"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 38,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>squad2</th>\n",
              "      <th>triviaQA</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>squad2</th>\n",
              "      <td>69.474265</td>\n",
              "      <td>41.755755</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>triviaQA</th>\n",
              "      <td>34.253412</td>\n",
              "      <td>56.193190</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "             squad2   triviaQA\n",
              "squad2    69.474265  41.755755\n",
              "triviaQA  34.253412  56.193190"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "def GetF1(accuracy_result):\n",
        "    # Result stored as list of strings.\n",
        "    dict_string = \" \".join(accuracy_result)\n",
        "    accuracy_dictionary = json.loads(dict_string)\n",
        "    return accuracy_dictionary['f1']\n",
        "\n",
        "squad_squad_f1 = GetF1(squad_squad_accuracy)\n",
        "squad_trivia_f1 = GetF1(squad_trivia_accuracy)\n",
        "trivia_trivia_f1 = GetF1(trivia_trivia_accuracy)\n",
        "trivia_squad_f1 = GetF1(trivia_squad_accuracy)\n",
        "\n",
        "dataset_names = ['squad2', 'triviaQA']\n",
        "results = pd.DataFrame([[squad_squad_f1, squad_trivia_f1],[trivia_squad_f1, trivia_trivia_f1]], dataset_names, dataset_names)\n",
        "display(results)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "# Zip saved models."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "!tar -zcvf saved_models.tar.gz ./saved_models"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "collapsed_sections": [],
      "name": "erg4.ipynb",
      "provenance": [],
      "toc_visible": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.8.8"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "03ae2a8acee7464c9845b485347cc5e1": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "0b0a34d3d3c6480b88180c5b4270d81c": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e8e47b9c09e34470ae79c813c8c424af",
            "placeholder": "​",
            "style": "IPY_MODEL_e122624b71be451ca95769ca8bc19403",
            "value": " 420M/420M [00:22&lt;00:00, 11.1MB/s]"
          }
        },
        "1223a56b9df047cda9e9284b5b64ed26": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3805feead565455ca4e56fbf93ad33b0": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_1223a56b9df047cda9e9284b5b64ed26",
            "max": 440473133,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_03ae2a8acee7464c9845b485347cc5e1",
            "value": 440473133
          }
        },
        "5b4b6c2556134b358bd2fa477c584cf9": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a92acf4e0f3546d5944f6f8029f0caf1": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ab1443ee2af1482ead4272414933e460": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a92acf4e0f3546d5944f6f8029f0caf1",
            "placeholder": "​",
            "style": "IPY_MODEL_ea1ad5057bbd47a3b6416f8d63ca5e70",
            "value": "Downloading: 100%"
          }
        },
        "b1d5b5f23b254db0b8ed713c9f68df3e": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_ab1443ee2af1482ead4272414933e460",
              "IPY_MODEL_3805feead565455ca4e56fbf93ad33b0",
              "IPY_MODEL_0b0a34d3d3c6480b88180c5b4270d81c"
            ],
            "layout": "IPY_MODEL_5b4b6c2556134b358bd2fa477c584cf9"
          }
        },
        "e122624b71be451ca95769ca8bc19403": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "e8e47b9c09e34470ae79c813c8c424af": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ea1ad5057bbd47a3b6416f8d63ca5e70": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}