02_Data_Augmentation.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "4cce1e79a3ed4477b5611802b54e4ece": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_20ea4da1c5c042b0995b42c2b7db0bce",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_5c7b9e5834c948d0978282b00fff8a27",
              "IPY_MODEL_b3d175109f484fb89459cb0bbd93b7a8",
              "IPY_MODEL_bcc4b6427b1649d7b0bddaa68a261445"
            ]
          }
        },
        "20ea4da1c5c042b0995b42c2b7db0bce": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "5c7b9e5834c948d0978282b00fff8a27": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_606369f010704aa1af0bce8d27ccac1f",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "Downloading: 100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_e5da8ea5e93f41c0b40101c07266765c"
          }
        },
        "b3d175109f484fb89459cb0bbd93b7a8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_27a226bd008f48098d736d307709d3be",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 825,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 825,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_ae59656d924e4c75be90c8035b2f10cb"
          }
        },
        "bcc4b6427b1649d7b0bddaa68a261445": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_20b14c89957a4ed997f3136558e38cfc",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 825/825 [00:00&lt;00:00, 20.9kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_621d4b7b9308455aab6ac892e41f8f54"
          }
        },
        "606369f010704aa1af0bce8d27ccac1f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "e5da8ea5e93f41c0b40101c07266765c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "27a226bd008f48098d736d307709d3be": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "ae59656d924e4c75be90c8035b2f10cb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "20b14c89957a4ed997f3136558e38cfc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "621d4b7b9308455aab6ac892e41f8f54": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/l43lu/NLP_Project_Assignment/blob/master/02_Data_Augmentation.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "18ns33DmZScj"
      },
      "source": [
        "##Mit der Backtranslation Funktion wird die ungleiche Menge an Artikel pro Artikel-Typ ausgeglichen."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bOxG4fahcYp1",
        "outputId": "58199a82-6a78-4493-dd4c-948d81c883c1"
      },
      "source": [
        "!pip install stopwordsiso"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting stopwordsiso\n",
            "  Downloading stopwordsiso-0.6.1-py3-none-any.whl (73 kB)\n",
            "\u001b[?25l\r\u001b[K     |████▌                           | 10 kB 28.2 MB/s eta 0:00:01\r\u001b[K     |█████████                       | 20 kB 31.8 MB/s eta 0:00:01\r\u001b[K     |█████████████▍                  | 30 kB 36.6 MB/s eta 0:00:01\r\u001b[K     |█████████████████▉              | 40 kB 30.2 MB/s eta 0:00:01\r\u001b[K     |██████████████████████▎         | 51 kB 18.6 MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▊     | 61 kB 15.3 MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▏| 71 kB 16.9 MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 73 kB 2.3 MB/s \n",
            "\u001b[?25hInstalling collected packages: stopwordsiso\n",
            "Successfully installed stopwordsiso-0.6.1\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wocBDpiocdse",
        "outputId": "232fe93f-53a5-4c36-ef33-41b6c33e2c2e"
      },
      "source": [
        "!pip install nlpaug"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting nlpaug\n",
            "  Downloading nlpaug-1.1.7-py3-none-any.whl (405 kB)\n",
            "\u001b[?25l\r\u001b[K     |▉                               | 10 kB 27.2 MB/s eta 0:00:01\r\u001b[K     |█▋                              | 20 kB 34.6 MB/s eta 0:00:01\r\u001b[K     |██▍                             | 30 kB 41.1 MB/s eta 0:00:01\r\u001b[K     |███▎                            | 40 kB 30.3 MB/s eta 0:00:01\r\u001b[K     |████                            | 51 kB 18.5 MB/s eta 0:00:01\r\u001b[K     |████▉                           | 61 kB 15.5 MB/s eta 0:00:01\r\u001b[K     |█████▋                          | 71 kB 14.5 MB/s eta 0:00:01\r\u001b[K     |██████▌                         | 81 kB 16.1 MB/s eta 0:00:01\r\u001b[K     |███████▎                        | 92 kB 14.9 MB/s eta 0:00:01\r\u001b[K     |████████                        | 102 kB 13.4 MB/s eta 0:00:01\r\u001b[K     |█████████                       | 112 kB 13.4 MB/s eta 0:00:01\r\u001b[K     |█████████▊                      | 122 kB 13.4 MB/s eta 0:00:01\r\u001b[K     |██████████▌                     | 133 kB 13.4 MB/s eta 0:00:01\r\u001b[K     |███████████▎                    | 143 kB 13.4 MB/s eta 0:00:01\r\u001b[K     |████████████▏                   | 153 kB 13.4 MB/s eta 0:00:01\r\u001b[K     |█████████████                   | 163 kB 13.4 MB/s eta 0:00:01\r\u001b[K     |█████████████▊                  | 174 kB 13.4 MB/s eta 0:00:01\r\u001b[K     |██████████████▋                 | 184 kB 13.4 MB/s eta 0:00:01\r\u001b[K     |███████████████▍                | 194 kB 13.4 MB/s eta 0:00:01\r\u001b[K     |████████████████▏               | 204 kB 13.4 MB/s eta 0:00:01\r\u001b[K     |█████████████████               | 215 kB 13.4 MB/s eta 0:00:01\r\u001b[K     |█████████████████▉              | 225 kB 13.4 MB/s eta 0:00:01\r\u001b[K     |██████████████████▋             | 235 kB 13.4 MB/s eta 0:00:01\r\u001b[K     |███████████████████▍            | 245 kB 13.4 MB/s eta 0:00:01\r\u001b[K     |████████████████████▎           | 256 kB 13.4 MB/s eta 0:00:01\r\u001b[K     |█████████████████████           | 266 kB 13.4 MB/s eta 0:00:01\r\u001b[K     |█████████████████████▉          | 276 kB 13.4 MB/s eta 0:00:01\r\u001b[K     |██████████████████████▋         | 286 kB 13.4 MB/s eta 0:00:01\r\u001b[K     |███████████████████████▌        | 296 kB 13.4 MB/s eta 0:00:01\r\u001b[K     |████████████████████████▎       | 307 kB 13.4 MB/s eta 0:00:01\r\u001b[K     |█████████████████████████       | 317 kB 13.4 MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▉      | 327 kB 13.4 MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▊     | 337 kB 13.4 MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▌    | 348 kB 13.4 MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▎   | 358 kB 13.4 MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████▏  | 368 kB 13.4 MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████  | 378 kB 13.4 MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▊ | 389 kB 13.4 MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▌| 399 kB 13.4 MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 405 kB 13.4 MB/s \n",
            "\u001b[?25hInstalling collected packages: nlpaug\n",
            "Successfully installed nlpaug-1.1.7\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7rMacqHHgyse",
        "outputId": "d011f06b-daa2-43ab-9e4d-1011ad505602"
      },
      "source": [
        "!pip install transformers"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting transformers\n",
            "  Downloading transformers-4.11.3-py3-none-any.whl (2.9 MB)\n",
            "\u001b[?25l\r\u001b[K     |▏                               | 10 kB 22.3 MB/s eta 0:00:01\r\u001b[K     |▎                               | 20 kB 31.1 MB/s eta 0:00:01\r\u001b[K     |▍                               | 30 kB 24.8 MB/s eta 0:00:01\r\u001b[K     |▌                               | 40 kB 14.2 MB/s eta 0:00:01\r\u001b[K     |▋                               | 51 kB 14.2 MB/s eta 0:00:01\r\u001b[K     |▊                               | 61 kB 16.1 MB/s eta 0:00:01\r\u001b[K     |▉                               | 71 kB 14.4 MB/s eta 0:00:01\r\u001b[K     |█                               | 81 kB 15.9 MB/s eta 0:00:01\r\u001b[K     |█                               | 92 kB 13.1 MB/s eta 0:00:01\r\u001b[K     |█▏                              | 102 kB 13.9 MB/s eta 0:00:01\r\u001b[K     |█▎                              | 112 kB 13.9 MB/s eta 0:00:01\r\u001b[K     |█▍                              | 122 kB 13.9 MB/s eta 0:00:01\r\u001b[K     |█▌                              | 133 kB 13.9 MB/s eta 0:00:01\r\u001b[K     |█▋                              | 143 kB 13.9 MB/s eta 0:00:01\r\u001b[K     |█▊                              | 153 kB 13.9 MB/s eta 0:00:01\r\u001b[K     |█▉                              | 163 kB 13.9 MB/s eta 0:00:01\r\u001b[K     |██                              | 174 kB 13.9 MB/s eta 0:00:01\r\u001b[K     |██                              | 184 kB 13.9 MB/s eta 0:00:01\r\u001b[K     |██▏                             | 194 kB 13.9 MB/s eta 0:00:01\r\u001b[K     |██▎                             | 204 kB 13.9 MB/s eta 0:00:01\r\u001b[K     |██▍                             | 215 kB 13.9 MB/s eta 0:00:01\r\u001b[K     |██▌                             | 225 kB 13.9 MB/s eta 0:00:01\r\u001b[K     |██▋                             | 235 kB 13.9 MB/s eta 0:00:01\r\u001b[K     |██▊                             | 245 kB 13.9 MB/s eta 0:00:01\r\u001b[K     |██▉                             | 256 kB 13.9 MB/s eta 0:00:01\r\u001b[K     |███                             | 266 kB 13.9 MB/s eta 0:00:01\r\u001b[K     |███                             | 276 kB 13.9 MB/s eta 0:00:01\r\u001b[K     |███▏                            | 286 kB 13.9 MB/s eta 0:00:01\r\u001b[K     |███▎                            | 296 kB 13.9 MB/s eta 0:00:01\r\u001b[K     |███▍                            | 307 kB 13.9 MB/s eta 0:00:01\r\u001b[K     |███▌                            | 317 kB 13.9 MB/s eta 0:00:01\r\u001b[K     |███▋                            | 327 kB 13.9 MB/s eta 0:00:01\r\u001b[K     |███▊                            | 337 kB 13.9 MB/s eta 0:00:01\r\u001b[K     |███▉                            | 348 kB 13.9 MB/s eta 0:00:01\r\u001b[K     |████                            | 358 kB 13.9 MB/s eta 0:00:01\r\u001b[K     |████                            | 368 kB 13.9 MB/s eta 0:00:01\r\u001b[K     |████▏                           | 378 kB 13.9 MB/s eta 0:00:01\r\u001b[K     |████▎                           | 389 kB 13.9 MB/s eta 0:00:01\r\u001b[K     |████▍                           | 399 kB 13.9 MB/s eta 0:00:01\r\u001b[K     |████▋                           | 409 kB 13.9 MB/s eta 0:00:01\r\u001b[K     |████▊                           | 419 kB 13.9 MB/s eta 0:00:01\r\u001b[K     |████▉                           | 430 kB 13.9 MB/s eta 0:00:01\r\u001b[K     |█████                           | 440 kB 13.9 MB/s eta 0:00:01\r\u001b[K     |█████                           | 450 kB 13.9 MB/s eta 0:00:01\r\u001b[K     |█████▏                          | 460 kB 13.9 MB/s eta 0:00:01\r\u001b[K     |█████▎                          | 471 kB 13.9 MB/s eta 0:00:01\r\u001b[K     |█████▍                          | 481 kB 13.9 MB/s eta 0:00:01\r\u001b[K     |█████▌                          | 491 kB 13.9 MB/s eta 0:00:01\r\u001b[K     |█████▋                          | 501 kB 13.9 MB/s eta 0:00:01\r\u001b[K     |█████▊                          | 512 kB 13.9 MB/s eta 0:00:01\r\u001b[K     |█████▉                          | 522 kB 13.9 MB/s eta 0:00:01\r\u001b[K     |██████                          | 532 kB 13.9 MB/s eta 0:00:01\r\u001b[K     |██████                          | 542 kB 13.9 MB/s eta 0:00:01\r\u001b[K     |██████▏                         | 552 kB 13.9 MB/s eta 0:00:01\r\u001b[K     |██████▎                         | 563 kB 13.9 MB/s eta 0:00:01\r\u001b[K     |██████▍                         | 573 kB 13.9 MB/s eta 0:00:01\r\u001b[K     |██████▌                         | 583 kB 13.9 MB/s eta 0:00:01\r\u001b[K     |██████▋                         | 593 kB 13.9 MB/s eta 0:00:01\r\u001b[K     |██████▊                         | 604 kB 13.9 MB/s eta 0:00:01\r\u001b[K     |██████▉                         | 614 kB 13.9 MB/s eta 0:00:01\r\u001b[K     |███████                         | 624 kB 13.9 MB/s eta 0:00:01\r\u001b[K     |███████                         | 634 kB 13.9 MB/s eta 0:00:01\r\u001b[K     |███████▏                        | 645 kB 13.9 MB/s eta 0:00:01\r\u001b[K     |███████▎                        | 655 kB 13.9 MB/s eta 0:00:01\r\u001b[K     |███████▍                        | 665 kB 13.9 MB/s eta 0:00:01\r\u001b[K     |███████▌                        | 675 kB 13.9 MB/s eta 0:00:01\r\u001b[K     |███████▋                        | 686 kB 13.9 MB/s eta 0:00:01\r\u001b[K     |███████▊                        | 696 kB 13.9 MB/s eta 0:00:01\r\u001b[K     |███████▉                        | 706 kB 13.9 MB/s eta 0:00:01\r\u001b[K     |████████                        | 716 kB 13.9 MB/s eta 0:00:01\r\u001b[K     |████████                        | 727 kB 13.9 MB/s eta 0:00:01\r\u001b[K     |████████▏                       | 737 kB 13.9 MB/s eta 0:00:01\r\u001b[K     |████████▎                       | 747 kB 13.9 MB/s eta 0:00:01\r\u001b[K     |████████▍                       | 757 kB 13.9 MB/s eta 0:00:01\r\u001b[K     |████████▌                       | 768 kB 13.9 MB/s eta 0:00:01\r\u001b[K     |████████▋                       | 778 kB 13.9 MB/s eta 0:00:01\r\u001b[K     |████████▊                       | 788 kB 13.9 MB/s eta 0:00:01\r\u001b[K     |████████▉                       | 798 kB 13.9 MB/s eta 0:00:01\r\u001b[K     |█████████                       | 808 kB 13.9 MB/s eta 0:00:01\r\u001b[K     |█████████▏                      | 819 kB 13.9 MB/s eta 0:00:01\r\u001b[K     |█████████▎                      | 829 kB 13.9 MB/s eta 0:00:01\r\u001b[K     |█████████▍                      | 839 kB 13.9 MB/s eta 0:00:01\r\u001b[K     |█████████▌                      | 849 kB 13.9 MB/s eta 0:00:01\r\u001b[K     |█████████▋                      | 860 kB 13.9 MB/s eta 0:00:01\r\u001b[K     |█████████▊                      | 870 kB 13.9 MB/s eta 0:00:01\r\u001b[K     |█████████▉                      | 880 kB 13.9 MB/s eta 0:00:01\r\u001b[K     |██████████                      | 890 kB 13.9 MB/s eta 0:00:01\r\u001b[K     |██████████                      | 901 kB 13.9 MB/s eta 0:00:01\r\u001b[K     |██████████▏                     | 911 kB 13.9 MB/s eta 0:00:01\r\u001b[K     |██████████▎                     | 921 kB 13.9 MB/s eta 0:00:01\r\u001b[K     |██████████▍                     | 931 kB 13.9 MB/s eta 0:00:01\r\u001b[K     |██████████▌                     | 942 kB 13.9 MB/s eta 0:00:01\r\u001b[K     |██████████▋                     | 952 kB 13.9 MB/s eta 0:00:01\r\u001b[K     |██████████▊                     | 962 kB 13.9 MB/s eta 0:00:01\r\u001b[K     |██████████▉                     | 972 kB 13.9 MB/s eta 0:00:01\r\u001b[K     |███████████                     | 983 kB 13.9 MB/s eta 0:00:01\r\u001b[K     |███████████                     | 993 kB 13.9 MB/s eta 0:00:01\r\u001b[K     |███████████▏                    | 1.0 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |███████████▎                    | 1.0 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |███████████▍                    | 1.0 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |███████████▌                    | 1.0 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |███████████▋                    | 1.0 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |███████████▊                    | 1.1 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |███████████▉                    | 1.1 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |████████████                    | 1.1 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |████████████                    | 1.1 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |████████████▏                   | 1.1 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |████████████▎                   | 1.1 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |████████████▍                   | 1.1 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |████████████▌                   | 1.1 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |████████████▋                   | 1.1 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |████████████▊                   | 1.1 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |████████████▉                   | 1.2 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |█████████████                   | 1.2 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |█████████████                   | 1.2 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |█████████████▏                  | 1.2 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |█████████████▎                  | 1.2 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |█████████████▍                  | 1.2 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |█████████████▌                  | 1.2 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |█████████████▊                  | 1.2 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |█████████████▉                  | 1.2 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |██████████████                  | 1.2 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |██████████████                  | 1.3 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |██████████████▏                 | 1.3 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |██████████████▎                 | 1.3 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |██████████████▍                 | 1.3 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |██████████████▌                 | 1.3 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |██████████████▋                 | 1.3 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |██████████████▊                 | 1.3 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |██████████████▉                 | 1.3 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |███████████████                 | 1.3 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |███████████████                 | 1.4 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |███████████████▏                | 1.4 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |███████████████▎                | 1.4 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |███████████████▍                | 1.4 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |███████████████▌                | 1.4 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |███████████████▋                | 1.4 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |███████████████▊                | 1.4 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |███████████████▉                | 1.4 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |████████████████                | 1.4 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |████████████████                | 1.4 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |████████████████▏               | 1.5 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |████████████████▎               | 1.5 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |████████████████▍               | 1.5 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |████████████████▌               | 1.5 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |████████████████▋               | 1.5 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |████████████████▊               | 1.5 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |████████████████▉               | 1.5 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |█████████████████               | 1.5 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |█████████████████               | 1.5 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |█████████████████▏              | 1.5 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |█████████████████▎              | 1.6 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |█████████████████▍              | 1.6 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |█████████████████▌              | 1.6 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |█████████████████▋              | 1.6 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |█████████████████▊              | 1.6 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |█████████████████▉              | 1.6 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |██████████████████              | 1.6 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |██████████████████▏             | 1.6 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |██████████████████▎             | 1.6 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |██████████████████▍             | 1.6 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |██████████████████▌             | 1.7 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |██████████████████▋             | 1.7 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |██████████████████▊             | 1.7 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |██████████████████▉             | 1.7 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |███████████████████             | 1.7 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |███████████████████             | 1.7 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |███████████████████▏            | 1.7 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |███████████████████▎            | 1.7 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |███████████████████▍            | 1.7 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |███████████████████▌            | 1.8 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |███████████████████▋            | 1.8 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |███████████████████▊            | 1.8 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |███████████████████▉            | 1.8 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |████████████████████            | 1.8 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |████████████████████            | 1.8 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |████████████████████▏           | 1.8 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |████████████████████▎           | 1.8 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |████████████████████▍           | 1.8 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |████████████████████▌           | 1.8 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |████████████████████▋           | 1.9 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |████████████████████▊           | 1.9 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |████████████████████▉           | 1.9 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |█████████████████████           | 1.9 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |█████████████████████           | 1.9 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |█████████████████████▏          | 1.9 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |█████████████████████▎          | 1.9 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |█████████████████████▍          | 1.9 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |█████████████████████▌          | 1.9 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |█████████████████████▋          | 1.9 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |█████████████████████▊          | 2.0 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |█████████████████████▉          | 2.0 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |██████████████████████          | 2.0 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |██████████████████████          | 2.0 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |██████████████████████▏         | 2.0 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |██████████████████████▎         | 2.0 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |██████████████████████▍         | 2.0 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |██████████████████████▌         | 2.0 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |██████████████████████▊         | 2.0 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |██████████████████████▉         | 2.0 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |███████████████████████         | 2.1 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |███████████████████████         | 2.1 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |███████████████████████▏        | 2.1 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |███████████████████████▎        | 2.1 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |███████████████████████▍        | 2.1 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |███████████████████████▌        | 2.1 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |███████████████████████▋        | 2.1 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |███████████████████████▊        | 2.1 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |███████████████████████▉        | 2.1 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |████████████████████████        | 2.2 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |████████████████████████        | 2.2 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |████████████████████████▏       | 2.2 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |████████████████████████▎       | 2.2 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |████████████████████████▍       | 2.2 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |████████████████████████▌       | 2.2 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |████████████████████████▋       | 2.2 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |████████████████████████▊       | 2.2 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |████████████████████████▉       | 2.2 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |█████████████████████████       | 2.2 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |█████████████████████████       | 2.3 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▏      | 2.3 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▎      | 2.3 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▍      | 2.3 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▌      | 2.3 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▋      | 2.3 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▊      | 2.3 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▉      | 2.3 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |██████████████████████████      | 2.3 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |██████████████████████████      | 2.3 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▏     | 2.4 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▎     | 2.4 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▍     | 2.4 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▌     | 2.4 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▋     | 2.4 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▊     | 2.4 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▉     | 2.4 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |███████████████████████████     | 2.4 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |███████████████████████████     | 2.4 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▎    | 2.4 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▍    | 2.5 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▌    | 2.5 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▋    | 2.5 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▊    | 2.5 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▉    | 2.5 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |████████████████████████████    | 2.5 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |████████████████████████████    | 2.5 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▏   | 2.5 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▎   | 2.5 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▍   | 2.5 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▌   | 2.6 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▋   | 2.6 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▊   | 2.6 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▉   | 2.6 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████   | 2.6 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████   | 2.6 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████▏  | 2.6 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████▎  | 2.6 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████▍  | 2.6 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████▌  | 2.7 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████▋  | 2.7 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████▊  | 2.7 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████▉  | 2.7 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████  | 2.7 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████  | 2.7 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▏ | 2.7 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▎ | 2.7 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▍ | 2.7 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▌ | 2.7 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▋ | 2.8 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▊ | 2.8 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▉ | 2.8 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████ | 2.8 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████ | 2.8 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▏| 2.8 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▎| 2.8 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▍| 2.8 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▌| 2.8 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▋| 2.8 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▉| 2.9 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 2.9 MB 13.9 MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 2.9 MB 13.9 MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (1.19.5)\n",
            "Collecting tokenizers<0.11,>=0.10.1\n",
            "  Downloading tokenizers-0.10.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (3.3 MB)\n",
            "\u001b[K     |████████████████████████████████| 3.3 MB 28.0 MB/s \n",
            "\u001b[?25hRequirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from transformers) (4.8.1)\n",
            "Collecting huggingface-hub>=0.0.17\n",
            "  Downloading huggingface_hub-0.0.19-py3-none-any.whl (56 kB)\n",
            "\u001b[K     |████████████████████████████████| 56 kB 6.2 MB/s \n",
            "\u001b[?25hRequirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.7/dist-packages (from transformers) (21.0)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (2019.12.20)\n",
            "Collecting sacremoses\n",
            "  Downloading sacremoses-0.0.46-py3-none-any.whl (895 kB)\n",
            "\u001b[K     |████████████████████████████████| 895 kB 57.5 MB/s \n",
            "\u001b[?25hCollecting pyyaml>=5.1\n",
            "  Downloading PyYAML-5.4.1-cp37-cp37m-manylinux1_x86_64.whl (636 kB)\n",
            "\u001b[K     |████████████████████████████████| 636 kB 67.1 MB/s \n",
            "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers) (3.3.0)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers) (2.23.0)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers) (4.62.3)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from huggingface-hub>=0.0.17->transformers) (3.7.4.3)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging>=20.0->transformers) (2.4.7)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->transformers) (3.6.0)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2021.5.30)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2.10)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (7.1.2)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.0.1)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.15.0)\n",
            "Installing collected packages: pyyaml, tokenizers, sacremoses, huggingface-hub, transformers\n",
            "  Attempting uninstall: pyyaml\n",
            "    Found existing installation: PyYAML 3.13\n",
            "    Uninstalling PyYAML-3.13:\n",
            "      Successfully uninstalled PyYAML-3.13\n",
            "Successfully installed huggingface-hub-0.0.19 pyyaml-5.4.1 sacremoses-0.0.46 tokenizers-0.10.3 transformers-4.11.3\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cqF-IUtLcIYz"
      },
      "source": [
        "import stopwordsiso as stopwords\n",
        "import unicodedata as unicode"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "unyab7oCcMVS"
      },
      "source": [
        "#%tensorflow_version 2.x\n",
        "import tensorflow as tf\n",
        "import tensorflow_datasets as tfds\n",
        "\n",
        "from tensorflow.keras import layers\n",
        "from tensorflow.keras import losses\n",
        "from tensorflow.keras import preprocessing\n",
        "from tensorflow.keras.layers.experimental.preprocessing import TextVectorization\n",
        "\n",
        "import os\n",
        "import errno\n",
        "import io\n",
        "tf.__version__\n",
        "\n",
        "from os import listdir\n",
        "from os.path import isfile, join, isdir"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "00X9XCEfcOsA"
      },
      "source": [
        "import nlpaug.augmenter.word as naw"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1_dbDeycejUm"
      },
      "source": [
        "import pandas as pd"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "T3wIFVNSp-di"
      },
      "source": [
        "from google.colab import drive"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8yaO3ViwqDUu",
        "outputId": "37dda1be-defe-47ac-c24a-62a0c2810e0a"
      },
      "source": [
        "drive.mount('drive')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at drive\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8446IFmsO9gX"
      },
      "source": [
        "train = pd.read_csv('/content/drive/MyDrive/Colab Notebooks/augmented_data/train.csv')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wgyD4TsOecm7"
      },
      "source": [
        "test  = pd.read_csv('https://raw.githubusercontent.com/tblock/10kGNAD/master/test.csv', sep=';', names=['sub','text_1','text_2'])\n",
        "#train = pd.read_csv('https://raw.githubusercontent.com/tblock/10kGNAD/master/train.csv', sep=';', names=['sub','text'])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3Dxa_Q6qdRtJ"
      },
      "source": [
        "test.fillna('', inplace=True)\n",
        "train.fillna('', inplace=True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Y-EjhrFFdTD6"
      },
      "source": [
        "test['text'] = test['text_1'] + test['text_2']"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GdYHyBMQdVYE"
      },
      "source": [
        "test.drop(['text_1','text_2'],inplace=True, axis=1)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "PMsbL_I0fHnj",
        "outputId": "7e0791ac-1cb6-410f-ec75-2c6198a132f7"
      },
      "source": [
        "test.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>sub</th>\n",
              "      <th>text</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Wirtschaft</td>\n",
              "      <td>'Die Gewerkschaft GPA-djp lanciert den \"All-in...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Sport</td>\n",
              "      <td>Franzosen verteidigen 2:1-Führung – Kritische ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Web</td>\n",
              "      <td>'Neues Video von Designern macht im Netz die R...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Sport</td>\n",
              "      <td>23-jähriger Brasilianer muss vier Spiele pausi...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>International</td>\n",
              "      <td>Aufständische verwendeten Chemikalie bei Gefec...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "             sub                                               text\n",
              "0     Wirtschaft  'Die Gewerkschaft GPA-djp lanciert den \"All-in...\n",
              "1          Sport  Franzosen verteidigen 2:1-Führung – Kritische ...\n",
              "2            Web  'Neues Video von Designern macht im Netz die R...\n",
              "3          Sport  23-jähriger Brasilianer muss vier Spiele pausi...\n",
              "4  International  Aufständische verwendeten Chemikalie bei Gefec..."
            ]
          },
          "metadata": {},
          "execution_count": 38
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kmXbIux5fIfB"
      },
      "source": [
        "text_type_train_list = train['sub'].unique()\n",
        "text_type_test_list = test['sub'].unique()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lHYN4DP-fin-"
      },
      "source": [
        "count_files_in_folder = {}\n",
        "inverse_count_files_in_folder = {}\n",
        "for text_type in text_type_train_list:\n",
        "    count_text_type = len(train.loc[train['sub'] == text_type])\n",
        "    \n",
        "    count_files_in_folder[text_type] = count_text_type\n",
        "    inverse_count_files_in_folder[count_text_type] = text_type"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WrmHHk88ZCQo"
      },
      "source": [
        "### Max Anzahl Text pro Artikel-Typ"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "oNaSWBxIfks1",
        "outputId": "f0135d7c-7681-46cd-c1f1-5c7b946c4d8e"
      },
      "source": [
        "all_values = count_files_in_folder.values()\n",
        "max_value = max(all_values)\n",
        "max_value"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1510"
            ]
          },
          "metadata": {},
          "execution_count": 41
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dxoWLJdUPWUX"
      },
      "source": [
        "#del count_files_in_folder['Sport']\n",
        "#del count_files_in_folder['Kultur']\n",
        "#del count_files_in_folder['Wirtschaft']\n",
        "#del count_files_in_folder['Web']\n",
        "#del count_files_in_folder['Etat']\n",
        "#del count_files_in_folder['Inland']\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GoWpuB2sfmLJ"
      },
      "source": [
        "del count_files_in_folder[inverse_count_files_in_folder[max_value]]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fDIBVrAqfnjE"
      },
      "source": [
        "### get diff to maxvalue for subjects\n",
        "for key in count_files_in_folder:\n",
        "    count_files_in_folder[key] = max_value - count_files_in_folder[key]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "BUK_dhfYfpTH",
        "outputId": "07e64b9d-18b8-4ea8-d05c-979e870df861"
      },
      "source": [
        "count_files_in_folder"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'Etat': 1147,\n",
              " 'Inland': 1510,\n",
              " 'International': 1510,\n",
              " 'Kultur': 1044,\n",
              " 'Panorama': 1510,\n",
              " 'Sport': 1510,\n",
              " 'Web': 1510,\n",
              " 'Wirtschaft': 1510,\n",
              " 'Wissenschaft': 1032}"
            ]
          },
          "metadata": {},
          "execution_count": 42
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZeFVtAluf1AP"
      },
      "source": [
        "## **Backtranslation**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zjfhX1jgfw1H"
      },
      "source": [
        "def generate_back_translation(df,subject_list=[],subject_to_generate_list=[], sub_column_name='sub', text_column_name='text'):\n",
        "    for i, subject in enumerate(subject_list):\n",
        "        amount_to_generate = subject_to_generate_list[i]\n",
        "        sub_df             = df.loc[df[sub_column_name] == subject]\n",
        "        print(subject)\n",
        "        if(i>0):\n",
        "          with open('/content/drive/MyDrive/Colab Notebooks/augmented_data/'+subject_list[i-1]+\".csv\", 'w') as f:\n",
        "            df.to_csv(f, sep=',', index=False) \n",
        "        \n",
        "        for index, row in sub_df.iloc[0:amount_to_generate].iterrows():\n",
        "            current_text = row['text']\n",
        "            text_aug     = back_translation_aug.augment(current_text)\n",
        "            new_row = {sub_column_name:subject, text_column_name:text_aug}\n",
        "            df = df.append(new_row, ignore_index=True)\n",
        "            # Write the DataFrame to CSV file.\n",
        "            with open('/content/drive/MyDrive/Colab Notebooks/augmented_data/train.csv', 'w') as f:\n",
        "              df.to_csv(f, sep=',', index=False) \n",
        "\n",
        "    \n",
        "    return df"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OV3IKRZmfz8v"
      },
      "source": [
        "counts = list(count_files_in_folder.values())\n",
        "subs   = list(count_files_in_folder.keys())"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 401,
          "referenced_widgets": [
            "4cce1e79a3ed4477b5611802b54e4ece",
            "20ea4da1c5c042b0995b42c2b7db0bce",
            "5c7b9e5834c948d0978282b00fff8a27",
            "b3d175109f484fb89459cb0bbd93b7a8",
            "bcc4b6427b1649d7b0bddaa68a261445",
            "606369f010704aa1af0bce8d27ccac1f",
            "e5da8ea5e93f41c0b40101c07266765c",
            "27a226bd008f48098d736d307709d3be",
            "ae59656d924e4c75be90c8035b2f10cb",
            "20b14c89957a4ed997f3136558e38cfc",
            "621d4b7b9308455aab6ac892e41f8f54",
            "e82e1bad28da43a38c38f87317c56c50",
            "c75f9b335b8b4038ac4de7ecdbb15b53",
            "28c094b291d2469ab6b4629e118aa41d",
            "91666ae98bee41229ab2eab9bdee609b",
            "158af526660b43aaa258434af0224539",
            "55f96bd9e013400bb6954add2148876a",
            "b8278308318d4c04aff2929a3b1e689b",
            "c5b7cee6b14c41aebe3b0d386faf149c",
            "26e81669a32e448aae3cdf30c6eba212",
            "18b65aba69634d57bb1290fbec6ec48d",
            "b8148d5a7a624aa8ada10cd35e759dbf"
          ]
        },
        "id": "HcYjXlV6f81d",
        "outputId": "6bd19786-1516-4699-b710-93e719bd2a27"
      },
      "source": [
        "back_translation_aug = naw.BackTranslationAug(\n",
        "    from_model_name='facebook/wmt19-de-en',\n",
        "    to_model_name='facebook/wmt19-en-de',\n",
        "    #device='cuda'\n",
        ")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "4cce1e79a3ed4477b5611802b54e4ece",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/825 [00:00<?, ?B/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "e82e1bad28da43a38c38f87317c56c50",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/1.00G [00:00<?, ?B/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "c75f9b335b8b4038ac4de7ecdbb15b53",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/825 [00:00<?, ?B/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "28c094b291d2469ab6b4629e118aa41d",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/1.00G [00:00<?, ?B/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "91666ae98bee41229ab2eab9bdee609b",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/67.0 [00:00<?, ?B/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "158af526660b43aaa258434af0224539",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/829k [00:00<?, ?B/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "55f96bd9e013400bb6954add2148876a",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/829k [00:00<?, ?B/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "b8278308318d4c04aff2929a3b1e689b",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/308k [00:00<?, ?B/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "c5b7cee6b14c41aebe3b0d386faf149c",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/67.0 [00:00<?, ?B/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "26e81669a32e448aae3cdf30c6eba212",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/829k [00:00<?, ?B/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "18b65aba69634d57bb1290fbec6ec48d",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/829k [00:00<?, ?B/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "b8148d5a7a624aa8ada10cd35e759dbf",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/308k [00:00<?, ?B/s]"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qHJAp1kvhDny"
      },
      "source": [
        "return_df = generate_back_translation(train,subs,counts)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qj3Imb03gihz"
      },
      "source": [
        "return_df.to_csv('backtranslated_train.csv', sep=';')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Mids_9rpq967"
      },
      "source": [
        "train  = pd.read_csv('backtranslated_train.csv', sep=';', names=['sub','text'])"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}