KcELECTRA_filtering.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "gpuType": "T4"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tZxGvWzz8S9d",
        "outputId": "80bb516e-8748-4e11-a105-2744bda08bd4"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: transformers[torch] in /usr/local/lib/python3.10/dist-packages (4.35.2)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from transformers[torch]) (3.13.1)\n",
            "Requirement already satisfied: huggingface-hub<1.0,>=0.16.4 in /usr/local/lib/python3.10/dist-packages (from transformers[torch]) (0.20.3)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from transformers[torch]) (1.25.2)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from transformers[torch]) (23.2)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from transformers[torch]) (6.0.1)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers[torch]) (2023.12.25)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from transformers[torch]) (2.31.0)\n",
            "Requirement already satisfied: tokenizers<0.19,>=0.14 in /usr/local/lib/python3.10/dist-packages (from transformers[torch]) (0.15.2)\n",
            "Requirement already satisfied: safetensors>=0.3.1 in /usr/local/lib/python3.10/dist-packages (from transformers[torch]) (0.4.2)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.10/dist-packages (from transformers[torch]) (4.66.2)\n",
            "Requirement already satisfied: torch!=1.12.0,>=1.10 in /usr/local/lib/python3.10/dist-packages (from transformers[torch]) (2.1.0+cu121)\n",
            "Requirement already satisfied: accelerate>=0.20.3 in /usr/local/lib/python3.10/dist-packages (from transformers[torch]) (0.27.2)\n",
            "Requirement already satisfied: psutil in /usr/local/lib/python3.10/dist-packages (from accelerate>=0.20.3->transformers[torch]) (5.9.5)\n",
            "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.16.4->transformers[torch]) (2023.6.0)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.16.4->transformers[torch]) (4.9.0)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch!=1.12.0,>=1.10->transformers[torch]) (1.12)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch!=1.12.0,>=1.10->transformers[torch]) (3.2.1)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch!=1.12.0,>=1.10->transformers[torch]) (3.1.3)\n",
            "Requirement already satisfied: triton==2.1.0 in /usr/local/lib/python3.10/dist-packages (from torch!=1.12.0,>=1.10->transformers[torch]) (2.1.0)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->transformers[torch]) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->transformers[torch]) (3.6)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->transformers[torch]) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->transformers[torch]) (2024.2.2)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch!=1.12.0,>=1.10->transformers[torch]) (2.1.5)\n",
            "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch!=1.12.0,>=1.10->transformers[torch]) (1.3.0)\n",
            "fatal: destination path 'korean-malicious-comments-dataset' already exists and is not an empty directory.\n",
            "device: cuda:0\n"
          ]
        }
      ],
      "source": [
        "!pip install transformers[torch]\n",
        "!git clone https://github.com/ZIZUN/korean-malicious-comments-dataset.git\n",
        "import pandas as pd\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "import torch\n",
        "from transformers import AutoTokenizer, AutoModelForSequenceClassification, TrainingArguments, Trainer\n",
        "from sklearn.metrics import precision_recall_fscore_support, accuracy_score\n",
        "\n",
        "device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')\n",
        "print(\"device:\", device)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df = pd.read_csv(\"/content/korean-malicious-comments-dataset/Dataset.csv\", sep=\"\\t\")\n",
        "df.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "xh-Kmf1D83SI",
        "outputId": "43a26299-6100-47e0-c765-d496fb12d9d8"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                             content  lable\n",
              "0  이종석 한효주 나오는 드라마 이후로 드라마 안봤다. 2년전인가?? 좀 신선했었지. ...    0.0\n",
              "1                    씨바알..노무노무 술프노... 오늘 저녁은 꽂등심이다ㅠㅜ    0.0\n",
              "2                                           짱깨 꺼라ㅡ패쓰    0.0\n",
              "3  그들의 사생활 ~ 고인이된 설리를 위해서라도 모두 조용하길 지금 누굴 탓한다고 무슨...    1.0\n",
              "4  아무리 법이 뭣같아도 무슨 자격으로 개인의 신상정보를 불특정 다수에게 공개하는지 도...    1.0"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-903d71ab-2ba1-45f2-aa0c-fbb10caf4f18\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>content</th>\n",
              "      <th>lable</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>이종석 한효주 나오는 드라마 이후로 드라마 안봤다. 2년전인가?? 좀 신선했었지. ...</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>씨바알..노무노무 술프노... 오늘 저녁은 꽂등심이다ㅠㅜ</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>짱깨 꺼라ㅡ패쓰</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>그들의 사생활 ~ 고인이된 설리를 위해서라도 모두 조용하길 지금 누굴 탓한다고 무슨...</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>아무리 법이 뭣같아도 무슨 자격으로 개인의 신상정보를 불특정 다수에게 공개하는지 도...</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-903d71ab-2ba1-45f2-aa0c-fbb10caf4f18')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-903d71ab-2ba1-45f2-aa0c-fbb10caf4f18 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-903d71ab-2ba1-45f2-aa0c-fbb10caf4f18');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-c51b3f01-52bf-44fb-97c4-eccdec6d6d4b\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-c51b3f01-52bf-44fb-97c4-eccdec6d6d4b')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-c51b3f01-52bf-44fb-97c4-eccdec6d6d4b button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "variable_name": "df",
              "summary": "{\n  \"name\": \"df\",\n  \"rows\": 10000,\n  \"fields\": [\n    {\n      \"column\": \"content\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"samples\": [\n          \"\\uc774\\ud544\\ubaa8 \\uc88b\\uaca0\\ub2e4 \\uc11c\\uc218\\uc5f0 \\uac19\\uc740 \\ubbf8\\ubaa8\\uc758 \\uc7ac\\uc6d0\\uacfc \\uacb0\\ud63c\\ud558\\ub2c8...\",\n          \"\\uc62c\\ub824\\uce58\\uae30 \\uac00\\uc988\\uc544\",\n          \"\\ub545\\ucf69 \\uc0ac\\ubb34\\uc7a5\\uc591\\ubc18 \\uc911\\uac04\\uc5d0 \\ub091\\uae30\\uac8c \\ud55c\\uac8c \\ub354\\uc584\\ubbf8\\uc6c0\"\n        ],\n        \"num_unique_values\": 9987,\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"lable\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.5000248610146563,\n        \"min\": 0.0,\n        \"max\": 1.0,\n        \"samples\": [\n          1.0,\n          0.0\n        ],\n        \"num_unique_values\": 2,\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 2
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df.info()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nZZlIDzx85vY",
        "outputId": "f9a12dad-7a77-4ba1-9501-deaf22a18c38"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "<class 'pandas.core.frame.DataFrame'>\n",
            "RangeIndex: 10000 entries, 0 to 9999\n",
            "Data columns (total 2 columns):\n",
            " #   Column   Non-Null Count  Dtype  \n",
            "---  ------   --------------  -----  \n",
            " 0   content  10000 non-null  object \n",
            " 1   lable    9975 non-null   float64\n",
            "dtypes: float64(1), object(1)\n",
            "memory usage: 156.4+ KB\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "null_idx = df[df.lable.isnull()].index\n",
        "df.loc[null_idx, \"content\"]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1SitfDxj87_b",
        "outputId": "c3fedcaa-540f-44f2-917e-784a9ce30e31"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1602    응애 응애 엄마 저 맘에 안들죠? ........아들 ?? \" 너 내가 우스워 보이...\n",
              "1654           토니스타크 평소 \"아이엠그루트\"라는 유행어를 부러워했다는게 학계의 정설\\t1\n",
              "1992    \"13일 현대차에 따르면 올 들어 국내 소비자들의 수입차 구매의향률이 3년 만에 하...\n",
              "2920                 에이프릴이 한마디 합니다 \"예쁜게 죄\" 구하라님 \"무기징역\"\\t1\n",
              "3720          답글 글씨체를 봐라 저게 애새끼가 쓴거냐?\"빨갱이새끼가 쓴거지 ㅁㅈㅎㅉㅉ\\t0\n",
              "3807    알겠다이기ㅋㅋ 딱 채찍쳐맞는거 좋아하는 한국식 마인드네. 노예마인드. 조금만 성공한...\n",
              "3908           이래서 스스로 걸리거든 \"죄인들이\"~ㅎㅎㅎ 재미보고 털리고 그치~~~?\\t0\n",
              "4241    아버지는 내재된 악마들을 다룰 정신적 힘을 가지고 있지 않았다.\" 이 말한마디가 사...\n",
              "4283    댓글 중 \"선동 당해서 촞불든 개돼지 홍어들도 단죄를 받아야 할 공범자들이다\"에10...\n",
              "5000    스파이 제안받고 살해 안당하는 법1. 처음에 스파이 제안을 받았을때 \"중국을 위해서...\n",
              "5521    \"국방부 \"까지 ㅡㄱ ㅐ 엿같은 ㅈ ㅣ랄주댕이...좌빨에서 ㅡ인민군대로 ㅡ가려는건가...\n",
              "5866    쌩뚱맞게 60대최반엌 치매라니 그것도 곱게 사는 사모님이- -\" 알콜중독도 아니고 ...\n",
              "6477    페미메퇘지쿵쾅년인 메갈페미들은 니들이 좋아하는 싫어요 ㄱㄱ제발부탁해~~\"일반 여성\"...\n",
              "6538    아니 ㅆㅂ 그런 \"카더라\"가 넘쳐난다고 그거에 대해서 혹시 댓글게이는 뭔가 아는거 ...\n",
              "6771    저 때 투니버스에서 코요태 짧게 인터뷰 했었는데 김종민이 \"노래는 뭐 신지가 다 하...\n",
              "6932               개 족 가튼 국방부의 \"휴기연장콜센터\"발족을 축하한다 ㅆ ㅂ..\\t0\n",
              "7199    민족적 자존심과 애국심을 갖고 국산품 이용합시다 . . . \"겸손\"한 마음으로 재산...\n",
              "7252    아나운서는 목표가 아니었지ㅋㅋ재벌하고 결혼하자마자 바로 은퇴하네ㅋㅋ무슨 인터뷰한 거...\n",
              "7270    결국 준영과 다솜은 바람을 피게되고 무인도로 떠난다에 한표 ㅋㅋㅋ 자연인이 되어 \"...\n",
              "7480    지금 연락하는 여자랑 폰섹 엄청 많이했는데만나서 호텔 들어가서침대에 서로 마주보고 ...\n",
              "7499    몽골한테 \"최근에\" 250년간 지배당하고 집단강간을 당했는데 동양피가 하나도 안섞였...\n",
              "7887    뭐 선천적으로 여성스럽거나 여자역할을 하고 싶어하는 동성애자들 그럴 수 있다고는 생...\n",
              "9666         ㄹㅇ 시발 그냥 \"다른 진로 생각해 보세요\"라고만 했어도 욕 안처 먹었지.\\t0\n",
              "9698                              간만에 이단어가 떠오르는군 \"이뭐병\"\\t0\n",
              "9875    노라조 \"형\"이란 노래로 힘들 때 위로를 받곤 했습니다. 앞으로도 노라조라는 이름으...\n",
              "Name: content, dtype: object"
            ]
          },
          "metadata": {},
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df.loc[null_idx, \"lable\"] = df.loc[null_idx, \"content\"].apply(lambda x: x[-1])\n",
        "df.loc[null_idx, \"content\"] = df.loc[null_idx, \"content\"].apply(lambda x: x[:-2])"
      ],
      "metadata": {
        "id": "-qqStNIF8_Aa"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df = df.astype({\"lable\":\"int\"})\n",
        "df.info()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fU9Sevg79CJc",
        "outputId": "75eeea35-37f2-4165-8be6-2945935db620"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "<class 'pandas.core.frame.DataFrame'>\n",
            "RangeIndex: 10000 entries, 0 to 9999\n",
            "Data columns (total 2 columns):\n",
            " #   Column   Non-Null Count  Dtype \n",
            "---  ------   --------------  ----- \n",
            " 0   content  10000 non-null  object\n",
            " 1   lable    10000 non-null  int64 \n",
            "dtypes: int64(1), object(1)\n",
            "memory usage: 156.4+ KB\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## 데이터셋 준비"
      ],
      "metadata": {
        "id": "5kOgPAUjIRGt"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "train_data = df.sample(frac=0.8, random_state=42)\n",
        "test_data = df.drop(train_data.index)"
      ],
      "metadata": {
        "id": "glFBls5y9Kw-"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# 데이터셋 갯수 확인\n",
        "print('중복 제거 전 학습 데이터셋 : {}'.format(len(train_data)))\n",
        "print('중복 제거 전 테스트 데이터셋 : {}'.format(len(test_data)))\n",
        "\n",
        "# 중복 데이터 제거\n",
        "train_data.drop_duplicates(subset=[\"content\"], inplace= True)\n",
        "test_data.drop_duplicates(subset=[\"content\"], inplace= True)\n",
        "\n",
        "# 데이터셋 갯수 확인\n",
        "print('중복 제거 후 학습 데이터셋 : {}'.format(len(train_data)))\n",
        "print('중복 제거 후 테스트 데이터셋 : {}'.format(len(test_data)))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2yv5swEZ9Ljh",
        "outputId": "2a2c6d12-1242-4565-9b2c-4e0280f1105f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "중복 제거 전 학습 데이터셋 : 8000\n",
            "중복 제거 전 테스트 데이터셋 : 2000\n",
            "중복 제거 후 학습 데이터셋 : 7992\n",
            "중복 제거 후 테스트 데이터셋 : 2000\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "MODEL_NAME = \"beomi/KcELECTRA-base\"\n",
        "tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mxjJSdQa9NTJ",
        "outputId": "26b0bcb6-3872-4a24-e5ce-08975ff72405"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/huggingface_hub/utils/_token.py:88: UserWarning: \n",
            "The secret `HF_TOKEN` does not exist in your Colab secrets.\n",
            "To authenticate with the Hugging Face Hub, create a token in your settings tab (https://huggingface.co/settings/tokens), set it as secret in your Google Colab and restart your session.\n",
            "You will be able to reuse this secret in all of your notebooks.\n",
            "Please note that authentication is recommended but still optional to access public models or datasets.\n",
            "  warnings.warn(\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "tokenized_train_sentences = tokenizer(\n",
        "    list(train_data[\"content\"]),\n",
        "    return_tensors=\"pt\",\n",
        "    max_length=128,\n",
        "    padding=True,\n",
        "    truncation=True,\n",
        "    add_special_tokens=True,\n",
        "    )"
      ],
      "metadata": {
        "id": "SYHen5k49Ph-"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(tokenized_train_sentences[0])\n",
        "print(tokenized_train_sentences[0].tokens)\n",
        "print(tokenized_train_sentences[0].ids)\n",
        "print(tokenized_train_sentences[0].attention_mask)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "r1ocm16n9Uw1",
        "outputId": "bf1b9aab-9971-4de4-c7da-6720504fc50e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Encoding(num_tokens=128, attributes=[ids, type_ids, tokens, offsets, attention_mask, special_tokens_mask, overflowing])\n",
            "['ĠêµŃë°©ë¶Ģ', '~~', 'ìłĦ', 'íĻĶë¡ľ', 'Ġíľ´ê°Ģ', 'ìĹ°', 'ìŀ¥ìĿĦ', 'Ġíķľ', 'Ġë³ĳ', 'ìĤ¬ëĵ¤', 'ĠëªĩìĿ´ëĤĺ', 'ĠëĲĺëĬĶì§Ģ', 'Ġê³µê°ľíķ´ëĿ¼', '~', 'Ġìĸ´ëĬĲ', 'ĠíĽĮë¥Ńíķľ', 'Ġì§ĳìķĪ', 'ìĿĺ', 'ĠìŀĲìłľ', 'ë¶Ħëĵ¤', 'ìĿ¸ì§ĢëıĦ', 'Ġê°ĻìĿ´', 'Ġê³µê°ľíķ´ëĿ¼', '~~', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "[7727, 424, 497, 9927, 6413, 712, 3101, 372, 1566, 5898, 16745, 9129, 11079, 96, 1952, 7846, 3628, 299, 8668, 3819, 10134, 1744, 11079, 424, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3]\n",
            "[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "tokenized_test_sentences = tokenizer(\n",
        "    list(test_data[\"content\"]),\n",
        "    return_tensors=\"pt\",\n",
        "    max_length=128,\n",
        "    padding=True,\n",
        "    truncation=True,\n",
        "    add_special_tokens=True,\n",
        "    )"
      ],
      "metadata": {
        "id": "O9X8jl0V9WBo"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "class CurseDataset(torch.utils.data.Dataset):\n",
        "    def __init__(self, encodings, labels):\n",
        "        self.encodings = encodings\n",
        "        self.labels = labels\n",
        "\n",
        "    def __getitem__(self, idx):\n",
        "        item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
        "        item[\"labels\"] = torch.tensor(self.labels[idx])\n",
        "        return item\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.labels)"
      ],
      "metadata": {
        "id": "tIm-iXkc9XR2"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "train_label = train_data[\"lable\"].values\n",
        "test_label = test_data[\"lable\"].values\n",
        "\n",
        "train_dataset = CurseDataset(tokenized_train_sentences, train_label)\n",
        "test_dataset = CurseDataset(tokenized_test_sentences, test_label)"
      ],
      "metadata": {
        "id": "O7ihkYE-9Ym-"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## 학습"
      ],
      "metadata": {
        "id": "2euuaFhWIOC-"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "model = AutoModelForSequenceClassification.from_pretrained(MODEL_NAME, num_labels=2)\n",
        "model.to(device)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KuRG1dlZ9bFl",
        "outputId": "89372bd6-ad66-4a2f-8722-d7c43b3110b0"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Some weights of ElectraForSequenceClassification were not initialized from the model checkpoint at beomi/KcELECTRA-base and are newly initialized: ['classifier.dense.weight', 'classifier.dense.bias', 'classifier.out_proj.weight', 'classifier.out_proj.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "ElectraForSequenceClassification(\n",
              "  (electra): ElectraModel(\n",
              "    (embeddings): ElectraEmbeddings(\n",
              "      (word_embeddings): Embedding(30000, 768, padding_idx=3)\n",
              "      (position_embeddings): Embedding(512, 768)\n",
              "      (token_type_embeddings): Embedding(2, 768)\n",
              "      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "      (dropout): Dropout(p=0.1, inplace=False)\n",
              "    )\n",
              "    (encoder): ElectraEncoder(\n",
              "      (layer): ModuleList(\n",
              "        (0-11): 12 x ElectraLayer(\n",
              "          (attention): ElectraAttention(\n",
              "            (self): ElectraSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): ElectraSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): ElectraIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): ElectraOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "    )\n",
              "  )\n",
              "  (classifier): ElectraClassificationHead(\n",
              "    (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "    (dropout): Dropout(p=0.1, inplace=False)\n",
              "    (out_proj): Linear(in_features=768, out_features=2, bias=True)\n",
              "  )\n",
              ")"
            ]
          },
          "metadata": {},
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "training_args = TrainingArguments(\n",
        "    output_dir='./',\n",
        "    num_train_epochs=10,\n",
        "    per_device_train_batch_size=8,\n",
        "    per_device_eval_batch_size=64,\n",
        "    logging_dir='./logs',\n",
        "    logging_steps=500,\n",
        "    save_total_limit=2,\n",
        ")"
      ],
      "metadata": {
        "id": "1wJbh3Es9b1C"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def compute_metrics(pred):\n",
        "    labels = pred.label_ids\n",
        "    preds = pred.predictions.argmax(-1)\n",
        "    precision, recall, f1, _ = precision_recall_fscore_support(labels, preds, average='binary')\n",
        "    acc = accuracy_score(labels, preds)\n",
        "    return {\n",
        "        'accuracy': acc,\n",
        "        'f1': f1,\n",
        "        'precision': precision,\n",
        "        'recall': recall\n",
        "    }"
      ],
      "metadata": {
        "id": "Yga3rEXJ9iA2"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "trainer = Trainer(\n",
        "    model=model,\n",
        "    args=training_args,\n",
        "    train_dataset=train_dataset,\n",
        "    eval_dataset=test_dataset,\n",
        "    compute_metrics=compute_metrics,\n",
        ")"
      ],
      "metadata": {
        "id": "74cPgLjy-fwa"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "trainer.train()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1441
        },
        "id": "Iqe3tU8m-ir0",
        "outputId": "f3aaabd5-0f80-45ae-938c-4f2f0afdc88a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "<ipython-input-14-b5e02a6514c5>:7: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
            "  item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='9990' max='9990' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [9990/9990 36:42, Epoch 10/10]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Step</th>\n",
              "      <th>Training Loss</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>500</td>\n",
              "      <td>0.417600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1000</td>\n",
              "      <td>0.332200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1500</td>\n",
              "      <td>0.236400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2000</td>\n",
              "      <td>0.237400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2500</td>\n",
              "      <td>0.175400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3000</td>\n",
              "      <td>0.168200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3500</td>\n",
              "      <td>0.128000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4000</td>\n",
              "      <td>0.136200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4500</td>\n",
              "      <td>0.099500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5000</td>\n",
              "      <td>0.089700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5500</td>\n",
              "      <td>0.073900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6000</td>\n",
              "      <td>0.076600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6500</td>\n",
              "      <td>0.057600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7000</td>\n",
              "      <td>0.070400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7500</td>\n",
              "      <td>0.038200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8000</td>\n",
              "      <td>0.043200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8500</td>\n",
              "      <td>0.024800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9000</td>\n",
              "      <td>0.041800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9500</td>\n",
              "      <td>0.033100</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "<ipython-input-14-b5e02a6514c5>:7: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
            "  item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
            "<ipython-input-14-b5e02a6514c5>:7: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
            "  item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
            "<ipython-input-14-b5e02a6514c5>:7: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
            "  item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
            "<ipython-input-14-b5e02a6514c5>:7: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
            "  item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
            "<ipython-input-14-b5e02a6514c5>:7: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
            "  item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
            "<ipython-input-14-b5e02a6514c5>:7: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
            "  item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
            "<ipython-input-14-b5e02a6514c5>:7: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
            "  item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
            "<ipython-input-14-b5e02a6514c5>:7: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
            "  item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
            "<ipython-input-14-b5e02a6514c5>:7: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
            "  item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
            "<ipython-input-14-b5e02a6514c5>:7: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
            "  item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
            "<ipython-input-14-b5e02a6514c5>:7: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
            "  item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
            "<ipython-input-14-b5e02a6514c5>:7: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
            "  item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
            "<ipython-input-14-b5e02a6514c5>:7: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
            "  item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
            "<ipython-input-14-b5e02a6514c5>:7: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
            "  item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
            "<ipython-input-14-b5e02a6514c5>:7: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
            "  item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
            "<ipython-input-14-b5e02a6514c5>:7: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
            "  item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
            "<ipython-input-14-b5e02a6514c5>:7: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
            "  item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
            "<ipython-input-14-b5e02a6514c5>:7: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
            "  item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
            "<ipython-input-14-b5e02a6514c5>:7: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
            "  item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "TrainOutput(global_step=9990, training_loss=0.1253599260423754, metrics={'train_runtime': 2206.0514, 'train_samples_per_second': 36.228, 'train_steps_per_second': 4.528, 'total_flos': 5256958886092800.0, 'train_loss': 0.1253599260423754, 'epoch': 10.0})"
            ]
          },
          "metadata": {},
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## 평가"
      ],
      "metadata": {
        "id": "Dz7U7nLkIK9q"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "trainer.evaluate(eval_dataset=test_dataset)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 253
        },
        "id": "4decK5q_-mDF",
        "outputId": "0b582533-f204-49e2-8c7e-f878d078ff5b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "<ipython-input-14-b5e02a6514c5>:7: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
            "  item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='32' max='32' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [32/32 00:14]\n",
              "    </div>\n",
              "    "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'eval_loss': 0.5940119624137878,\n",
              " 'eval_accuracy': 0.9055,\n",
              " 'eval_f1': 0.904977375565611,\n",
              " 'eval_precision': 0.8973080757726819,\n",
              " 'eval_recall': 0.9127789046653144,\n",
              " 'eval_runtime': 15.1354,\n",
              " 'eval_samples_per_second': 132.141,\n",
              " 'eval_steps_per_second': 2.114,\n",
              " 'epoch': 10.0}"
            ]
          },
          "metadata": {},
          "execution_count": 23
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# 0: curse, 1: non_curse\n",
        "def sentence_predict(sent):\n",
        "    model.eval()\n",
        "\n",
        "    tokenized_sent = tokenizer(\n",
        "        sent,\n",
        "        return_tensors=\"pt\",\n",
        "        truncation=True,\n",
        "        add_special_tokens=True,\n",
        "        max_length=128\n",
        "    )\n",
        "\n",
        "    tokenized_sent.to(device)\n",
        "\n",
        "    with torch.no_grad():\n",
        "        outputs = model(\n",
        "            input_ids=tokenized_sent[\"input_ids\"],\n",
        "            attention_mask=tokenized_sent[\"attention_mask\"],\n",
        "            token_type_ids=tokenized_sent[\"token_type_ids\"]\n",
        "            )\n",
        "\n",
        "    logits = outputs[0]\n",
        "    logits = logits.detach().cpu()\n",
        "    result = logits.argmax(-1)\n",
        "    if result == 0:\n",
        "        result = \" >> 악성댓글 👿\"\n",
        "    elif result == 1:\n",
        "        result = \" >> 정상댓글 😀\"\n",
        "    return result"
      ],
      "metadata": {
        "id": "Bmc6Blec-p5m"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "torch.save(model.state_dict(), 'dict.pt')"
      ],
      "metadata": {
        "id": "wm8gVacaHGW9"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "torch.save(model, 'model.pt')"
      ],
      "metadata": {
        "id": "Tfdgxj4EHVvJ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "RxLA_qBSIjwN"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}