비속어모델.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyPbP24sKQKFAenBrlpf5vHt",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "gpuClass": "standard",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "d7bf8b241cb04012a6fe07d662397d8e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_3b079a887856469c99a7c74513a78530",
              "IPY_MODEL_59e9f08c63854a69934f8c1ce04914e4",
              "IPY_MODEL_0cea260e0280430e87990fa525fc7d4b"
            ],
            "layout": "IPY_MODEL_a16d2f20b83e46148b8134c98f3d9de7"
          }
        },
        "3b079a887856469c99a7c74513a78530": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_3b175264eb5846e8bcf58e139607ac28",
            "placeholder": "​",
            "style": "IPY_MODEL_cc2aa2f4a96f4016b1ae8dae9e6306a4",
            "value": "100%"
          }
        },
        "59e9f08c63854a69934f8c1ce04914e4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ceaaaaa584a145d983e393c0a2c91805",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_70818f00ca824b9a958052dff529f749",
            "value": 1
          }
        },
        "0cea260e0280430e87990fa525fc7d4b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_adc8bccc1a6a436db760cb32b046b831",
            "placeholder": "​",
            "style": "IPY_MODEL_7da31dc6c7ad41ea990faf21c0b0052b",
            "value": " 1/1 [00:00&lt;00:00, 56.54it/s]"
          }
        },
        "a16d2f20b83e46148b8134c98f3d9de7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3b175264eb5846e8bcf58e139607ac28": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "cc2aa2f4a96f4016b1ae8dae9e6306a4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "ceaaaaa584a145d983e393c0a2c91805": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "70818f00ca824b9a958052dff529f749": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "adc8bccc1a6a436db760cb32b046b831": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7da31dc6c7ad41ea990faf21c0b0052b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/CAU-CAPSTONE-2-Friendly-Keyboard/Models/blob/main/%EB%B9%84%EC%86%8D%EC%96%B4%EB%AA%A8%EB%8D%B8.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## 설치"
      ],
      "metadata": {
        "id": "INKy0wxMngqK"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install transformers==4.1.1\n",
        "!pip install datasets"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hsqINtBY6qs7",
        "outputId": "99ec4ed6-376e-40c1-b796-8e3acf2a63ba"
      },
      "execution_count": 38,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: transformers==4.1.1 in /usr/local/lib/python3.9/dist-packages (4.1.1)\n",
            "Requirement already satisfied: sacremoses in /usr/local/lib/python3.9/dist-packages (from transformers==4.1.1) (0.0.53)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.9/dist-packages (from transformers==4.1.1) (23.0)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.9/dist-packages (from transformers==4.1.1) (1.22.4)\n",
            "Requirement already satisfied: tokenizers==0.9.4 in /usr/local/lib/python3.9/dist-packages (from transformers==4.1.1) (0.9.4)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.9/dist-packages (from transformers==4.1.1) (2.27.1)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.9/dist-packages (from transformers==4.1.1) (4.65.0)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.9/dist-packages (from transformers==4.1.1) (3.10.1)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.9/dist-packages (from transformers==4.1.1) (2022.10.31)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.9/dist-packages (from requests->transformers==4.1.1) (2022.12.7)\n",
            "Requirement already satisfied: charset-normalizer~=2.0.0 in /usr/local/lib/python3.9/dist-packages (from requests->transformers==4.1.1) (2.0.12)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.9/dist-packages (from requests->transformers==4.1.1) (3.4)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.9/dist-packages (from requests->transformers==4.1.1) (1.26.15)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.9/dist-packages (from sacremoses->transformers==4.1.1) (8.1.3)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.9/dist-packages (from sacremoses->transformers==4.1.1) (1.16.0)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.9/dist-packages (from sacremoses->transformers==4.1.1) (1.1.1)\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: datasets in /usr/local/lib/python3.9/dist-packages (2.10.1)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.9/dist-packages (from datasets) (1.4.4)\n",
            "Requirement already satisfied: pyarrow>=6.0.0 in /usr/local/lib/python3.9/dist-packages (from datasets) (9.0.0)\n",
            "Requirement already satisfied: multiprocess in /usr/local/lib/python3.9/dist-packages (from datasets) (0.70.14)\n",
            "Requirement already satisfied: xxhash in /usr/local/lib/python3.9/dist-packages (from datasets) (3.2.0)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.9/dist-packages (from datasets) (1.22.4)\n",
            "Requirement already satisfied: fsspec[http]>=2021.11.1 in /usr/local/lib/python3.9/dist-packages (from datasets) (2023.3.0)\n",
            "Requirement already satisfied: huggingface-hub<1.0.0,>=0.2.0 in /usr/local/lib/python3.9/dist-packages (from datasets) (0.13.3)\n",
            "Requirement already satisfied: aiohttp in /usr/local/lib/python3.9/dist-packages (from datasets) (3.8.4)\n",
            "Requirement already satisfied: responses<0.19 in /usr/local/lib/python3.9/dist-packages (from datasets) (0.18.0)\n",
            "Requirement already satisfied: dill<0.3.7,>=0.3.0 in /usr/local/lib/python3.9/dist-packages (from datasets) (0.3.6)\n",
            "Requirement already satisfied: requests>=2.19.0 in /usr/local/lib/python3.9/dist-packages (from datasets) (2.27.1)\n",
            "Requirement already satisfied: tqdm>=4.62.1 in /usr/local/lib/python3.9/dist-packages (from datasets) (4.65.0)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.9/dist-packages (from datasets) (6.0)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.9/dist-packages (from datasets) (23.0)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.9/dist-packages (from aiohttp->datasets) (1.3.1)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.9/dist-packages (from aiohttp->datasets) (22.2.0)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.9/dist-packages (from aiohttp->datasets) (1.3.3)\n",
            "Requirement already satisfied: charset-normalizer<4.0,>=2.0 in /usr/local/lib/python3.9/dist-packages (from aiohttp->datasets) (2.0.12)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.9/dist-packages (from aiohttp->datasets) (1.8.2)\n",
            "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.9/dist-packages (from aiohttp->datasets) (4.0.2)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.9/dist-packages (from aiohttp->datasets) (6.0.4)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.9/dist-packages (from huggingface-hub<1.0.0,>=0.2.0->datasets) (4.5.0)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.9/dist-packages (from huggingface-hub<1.0.0,>=0.2.0->datasets) (3.10.1)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.9/dist-packages (from requests>=2.19.0->datasets) (2022.12.7)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.9/dist-packages (from requests>=2.19.0->datasets) (3.4)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.9/dist-packages (from requests>=2.19.0->datasets) (1.26.15)\n",
            "Requirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.9/dist-packages (from pandas->datasets) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.9/dist-packages (from pandas->datasets) (2022.7.1)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.9/dist-packages (from python-dateutil>=2.8.1->pandas->datasets) (1.16.0)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Importing"
      ],
      "metadata": {
        "id": "mSbORAoWnn0_"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import math\n",
        "from datasets import load_dataset\n",
        "import pandas as pd\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "from transformers import AutoModelForSequenceClassification, AutoTokenizer"
      ],
      "metadata": {
        "id": "Q0RbrfwSnqWN"
      },
      "execution_count": 39,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## DataLoader"
      ],
      "metadata": {
        "id": "4PDYa1f3g2lG"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "dataset = load_dataset(\"csv\",data_files='./sample_data/dataset.txt',sep='|')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 87,
          "referenced_widgets": [
            "d7bf8b241cb04012a6fe07d662397d8e",
            "3b079a887856469c99a7c74513a78530",
            "59e9f08c63854a69934f8c1ce04914e4",
            "0cea260e0280430e87990fa525fc7d4b",
            "a16d2f20b83e46148b8134c98f3d9de7",
            "3b175264eb5846e8bcf58e139607ac28",
            "cc2aa2f4a96f4016b1ae8dae9e6306a4",
            "ceaaaaa584a145d983e393c0a2c91805",
            "70818f00ca824b9a958052dff529f749",
            "adc8bccc1a6a436db760cb32b046b831",
            "7da31dc6c7ad41ea990faf21c0b0052b"
          ]
        },
        "id": "_uwrw8SkgH0l",
        "outputId": "0090ce20-a4ec-4098-9fe4-9c897dd930d0"
      },
      "execution_count": 40,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "WARNING:datasets.builder:Found cached dataset csv (/root/.cache/huggingface/datasets/csv/default-b81ec849c007f29b/0.0.0/6b34fb8fcf56f7c8ba51dc895bfa2bfbe43546f190a60fcf74bb5e8afdcc2317)\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0/1 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "d7bf8b241cb04012a6fe07d662397d8e"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "dataset['train'][0]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pEW-iLRkd9uG",
        "outputId": "bec5ba1f-8cdb-48ed-f2e1-d27ee22e9b17"
      },
      "execution_count": 41,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'text': '좌배 까는건 ㅇㅂ', 'label': 1}"
            ]
          },
          "metadata": {},
          "execution_count": 41
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "Lm_TeGq7g2YU"
      },
      "execution_count": 41,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## pretrained Model 불러오기"
      ],
      "metadata": {
        "id": "7Jg6sb9dniji"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 42,
      "metadata": {
        "id": "yVnWl9cwa5IE"
      },
      "outputs": [],
      "source": [
        "\n",
        "\n",
        "model = AutoModelForSequenceClassification.from_pretrained('JminJ/kcElectra_base_Bad_Sentence_Classifier')\n",
        "tokenizer = AutoTokenizer.from_pretrained('JminJ/kcElectra_base_Bad_Sentence_Classifier')\n"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## 데이터 전처리"
      ],
      "metadata": {
        "id": "zhrbHe9Phqi3"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "tokenizer"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "epzzhgR8qHKR",
        "outputId": "1c54bfa9-bf1e-47bc-e586-ab9ab92b623b"
      },
      "execution_count": 43,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "PreTrainedTokenizerFast(name_or_path='JminJ/kcElectra_base_Bad_Sentence_Classifier', vocab_size=50135, model_max_len=512, is_fast=True, padding_side='right', special_tokens={'unk_token': '[UNK]', 'sep_token': '[SEP]', 'pad_token': '[PAD]', 'cls_token': '[CLS]', 'mask_token': '[MASK]'})"
            ]
          },
          "metadata": {},
          "execution_count": 43
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def preprocessing(data):\n",
        "  return tokenizer(data['text'])"
      ],
      "metadata": {
        "id": "zEYwqvitp7T8"
      },
      "execution_count": 44,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "%time tokenized_dataset = dataset.map(preprocessing,batched=True)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1BtjnbUIvCNk",
        "outputId": "7001bfc2-bf4d-47ff-8084-cde55f6c890a"
      },
      "execution_count": 45,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "WARNING:datasets.arrow_dataset:Loading cached processed dataset at /root/.cache/huggingface/datasets/csv/default-b81ec849c007f29b/0.0.0/6b34fb8fcf56f7c8ba51dc895bfa2bfbe43546f190a60fcf74bb5e8afdcc2317/cache-274cc91e442d5f39.arrow\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "CPU times: user 17.3 ms, sys: 875 µs, total: 18.1 ms\n",
            "Wall time: 26.5 ms\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "tokenized_dataset['train'][0]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gTldL60zrTKl",
        "outputId": "484d2dce-c2d7-444f-ea84-df1a063294cd"
      },
      "execution_count": 46,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'text': '좌배 까는건 ㅇㅂ',\n",
              " 'label': 1,\n",
              " 'input_ids': [2, 2896, 4225, 14257, 4030, 116, 4770, 3],\n",
              " 'token_type_ids': [0, 0, 0, 0, 0, 0, 0, 0],\n",
              " 'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1]}"
            ]
          },
          "metadata": {},
          "execution_count": 46
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Attention을 이용한 Masking 연결을 위한 기존 모델 attention 구조 및 연산 확인"
      ],
      "metadata": {
        "id": "IWbAL4S5o3lU"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "layernorm = nn.LayerNorm(768)"
      ],
      "metadata": {
        "id": "_frhZHKGhAj2"
      },
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "layers = model.electra.encoder.layer\n",
        "tmp_query = layers[0].attention.self.query.weight\n",
        "for layer in layers[1:]:\n",
        "  tmp_query = torch.add(tmp_query,layer.attention.self.query.weight)\n",
        "  print(tmp_query)\n",
        "query_result = layernorm(tmp_query)\n",
        "\n",
        "print(query_result.shape)\n",
        "print(query_result)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uTu1XUe5QTIX",
        "outputId": "4a57ff85-7ab8-47f5-bbd9-1ba90eb5946a"
      },
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "tensor([[-0.0313,  0.0637,  0.0118,  ...,  0.0840,  0.0239, -0.0226],\n",
            "        [ 0.1325, -0.0024, -0.0972,  ...,  0.0238,  0.0708, -0.1140],\n",
            "        [ 0.0134, -0.0985,  0.0316,  ...,  0.0356,  0.1798, -0.0309],\n",
            "        ...,\n",
            "        [ 0.1682,  0.0282, -0.0483,  ..., -0.0115, -0.0993,  0.0424],\n",
            "        [ 0.0239, -0.0051,  0.1035,  ..., -0.0839, -0.0149, -0.1470],\n",
            "        [ 0.0264, -0.0629,  0.1732,  ...,  0.1792,  0.0791, -0.0692]],\n",
            "       grad_fn=<AddBackward0>)\n",
            "tensor([[-0.0224,  0.0687,  0.0400,  ...,  0.1129,  0.0041, -0.0162],\n",
            "        [ 0.1064, -0.0062, -0.1138,  ...,  0.0996,  0.1383, -0.1605],\n",
            "        [ 0.1204, -0.1723,  0.0691,  ...,  0.0011,  0.1476, -0.0410],\n",
            "        ...,\n",
            "        [ 0.1530,  0.0350, -0.1209,  ..., -0.0819,  0.0037, -0.0195],\n",
            "        [ 0.0744,  0.0670,  0.0794,  ..., -0.0786, -0.0490, -0.2046],\n",
            "        [ 0.0226, -0.0184,  0.0607,  ...,  0.1737,  0.0227,  0.0250]],\n",
            "       grad_fn=<AddBackward0>)\n",
            "tensor([[-0.0212,  0.1165,  0.2464,  ...,  0.1626,  0.1504,  0.0431],\n",
            "        [ 0.1030,  0.0288, -0.1332,  ...,  0.1105,  0.1557, -0.1921],\n",
            "        [ 0.0987, -0.2093,  0.1036,  ...,  0.0778,  0.1056, -0.0298],\n",
            "        ...,\n",
            "        [ 0.1250,  0.0361, -0.1252,  ..., -0.1200,  0.0585, -0.0328],\n",
            "        [ 0.0872,  0.1497,  0.1694,  ..., -0.0857, -0.0719, -0.1993],\n",
            "        [-0.0656, -0.0304,  0.1303,  ...,  0.1783,  0.0571, -0.0613]],\n",
            "       grad_fn=<AddBackward0>)\n",
            "tensor([[-0.1481,  0.0427,  0.2985,  ...,  0.1345,  0.2334,  0.0809],\n",
            "        [ 0.1773,  0.0082, -0.1548,  ...,  0.0857,  0.0676, -0.1991],\n",
            "        [ 0.0289, -0.1220,  0.2302,  ...,  0.1249,  0.1019,  0.0084],\n",
            "        ...,\n",
            "        [ 0.1008,  0.0768, -0.0461,  ..., -0.2976, -0.1416, -0.1053],\n",
            "        [ 0.1707,  0.1761,  0.1794,  ..., -0.1248, -0.0972, -0.2483],\n",
            "        [-0.0496, -0.0549,  0.2139,  ...,  0.0866,  0.0763, -0.0437]],\n",
            "       grad_fn=<AddBackward0>)\n",
            "tensor([[-0.1004,  0.0025,  0.3408,  ...,  0.0906,  0.2461,  0.0200],\n",
            "        [ 0.2458,  0.0127, -0.1355,  ...,  0.0266,  0.1450, -0.1961],\n",
            "        [ 0.0369, -0.1087,  0.2987,  ...,  0.1041,  0.1945,  0.0425],\n",
            "        ...,\n",
            "        [ 0.1728,  0.0976,  0.0830,  ..., -0.1562, -0.2351, -0.0866],\n",
            "        [ 0.2046,  0.2702,  0.2201,  ..., -0.3101, -0.1705, -0.3005],\n",
            "        [ 0.0535, -0.0300,  0.3321,  ..., -0.0540,  0.0119, -0.0699]],\n",
            "       grad_fn=<AddBackward0>)\n",
            "tensor([[-0.1568, -0.0618,  0.1209,  ...,  0.0154,  0.1909,  0.0532],\n",
            "        [ 0.2050,  0.1010, -0.1152,  ...,  0.0145,  0.0755, -0.2340],\n",
            "        [-0.0122, -0.1360,  0.2636,  ...,  0.1590,  0.3137,  0.1033],\n",
            "        ...,\n",
            "        [ 0.1433,  0.0617,  0.2477,  ..., -0.1964, -0.2271, -0.1461],\n",
            "        [ 0.2043,  0.1362,  0.1489,  ..., -0.3164, -0.2277, -0.3088],\n",
            "        [ 0.1219, -0.0529,  0.2666,  ..., -0.0999, -0.0454, -0.0383]],\n",
            "       grad_fn=<AddBackward0>)\n",
            "tensor([[-0.1926, -0.0989,  0.1917,  ...,  0.0433,  0.2159,  0.1056],\n",
            "        [ 0.3177,  0.0974, -0.0573,  ...,  0.0738,  0.1540, -0.2477],\n",
            "        [ 0.0319, -0.0523,  0.2039,  ...,  0.1262,  0.3711,  0.1170],\n",
            "        ...,\n",
            "        [ 0.0287, -0.0736,  0.2787,  ..., -0.2465, -0.1113, -0.0503],\n",
            "        [ 0.2627,  0.1868,  0.2319,  ..., -0.3506, -0.3324, -0.3617],\n",
            "        [ 0.0433, -0.0332,  0.2503,  ..., -0.0780, -0.0436,  0.0974]],\n",
            "       grad_fn=<AddBackward0>)\n",
            "tensor([[ 0.0193, -0.1942, -0.1119,  ..., -0.2183,  0.2148,  0.0931],\n",
            "        [ 0.3180,  0.1253,  0.1201,  ...,  0.1208,  0.1448, -0.3639],\n",
            "        [ 0.1119, -0.0298, -0.2039,  ..., -0.0210,  0.4866,  0.0644],\n",
            "        ...,\n",
            "        [ 0.0572, -0.0054,  0.2915,  ..., -0.2434, -0.1028, -0.0410],\n",
            "        [ 0.2213,  0.1752,  0.3050,  ..., -0.3780, -0.2453, -0.4070],\n",
            "        [ 0.0133,  0.0029,  0.2915,  ..., -0.1578, -0.0644,  0.0957]],\n",
            "       grad_fn=<AddBackward0>)\n",
            "tensor([[ 0.0107, -0.2928, -0.1048,  ..., -0.2739,  0.1796,  0.1726],\n",
            "        [ 0.2625,  0.1020,  0.1213,  ...,  0.1160,  0.2548, -0.4156],\n",
            "        [ 0.0404, -0.0527, -0.1704,  ..., -0.0474,  0.5971,  0.0483],\n",
            "        ...,\n",
            "        [ 0.1249, -0.1018,  0.3060,  ..., -0.2245, -0.1139,  0.0497],\n",
            "        [ 0.2916,  0.2957,  0.2425,  ..., -0.4427, -0.1925, -0.4425],\n",
            "        [-0.0176,  0.0133,  0.3701,  ..., -0.2439, -0.1731,  0.0641]],\n",
            "       grad_fn=<AddBackward0>)\n",
            "tensor([[ 0.0537, -0.4086, -0.1468,  ..., -0.3010,  0.0520,  0.0983],\n",
            "        [ 0.1655,  0.1688, -0.0109,  ...,  0.0610,  0.2204, -0.4892],\n",
            "        [-0.0519, -0.1527, -0.2720,  ..., -0.0713,  0.6026,  0.0879],\n",
            "        ...,\n",
            "        [ 0.0567, -0.1066,  0.3377,  ..., -0.2339, -0.1044,  0.1459],\n",
            "        [ 0.2224,  0.3928,  0.2374,  ..., -0.4471, -0.2706, -0.3859],\n",
            "        [ 0.0721,  0.0388,  0.2842,  ..., -0.3994, -0.2206,  0.0281]],\n",
            "       grad_fn=<AddBackward0>)\n",
            "tensor([[ 0.0549, -0.4894, -0.1447,  ..., -0.4570,  0.1267,  0.0236],\n",
            "        [ 0.1510,  0.2087,  0.0312,  ...,  0.0799,  0.2150, -0.5400],\n",
            "        [-0.1640, -0.1374, -0.3568,  ..., -0.0136,  0.5914,  0.0429],\n",
            "        ...,\n",
            "        [ 0.1348, -0.0302,  0.3605,  ..., -0.1952, -0.1221,  0.1642],\n",
            "        [ 0.2239,  0.2880,  0.2230,  ..., -0.3005, -0.2298, -0.4810],\n",
            "        [ 0.1238,  0.0756,  0.2655,  ..., -0.5339, -0.2337, -0.0122]],\n",
            "       grad_fn=<AddBackward0>)\n",
            "torch.Size([768, 768])\n",
            "tensor([[ 0.2457, -1.8326, -0.5165,  ..., -1.7092,  0.5195,  0.1261],\n",
            "        [ 0.6520,  0.8767,  0.1862,  ...,  0.3754,  0.9010, -2.0360],\n",
            "        [-0.6262, -0.5221, -1.3819,  ..., -0.0370,  2.3333,  0.1844],\n",
            "        ...,\n",
            "        [ 0.5983, -0.1335,  1.5990,  ..., -0.8651, -0.5412,  0.7283],\n",
            "        [ 0.9846,  1.2765,  0.9804,  ..., -1.4039, -1.0820, -2.2261],\n",
            "        [ 0.4564,  0.2572,  1.0414,  ..., -2.2593, -1.0198, -0.1053]],\n",
            "       grad_fn=<NativeLayerNormBackward0>)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## 모델"
      ],
      "metadata": {
        "id": "ebVQS907pFx6"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "class MaskingAttention(nn.Module):\n",
        "  def __init__(self,config):\n",
        "    self.num_att_heads = config.num_att_heads\n",
        "    self.size_att_head = int(config.hidden_size/config.num_att_heads)\n",
        "    self.size_all_head = self.num_att_heads * self.size_att_head\n",
        "    self.query = nn.Linear(config.hidden_size, config.hidden_size)\n",
        "    self.key = nn.Linear(config.hidden_size, config.hidden_size)\n",
        "  \n",
        "  def transpose_score(self,x):\n",
        "    new_shape = x.size()[:-1] + (self.num_att_heads, self.size_att_head)\n",
        "    x = x.view(*new_shape)\n",
        "    return x.permute(0,2,1,3)\n",
        "\n",
        "  def forward(self, query_hidden_states, key_hidden_states, value_hidden_states, attention_mask,\n",
        "                query_att=False, key_att=False, multi_head=True):\n",
        "    # to save weight information\n",
        "    cls_info = {}\n",
        "\n",
        "    # choose to use query_attention matrix when update attention_probs\n",
        "    # all of hidden_states sizes are [32, 128, 768] and it belongs to selected encoder layers output\n",
        "    if query_att:\n",
        "        mixed_query_layer = self.query(query_hidden_states)\n",
        "    else:\n",
        "        mixed_query_layer = query_hidden_states[:, :, :]\n",
        "    if key_att:\n",
        "        mixed_key_layer = self.key(key_hidden_states)\n",
        "    else:\n",
        "        mixed_key_layer = key_hidden_states[:, :, :]\n",
        "    mixed_value_layer = value_hidden_states[:, :, :]\n",
        "\n",
        "    # block attention score cls to itself\n",
        "    attention_mask[:, :, :, 0] -= 10000\n",
        "\n",
        "    # apply multi-head attention\n",
        "    if multi_head:\n",
        "        query_layer = self.transpose_score(mixed_query_layer)\n",
        "        key_layer = self.transpose_score(mixed_key_layer)\n",
        "        value_layer = self.transpose_score(mixed_value_layer)\n",
        "\n",
        "        # Take the dot product between \"query\" and \"key\" to get the raw attention scores.\n",
        "        attention_scores = torch.matmul(query_layer, key_layer.transpose(-1, -2))\n",
        "        # attention_scores size becomes [batch, num_head, 1(cls), seq_length]\n",
        "        attention_scores = attention_scores[:, :, 0:1, :]\n",
        "        attention_scores = attention_scores / math.sqrt(self.size_att_head)\n",
        "        # Apply the attention mask is (precomputed for all layers in BertModel forward() function)\n",
        "        attention_scores = attention_scores + attention_mask\n",
        "        # Normalize the attention scores to probabilities.\n",
        "        attention_probs = nn.Softmax(dim=-1)(attention_scores)\n",
        "\n",
        "        # context_layer size is [batch, num_head, 1(cls), head_size]\n",
        "        context_layer = torch.matmul(attention_probs, value_layer)\n",
        "        context_layer = context_layer.permute(0, 2, 1, 3).contiguous()\n",
        "        new_context_layer_shape = context_layer.size()[:-2] + (self.all_head_size,)\n",
        "        # context_layer size is [batch, 1(cls), all_head_size]\n",
        "        attention_output = context_layer.view(*new_context_layer_shape)\n",
        "\n",
        "        # attention size becomes [batch, 1(cls), num_head, seq_length]\n",
        "        attention_probs = attention_probs.permute(0, 2, 1, 3).contiguous()\n",
        "        attention_scores = attention_scores.permute(0, 2, 1, 3).contiguous()\n",
        "\n",
        "    # apply single-head attention\n",
        "    else:\n",
        "        # Take the dot product between \"query\" and \"key\" to get the raw attention scores.\n",
        "        attention_scores = torch.matmul(mixed_query_layer, mixed_key_layer.transpose(-1, -2))\n",
        "        # attention_scores size becomes [batch, 1(cls), seq_length]\n",
        "        attention_scores = attention_scores[:, 0:1, :]\n",
        "        attention_scores = attention_scores / math.sqrt(self.size_att_head)\n",
        "        # reduce rank of attention mask\n",
        "        attention_mask = attention_mask.squeeze(1)\n",
        "        # Apply the attention mask is (precomputed for all layers in BertModel forward() function)\n",
        "        attention_scores = attention_scores + attention_mask\n",
        "        # Normalize the attention scores to probabilities.\n",
        "        attention_probs = nn.Softmax(dim=-1)(attention_scores)\n",
        "\n",
        "        # attention_output size is [batch, 1(cls), all_head_size]\n",
        "        attention_output = torch.matmul(attention_probs, mixed_value_layer)\n",
        "\n",
        "    cls_info['scores'] = attention_scores\n",
        "    cls_info['probs'] = attention_probs\n",
        "\n",
        "    # # reduce the rank of attention_output\n",
        "    # attention_output = attention_output[:, 0]\n",
        "\n",
        "    return attention_output, cls_info"
      ],
      "metadata": {
        "id": "KChVbEJgreus"
      },
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "class FriendlyModel(nn.Module):\n",
        "  def __init__(self,config):\n",
        "    super().__init__(config)\n",
        "    self.num_labels = config.num_labels\n",
        "    self.model = AutoModelForSequenceClassification.from_pretrained('JminJ/kcElectra_base_Bad_Sentence_Classifier')\n",
        "    self.layerNorm = nn.LayerNorm\n",
        "    self.maskingAttention = MaskingAttention(config)\n",
        "    self.classifier = model.classifier\n",
        "\n",
        "  def mean_layer(self,selected):\n",
        "    layers = model.electra.encoder.layer\n",
        "    tmp_query = layers[0].attention[selected].weight\n",
        "    for layer in layers[1:]:\n",
        "      tmp_query = torch.add(tmp_query,layer.attention[selected].weight)\n",
        "    query_result = self.layernorm(tmp_query)\n",
        "    return query_result\n",
        "\n",
        "  def forward(self,x):\n",
        "    return x\n"
      ],
      "metadata": {
        "id": "FoZf1CkunfmO"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "7nhPfEmoEsVT"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}