Chapter_8_2022_04_11.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Chapter_8_2022_04_11.ipynb",
      "provenance": [],
      "authorship_tag": "ABX9TyO3JY0kEvevpI77jvysIwxq"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "4e49ed4edaea4b82adda8e000ab89379": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_987037d831b64c269587453d97a74c6e",
              "IPY_MODEL_26fdef092bc74fb1aaf5f66858ba120a",
              "IPY_MODEL_c1aeacafa177464f90169c4405d0a9eb"
            ],
            "layout": "IPY_MODEL_ecfcb519af0141c98a0273d480b0d4ee"
          }
        },
        "987037d831b64c269587453d97a74c6e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_8d9fcdf4646e4d99922a33e7a2534c63",
            "placeholder": "​",
            "style": "IPY_MODEL_cb05fae7161741bdad65f92f806588fd",
            "value": "Downloading: 100%"
          }
        },
        "26fdef092bc74fb1aaf5f66858ba120a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_1c5baebcf7f9461283eaf3784be3e567",
            "max": 2825034,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_8df458ce1ff64cc3badc0d11211bc0c1",
            "value": 2825034
          }
        },
        "c1aeacafa177464f90169c4405d0a9eb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f80ab9a3c78b4e319cc5ef601d1ffd1b",
            "placeholder": "​",
            "style": "IPY_MODEL_870c204d21d143fd97ef849f44a5f9d1",
            "value": " 2.83M/2.83M [00:01&lt;00:00, 2.47MB/s]"
          }
        },
        "ecfcb519af0141c98a0273d480b0d4ee": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8d9fcdf4646e4d99922a33e7a2534c63": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "cb05fae7161741bdad65f92f806588fd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "1c5baebcf7f9461283eaf3784be3e567": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8df458ce1ff64cc3badc0d11211bc0c1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "f80ab9a3c78b4e319cc5ef601d1ffd1b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "870c204d21d143fd97ef849f44a5f9d1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "f1ea165980c042b0af0cc193bec75bb6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_69bd862713e14acc9238374e8d68e5e9",
              "IPY_MODEL_4301c42e111f43258274dd85f2944678",
              "IPY_MODEL_f8aeb0b299fd449c9a53ce9f55fa1573"
            ],
            "layout": "IPY_MODEL_8d8ef4c2150f4431b997c6e1e83aca25"
          }
        },
        "69bd862713e14acc9238374e8d68e5e9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c120d554556a46d6a0751e987f04163e",
            "placeholder": "​",
            "style": "IPY_MODEL_a437db6750ee4ed28d8436ee34e00856",
            "value": "Downloading: 100%"
          }
        },
        "4301c42e111f43258274dd85f2944678": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_893347a946594403812d7f1e959ca899",
            "max": 1000,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_27b47a32b5234ae5a04636eea2830826",
            "value": 1000
          }
        },
        "f8aeb0b299fd449c9a53ce9f55fa1573": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5319efcb27064561846bf6be9e700eb2",
            "placeholder": "​",
            "style": "IPY_MODEL_677cbe407cbf4747873daf9ff7fcdd3b",
            "value": " 1.00k/1.00k [00:00&lt;00:00, 24.6kB/s]"
          }
        },
        "8d8ef4c2150f4431b997c6e1e83aca25": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c120d554556a46d6a0751e987f04163e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a437db6750ee4ed28d8436ee34e00856": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "893347a946594403812d7f1e959ca899": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "27b47a32b5234ae5a04636eea2830826": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "5319efcb27064561846bf6be9e700eb2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "677cbe407cbf4747873daf9ff7fcdd3b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "## <8-2 문장 생성 모델 파인튜닝하기>   \n",
        "\n",
        "이번 절에서는 문장 생성 모델의 데이터 전처리와 파인튜닝 과정을 실습해 보겠습니다."
      ],
      "metadata": {
        "id": "Unmu-RSBToEE"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### 문장 생성 모델 만들기  \n",
        "\n",
        "<**1단계**> **코랩 노트북 초기화하기**   \n",
        "이번 실습에서는 이전 실습과 마찬가지로 코랩에서 `[내 드라이브에 복사]`를 진행합니다. 그리고 코랩의 메뉴에서 [런타임 $\t\\rightarrow$ `런타임 유형 변경`]을 클릭하고 `[GPU]`나 `[TPU]` 둘 중 하나를 선택(GPU 권장)해 하드웨어 가속을 사용합니다."
      ],
      "metadata": {
        "id": "9rrZS88PTpLA"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "<**2단계**> **각종 설정하기**   \n",
        "다음 코드를 차례로 실행해 TPU 관련 라이브러리와 그 외에 의존성 있는 패키지를 설치하고 자신의 구글 드라이브를 코랩 노트북과 연결합니다. 다만 1단계에서 GPU를 선택했다면 코드 8-1은 생략합니다."
      ],
      "metadata": {
        "id": "sX8ioFmnTpPd"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#TPU 관련 패키지 설치\n",
        "#!pip install cloud-tpu-client==0.10 https://storage.googleapis.com/tpu-pytorch/wheels/torch_xla-1.9-cp37-cp37m-linux_x86_64.whl"
      ],
      "metadata": {
        "id": "vVniHiGtVqd4"
      },
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#의존성 패키지 설치\n",
        "!pip install ratsnlp"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Xfpt40ZwVqgb",
        "outputId": "883c1943-0476-4c07-ed23-9fdd8df7527c"
      },
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting ratsnlp\n",
            "  Downloading ratsnlp-1.0.1-py3-none-any.whl (42 kB)\n",
            "\u001b[?25l\r\u001b[K     |███████▊                        | 10 kB 24.9 MB/s eta 0:00:01\r\u001b[K     |███████████████▌                | 20 kB 11.4 MB/s eta 0:00:01\r\u001b[K     |███████████████████████▎        | 30 kB 9.0 MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████ | 40 kB 7.8 MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 42 kB 423 kB/s \n",
            "\u001b[?25hCollecting flask-ngrok>=0.0.25\n",
            "  Downloading flask_ngrok-0.0.25-py3-none-any.whl (3.1 kB)\n",
            "Collecting transformers==4.10.0\n",
            "  Downloading transformers-4.10.0-py3-none-any.whl (2.8 MB)\n",
            "\u001b[K     |████████████████████████████████| 2.8 MB 6.8 MB/s \n",
            "\u001b[?25hCollecting flask-cors>=3.0.10\n",
            "  Downloading Flask_Cors-3.0.10-py2.py3-none-any.whl (14 kB)\n",
            "Requirement already satisfied: torch>=1.9.0 in /usr/local/lib/python3.7/dist-packages (from ratsnlp) (1.10.0+cu111)\n",
            "Requirement already satisfied: flask>=1.1.4 in /usr/local/lib/python3.7/dist-packages (from ratsnlp) (1.1.4)\n",
            "Collecting Korpora>=0.2.0\n",
            "  Downloading Korpora-0.2.0-py3-none-any.whl (57 kB)\n",
            "\u001b[K     |████████████████████████████████| 57 kB 4.2 MB/s \n",
            "\u001b[?25hCollecting pytorch-lightning==1.3.4\n",
            "  Downloading pytorch_lightning-1.3.4-py3-none-any.whl (806 kB)\n",
            "\u001b[K     |████████████████████████████████| 806 kB 40.9 MB/s \n",
            "\u001b[?25hRequirement already satisfied: tqdm>=4.41.0 in /usr/local/lib/python3.7/dist-packages (from pytorch-lightning==1.3.4->ratsnlp) (4.63.0)\n",
            "Collecting pyDeprecate==0.3.0\n",
            "  Downloading pyDeprecate-0.3.0-py3-none-any.whl (10 kB)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from pytorch-lightning==1.3.4->ratsnlp) (21.3)\n",
            "Collecting fsspec[http]>=2021.4.0\n",
            "  Downloading fsspec-2022.3.0-py3-none-any.whl (136 kB)\n",
            "\u001b[K     |████████████████████████████████| 136 kB 54.4 MB/s \n",
            "\u001b[?25hCollecting torchmetrics>=0.2.0\n",
            "  Downloading torchmetrics-0.7.3-py3-none-any.whl (398 kB)\n",
            "\u001b[K     |████████████████████████████████| 398 kB 43.5 MB/s \n",
            "\u001b[?25hCollecting future>=0.17.1\n",
            "  Downloading future-0.18.2.tar.gz (829 kB)\n",
            "\u001b[K     |████████████████████████████████| 829 kB 56.3 MB/s \n",
            "\u001b[?25hRequirement already satisfied: tensorboard!=2.5.0,>=2.2.0 in /usr/local/lib/python3.7/dist-packages (from pytorch-lightning==1.3.4->ratsnlp) (2.8.0)\n",
            "Requirement already satisfied: numpy>=1.17.2 in /usr/local/lib/python3.7/dist-packages (from pytorch-lightning==1.3.4->ratsnlp) (1.21.5)\n",
            "Collecting PyYAML<=5.4.1,>=5.1\n",
            "  Downloading PyYAML-5.4.1-cp37-cp37m-manylinux1_x86_64.whl (636 kB)\n",
            "\u001b[K     |████████████████████████████████| 636 kB 17.9 MB/s \n",
            "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers==4.10.0->ratsnlp) (3.6.0)\n",
            "Collecting huggingface-hub>=0.0.12\n",
            "  Downloading huggingface_hub-0.5.1-py3-none-any.whl (77 kB)\n",
            "\u001b[K     |████████████████████████████████| 77 kB 4.7 MB/s \n",
            "\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers==4.10.0->ratsnlp) (2.23.0)\n",
            "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from transformers==4.10.0->ratsnlp) (4.11.3)\n",
            "Collecting sacremoses\n",
            "  Downloading sacremoses-0.0.49-py3-none-any.whl (895 kB)\n",
            "\u001b[K     |████████████████████████████████| 895 kB 63.3 MB/s \n",
            "\u001b[?25hCollecting tokenizers<0.11,>=0.10.1\n",
            "  Downloading tokenizers-0.10.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (3.3 MB)\n",
            "\u001b[K     |████████████████████████████████| 3.3 MB 37.5 MB/s \n",
            "\u001b[?25hRequirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers==4.10.0->ratsnlp) (2019.12.20)\n",
            "Requirement already satisfied: itsdangerous<2.0,>=0.24 in /usr/local/lib/python3.7/dist-packages (from flask>=1.1.4->ratsnlp) (1.1.0)\n",
            "Requirement already satisfied: click<8.0,>=5.1 in /usr/local/lib/python3.7/dist-packages (from flask>=1.1.4->ratsnlp) (7.1.2)\n",
            "Requirement already satisfied: Jinja2<3.0,>=2.10.1 in /usr/local/lib/python3.7/dist-packages (from flask>=1.1.4->ratsnlp) (2.11.3)\n",
            "Requirement already satisfied: Werkzeug<2.0,>=0.15 in /usr/local/lib/python3.7/dist-packages (from flask>=1.1.4->ratsnlp) (1.0.1)\n",
            "Requirement already satisfied: Six in /usr/local/lib/python3.7/dist-packages (from flask-cors>=3.0.10->ratsnlp) (1.15.0)\n",
            "Collecting aiohttp\n",
            "  Downloading aiohttp-3.8.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (1.1 MB)\n",
            "\u001b[K     |████████████████████████████████| 1.1 MB 50.6 MB/s \n",
            "\u001b[?25hRequirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.7/dist-packages (from huggingface-hub>=0.0.12->transformers==4.10.0->ratsnlp) (3.10.0.2)\n",
            "Requirement already satisfied: MarkupSafe>=0.23 in /usr/local/lib/python3.7/dist-packages (from Jinja2<3.0,>=2.10.1->flask>=1.1.4->ratsnlp) (2.0.1)\n",
            "Collecting xlrd>=1.2.0\n",
            "  Downloading xlrd-2.0.1-py2.py3-none-any.whl (96 kB)\n",
            "\u001b[K     |████████████████████████████████| 96 kB 4.5 MB/s \n",
            "\u001b[?25hCollecting dataclasses>=0.6\n",
            "  Downloading dataclasses-0.6-py3-none-any.whl (14 kB)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->pytorch-lightning==1.3.4->ratsnlp) (3.0.7)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers==4.10.0->ratsnlp) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers==4.10.0->ratsnlp) (2021.10.8)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers==4.10.0->ratsnlp) (1.24.3)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers==4.10.0->ratsnlp) (2.10)\n",
            "Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard!=2.5.0,>=2.2.0->pytorch-lightning==1.3.4->ratsnlp) (1.8.1)\n",
            "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.7/dist-packages (from tensorboard!=2.5.0,>=2.2.0->pytorch-lightning==1.3.4->ratsnlp) (3.3.6)\n",
            "Requirement already satisfied: google-auth<3,>=1.6.3 in /usr/local/lib/python3.7/dist-packages (from tensorboard!=2.5.0,>=2.2.0->pytorch-lightning==1.3.4->ratsnlp) (1.35.0)\n",
            "Requirement already satisfied: tensorboard-data-server<0.7.0,>=0.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard!=2.5.0,>=2.2.0->pytorch-lightning==1.3.4->ratsnlp) (0.6.1)\n",
            "Requirement already satisfied: wheel>=0.26 in /usr/local/lib/python3.7/dist-packages (from tensorboard!=2.5.0,>=2.2.0->pytorch-lightning==1.3.4->ratsnlp) (0.37.1)\n",
            "Requirement already satisfied: setuptools>=41.0.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard!=2.5.0,>=2.2.0->pytorch-lightning==1.3.4->ratsnlp) (57.4.0)\n",
            "Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /usr/local/lib/python3.7/dist-packages (from tensorboard!=2.5.0,>=2.2.0->pytorch-lightning==1.3.4->ratsnlp) (0.4.6)\n",
            "Requirement already satisfied: absl-py>=0.4 in /usr/local/lib/python3.7/dist-packages (from tensorboard!=2.5.0,>=2.2.0->pytorch-lightning==1.3.4->ratsnlp) (1.0.0)\n",
            "Requirement already satisfied: grpcio>=1.24.3 in /usr/local/lib/python3.7/dist-packages (from tensorboard!=2.5.0,>=2.2.0->pytorch-lightning==1.3.4->ratsnlp) (1.44.0)\n",
            "Requirement already satisfied: protobuf>=3.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard!=2.5.0,>=2.2.0->pytorch-lightning==1.3.4->ratsnlp) (3.17.3)\n",
            "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.7/dist-packages (from google-auth<3,>=1.6.3->tensorboard!=2.5.0,>=2.2.0->pytorch-lightning==1.3.4->ratsnlp) (0.2.8)\n",
            "Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.7/dist-packages (from google-auth<3,>=1.6.3->tensorboard!=2.5.0,>=2.2.0->pytorch-lightning==1.3.4->ratsnlp) (4.8)\n",
            "Requirement already satisfied: cachetools<5.0,>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from google-auth<3,>=1.6.3->tensorboard!=2.5.0,>=2.2.0->pytorch-lightning==1.3.4->ratsnlp) (4.2.4)\n",
            "Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.7/dist-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard!=2.5.0,>=2.2.0->pytorch-lightning==1.3.4->ratsnlp) (1.3.1)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->transformers==4.10.0->ratsnlp) (3.7.0)\n",
            "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /usr/local/lib/python3.7/dist-packages (from pyasn1-modules>=0.2.1->google-auth<3,>=1.6.3->tensorboard!=2.5.0,>=2.2.0->pytorch-lightning==1.3.4->ratsnlp) (0.4.8)\n",
            "Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.7/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard!=2.5.0,>=2.2.0->pytorch-lightning==1.3.4->ratsnlp) (3.2.0)\n",
            "Requirement already satisfied: charset-normalizer<3.0,>=2.0 in /usr/local/lib/python3.7/dist-packages (from aiohttp->fsspec[http]>=2021.4.0->pytorch-lightning==1.3.4->ratsnlp) (2.0.12)\n",
            "Collecting async-timeout<5.0,>=4.0.0a3\n",
            "  Downloading async_timeout-4.0.2-py3-none-any.whl (5.8 kB)\n",
            "Collecting frozenlist>=1.1.1\n",
            "  Downloading frozenlist-1.3.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (144 kB)\n",
            "\u001b[K     |████████████████████████████████| 144 kB 56.3 MB/s \n",
            "\u001b[?25hCollecting multidict<7.0,>=4.5\n",
            "  Downloading multidict-6.0.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (94 kB)\n",
            "\u001b[K     |████████████████████████████████| 94 kB 2.9 MB/s \n",
            "\u001b[?25hCollecting aiosignal>=1.1.2\n",
            "  Downloading aiosignal-1.2.0-py3-none-any.whl (8.2 kB)\n",
            "Collecting asynctest==0.13.0\n",
            "  Downloading asynctest-0.13.0-py3-none-any.whl (26 kB)\n",
            "Collecting yarl<2.0,>=1.0\n",
            "  Downloading yarl-1.7.2-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (271 kB)\n",
            "\u001b[K     |████████████████████████████████| 271 kB 59.1 MB/s \n",
            "\u001b[?25hRequirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.7/dist-packages (from aiohttp->fsspec[http]>=2021.4.0->pytorch-lightning==1.3.4->ratsnlp) (21.4.0)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers==4.10.0->ratsnlp) (1.1.0)\n",
            "Building wheels for collected packages: future\n",
            "  Building wheel for future (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for future: filename=future-0.18.2-py3-none-any.whl size=491070 sha256=c913722187da33b9189ac52e5a2b30476985a7c5e90f203c54627a5b494e97f7\n",
            "  Stored in directory: /root/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0\n",
            "Successfully built future\n",
            "Installing collected packages: multidict, frozenlist, yarl, asynctest, async-timeout, aiosignal, PyYAML, pyDeprecate, fsspec, aiohttp, xlrd, torchmetrics, tokenizers, sacremoses, huggingface-hub, future, dataclasses, transformers, pytorch-lightning, Korpora, flask-ngrok, flask-cors, ratsnlp\n",
            "  Attempting uninstall: PyYAML\n",
            "    Found existing installation: PyYAML 3.13\n",
            "    Uninstalling PyYAML-3.13:\n",
            "      Successfully uninstalled PyYAML-3.13\n",
            "  Attempting uninstall: xlrd\n",
            "    Found existing installation: xlrd 1.1.0\n",
            "    Uninstalling xlrd-1.1.0:\n",
            "      Successfully uninstalled xlrd-1.1.0\n",
            "  Attempting uninstall: future\n",
            "    Found existing installation: future 0.16.0\n",
            "    Uninstalling future-0.16.0:\n",
            "      Successfully uninstalled future-0.16.0\n",
            "Successfully installed Korpora-0.2.0 PyYAML-5.4.1 aiohttp-3.8.1 aiosignal-1.2.0 async-timeout-4.0.2 asynctest-0.13.0 dataclasses-0.6 flask-cors-3.0.10 flask-ngrok-0.0.25 frozenlist-1.3.0 fsspec-2022.3.0 future-0.18.2 huggingface-hub-0.5.1 multidict-6.0.2 pyDeprecate-0.3.0 pytorch-lightning-1.3.4 ratsnlp-1.0.1 sacremoses-0.0.49 tokenizers-0.10.3 torchmetrics-0.7.3 transformers-4.10.0 xlrd-2.0.1 yarl-1.7.2\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#구글 드라이브와 연결\n",
        "from google.colab import drive\n",
        "drive.mount('/gdrive', force_remount=True)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "h1zi76IkVqif",
        "outputId": "a99e1418-295f-4c8f-d5dc-98ef73eb2e56"
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /gdrive\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "이번 실습에서는 SK텔레콤이 공개한 KoGPT2 모델(skt/kogpt2-base-v2)을 NSMC로 파인튜닝해 볼 예정입니다. 다음 코드를 실행하면 관련 설정을 할 수 있습니다. `GenerationTrainArguments`의 인자와 역할은 <4-2>절 문서 분류 모델을 실습할 때 사용한 `GenerationTrainArguments`와 거의 비슷하므로 해당 절을 참고 바랍니다."
      ],
      "metadata": {
        "id": "5_NKk00PTpRS"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "id": "BVSan4zrTVRs"
      },
      "outputs": [],
      "source": [
        "#모델 환경 설정\n",
        "import torch\n",
        "from ratsnlp.nlpbook.generation import GenerationTrainArguments\n",
        "args = GenerationTrainArguments(\n",
        "    pretrained_model_name=\"skt/kogpt2-base-v2\",\n",
        "    downstream_corpus_name=\"nsmc\",\n",
        "    downstream_model_dir=\"/gdrive/My Drive/nlpbook/checkpoint-generation\",\n",
        "    max_seq_length=32,\n",
        "    batch_size=32 if torch.cuda.is_available() else 4,\n",
        "    learning_rate=5e-5,\n",
        "    epochs=3,\n",
        "    tpu_cores=0 if torch.cuda.is_available() else 8,\n",
        "    seed=7,\n",
        ")"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "이어서 다음 코드를 차례로 실행해 랜덤 시드를 설정하고 각종 기록을 출력하는 로거를 설정합니다."
      ],
      "metadata": {
        "id": "MGX-EOnSToHF"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#랜덤 시드 고정\n",
        "from ratsnlp import nlpbook\n",
        "nlpbook.set_seed(args)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XPWwBZ2ZX7or",
        "outputId": "065db5cd-8a52-4422-edd2-35fe545f9fdf"
      },
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "set seed: 7\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#로거 설정\n",
        "nlpbook.set_logger(args)"
      ],
      "metadata": {
        "id": "AW20k9zzYEDL"
      },
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "<**3단계**> **말뭉치 내려받기**  \n",
        "다음 코드를 실행하면 NSMC 말뭉치를 내려받습니다. 데이터를 내려받는 도구로 오픈소스 패키지 `Korpora`를 사용해 `corpus_name`에 해당하는 말뭉치(`nsmc`)를 `root_dir`(/root/Korpora) 아래에 저장해 둡시다.   "
      ],
      "metadata": {
        "id": "p1Agl2lOYKXD"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#말뭉치 다운로드\n",
        "from Korpora import Korpora\n",
        "Korpora.fetch(\n",
        "    corpus_name=args.downstream_corpus_name,\n",
        "    root_dir=args.downstream_corpus_root_dir,\n",
        "    force_download=args.force_download,\n",
        ")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dF1vDbIqYHqM",
        "outputId": "b9087046-a0b9-40dc-9c01-738cd1de7a7c"
      },
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "[nsmc] download ratings_train.txt: 14.6MB [00:00, 81.9MB/s]                           \n",
            "[nsmc] download ratings_test.txt: 4.90MB [00:00, 15.7MB/s]                            \n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "<**4단계**> **토크나이저 준비하기**   \n",
        "\n",
        "다음 코드를 실행해 KoGPT2 모델(`skt/kogpt2-base-v2`)이 사용하는 토크나이저를 선언합니다.  \n",
        "`eos_token`은 문장 마지막(end of sentence)에 붙이는 스페셜 토큰으로, SKT가 모델을 프리트레인할 때 이렇게 지정했기 때문에 같은 방식으로 사용합니다."
      ],
      "metadata": {
        "id": "MkcAkIonZDKu"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#토크나이저 준비\n",
        "from transformers import PreTrainedTokenizerFast\n",
        "tokenizer = PreTrainedTokenizerFast.from_pretrained(\n",
        "    args.pretrained_model_name,\n",
        "    eos_token=\"</s>\",\n",
        ")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 134,
          "referenced_widgets": [
            "4e49ed4edaea4b82adda8e000ab89379",
            "987037d831b64c269587453d97a74c6e",
            "26fdef092bc74fb1aaf5f66858ba120a",
            "c1aeacafa177464f90169c4405d0a9eb",
            "ecfcb519af0141c98a0273d480b0d4ee",
            "8d9fcdf4646e4d99922a33e7a2534c63",
            "cb05fae7161741bdad65f92f806588fd",
            "1c5baebcf7f9461283eaf3784be3e567",
            "8df458ce1ff64cc3badc0d11211bc0c1",
            "f80ab9a3c78b4e319cc5ef601d1ffd1b",
            "870c204d21d143fd97ef849f44a5f9d1",
            "f1ea165980c042b0af0cc193bec75bb6",
            "69bd862713e14acc9238374e8d68e5e9",
            "4301c42e111f43258274dd85f2944678",
            "f8aeb0b299fd449c9a53ce9f55fa1573",
            "8d8ef4c2150f4431b997c6e1e83aca25",
            "c120d554556a46d6a0751e987f04163e",
            "a437db6750ee4ed28d8436ee34e00856",
            "893347a946594403812d7f1e959ca899",
            "27b47a32b5234ae5a04636eea2830826",
            "5319efcb27064561846bf6be9e700eb2",
            "677cbe407cbf4747873daf9ff7fcdd3b"
          ]
        },
        "id": "XhYsbXW_ZBYr",
        "outputId": "8b538b21-7a35-4086-f88a-caa18ef31334"
      },
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading:   0%|          | 0.00/2.83M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "4e49ed4edaea4b82adda8e000ab89379"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading:   0%|          | 0.00/1.00k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "f1ea165980c042b0af0cc193bec75bb6"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "The tokenizer class you load from this checkpoint is not the same type as the class this function is called from. It may result in unexpected tokenization. \n",
            "The tokenizer class you load from this checkpoint is 'GPT2Tokenizer'. \n",
            "The class this function is called from is 'PreTrainedTokenizerFast'.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        ""
      ],
      "metadata": {
        "id": "sOjSHxVcZy_4"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}