05_06_adv_nlp_end.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "645908d710ec471b99614b3f90852d61": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_40a08a72f4944f659a604f638ad17387",
              "IPY_MODEL_009c5bcbbbc044338d990d73c43eaa95",
              "IPY_MODEL_964f1781f3df4538b4ab5b2d1b81c7e7"
            ],
            "layout": "IPY_MODEL_232314cd650f404fb9d82d6eda092bb6"
          }
        },
        "40a08a72f4944f659a604f638ad17387": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b7cd44aad14049db87f1d1dbe98341a9",
            "placeholder": "​",
            "style": "IPY_MODEL_94300388ece84fa7b93f1e5ac345c876",
            "value": "config.json: 100%"
          }
        },
        "009c5bcbbbc044338d990d73c43eaa95": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_cd495853227b48d2ab4b193eb4e7fbde",
            "max": 483,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_67ac478ad1134a50ab2d3ddfcd85a1a8",
            "value": 483
          }
        },
        "964f1781f3df4538b4ab5b2d1b81c7e7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_209043ea99c441edbcc90c31312c7ab3",
            "placeholder": "​",
            "style": "IPY_MODEL_f4216562dd2e4724a4784a31bab14ec6",
            "value": " 483/483 [00:00&lt;00:00, 13.3kB/s]"
          }
        },
        "232314cd650f404fb9d82d6eda092bb6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b7cd44aad14049db87f1d1dbe98341a9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "94300388ece84fa7b93f1e5ac345c876": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "cd495853227b48d2ab4b193eb4e7fbde": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "67ac478ad1134a50ab2d3ddfcd85a1a8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "209043ea99c441edbcc90c31312c7ab3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f4216562dd2e4724a4784a31bab14ec6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "8ca31e8c11794aa6b879fa3d9b0986fa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_eb16c62393a947479132681829cee565",
              "IPY_MODEL_add4bab838d946d89205d5c28493492f",
              "IPY_MODEL_2f61a64c7c9d4310a7551759acd1a174"
            ],
            "layout": "IPY_MODEL_04b7215f1d964266aa9a28e89feac41a"
          }
        },
        "eb16c62393a947479132681829cee565": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f35147236ed0473e8aac470ee586382f",
            "placeholder": "​",
            "style": "IPY_MODEL_5f0a8044bb2040ae8771f1e9fd824600",
            "value": "model.safetensors: 100%"
          }
        },
        "add4bab838d946d89205d5c28493492f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_20697f2d2f7b4e94b10de2ecbe00877f",
            "max": 267954768,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_5643e861091f439c91da7ecc1f0f933e",
            "value": 267954768
          }
        },
        "2f61a64c7c9d4310a7551759acd1a174": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_83fe9ae4af094e0dad9f78ed083cb870",
            "placeholder": "​",
            "style": "IPY_MODEL_a625b9d370cc4b6dbd6b393666610a42",
            "value": " 268M/268M [00:06&lt;00:00, 49.2MB/s]"
          }
        },
        "04b7215f1d964266aa9a28e89feac41a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f35147236ed0473e8aac470ee586382f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5f0a8044bb2040ae8771f1e9fd824600": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "20697f2d2f7b4e94b10de2ecbe00877f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5643e861091f439c91da7ecc1f0f933e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "83fe9ae4af094e0dad9f78ed083cb870": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a625b9d370cc4b6dbd6b393666610a42": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "c6526b96fa3d4fb681cfc079dd383614": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_487540bcdf19407d9feba8e8bb96895d",
              "IPY_MODEL_3568de793afa458aa2d8ddba862d1ec1",
              "IPY_MODEL_c4a5fdb2ac9e4a82a28d48dd081725fc"
            ],
            "layout": "IPY_MODEL_f86ac528c0b14c3fbe15493c082192a6"
          }
        },
        "487540bcdf19407d9feba8e8bb96895d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_8e29950259d94cc2b5d58354370d5337",
            "placeholder": "​",
            "style": "IPY_MODEL_cbc6f4244ef04ab6961d5bd7c8c54374",
            "value": "tokenizer_config.json: 100%"
          }
        },
        "3568de793afa458aa2d8ddba862d1ec1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7cef656410744d119f791eba0c5ffef0",
            "max": 28,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_6043d820cd474b7bb57abb32d54081b4",
            "value": 28
          }
        },
        "c4a5fdb2ac9e4a82a28d48dd081725fc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_76d4733366b94f06aa280451c43fd402",
            "placeholder": "​",
            "style": "IPY_MODEL_b9069456c8a24bfb8587c58a67491e87",
            "value": " 28.0/28.0 [00:00&lt;00:00, 696B/s]"
          }
        },
        "f86ac528c0b14c3fbe15493c082192a6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8e29950259d94cc2b5d58354370d5337": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "cbc6f4244ef04ab6961d5bd7c8c54374": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "7cef656410744d119f791eba0c5ffef0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6043d820cd474b7bb57abb32d54081b4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "76d4733366b94f06aa280451c43fd402": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b9069456c8a24bfb8587c58a67491e87": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "50f49dcad596434f8a0d87f7abada4eb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_22d87df416734d5ca19dee793dd428c9",
              "IPY_MODEL_d48d0a4c952242a7b4c041a0244a729d",
              "IPY_MODEL_26817a18ed534b55895a533a8784f3fc"
            ],
            "layout": "IPY_MODEL_8b95686df8e94cd39dd1877ddeb2befd"
          }
        },
        "22d87df416734d5ca19dee793dd428c9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_fbeae70bf35a4c47a00b901d8eeb5110",
            "placeholder": "​",
            "style": "IPY_MODEL_6074ff5fbcea4bc99304b1a63ccb0269",
            "value": "vocab.txt: 100%"
          }
        },
        "d48d0a4c952242a7b4c041a0244a729d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b5ebb388045f42e39c85cb847b4c7110",
            "max": 231508,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_2f0a699c470041e09cf2376d676debe9",
            "value": 231508
          }
        },
        "26817a18ed534b55895a533a8784f3fc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_72ff38ebd8a34ea289eb0008a0abaa75",
            "placeholder": "​",
            "style": "IPY_MODEL_71d0350ad4f24e89afbc870fce0822cc",
            "value": " 232k/232k [00:00&lt;00:00, 465kB/s]"
          }
        },
        "8b95686df8e94cd39dd1877ddeb2befd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "fbeae70bf35a4c47a00b901d8eeb5110": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6074ff5fbcea4bc99304b1a63ccb0269": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "b5ebb388045f42e39c85cb847b4c7110": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2f0a699c470041e09cf2376d676debe9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "72ff38ebd8a34ea289eb0008a0abaa75": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "71d0350ad4f24e89afbc870fce0822cc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "df0cdf1b7f60453e8306823896a61444": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_a41d35005fa5436d983721943017b0d2",
              "IPY_MODEL_a12dfb2e292740cc8767f4b5ad158d77",
              "IPY_MODEL_ac54d4922cda46c3b4d638363ed918e0"
            ],
            "layout": "IPY_MODEL_1668713b97404f85b747e2b06e58ecf8"
          }
        },
        "a41d35005fa5436d983721943017b0d2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f0afa66001e049fbb419ce2d2c93175d",
            "placeholder": "​",
            "style": "IPY_MODEL_507599bcfbd0489ea4b30a9794173201",
            "value": "tokenizer.json: 100%"
          }
        },
        "a12dfb2e292740cc8767f4b5ad158d77": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_95bf033f0e2a4bf4a4020206a4e45fd4",
            "max": 466062,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_f46447f11bdb49ceb973508593ab50f4",
            "value": 466062
          }
        },
        "ac54d4922cda46c3b4d638363ed918e0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_3a14a563534f4caf8b281efa6b25c2c8",
            "placeholder": "​",
            "style": "IPY_MODEL_c807789537f64d978fb821a2a1b815bf",
            "value": " 466k/466k [00:00&lt;00:00, 5.95MB/s]"
          }
        },
        "1668713b97404f85b747e2b06e58ecf8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f0afa66001e049fbb419ce2d2c93175d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "507599bcfbd0489ea4b30a9794173201": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "95bf033f0e2a4bf4a4020206a4e45fd4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f46447f11bdb49ceb973508593ab50f4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "3a14a563534f4caf8b281efa6b25c2c8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c807789537f64d978fb821a2a1b815bf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "2adf07c3ed7e4db69675a94a53e9671a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_4983a3b9d57c46a1bea5e029e6bb397c",
              "IPY_MODEL_02d1295785724218bad13fcd5657d80b",
              "IPY_MODEL_7e9439093fd846dcb1a91181dea6547b"
            ],
            "layout": "IPY_MODEL_a0604634bcac4fa389f8b55d8a480de1"
          }
        },
        "4983a3b9d57c46a1bea5e029e6bb397c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2efb175f8c644ef4a75345e8d1c36eda",
            "placeholder": "​",
            "style": "IPY_MODEL_d64f030d434a48229c884c407b816462",
            "value": "config.json: 100%"
          }
        },
        "02d1295785724218bad13fcd5657d80b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d5a5ff74fab241ec8527ea1043a435b3",
            "max": 451,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_0de8950e790c4902aed5e4f2218e6bbf",
            "value": 451
          }
        },
        "7e9439093fd846dcb1a91181dea6547b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f5cb2645ed8040469cce3d5bde65475d",
            "placeholder": "​",
            "style": "IPY_MODEL_4054f8ce21474ed5a2b425051c6109e4",
            "value": " 451/451 [00:00&lt;00:00, 7.65kB/s]"
          }
        },
        "a0604634bcac4fa389f8b55d8a480de1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2efb175f8c644ef4a75345e8d1c36eda": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d64f030d434a48229c884c407b816462": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "d5a5ff74fab241ec8527ea1043a435b3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0de8950e790c4902aed5e4f2218e6bbf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "f5cb2645ed8040469cce3d5bde65475d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4054f8ce21474ed5a2b425051c6109e4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "3df9e923b95740119b47f23d67a6510a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_218ec2bd10704946b81ee4cf676f8a4a",
              "IPY_MODEL_75f4e300d50a476987e258ad7df60977",
              "IPY_MODEL_791358bf4a9a46afbe7c6051ccc8528d"
            ],
            "layout": "IPY_MODEL_211cde38356847258a32a0a3915a81bd"
          }
        },
        "218ec2bd10704946b81ee4cf676f8a4a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_88991280e20d4655849f644c3dc44bca",
            "placeholder": "​",
            "style": "IPY_MODEL_2f20a9347c064533a11e51c057754121",
            "value": "model.safetensors: 100%"
          }
        },
        "75f4e300d50a476987e258ad7df60977": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b47d638ccdb542a3987fbaff80dcdea9",
            "max": 265470036,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_7932b1ac40ae4b60b2be9810f55f2844",
            "value": 265470036
          }
        },
        "791358bf4a9a46afbe7c6051ccc8528d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_4c80492616d647c39ef966e998fd85e0",
            "placeholder": "​",
            "style": "IPY_MODEL_a1296ed31718414bb16fcbfa432fbbee",
            "value": " 265M/265M [00:03&lt;00:00, 63.5MB/s]"
          }
        },
        "211cde38356847258a32a0a3915a81bd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "88991280e20d4655849f644c3dc44bca": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2f20a9347c064533a11e51c057754121": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "b47d638ccdb542a3987fbaff80dcdea9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7932b1ac40ae4b60b2be9810f55f2844": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "4c80492616d647c39ef966e998fd85e0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a1296ed31718414bb16fcbfa432fbbee": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "3d2523c23d104077aa8a7e70b70fd23e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_ca266a80ee06429f88029efed00cab38",
              "IPY_MODEL_5b7be193d93f4b7cbc5deeed4a3fa23a",
              "IPY_MODEL_5938660cc9a84373a3d0bda118ff5524"
            ],
            "layout": "IPY_MODEL_f8b55da9f46841779ebbfe1fd15c0742"
          }
        },
        "ca266a80ee06429f88029efed00cab38": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c625a6c041f4466db3533b79f928908c",
            "placeholder": "​",
            "style": "IPY_MODEL_f1c0b88c0871491db9cd45bbbb683111",
            "value": "tokenizer_config.json: 100%"
          }
        },
        "5b7be193d93f4b7cbc5deeed4a3fa23a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_fbcd14631caf497ba4a311ccb434c338",
            "max": 28,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_af9d41d330b2445893e3ce25c440fa82",
            "value": 28
          }
        },
        "5938660cc9a84373a3d0bda118ff5524": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7538aa65c3c44924906f0b61e1b487f3",
            "placeholder": "​",
            "style": "IPY_MODEL_e1fcc8d9e4064e8fac3803f5f22e4f55",
            "value": " 28.0/28.0 [00:00&lt;00:00, 1.94kB/s]"
          }
        },
        "f8b55da9f46841779ebbfe1fd15c0742": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c625a6c041f4466db3533b79f928908c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f1c0b88c0871491db9cd45bbbb683111": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "fbcd14631caf497ba4a311ccb434c338": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "af9d41d330b2445893e3ce25c440fa82": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "7538aa65c3c44924906f0b61e1b487f3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e1fcc8d9e4064e8fac3803f5f22e4f55": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "f20c7985bc834c00a5ed778a170efbda": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_303f782f9cf04dcc80b851cb468297b7",
              "IPY_MODEL_68ce75124cbf400abe6581656fa7c3fb",
              "IPY_MODEL_23181efd60d248eb8e3c2680e1c5cc43"
            ],
            "layout": "IPY_MODEL_926a1f7be5d74771afc361aad1da490a"
          }
        },
        "303f782f9cf04dcc80b851cb468297b7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_deab158c3a92492ab2d575809f5c5abd",
            "placeholder": "​",
            "style": "IPY_MODEL_78cffeeb4aad49af83d9b3f83530324e",
            "value": "vocab.txt: 100%"
          }
        },
        "68ce75124cbf400abe6581656fa7c3fb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a2b023d625c4442787d2636d8c6bc097",
            "max": 231508,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_e9edd12aa38b4595a5169294dbb7617f",
            "value": 231508
          }
        },
        "23181efd60d248eb8e3c2680e1c5cc43": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e4ac98859771482da96a330a02741718",
            "placeholder": "​",
            "style": "IPY_MODEL_ee5b3988a31e4452bb313b06695ee4e4",
            "value": " 232k/232k [00:00&lt;00:00, 987kB/s]"
          }
        },
        "926a1f7be5d74771afc361aad1da490a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "deab158c3a92492ab2d575809f5c5abd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "78cffeeb4aad49af83d9b3f83530324e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "a2b023d625c4442787d2636d8c6bc097": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e9edd12aa38b4595a5169294dbb7617f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "e4ac98859771482da96a330a02741718": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ee5b3988a31e4452bb313b06695ee4e4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "297232316a22434f961fa0be3e34f23c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_db728172fc994f7db4b5a109feace62a",
              "IPY_MODEL_9157948e0de94670be3224774a58f737",
              "IPY_MODEL_f9bb8fe36ff347faa6544d370337c9ea"
            ],
            "layout": "IPY_MODEL_a38143786f6a4c0582f8016cea29c63e"
          }
        },
        "db728172fc994f7db4b5a109feace62a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_329c4f3cadec4ee7a2f8c37a3fe6e516",
            "placeholder": "​",
            "style": "IPY_MODEL_75b37031eeb946d99d1aa1ba5e516486",
            "value": "tokenizer.json: 100%"
          }
        },
        "9157948e0de94670be3224774a58f737": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_47801102d63b4e3f888e8fa63fdbd59a",
            "max": 466062,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_c9617a5645ab4b3cb62fd59c36cd9fec",
            "value": 466062
          }
        },
        "f9bb8fe36ff347faa6544d370337c9ea": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_073c69aa138d4860980262bbbb9b8d26",
            "placeholder": "​",
            "style": "IPY_MODEL_32a6df8150834c41b77b28163d84affd",
            "value": " 466k/466k [00:00&lt;00:00, 24.3MB/s]"
          }
        },
        "a38143786f6a4c0582f8016cea29c63e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "329c4f3cadec4ee7a2f8c37a3fe6e516": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "75b37031eeb946d99d1aa1ba5e516486": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "47801102d63b4e3f888e8fa63fdbd59a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c9617a5645ab4b3cb62fd59c36cd9fec": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "073c69aa138d4860980262bbbb9b8d26": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "32a6df8150834c41b77b28163d84affd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "0c44052698904deea17895d8c344afab": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_43060b3bc1de41f0867498b356531175",
              "IPY_MODEL_cee78aa0b2e1443691d37af57a8b69b3",
              "IPY_MODEL_513c0fbe4be24c4880adc40e337ea67d"
            ],
            "layout": "IPY_MODEL_e0096b48ff24408e86458635f32e352d"
          }
        },
        "43060b3bc1de41f0867498b356531175": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d9ea534382814b0d9e11e97b2b2373a7",
            "placeholder": "​",
            "style": "IPY_MODEL_87b78513bd674fa2ac44bcf776abc28d",
            "value": "config.json: 100%"
          }
        },
        "cee78aa0b2e1443691d37af57a8b69b3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d90810023dbc493ca9698b357189ac9e",
            "max": 665,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_00aa476a88f943a997a3bf3a29c24809",
            "value": 665
          }
        },
        "513c0fbe4be24c4880adc40e337ea67d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_3be01fe9bb2549ce8f3f2eca15e1a8aa",
            "placeholder": "​",
            "style": "IPY_MODEL_40e05de6e45b45828e5b6d5099b9353a",
            "value": " 665/665 [00:00&lt;00:00, 27.7kB/s]"
          }
        },
        "e0096b48ff24408e86458635f32e352d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d9ea534382814b0d9e11e97b2b2373a7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "87b78513bd674fa2ac44bcf776abc28d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "d90810023dbc493ca9698b357189ac9e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "00aa476a88f943a997a3bf3a29c24809": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "3be01fe9bb2549ce8f3f2eca15e1a8aa": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "40e05de6e45b45828e5b6d5099b9353a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "63f4fd0a10a64c69a4842b478f4f2227": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_72ad3cfc11a140d7b04c458c710ced7c",
              "IPY_MODEL_35e7c212ba054f47ae6cc84f3aa0a77c",
              "IPY_MODEL_7b749c3536aa453aa1c4ac09422328fe"
            ],
            "layout": "IPY_MODEL_65281acf40934b9aaffaeedf35210319"
          }
        },
        "72ad3cfc11a140d7b04c458c710ced7c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_06e82b0fa9244f86b035368c4d19f334",
            "placeholder": "​",
            "style": "IPY_MODEL_65942244bed2418e8497f8fabbe71136",
            "value": "model.safetensors: 100%"
          }
        },
        "35e7c212ba054f47ae6cc84f3aa0a77c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_6ac2966128684dd1b358b0a0355fed52",
            "max": 548105171,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_a448704799a941c4a124824c7fccc78e",
            "value": 548105171
          }
        },
        "7b749c3536aa453aa1c4ac09422328fe": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ada5eb06591546dbae920a562e15419c",
            "placeholder": "​",
            "style": "IPY_MODEL_a759fbbd7ea54b7b82e9d9f3b62aee7d",
            "value": " 548M/548M [00:05&lt;00:00, 248MB/s]"
          }
        },
        "65281acf40934b9aaffaeedf35210319": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "06e82b0fa9244f86b035368c4d19f334": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "65942244bed2418e8497f8fabbe71136": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "6ac2966128684dd1b358b0a0355fed52": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a448704799a941c4a124824c7fccc78e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "ada5eb06591546dbae920a562e15419c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a759fbbd7ea54b7b82e9d9f3b62aee7d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "1a19f14282924a31b152cd77952e9a8f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_3e8634c6200d43c4be1f5f95a94131f5",
              "IPY_MODEL_f16f472e249c48dfab3e26d0e34d5ff9",
              "IPY_MODEL_d5fbf04da152438c84afa4b9edebffe9"
            ],
            "layout": "IPY_MODEL_632a03b6e2834c97828f6da4bdc58ceb"
          }
        },
        "3e8634c6200d43c4be1f5f95a94131f5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2290c75f243449a3bb2c3753c76a56ef",
            "placeholder": "​",
            "style": "IPY_MODEL_bcde98ad1c3c4e7aab1069e1188fda10",
            "value": "generation_config.json: 100%"
          }
        },
        "f16f472e249c48dfab3e26d0e34d5ff9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7c5da0084ab4408cbdb66225dbd87c23",
            "max": 124,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_e7df309fd349445ba7965a047f79de28",
            "value": 124
          }
        },
        "d5fbf04da152438c84afa4b9edebffe9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_92dfa3e5f2d44758a8f4e18b7c4eba7f",
            "placeholder": "​",
            "style": "IPY_MODEL_2876538b77a04ed8be0e448bfbcb56af",
            "value": " 124/124 [00:00&lt;00:00, 9.09kB/s]"
          }
        },
        "632a03b6e2834c97828f6da4bdc58ceb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2290c75f243449a3bb2c3753c76a56ef": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "bcde98ad1c3c4e7aab1069e1188fda10": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "7c5da0084ab4408cbdb66225dbd87c23": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e7df309fd349445ba7965a047f79de28": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "92dfa3e5f2d44758a8f4e18b7c4eba7f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2876538b77a04ed8be0e448bfbcb56af": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "fe90c90391d241879e7f07f298a56565": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_fde8ce15f9f94348bbc66efcd192a403",
              "IPY_MODEL_983d32ede7a9476e85ef7cb169abe6b5",
              "IPY_MODEL_06fe33a36232470f8f9c24e7210b4ae9"
            ],
            "layout": "IPY_MODEL_172a6d15ec874c2086552e8808d71ae9"
          }
        },
        "fde8ce15f9f94348bbc66efcd192a403": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_3728454371b94e6d9a6ad452e3e49825",
            "placeholder": "​",
            "style": "IPY_MODEL_b09143395771454a81dce318b793fe25",
            "value": "vocab.json: 100%"
          }
        },
        "983d32ede7a9476e85ef7cb169abe6b5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_1aacdb4eea2e465c8049e250dabce6d3",
            "max": 1042301,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_5024b5a4cc7743269d88efa1766fccc9",
            "value": 1042301
          }
        },
        "06fe33a36232470f8f9c24e7210b4ae9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_cc51a4e5936540d1b7f8bee913aab0c0",
            "placeholder": "​",
            "style": "IPY_MODEL_9e66e23b275942739db0be9914d9f0f9",
            "value": " 1.04M/1.04M [00:00&lt;00:00, 4.18MB/s]"
          }
        },
        "172a6d15ec874c2086552e8808d71ae9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3728454371b94e6d9a6ad452e3e49825": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b09143395771454a81dce318b793fe25": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "1aacdb4eea2e465c8049e250dabce6d3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5024b5a4cc7743269d88efa1766fccc9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "cc51a4e5936540d1b7f8bee913aab0c0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9e66e23b275942739db0be9914d9f0f9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "bcbe7837620a4126aaeb51b0fbeda667": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_e981a6d36c1b4e0baade56aa8640848e",
              "IPY_MODEL_4fa4731c46344a418a39150f9cda5e96",
              "IPY_MODEL_798d1482850643e18c16440d56af88c5"
            ],
            "layout": "IPY_MODEL_a364b0a0cc0346b385d45f393a94d612"
          }
        },
        "e981a6d36c1b4e0baade56aa8640848e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_73d5a52321174b74ab52c7846b8b4f07",
            "placeholder": "​",
            "style": "IPY_MODEL_011bc5791c10401190b020066c39d7a3",
            "value": "merges.txt: 100%"
          }
        },
        "4fa4731c46344a418a39150f9cda5e96": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b4945e1893884a72a8c856e34c5fb8af",
            "max": 456318,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_b31047b1a5dc4ac69d100b28e2a8c10b",
            "value": 456318
          }
        },
        "798d1482850643e18c16440d56af88c5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9ffec65328c742e19a91fbf4403e646f",
            "placeholder": "​",
            "style": "IPY_MODEL_d218810e7d80493f8c3d12599e28ce33",
            "value": " 456k/456k [00:00&lt;00:00, 1.91MB/s]"
          }
        },
        "a364b0a0cc0346b385d45f393a94d612": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "73d5a52321174b74ab52c7846b8b4f07": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "011bc5791c10401190b020066c39d7a3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "b4945e1893884a72a8c856e34c5fb8af": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b31047b1a5dc4ac69d100b28e2a8c10b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "9ffec65328c742e19a91fbf4403e646f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d218810e7d80493f8c3d12599e28ce33": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "27377480c46a47c085e9f659b7d31629": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_1bbd6f79860c47ed8785e14b0eab9fc8",
              "IPY_MODEL_469143e8f3a248c28a6cd7fcd3449fa5",
              "IPY_MODEL_af29d8007eae43639dd17cf793a7aea3"
            ],
            "layout": "IPY_MODEL_885945a372274fb1b569edddf5aa2cdb"
          }
        },
        "1bbd6f79860c47ed8785e14b0eab9fc8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_00b8b6c12aa34cb98b9bcfaaf0a1b60a",
            "placeholder": "​",
            "style": "IPY_MODEL_f386a981d651442ead1c286176f21c6d",
            "value": "tokenizer.json: 100%"
          }
        },
        "469143e8f3a248c28a6cd7fcd3449fa5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_07f19db2ad664cf986025bc442399cc5",
            "max": 1355256,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_08f7cba4c2064d6a9c39e02c61dfcc36",
            "value": 1355256
          }
        },
        "af29d8007eae43639dd17cf793a7aea3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b11cccfce306417b9d05249447631685",
            "placeholder": "​",
            "style": "IPY_MODEL_a901ccf6746b41afb3c3e5a64105e3d2",
            "value": " 1.36M/1.36M [00:00&lt;00:00, 1.83MB/s]"
          }
        },
        "885945a372274fb1b569edddf5aa2cdb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "00b8b6c12aa34cb98b9bcfaaf0a1b60a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f386a981d651442ead1c286176f21c6d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "07f19db2ad664cf986025bc442399cc5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "08f7cba4c2064d6a9c39e02c61dfcc36": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "b11cccfce306417b9d05249447631685": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a901ccf6746b41afb3c3e5a64105e3d2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/gstripling/Generative_AI/blob/main/05_06_adv_nlp_end.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "metadata": {
        "id": "SCZWZtKxObjh"
      },
      "cell_type": "markdown",
      "source": [
        "Next, we need to preprocess our data so that it matches the data BERT was trained on. For this, we'll need to do a couple of things (but don't worry--this is also included in the Python library):\n",
        "\n",
        "\n",
        "1. Lowercase our text (if we're using a BERT lowercase model)\n",
        "2. Tokenize it (i.e. \"sally says hi\" -> [\"sally\", \"says\", \"hi\"])\n",
        "3. Break words into WordPieces (i.e. \"calling\" -> [\"call\", \"##ing\"])\n",
        "4. Map our words to indexes using a vocab file that BERT provides\n",
        "5. Add special \"CLS\" and \"SEP\" tokens (see the [readme](https://github.com/google-research/bert))\n",
        "6. Append \"index\" and \"segment\" tokens to each input (see the [BERT paper](https://arxiv.org/pdf/1810.04805.pdf))\n",
        "\n",
        "Happily, we don't have to worry about most of these details.\n",
        "\n",
        "\n"
      ]
    },
    {
      "metadata": {
        "id": "qMWiDtpyQSoU"
      },
      "cell_type": "markdown",
      "source": [
        "To start, we'll need to load a vocabulary file and lowercasing information directly from the BERT tf hub module:"
      ]
    },
    {
      "metadata": {
        "id": "z4oFkhpZBDKm"
      },
      "cell_type": "markdown",
      "source": [
        "Great--we just learned that the BERT model we're using expects lowercase data (that's what stored in tokenization_info[\"do_lower_case\"]) and we also loaded BERT's vocab file. We also created a tokenizer, which breaks words into word pieces:"
      ]
    },
    {
      "metadata": {
        "id": "0OEzfFIt6GIc"
      },
      "cell_type": "markdown",
      "source": [
        "Using our tokenizer, we'll call `run_classifier.convert_examples_to_features` on our InputExamples to convert them into features BERT understands."
      ]
    },
    {
      "metadata": {
        "id": "-thbodgih_VJ"
      },
      "cell_type": "code",
      "source": [
        "pred_sentences = [\n",
        "  \"That movie was absolutely awful\",\n",
        "  \"The acting was a bit lacking\",\n",
        "  \"The film was creative and surprising\",\n",
        "  \"Absolutely fantastic!\"\n",
        "]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "#NEW EXERCISE - TOKENIZING"
      ],
      "metadata": {
        "id": "NFECgxxxjMqk"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "To import the DistilBERT model in Python, you can use the Hugging Face transformers library, which provides a convenient interface to work with various pre-trained transformer models, including DistilBERT. First, you need to install the library:"
      ],
      "metadata": {
        "id": "zNKOVb9ijO9R"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "In this example, DistilBertModel.from_pretrained loads the pre-trained DistilBERT model, and DistilBertTokenizer.from_pretrained loads the corresponding tokenizer. You can replace 'distilbert-base-uncased' with other DistilBERT variants or models fine-tuned for specific tasks, depending on your requirements.\n",
        "\n",
        "After loading the model and tokenizer, you can use them for various natural language processing tasks such as text classification, sentiment analysis, or embedding extraction.\n",
        "\n",
        "Note: Depending on your use case, you might want to choose a specific DistilBERT variant that suits your needs. The model variant in the example, 'distilbert-base-uncased', is a commonly used base version trained on English text in an uncased format. There are other variants and models available, and you can explore them in the Hugging Face model hub: DistilBERT Models.\n",
        "\n",
        "\n",
        "\n",
        "\n"
      ],
      "metadata": {
        "id": "bhn3ErqfjXKK"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import DistilBertModel, DistilBertTokenizer"
      ],
      "metadata": {
        "id": "zT4NCPNmiHXS"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Load pre-trained DistilBERT model and tokenizer\n",
        "model = DistilBertModel.from_pretrained('distilbert-base-uncased')\n",
        "tokenizer = DistilBertTokenizer.from_pretrained('distilbert-base-uncased')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 301,
          "referenced_widgets": [
            "645908d710ec471b99614b3f90852d61",
            "40a08a72f4944f659a604f638ad17387",
            "009c5bcbbbc044338d990d73c43eaa95",
            "964f1781f3df4538b4ab5b2d1b81c7e7",
            "232314cd650f404fb9d82d6eda092bb6",
            "b7cd44aad14049db87f1d1dbe98341a9",
            "94300388ece84fa7b93f1e5ac345c876",
            "cd495853227b48d2ab4b193eb4e7fbde",
            "67ac478ad1134a50ab2d3ddfcd85a1a8",
            "209043ea99c441edbcc90c31312c7ab3",
            "f4216562dd2e4724a4784a31bab14ec6",
            "8ca31e8c11794aa6b879fa3d9b0986fa",
            "eb16c62393a947479132681829cee565",
            "add4bab838d946d89205d5c28493492f",
            "2f61a64c7c9d4310a7551759acd1a174",
            "04b7215f1d964266aa9a28e89feac41a",
            "f35147236ed0473e8aac470ee586382f",
            "5f0a8044bb2040ae8771f1e9fd824600",
            "20697f2d2f7b4e94b10de2ecbe00877f",
            "5643e861091f439c91da7ecc1f0f933e",
            "83fe9ae4af094e0dad9f78ed083cb870",
            "a625b9d370cc4b6dbd6b393666610a42",
            "c6526b96fa3d4fb681cfc079dd383614",
            "487540bcdf19407d9feba8e8bb96895d",
            "3568de793afa458aa2d8ddba862d1ec1",
            "c4a5fdb2ac9e4a82a28d48dd081725fc",
            "f86ac528c0b14c3fbe15493c082192a6",
            "8e29950259d94cc2b5d58354370d5337",
            "cbc6f4244ef04ab6961d5bd7c8c54374",
            "7cef656410744d119f791eba0c5ffef0",
            "6043d820cd474b7bb57abb32d54081b4",
            "76d4733366b94f06aa280451c43fd402",
            "b9069456c8a24bfb8587c58a67491e87",
            "50f49dcad596434f8a0d87f7abada4eb",
            "22d87df416734d5ca19dee793dd428c9",
            "d48d0a4c952242a7b4c041a0244a729d",
            "26817a18ed534b55895a533a8784f3fc",
            "8b95686df8e94cd39dd1877ddeb2befd",
            "fbeae70bf35a4c47a00b901d8eeb5110",
            "6074ff5fbcea4bc99304b1a63ccb0269",
            "b5ebb388045f42e39c85cb847b4c7110",
            "2f0a699c470041e09cf2376d676debe9",
            "72ff38ebd8a34ea289eb0008a0abaa75",
            "71d0350ad4f24e89afbc870fce0822cc",
            "df0cdf1b7f60453e8306823896a61444",
            "a41d35005fa5436d983721943017b0d2",
            "a12dfb2e292740cc8767f4b5ad158d77",
            "ac54d4922cda46c3b4d638363ed918e0",
            "1668713b97404f85b747e2b06e58ecf8",
            "f0afa66001e049fbb419ce2d2c93175d",
            "507599bcfbd0489ea4b30a9794173201",
            "95bf033f0e2a4bf4a4020206a4e45fd4",
            "f46447f11bdb49ceb973508593ab50f4",
            "3a14a563534f4caf8b281efa6b25c2c8",
            "c807789537f64d978fb821a2a1b815bf"
          ]
        },
        "id": "BZueRByKiHfs",
        "outputId": "afd23157-8cc3-46f4-c694-8c57d8a02384"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/huggingface_hub/utils/_token.py:88: UserWarning: \n",
            "The secret `HF_TOKEN` does not exist in your Colab secrets.\n",
            "To authenticate with the Hugging Face Hub, create a token in your settings tab (https://huggingface.co/settings/tokens), set it as secret in your Google Colab and restart your session.\n",
            "You will be able to reuse this secret in all of your notebooks.\n",
            "Please note that authentication is recommended but still optional to access public models or datasets.\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "config.json:   0%|          | 0.00/483 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "645908d710ec471b99614b3f90852d61"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "model.safetensors:   0%|          | 0.00/268M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "8ca31e8c11794aa6b879fa3d9b0986fa"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "tokenizer_config.json:   0%|          | 0.00/28.0 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "c6526b96fa3d4fb681cfc079dd383614"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "vocab.txt:   0%|          | 0.00/232k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "50f49dcad596434f8a0d87f7abada4eb"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "tokenizer.json:   0%|          | 0.00/466k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "df0cdf1b7f60453e8306823896a61444"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Example: Tokenizing and encoding text\n",
        "text = \"Hello, how are you doing today?\"\n",
        "tokens = tokenizer(text, return_tensors='pt')\n",
        "outputs = model(**tokens)"
      ],
      "metadata": {
        "id": "ek2HTX8oiQqb"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(outputs)\n",
        "#Notice six words shown."
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6vSMSaZ_imfz",
        "outputId": "6970e2e2-01ae-49e8-994b-8887302a273f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "BaseModelOutput(last_hidden_state=tensor([[[-0.1287, -0.1419, -0.1345,  ..., -0.0144,  0.5856,  0.3436],\n",
            "         [ 0.1180, -0.1199,  0.3374,  ...,  0.1191,  0.7489, -0.0225],\n",
            "         [-0.5806,  0.1913,  0.5075,  ..., -0.1679,  0.4484,  0.1599],\n",
            "         ...,\n",
            "         [-0.6880, -0.7057, -0.5547,  ..., -0.1592,  0.1028, -0.4917],\n",
            "         [-0.3022, -0.6085, -0.4973,  ..., -0.1851,  0.6125,  0.1633],\n",
            "         [ 0.7487,  0.1659, -0.3929,  ...,  0.3324, -0.2238, -0.3643]]],\n",
            "       grad_fn=<NativeLayerNormBackward0>), hidden_states=None, attentions=None)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(tokens)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "TqjL418Pi5kM",
        "outputId": "e825405d-28e8-42c6-e4bb-72f67be2cf95"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{'input_ids': tensor([[ 101, 7592, 1010, 2129, 2024, 2017, 2725, 2651, 1029,  102]]), 'attention_mask': tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1]])}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(text)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "geJSR2dSi-40",
        "outputId": "40ed3131-9bdc-4587-deeb-a0aae3a60399"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Hello, how are you doing today?\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "tokenizer.tokenize(\"This here's an example of using the BERT tokenizer\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7hoS_P45p_zs",
        "outputId": "a8042377-11b4-40bf-d9d5-a343d68e3f8c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['this',\n",
              " 'here',\n",
              " \"'\",\n",
              " 's',\n",
              " 'an',\n",
              " 'example',\n",
              " 'of',\n",
              " 'using',\n",
              " 'the',\n",
              " 'bert',\n",
              " 'token',\n",
              " '##izer']"
            ]
          },
          "metadata": {},
          "execution_count": 25
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "tokenizer.tokenize(\"That movie was absolutely awful\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "06kI8ZzCqQZR",
        "outputId": "6fb03e01-1d6a-404f-f897-9bfdeb89fb01"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['that', 'movie', 'was', 'absolutely', 'awful']"
            ]
          },
          "metadata": {},
          "execution_count": 30
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [],
      "metadata": {
        "id": "ZNsDoBfBjgkM"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "#NEW EXERCISE"
      ],
      "metadata": {
        "id": "vfEDnEeMjgm1"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Certainly! Let's extend the previous example to perform sentiment analysis using a pre-trained DistilBERT model. For this, I'll use the pipeline module from the transformers library, which provides a convenient way to perform various NLP tasks, including sentiment analysis:"
      ],
      "metadata": {
        "id": "-aOqzJvsjocv"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import pipeline, DistilBertModel, DistilBertTokenizer\n",
        "\n",
        "# Load pre-trained DistilBERT model and tokenizer\n",
        "model = DistilBertModel.from_pretrained('distilbert-base-uncased')\n",
        "tokenizer = DistilBertTokenizer.from_pretrained('distilbert-base-uncased')\n",
        "\n",
        "# Example text for sentiment analysis\n",
        "text = \"I really enjoyed watching the movie. It was fantastic!\"\n",
        "\n",
        "# Tokenize and encode the text\n",
        "tokens = tokenizer(text, return_tensors='pt')\n",
        "\n",
        "# Get the model's output\n",
        "outputs = model(**tokens)\n",
        "\n",
        "# Access the output embeddings or other information from 'outputs'\n",
        "# For example, you can extract the embeddings for the [CLS] token\n",
        "cls_embedding = outputs.last_hidden_state[:, 0, :]\n",
        "\n",
        "# Print the embeddings (you can use these for your custom sentiment analysis task)\n",
        "print(\"Embeddings:\", cls_embedding)\n",
        "\n",
        "# Perform sentiment analysis using the pipeline\n",
        "sentiment_analysis = pipeline('sentiment-analysis', model='distilbert-base-uncased')\n",
        "result = sentiment_analysis(text)\n",
        "\n",
        "# Print the sentiment analysis result\n",
        "print(\"Sentiment Analysis Result:\", result)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "iGOGBMpgjkzu",
        "outputId": "071c9ec1-87ca-469a-d90f-d19d9b498fa1"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Embeddings: tensor([[ 2.0706e-01, -8.8669e-02, -4.8918e-02, -1.3041e-01,  4.7975e-02,\n",
            "         -3.9025e-01,  1.2626e-01,  6.5851e-01, -3.6693e-02, -8.2247e-02,\n",
            "          8.6010e-02, -1.0300e-01,  5.3220e-02,  6.7070e-01,  1.2688e-01,\n",
            "          1.5185e-01, -8.8836e-02,  2.7484e-01,  7.9578e-02, -1.2691e-01,\n",
            "         -1.5399e-02, -4.4519e-01, -1.8450e-02,  1.7430e-01,  6.2543e-02,\n",
            "         -1.3739e-01,  2.3637e-02, -7.6291e-02,  5.0763e-01, -1.4827e-02,\n",
            "          6.5497e-02,  3.6562e-02, -2.3892e-01, -2.3559e-01, -2.7979e-02,\n",
            "         -1.1920e-01, -2.0597e-01, -1.3350e-01, -2.5894e-01,  7.3608e-02,\n",
            "         -1.7778e-01, -4.5025e-03,  2.7264e-01, -1.3120e-01, -1.3571e-01,\n",
            "         -2.6281e-01, -2.4850e+00, -2.9151e-02,  2.1403e-02, -2.4612e-01,\n",
            "          4.5534e-01,  5.5840e-02,  7.8191e-02,  2.5053e-01,  3.8298e-01,\n",
            "          3.6595e-01, -4.9592e-01,  3.6923e-01, -2.2526e-01, -6.0456e-02,\n",
            "          3.1174e-01,  1.5549e-01, -1.9369e-01, -2.1970e-01,  3.8572e-02,\n",
            "         -4.1085e-02,  2.4707e-02,  1.4617e-01,  6.2864e-02,  2.8154e-01,\n",
            "         -1.0181e-01, -1.2331e-01,  1.8897e-01, -2.2510e-01, -6.2209e-02,\n",
            "          2.4534e-02, -6.8069e-02,  1.7248e-01,  2.3165e-02,  1.6548e-01,\n",
            "          1.2908e-01,  5.5311e-01,  3.3488e-01,  2.6752e-01,  8.2457e-02,\n",
            "          2.7306e-01, -3.6223e-01, -2.9196e-01,  3.1387e-01,  3.8054e-01,\n",
            "         -4.8587e-01,  5.5267e-02,  1.1312e-01,  1.8239e-01,  6.8425e-01,\n",
            "         -3.4050e-01,  1.9306e-02, -1.2614e-01,  6.8236e-02,  2.1131e-01,\n",
            "          2.3281e-01, -2.7559e-01, -9.3980e-02, -5.3529e-01,  8.6103e-02,\n",
            "          5.4234e-03,  2.1097e-01, -5.1999e-01,  1.2880e-01, -2.3327e+00,\n",
            "          2.8214e-01,  1.4685e-01, -1.8600e-01, -5.5324e-01, -2.2334e-01,\n",
            "          6.7317e-01,  3.5680e-01, -7.0620e-02,  1.0790e-01,  1.9131e-01,\n",
            "         -1.9220e-01,  2.7099e-01, -1.3977e-01,  6.9052e-02, -6.3412e-02,\n",
            "          4.1788e-01, -1.2854e-01, -4.1270e-01, -5.7345e-02,  5.1834e-02,\n",
            "          4.0118e-01,  5.6316e-01,  1.6877e-01, -2.6576e-01, -2.1654e-01,\n",
            "          1.0870e-01, -7.4192e-03, -1.9570e-01, -3.7734e-01, -1.6077e-02,\n",
            "         -8.4996e-02, -1.5673e-01, -3.1582e+00,  4.4820e-01,  3.0782e-01,\n",
            "          6.9475e-02,  8.6786e-02, -6.6694e-02, -1.0429e-01,  6.2008e-02,\n",
            "          1.7896e-01,  5.1430e-02, -5.3639e-01, -1.4609e-01, -2.0126e-01,\n",
            "          1.4227e-01, -3.9454e-01, -8.7160e-02,  3.1460e-01,  1.7651e-01,\n",
            "          2.9077e-01, -7.6356e-02, -9.2473e-02, -3.6427e-02, -1.1243e-01,\n",
            "          3.2692e-01,  2.9333e-01,  2.9094e-01, -8.0394e-02, -2.9835e-02,\n",
            "         -1.0763e-01,  2.6380e-01,  2.8374e-01, -5.2889e-02,  8.7276e-02,\n",
            "         -1.6420e-01,  2.9141e-01,  4.5129e-01,  1.0565e-01, -1.6464e-01,\n",
            "          2.9189e-02,  3.1959e-01,  2.8125e-01,  2.6986e-01,  2.5944e-01,\n",
            "          1.9278e-02,  4.1811e-01, -3.0498e-01, -1.2579e-01,  2.8649e-01,\n",
            "         -8.0526e-02, -2.3461e-01, -8.6629e-02, -3.3086e-01,  3.9412e-01,\n",
            "         -5.2155e-02,  1.5204e-01, -5.0489e-01,  3.0596e-01,  2.1820e-01,\n",
            "         -3.2314e-01,  2.8805e-02,  1.8898e-01,  3.1090e-02, -3.7318e-01,\n",
            "          3.4340e+00,  1.2818e-01,  7.5815e-02,  1.4749e-01,  1.0946e-02,\n",
            "         -2.5986e-01,  2.6052e-01, -4.4740e-02, -2.2305e-01, -8.9524e-02,\n",
            "          2.3338e-01,  7.5105e-02,  1.6383e-01,  2.3412e-01, -2.8108e-01,\n",
            "          9.4963e-02,  2.0160e-01, -3.6509e-01, -6.2761e-02,  3.0960e-02,\n",
            "          2.4025e-01, -2.0667e-01,  3.3669e-01,  3.4695e-02, -1.4206e+00,\n",
            "         -2.1103e-01,  4.2617e-03, -2.3335e-01,  2.8209e-01, -1.9113e-01,\n",
            "          1.1402e-02,  5.0549e-02, -7.7772e-02,  9.2771e-02,  3.2389e-02,\n",
            "          4.1144e-03,  2.4044e-01,  1.3504e-01,  1.9931e-01, -2.5447e-01,\n",
            "          3.8895e-01,  3.2603e-01,  1.3565e-01,  1.8842e-01, -3.9112e-02,\n",
            "          1.7901e-01, -1.3939e-01,  8.0536e-02, -1.7066e-01,  2.4761e-01,\n",
            "          8.5492e-02, -3.4393e-02,  4.3791e-02, -3.6303e-01, -1.1356e-01,\n",
            "         -1.0639e-01, -1.2420e-01,  1.8264e-01,  1.9134e-01, -5.7426e-01,\n",
            "         -2.3360e-01,  2.9841e-02, -3.7223e-01,  1.6197e-01,  1.2952e-01,\n",
            "         -1.5846e-01, -6.4124e-02, -2.0768e-01, -3.5157e+00,  1.5543e-01,\n",
            "         -3.1144e-01,  6.9244e-02,  9.3899e-02,  5.6945e-02,  2.0346e-01,\n",
            "          4.0453e-02,  1.8313e-01, -5.5370e-01,  1.4641e-01,  1.4070e-01,\n",
            "         -1.3589e-01, -2.0563e-02, -2.8124e-01,  3.5957e-01, -2.1818e-02,\n",
            "         -8.0351e-02, -2.6408e-01, -1.3934e-01, -2.0383e-01,  3.5412e-01,\n",
            "         -2.1363e-01,  4.5684e-01,  1.4611e-01, -7.7507e-02,  1.0856e-01,\n",
            "         -3.3337e-01,  2.8810e-01,  9.0313e-02, -4.1050e-02, -1.4498e-01,\n",
            "          2.9153e-01, -2.1550e-01, -3.1540e-01, -2.6645e+00, -2.9730e-02,\n",
            "          3.9498e-02, -1.7731e-01, -9.1091e-02,  4.9540e-02,  2.4252e-01,\n",
            "         -2.1940e-01, -2.6254e-01,  1.8029e-01,  2.0403e-01,  4.7933e-02,\n",
            "         -2.0241e-01,  3.1773e-01,  4.6945e-01, -7.4159e-02,  2.8996e-01,\n",
            "         -9.8550e-03,  2.8337e-01,  3.4332e-01, -1.3244e-01, -1.2418e-01,\n",
            "         -5.6962e-02, -1.5424e-01,  3.6812e-01,  2.6528e-01, -3.4925e-01,\n",
            "         -6.2518e-02, -1.7110e-01,  9.1046e-02, -1.7882e-01, -3.2408e-01,\n",
            "          1.4057e-01, -1.8963e-02, -1.4231e-01, -9.0090e-02,  1.0936e-01,\n",
            "         -2.0050e-01,  1.2274e-01, -8.1322e-02, -6.8242e-02,  3.2770e-01,\n",
            "          5.3927e-02,  4.5093e-02,  5.1561e-01,  3.4527e-02,  2.8920e-01,\n",
            "          8.7371e-02, -4.6426e-02, -5.3604e-02,  1.3548e-01,  1.8145e-01,\n",
            "          1.4620e+00, -2.9299e-02, -2.7658e-02, -4.3503e-01,  3.4446e-01,\n",
            "          1.3984e-01,  1.3671e-01,  2.2487e-01,  4.1597e-01, -2.6563e-01,\n",
            "          2.5840e-01, -2.8250e-01, -6.4824e-02, -1.9823e-01,  2.0070e-01,\n",
            "         -3.6015e-01, -1.0033e-01,  2.1484e-01,  1.4613e-01, -1.5889e-01,\n",
            "         -8.3516e-02, -8.2461e-01, -4.4122e-01,  1.1401e-01,  2.3718e-02,\n",
            "         -4.8873e-02, -1.2253e-01,  9.6639e-02, -1.8347e-01, -1.2157e-01,\n",
            "         -1.3178e-02,  1.6749e-01, -1.9584e-01, -6.1176e-02, -3.6947e-01,\n",
            "         -8.5497e-02, -4.3735e-01, -2.7659e-02,  9.6718e-02,  3.5329e-02,\n",
            "         -2.2810e-02,  2.9518e-01, -2.1224e-02, -5.1430e-02,  1.3923e-01,\n",
            "         -6.0178e-01,  4.3357e-01, -1.3916e-01, -1.9754e-01, -7.5599e-02,\n",
            "         -5.4211e-02,  1.8313e-01, -9.9159e-02, -1.5055e-01, -9.7802e-02,\n",
            "          1.4422e-01, -4.8247e-02,  1.3089e-01, -1.8053e-01,  1.2904e-01,\n",
            "          2.3001e-01,  2.9300e-01,  1.0460e+00, -3.7456e-02,  7.8900e-02,\n",
            "          1.6114e-01, -3.3765e-02,  2.6561e-01,  8.7403e-02,  7.0614e-03,\n",
            "         -5.3969e-01,  2.2455e-01, -1.2090e-01, -4.8699e-02,  2.3640e-02,\n",
            "         -2.6531e-01, -2.6001e-01, -1.6938e-01,  1.8373e-01, -3.4430e-02,\n",
            "         -2.8313e-01, -3.0912e-01,  3.4920e-02, -2.5350e-01, -1.2379e-01,\n",
            "         -2.9635e-01,  2.1274e-01,  2.9120e-01,  3.6022e-01,  2.8128e-01,\n",
            "         -1.5771e-01,  5.8585e-01, -2.2324e-01,  5.1394e-01, -1.2465e-02,\n",
            "          1.7164e-01, -1.1776e-01,  2.6445e-01, -3.0432e-01, -8.9432e-03,\n",
            "         -6.9918e-02, -3.0968e-01,  2.0411e-01,  2.2216e-01,  2.0822e-01,\n",
            "         -2.3593e-02,  6.5700e-02, -7.0042e-02,  4.1932e-01,  4.1913e-02,\n",
            "         -1.4653e+00,  4.3634e-01,  1.8057e-01,  1.0761e-01, -1.3262e-02,\n",
            "         -4.2241e-02, -7.4130e-02,  2.9911e-01,  4.2799e-01,  2.6741e-01,\n",
            "         -5.2421e-02,  5.0427e-03, -6.5900e-02,  5.1127e-02, -4.3086e-02,\n",
            "          1.4547e-01, -1.3473e-01, -8.3071e-02,  1.6098e-02, -3.3640e-02,\n",
            "          1.3555e-01,  2.0440e-01, -1.1039e-01,  1.0662e-01, -1.9347e-01,\n",
            "          5.7161e-03,  3.9429e-03,  2.2294e-01, -9.7271e-02,  3.1727e-01,\n",
            "         -1.7249e-01, -4.3268e-01, -7.4320e-01, -2.8565e-01,  8.0231e-02,\n",
            "         -2.8640e-01, -1.1530e-03, -1.7355e-02,  3.1990e-01,  1.0784e-01,\n",
            "         -5.6854e-01,  3.9452e-01,  2.2379e-01, -5.3480e-02,  5.5336e-01,\n",
            "          2.8201e-01,  4.9276e-02,  2.7171e-01, -1.4759e-02, -1.9672e-01,\n",
            "          8.2045e-02, -1.0493e-01, -1.6739e-01,  7.4037e-02,  2.7259e-01,\n",
            "         -1.7200e-01, -7.4944e-03,  1.5834e-01, -4.7366e-01,  3.8035e-02,\n",
            "          2.1083e-01, -1.8494e-01, -4.3270e-01,  1.6093e-01, -4.0325e-01,\n",
            "         -5.5749e-01, -3.0438e-01, -4.3697e-01, -1.3194e-01,  1.3136e-01,\n",
            "          4.9756e-01,  1.5926e-02, -4.2530e-01,  1.2379e-01, -1.7837e-01,\n",
            "          5.3540e-02,  3.0557e-01,  2.8421e-01,  2.2985e-01, -2.9800e-01,\n",
            "          1.3872e-01,  1.0640e-01, -2.2652e-01,  1.0748e-01, -1.6571e-01,\n",
            "          2.8296e-01, -1.3725e-01, -2.4731e-01, -8.0663e-03, -8.5202e-02,\n",
            "         -3.6983e-01, -5.4197e-02, -2.2700e-01, -2.6223e-01,  4.7880e-02,\n",
            "          1.5143e-01,  2.4051e-01, -1.4527e-01, -2.3066e-01, -5.5557e-02,\n",
            "         -4.3412e-01,  3.9486e-01,  5.2602e-01,  1.9910e-01,  1.3133e-01,\n",
            "          1.5394e-01,  1.6574e-01,  3.6346e-02, -3.0235e-01, -6.2884e-02,\n",
            "         -7.9558e-02,  1.2774e-01,  2.4704e-01,  5.3429e-02, -4.2546e-03,\n",
            "         -4.5961e-01,  1.0305e-01, -4.4691e-01,  1.7984e+00,  4.8555e-01,\n",
            "          2.8570e-01, -9.1507e-02,  3.2502e-01, -1.8614e-01, -2.7716e-01,\n",
            "          5.0319e-01, -1.0816e-01,  4.9738e-01, -2.1839e-01,  2.5104e-01,\n",
            "         -1.6620e-01, -3.4126e-02,  4.8728e-01, -8.4591e-02,  2.7863e-01,\n",
            "         -6.6855e-02, -6.3182e-01, -1.1332e-01, -4.5212e-01, -1.8627e-02,\n",
            "          2.3054e-01, -1.4194e-01, -1.4265e-01,  2.1068e-01,  1.3754e-01,\n",
            "         -7.5238e-02,  8.6938e-02,  2.7379e-01,  3.4218e-02, -6.2501e-02,\n",
            "         -1.8972e-01,  2.7638e-01, -3.2890e-01, -7.0843e-02,  2.2255e-01,\n",
            "         -1.8095e-01, -5.8833e-02, -1.3913e-01,  7.4171e-02, -3.4258e-01,\n",
            "          1.8371e-01, -2.4334e-01, -9.4516e-03,  5.3453e-01, -8.7182e-02,\n",
            "         -2.2759e-01,  2.7197e-01,  3.1159e-01, -1.2399e-02, -1.2019e-01,\n",
            "         -2.1714e-01,  2.5210e-01,  6.3619e-02, -3.4635e-01, -6.5211e-02,\n",
            "         -6.0126e-02,  5.8262e-02,  5.0974e-02, -4.3526e-02,  3.9306e-01,\n",
            "          2.2101e-01,  2.3378e-01,  1.0313e-01,  2.7017e-01, -2.8816e-01,\n",
            "         -7.2462e-02,  1.1311e-01, -5.1148e-01, -1.7523e-02,  3.6645e-01,\n",
            "          4.8162e-01,  9.1628e-02,  4.9514e-01,  1.9019e-01,  5.1925e-01,\n",
            "          3.8016e-02,  1.3103e-01, -2.5627e+00,  1.7707e-01,  3.0159e-02,\n",
            "          1.4305e-01, -9.6449e-03,  3.7251e-01,  3.9124e-01, -1.0542e-01,\n",
            "          2.6524e-01, -2.6068e-01,  1.0819e-01,  2.0877e-01,  3.4482e-01,\n",
            "          1.5284e-01,  4.8359e-02,  9.5537e-02,  2.2316e-01, -2.5754e-01,\n",
            "         -3.3091e-01,  1.9268e-01,  2.8610e-02,  2.4588e-01,  1.1224e-01,\n",
            "         -3.5953e-01, -4.1324e-01,  2.3263e-01, -5.7211e-02,  6.6348e-03,\n",
            "         -5.9350e-02,  1.6318e-01, -9.8257e-02,  5.2425e-01, -3.3782e-01,\n",
            "         -6.7074e-02, -1.1744e-01, -1.4769e-01, -3.3717e-01, -1.6037e-01,\n",
            "          1.5975e-01, -3.0458e-02, -1.1528e-01,  4.7169e-01,  8.9324e-02,\n",
            "          9.8143e-02, -2.7237e-02,  1.8024e-02,  3.6843e-01, -2.3917e-01,\n",
            "          1.3618e-01, -3.2476e-01, -1.5165e-01,  7.6218e-02,  3.7780e-01,\n",
            "          2.2197e-02,  6.9287e-02,  2.8588e-02, -9.2082e-02,  1.6636e-01,\n",
            "          2.8576e-01, -1.3112e-01, -1.5810e-01,  3.8875e-02, -6.4142e-02,\n",
            "         -1.4454e-01,  1.8612e-01, -1.9644e-01, -1.1561e-03,  7.8402e-02,\n",
            "         -1.4660e-01, -1.2260e-01, -1.8415e-01, -1.7009e-01,  4.5599e-01,\n",
            "          6.2600e-02,  1.5132e-01, -1.6927e-01,  3.6136e-01,  3.9457e-01,\n",
            "          9.8191e-02, -1.2537e-02, -2.0895e-04, -1.7373e-01,  1.5355e-01,\n",
            "         -1.5953e-01, -9.3504e-02, -7.1168e+00, -2.7053e-01,  2.0430e-03,\n",
            "         -4.9714e-03,  4.0080e-03, -4.6965e-01, -2.5748e-02, -2.8648e-01,\n",
            "          1.3834e-01, -2.7800e-01,  3.3612e-02,  1.5357e-01, -6.0711e-02,\n",
            "         -1.0130e-02,  3.8116e-01,  2.2861e-01]], grad_fn=<SliceBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Some weights of DistilBertForSequenceClassification were not initialized from the model checkpoint at distilbert-base-uncased and are newly initialized: ['pre_classifier.bias', 'classifier.bias', 'classifier.weight', 'pre_classifier.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Sentiment Analysis Result: [{'label': 'LABEL_0', 'score': 0.5327839255332947}]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "For example above:\n",
        "In this extended example:\n",
        "\n",
        "We tokenize and encode the input text using the DistilBERT tokenizer.\n",
        "\n",
        "We pass the tokens through the DistilBERT model to obtain the model's output embeddings. In this example, we extract the embeddings for the [CLS] token, which is commonly used for sentence-level tasks.\n",
        "\n",
        "We print the embeddings. You can use these embeddings for custom sentiment analysis or other downstream tasks.\n",
        "\n",
        "We use the pipeline module to perform sentiment analysis directly. The result is printed, showing the sentiment label ('POSITIVE', 'NEGATIVE', 'NEUTRAL') and the confidence score.\n",
        "\n",
        "Note: The pipeline module simplifies the process of using pre-trained models for specific tasks. In this case, we use the 'sentiment-analysis' pipeline, which is designed for sentiment analysis tasks. You can customize the example based on your specific use case and requirements.\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n"
      ],
      "metadata": {
        "id": "oIbkRv4zjzMr"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#Running again in a different way\n",
        "\n",
        "from transformers import pipeline, DistilBertModel, DistilBertTokenizer\n",
        "\n",
        "# Load pre-trained DistilBERT model and tokenizer\n",
        "model = DistilBertModel.from_pretrained('distilbert-base-uncased')\n",
        "tokenizer = DistilBertTokenizer.from_pretrained('distilbert-base-uncased')\n",
        "\n",
        "# Example text for sentiment analysis\n",
        "#text = \"I really enjoyed watching the movie. It was fantastic!\"\n",
        "text = \"That movie was absolutely awful\"\n",
        "\n",
        "# Tokenize and encode the text\n",
        "tokens = tokenizer(text, return_tensors='pt')\n",
        "\n",
        "# Get the model's output\n",
        "outputs = model(**tokens)\n",
        "\n",
        "# Access the output embeddings or other information from 'outputs'\n",
        "# For example, you can extract the embeddings for the [CLS] token\n",
        "cls_embedding = outputs.last_hidden_state[:, 0, :]\n",
        "\n",
        "# Print the embeddings (you can use these for your custom sentiment analysis task)\n",
        "print(\"Embeddings:\", cls_embedding)\n",
        "\n",
        "# Perform sentiment analysis using the pipeline\n",
        "sentiment_analysis = pipeline('sentiment-analysis', model='distilbert-base-uncased')\n",
        "result = sentiment_analysis(text)\n",
        "\n",
        "# Extract the sentiment label and confidence score\n",
        "sentiment_label = result[0]['label']\n",
        "\n",
        "#labels = ['Negative','Positive'] #(0:negative, 1:positive)\n",
        "confidence_score = result[0]['score']\n",
        "\n",
        "# Print the sentiment analysis result\n",
        "print(\"Sentiment Label:\", sentiment_label)\n",
        "print(\"Confidence Score:\", confidence_score)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GLSURb80k3Pk",
        "outputId": "0232b375-5f40-4033-9de6-da942a6f246b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Embeddings: tensor([[ 7.0210e-02,  6.8084e-02,  7.6470e-02,  1.9690e-02,  2.8007e-02,\n",
            "         -1.0710e-01,  1.2325e-01,  4.8434e-01, -1.9508e-02,  2.3242e-02,\n",
            "          5.9509e-02,  3.7812e-03, -7.5885e-02,  4.8900e-01,  9.0280e-02,\n",
            "          1.4583e-01, -1.8004e-01,  1.1228e-01,  9.8468e-02, -1.6458e-01,\n",
            "         -2.2962e-01, -2.3786e-01, -3.1089e-02,  1.1482e-02, -1.1495e-01,\n",
            "          2.2647e-02,  3.7191e-02, -9.5181e-02,  1.8810e-01,  7.0893e-02,\n",
            "          1.1936e-01, -5.1023e-02, -9.5914e-02, -4.0946e-03, -1.4172e-02,\n",
            "         -4.6111e-02, -5.8164e-02, -6.1541e-02, -9.2024e-02, -5.6487e-02,\n",
            "         -1.1766e-01,  4.8398e-02,  1.3649e-01, -1.2492e-01, -1.7980e-02,\n",
            "         -1.1209e-02, -1.8811e+00, -1.3138e-02,  4.5990e-02, -2.3842e-01,\n",
            "          2.7772e-01, -3.2499e-02,  1.7736e-01,  2.7165e-01,  3.0734e-01,\n",
            "          2.1610e-01, -1.3272e-01,  3.9919e-01, -6.1820e-02,  1.8868e-02,\n",
            "          1.9426e-01, -2.3554e-02, -5.8178e-02, -5.3716e-02, -1.0974e-02,\n",
            "          5.5273e-02,  2.6712e-02,  1.0244e-01, -1.6141e-01,  1.5072e-01,\n",
            "         -1.7391e-01, -2.6420e-01,  2.2884e-01, -9.5202e-02,  3.2873e-03,\n",
            "         -1.1645e-01,  4.6312e-02,  7.6179e-03,  9.2150e-04,  1.0870e-01,\n",
            "          7.4153e-02,  3.1126e-01,  1.5149e-01,  7.8298e-02, -8.4716e-02,\n",
            "          1.4393e-01, -1.2629e-01, -1.6662e-01,  1.2463e-01,  3.6217e-01,\n",
            "         -3.3603e-01,  5.0909e-02,  1.2921e-01,  2.6329e-01,  2.8940e-01,\n",
            "         -2.6184e-01,  5.4341e-02, -6.1922e-02,  2.2280e-01,  2.4691e-01,\n",
            "          7.3170e-03, -2.2587e-01, -6.4404e-02, -2.1725e-01, -3.4282e-02,\n",
            "         -7.4080e-02,  1.4955e-01, -1.0685e-01,  1.7219e-01, -2.5439e+00,\n",
            "          1.9954e-01,  1.8210e-01, -1.2624e-01, -4.9150e-01, -1.6287e-01,\n",
            "          4.0989e-01,  3.4003e-03, -9.3138e-02,  1.0669e-01,  8.7394e-02,\n",
            "         -5.3071e-02,  2.6903e-01, -8.2266e-02, -2.1511e-01,  8.1380e-02,\n",
            "          2.5792e-01,  4.8620e-04, -1.1205e-01, -8.7534e-02,  1.2530e-01,\n",
            "          3.2873e-01,  4.5060e-01,  4.8835e-02,  2.1659e-02, -2.2686e-01,\n",
            "         -1.8355e-02,  2.1946e-01, -8.6457e-02, -1.3597e-01, -1.1141e-01,\n",
            "         -1.0518e-01,  4.6651e-03, -3.3029e+00,  4.4642e-01,  1.6015e-01,\n",
            "         -6.6687e-02, -5.3272e-02, -3.9121e-02,  6.6998e-02,  4.5202e-02,\n",
            "          1.7310e-01,  2.0435e-01, -1.7824e-01, -9.5743e-02, -1.3881e-01,\n",
            "          6.6837e-02,  1.6904e-02, -9.5584e-02,  2.9577e-01,  5.4376e-02,\n",
            "          1.7277e-01, -2.0242e-02, -1.4180e-01, -1.8642e-01, -4.0464e-02,\n",
            "         -1.8090e-02,  8.0317e-02,  1.9451e-01,  4.7899e-02, -6.2248e-02,\n",
            "         -7.3494e-03, -4.2843e-02,  1.7671e-01, -3.7314e-02,  8.8881e-02,\n",
            "         -1.2693e-01,  2.2842e-01,  3.2210e-01,  4.8474e-02, -1.4896e-01,\n",
            "          1.2412e-01,  5.0364e-01,  2.0439e-01,  2.1017e-01,  6.3660e-02,\n",
            "          6.5129e-02,  3.3880e-01, -1.1937e-01, -5.4780e-02,  3.6465e-01,\n",
            "          2.2739e-02, -1.4237e-01,  7.4659e-02, -1.0673e-01,  2.0002e-01,\n",
            "         -1.1965e-01, -2.8842e-02, -4.7533e-01,  3.7402e-01,  1.0944e-01,\n",
            "         -2.6974e-01,  4.4372e-02,  5.2608e-02, -4.1959e-02, -8.2398e-02,\n",
            "          3.5727e+00, -2.9040e-02,  3.2614e-02,  1.5512e-01,  1.2221e-01,\n",
            "          1.0003e-01,  9.9599e-02, -4.9769e-03, -1.1049e-01,  6.5973e-02,\n",
            "          2.6047e-02,  3.5683e-02,  3.6941e-02,  2.0850e-01, -1.9138e-01,\n",
            "          8.0428e-02,  1.6420e-01, -1.5171e-01, -1.2848e-01, -7.5413e-02,\n",
            "          2.3845e-01,  1.1243e-01,  1.6437e-01, -1.5891e-02, -1.1823e+00,\n",
            "         -3.9776e-02,  1.5999e-01, -3.0544e-01,  2.4809e-01, -4.5131e-03,\n",
            "         -1.2085e-01,  1.8269e-03, -1.1104e-03,  9.1960e-02,  1.9021e-01,\n",
            "         -1.0746e-02,  1.3543e-01,  1.8933e-01,  1.9844e-01, -2.4609e-01,\n",
            "          2.2704e-01,  1.8756e-01,  1.9820e-02, -8.3310e-02,  3.9763e-02,\n",
            "          1.3475e-01, -2.4862e-01, -1.3573e-01, -1.0175e-01,  7.5262e-02,\n",
            "          3.3437e-02, -7.8380e-02,  7.5527e-02, -1.5418e-01, -4.9511e-02,\n",
            "         -1.3396e-01, -1.2878e-02,  8.3121e-02,  4.5249e-02, -1.6847e-01,\n",
            "          2.7127e-02, -2.7690e-02, -1.6939e-01, -2.5761e-02,  2.3491e-01,\n",
            "         -1.4882e-01,  7.7250e-02, -1.7147e-01, -4.1981e+00,  1.8235e-02,\n",
            "          5.0425e-02,  1.5842e-01,  2.5545e-01,  6.9077e-03,  2.6726e-01,\n",
            "          9.2382e-02,  2.0444e-01, -3.6579e-01,  7.3434e-02,  1.7368e-01,\n",
            "         -2.5656e-02,  1.0155e-01, -2.9859e-01,  1.7681e-01, -6.7990e-03,\n",
            "         -1.7406e-01, -1.9978e-02,  7.1345e-03, -2.0362e-02,  2.1270e-01,\n",
            "         -2.1537e-01,  3.0113e-02, -3.6798e-02, -1.4119e-01,  1.4477e-02,\n",
            "         -1.5803e-01,  3.9264e-01, -8.7364e-02, -8.4698e-02, -1.0178e-01,\n",
            "          2.0696e-01, -1.4076e-01, -1.6393e-01, -2.4259e+00,  9.0164e-02,\n",
            "         -1.0545e-01, -1.4428e-01,  5.0900e-02, -1.0954e-01,  2.0034e-01,\n",
            "         -1.4811e-01, -3.6446e-01,  5.7970e-02,  1.4776e-01, -4.8381e-03,\n",
            "         -1.1046e-01,  1.9530e-01,  3.5268e-01, -1.7863e-01, -9.3188e-02,\n",
            "         -1.0548e-01,  1.6100e-01,  2.2590e-01, -3.6949e-02, -8.0652e-02,\n",
            "         -1.2417e-01, -9.9830e-02,  8.9419e-02,  3.0115e-01, -4.0292e-01,\n",
            "          6.3017e-02, -2.3965e-01, -3.6706e-02, -1.9132e-01, -1.1254e-01,\n",
            "         -1.0118e-02,  8.3296e-02, -2.3813e-02, -1.9389e-01,  1.4846e-01,\n",
            "          1.3152e-01,  9.0962e-02, -7.2862e-02, -1.5728e-01,  2.8305e-01,\n",
            "          1.3410e-03,  1.8496e-01,  3.3793e-01,  6.7186e-02,  1.8811e-01,\n",
            "         -2.2263e-01, -1.0653e-01, -6.2588e-02,  1.3543e-02,  1.3539e-01,\n",
            "          1.1174e+00,  1.3951e-01,  9.8153e-02, -1.6124e-01,  2.1697e-01,\n",
            "          1.0880e-01,  8.4445e-02,  6.0807e-02,  2.6460e-01,  1.0800e-02,\n",
            "          1.0969e-01,  6.7569e-02, -4.6410e-02, -9.1961e-02,  2.5364e-01,\n",
            "         -3.2124e-01,  8.7444e-02, -3.4521e-02,  1.6120e-02, -1.1213e-01,\n",
            "         -3.7869e-02, -6.0515e-01, -1.1362e-01,  1.7341e-01, -2.2033e-02,\n",
            "          5.9073e-02,  9.1664e-04,  4.4855e-02, -1.9531e-01, -1.8482e-02,\n",
            "         -1.5434e-02,  1.3454e-01, -7.4706e-02, -3.9060e-02, -2.9883e-01,\n",
            "          1.9909e-02, -1.0703e-03, -1.7694e-02,  1.2347e-01,  1.8095e-01,\n",
            "          2.7663e-02,  2.8166e-01,  9.8733e-02, -4.3489e-02,  1.8613e-01,\n",
            "         -6.2743e-01,  1.3214e-01, -1.2334e-01, -1.1310e-01, -3.0477e-01,\n",
            "         -1.3528e-01,  2.1427e-02, -1.4990e-01, -2.3053e-01, -7.6550e-02,\n",
            "          3.6503e-01, -8.4248e-02,  1.3625e-01,  1.4415e-01,  2.7531e-01,\n",
            "          9.2659e-02,  6.5783e-02,  7.8776e-01, -7.6453e-03,  6.3825e-02,\n",
            "          2.5603e-01, -1.3428e-01,  1.9215e-01,  1.1506e-01, -2.1528e-02,\n",
            "         -3.1555e-01,  1.3217e-02, -4.7009e-02,  2.8015e-02,  5.1112e-02,\n",
            "         -3.4055e-01, -1.6770e-01, -2.6179e-03,  1.7905e-02,  2.7129e-02,\n",
            "         -1.2722e-01, -5.0489e-01, -1.4500e-02, -1.6000e-01, -1.1678e-01,\n",
            "          2.7223e-02,  1.1158e-01,  1.3185e-01,  2.7755e-01,  1.4216e-01,\n",
            "         -1.7605e-02,  3.1379e-01, -8.2785e-02,  3.3493e-01,  7.1293e-02,\n",
            "          1.0065e-01, -7.3071e-02,  9.4425e-02, -1.8008e-01, -8.9302e-02,\n",
            "          5.1244e-02,  1.1781e-02,  6.9097e-04,  1.5878e-01,  9.6841e-02,\n",
            "          3.2169e-02,  8.6110e-03,  1.0361e-01,  1.1567e-01,  1.6937e-01,\n",
            "         -1.1914e+00,  1.4822e-01,  2.8572e-01,  7.5464e-02,  8.1584e-02,\n",
            "          7.4819e-03, -1.5111e-01,  3.5056e-01,  1.4812e-01,  1.9560e-01,\n",
            "         -1.9797e-02,  2.1943e-01, -1.6503e-01,  1.2605e-02, -1.1448e-01,\n",
            "          3.8977e-02,  1.0257e-01,  1.2431e-01,  1.3797e-01,  1.7945e-01,\n",
            "          5.6076e-02,  7.6177e-02,  9.1883e-03, -2.6519e-03, -3.5716e-01,\n",
            "          5.9442e-02, -1.4653e-01,  2.3702e-01, -8.9915e-03, -5.3730e-02,\n",
            "          1.2348e-01, -2.9663e-01, -4.6666e-01, -1.4099e-01,  6.5454e-02,\n",
            "         -1.0160e-02,  4.1447e-02,  1.2959e-01,  1.4755e-01,  6.2910e-02,\n",
            "         -4.0963e-01,  2.3763e-01,  5.3742e-02, -1.5305e-01,  3.8188e-01,\n",
            "          1.1367e-01, -9.1228e-02,  1.9869e-01, -1.8616e-02, -1.5775e-01,\n",
            "         -1.3848e-01, -2.7079e-02, -8.4361e-02, -1.9715e-02,  1.5783e-01,\n",
            "         -6.9875e-02, -4.1190e-02, -7.3246e-02, -2.1516e-01,  2.6322e-02,\n",
            "          1.1042e-01, -1.3714e-01, -2.2542e-01,  2.6560e-01, -2.2820e-01,\n",
            "         -5.7218e-01, -2.0098e-01, -1.2566e-01, -3.4150e-02,  1.4867e-01,\n",
            "          1.7417e-01,  1.9225e-02, -1.5201e-01, -1.0276e-01, -2.4116e-02,\n",
            "          2.4561e-01,  2.5155e-02,  1.8822e-01,  1.4568e-01, -3.1045e-01,\n",
            "         -9.7277e-03,  1.3263e-01, -2.6272e-03,  1.6513e-01,  5.7662e-02,\n",
            "          7.5814e-02, -7.9262e-02, -7.1345e-03, -1.9329e-01, -6.8105e-02,\n",
            "         -2.2094e-01, -2.6495e-01,  1.1473e-01, -8.5436e-02,  6.3126e-02,\n",
            "          1.9983e-01,  3.1201e-01, -3.3396e-02, -2.2894e-01,  3.5266e-02,\n",
            "         -1.9004e-01,  2.5305e-01,  1.6835e-01,  7.3160e-02,  1.8570e-01,\n",
            "          1.1164e-01,  2.1383e-01,  7.9331e-02, -1.3096e-01, -5.2251e-02,\n",
            "         -6.1062e-02,  1.3912e-01,  9.1585e-02, -7.9250e-02, -1.6307e-01,\n",
            "         -2.2670e-01,  1.3385e-01, -1.7534e-01,  2.1269e+00,  3.2471e-01,\n",
            "         -2.6211e-03, -2.6499e-01,  8.0447e-02, -5.5818e-02,  1.0285e-01,\n",
            "          2.2205e-01, -8.8225e-02,  2.8026e-01, -3.7674e-02,  1.6147e-01,\n",
            "          6.7287e-02,  9.5320e-03,  3.2739e-01, -4.6458e-02,  2.9608e-02,\n",
            "         -1.2217e-01, -3.8833e-01, -1.8401e-01, -3.1209e-01,  9.4043e-02,\n",
            "          2.2778e-01, -7.1277e-02, -7.4049e-02,  1.2115e-01,  2.2647e-02,\n",
            "         -1.0845e-01,  2.1890e-02,  9.0043e-02, -4.1176e-02, -7.5949e-02,\n",
            "         -8.4193e-02,  1.6922e-01, -1.5809e-01,  3.0788e-02,  2.0228e-01,\n",
            "         -9.2700e-02, -1.2381e-01, -9.2184e-02,  1.3180e-01, -2.2731e-01,\n",
            "          1.2643e-01, -7.1765e-02,  1.8819e-01,  1.6790e-01, -1.4177e-02,\n",
            "         -9.1120e-02,  3.1611e-01,  1.1484e-01, -6.1316e-02, -6.9189e-02,\n",
            "         -9.0090e-02,  9.2345e-02, -3.5097e-02, -2.4078e-01,  4.5961e-02,\n",
            "         -2.4792e-01,  1.6229e-01,  1.2906e-01, -5.6944e-02,  1.2793e-01,\n",
            "          2.6016e-01,  1.1673e-01,  1.0791e-01,  1.1238e-01, -1.2967e-01,\n",
            "         -3.3522e-02, -5.9857e-02, -1.3557e-01, -1.2545e-01,  3.9175e-02,\n",
            "          3.1633e-01, -7.8773e-02,  2.6089e-01,  1.1353e-01,  4.0131e-01,\n",
            "          9.2380e-02, -6.0576e-02, -2.9233e+00, -5.1876e-02, -3.5782e-02,\n",
            "          5.5098e-02,  4.4209e-02,  4.9394e-01,  3.7819e-01, -2.7030e-01,\n",
            "          1.1889e-01, -1.7410e-01,  1.1357e-01,  2.3427e-01,  5.7641e-02,\n",
            "          2.0041e-01, -7.9178e-03,  1.3818e-01,  1.8767e-01, -1.2235e-01,\n",
            "         -8.8928e-02,  4.8166e-02, -9.4844e-02,  1.2722e-01, -6.6497e-02,\n",
            "         -2.0509e-01, -2.3247e-01,  1.6571e-01, -1.1115e-01,  5.1732e-02,\n",
            "          2.2200e-02,  2.0382e-01, -1.5163e-01,  2.8294e-01, -1.3474e-01,\n",
            "          3.2639e-02, -3.4362e-02, -1.9795e-01, -1.1394e-01, -1.6481e-01,\n",
            "         -2.9483e-02, -2.6767e-02,  4.4265e-02,  3.6267e-01,  1.3865e-01,\n",
            "         -5.7352e-02,  1.2206e-01, -8.2945e-02,  3.4013e-01, -1.9555e-01,\n",
            "          1.7477e-01, -2.9413e-01, -4.1646e-02,  6.8508e-02,  1.7337e-01,\n",
            "          9.1838e-03,  1.9749e-01,  8.7209e-02,  6.8549e-02,  1.8619e-02,\n",
            "          4.0999e-02, -8.5639e-02,  9.7062e-02,  5.4662e-02,  8.1088e-03,\n",
            "         -7.9338e-02,  2.3481e-01, -5.2332e-02, -7.3106e-02,  8.6984e-02,\n",
            "         -6.3082e-02,  5.9330e-03, -2.2763e-01, -1.2831e-01,  1.8726e-01,\n",
            "          8.0006e-02,  1.5455e-01, -3.4822e-02,  3.2261e-01,  3.1409e-01,\n",
            "          1.3222e-01, -1.1085e-02,  5.8345e-02, -2.7428e-01,  3.7859e-03,\n",
            "         -2.7500e-01, -5.6834e-02, -8.0004e+00, -1.7681e-01,  1.3058e-02,\n",
            "         -1.6325e-01,  2.5703e-01,  1.7243e-02,  2.4462e-02, -2.4326e-01,\n",
            "          5.1957e-02, -2.0985e-01,  1.2111e-01,  1.5801e-01, -1.2476e-01,\n",
            "         -1.4627e-01,  4.1183e-01,  2.9897e-01]], grad_fn=<SliceBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Some weights of DistilBertForSequenceClassification were not initialized from the model checkpoint at distilbert-base-uncased and are newly initialized: ['pre_classifier.bias', 'classifier.bias', 'classifier.weight', 'pre_classifier.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Sentiment Label: LABEL_0\n",
            "Confidence Score: 0.5016996264457703\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#TEST\n",
        "# Example text for sentiment analysis\n",
        "#text = \"That movie was absolutely awful\"\n",
        "\n",
        "pred_sentences = [\n",
        "  \"That movie was absolutely awful\",\n",
        "  \"The acting was a bit lacking\",\n",
        "  \"The film was creative and surprising\",\n",
        "  \"Absolutely fantastic!\"\n",
        "]\n",
        "# Print the sentiment analysis result\n",
        "print(\"Sentiment Label:\", sentiment_label)\n",
        "print(\"Confidence Score:\", confidence_score)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "d-QCAahml2sq",
        "outputId": "a8baa498-d584-4f60-d55c-13b3ebae5e6d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Sentiment Label: LABEL_0\n",
            "Confidence Score: 0.5293472409248352\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#Running again in a different way\n",
        "\n",
        "from transformers import pipeline, DistilBertModel, DistilBertTokenizer\n",
        "\n",
        "# Load pre-trained DistilBERT model and tokenizer\n",
        "model = DistilBertModel.from_pretrained('distilbert-base-uncased')\n",
        "tokenizer = DistilBertTokenizer.from_pretrained('distilbert-base-uncased')\n",
        "\n",
        "# Example text for sentiment analysis\n",
        "text = [\"That movie was absolutely awful\"]\n",
        "\n",
        "# Tokenize and encode the text\n",
        "tokens = tokenizer(text, return_tensors='pt')\n",
        "\n",
        "# Get the model's output\n",
        "outputs = model(**tokens)\n",
        "\n",
        "# Access the output embeddings or other information from 'outputs'\n",
        "# For example, you can extract the embeddings for the [CLS] token\n",
        "cls_embedding = outputs.last_hidden_state[:, 0, :]\n",
        "\n",
        "# Print the embeddings (you can use these for your custom sentiment analysis task)\n",
        "print(\"Embeddings:\", cls_embedding)\n",
        "\n",
        "# Perform sentiment analysis using the pipeline\n",
        "sentiment_analysis = pipeline('sentiment-analysis', model='distilbert-base-uncased')\n",
        "result = sentiment_analysis(text)\n",
        "\n",
        "# Extract the sentiment label and confidence score\n",
        "sentiment_label = result[0]['label']\n",
        "\n",
        "#labels = ['Negative','Positive'] #(0:negative, 1:positive)\n",
        "confidence_score = result[0]['score']\n",
        "\n",
        "# Print the sentiment analysis result\n",
        "print(\"Sentiment Label:\", sentiment_label)\n",
        "print(\"Confidence Score:\", confidence_score)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hJ_A0wlgoZdD",
        "outputId": "0692d9f2-d44c-400d-9bf3-abc3478701eb"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Embeddings: tensor([[ 7.0210e-02,  6.8084e-02,  7.6470e-02,  1.9690e-02,  2.8007e-02,\n",
            "         -1.0710e-01,  1.2325e-01,  4.8434e-01, -1.9508e-02,  2.3242e-02,\n",
            "          5.9509e-02,  3.7812e-03, -7.5885e-02,  4.8900e-01,  9.0280e-02,\n",
            "          1.4583e-01, -1.8004e-01,  1.1228e-01,  9.8468e-02, -1.6458e-01,\n",
            "         -2.2962e-01, -2.3786e-01, -3.1089e-02,  1.1482e-02, -1.1495e-01,\n",
            "          2.2647e-02,  3.7191e-02, -9.5181e-02,  1.8810e-01,  7.0893e-02,\n",
            "          1.1936e-01, -5.1023e-02, -9.5914e-02, -4.0946e-03, -1.4172e-02,\n",
            "         -4.6111e-02, -5.8164e-02, -6.1541e-02, -9.2024e-02, -5.6487e-02,\n",
            "         -1.1766e-01,  4.8398e-02,  1.3649e-01, -1.2492e-01, -1.7980e-02,\n",
            "         -1.1209e-02, -1.8811e+00, -1.3138e-02,  4.5990e-02, -2.3842e-01,\n",
            "          2.7772e-01, -3.2499e-02,  1.7736e-01,  2.7165e-01,  3.0734e-01,\n",
            "          2.1610e-01, -1.3272e-01,  3.9919e-01, -6.1820e-02,  1.8868e-02,\n",
            "          1.9426e-01, -2.3554e-02, -5.8178e-02, -5.3716e-02, -1.0974e-02,\n",
            "          5.5273e-02,  2.6712e-02,  1.0244e-01, -1.6141e-01,  1.5072e-01,\n",
            "         -1.7391e-01, -2.6420e-01,  2.2884e-01, -9.5202e-02,  3.2873e-03,\n",
            "         -1.1645e-01,  4.6312e-02,  7.6179e-03,  9.2150e-04,  1.0870e-01,\n",
            "          7.4153e-02,  3.1126e-01,  1.5149e-01,  7.8298e-02, -8.4716e-02,\n",
            "          1.4393e-01, -1.2629e-01, -1.6662e-01,  1.2463e-01,  3.6217e-01,\n",
            "         -3.3603e-01,  5.0909e-02,  1.2921e-01,  2.6329e-01,  2.8940e-01,\n",
            "         -2.6184e-01,  5.4341e-02, -6.1922e-02,  2.2280e-01,  2.4691e-01,\n",
            "          7.3170e-03, -2.2587e-01, -6.4404e-02, -2.1725e-01, -3.4282e-02,\n",
            "         -7.4080e-02,  1.4955e-01, -1.0685e-01,  1.7219e-01, -2.5439e+00,\n",
            "          1.9954e-01,  1.8210e-01, -1.2624e-01, -4.9150e-01, -1.6287e-01,\n",
            "          4.0989e-01,  3.4003e-03, -9.3138e-02,  1.0669e-01,  8.7394e-02,\n",
            "         -5.3071e-02,  2.6903e-01, -8.2266e-02, -2.1511e-01,  8.1380e-02,\n",
            "          2.5792e-01,  4.8620e-04, -1.1205e-01, -8.7534e-02,  1.2530e-01,\n",
            "          3.2873e-01,  4.5060e-01,  4.8835e-02,  2.1659e-02, -2.2686e-01,\n",
            "         -1.8355e-02,  2.1946e-01, -8.6457e-02, -1.3597e-01, -1.1141e-01,\n",
            "         -1.0518e-01,  4.6651e-03, -3.3029e+00,  4.4642e-01,  1.6015e-01,\n",
            "         -6.6687e-02, -5.3272e-02, -3.9121e-02,  6.6998e-02,  4.5202e-02,\n",
            "          1.7310e-01,  2.0435e-01, -1.7824e-01, -9.5743e-02, -1.3881e-01,\n",
            "          6.6837e-02,  1.6904e-02, -9.5584e-02,  2.9577e-01,  5.4376e-02,\n",
            "          1.7277e-01, -2.0242e-02, -1.4180e-01, -1.8642e-01, -4.0464e-02,\n",
            "         -1.8090e-02,  8.0317e-02,  1.9451e-01,  4.7899e-02, -6.2248e-02,\n",
            "         -7.3494e-03, -4.2843e-02,  1.7671e-01, -3.7314e-02,  8.8881e-02,\n",
            "         -1.2693e-01,  2.2842e-01,  3.2210e-01,  4.8474e-02, -1.4896e-01,\n",
            "          1.2412e-01,  5.0364e-01,  2.0439e-01,  2.1017e-01,  6.3660e-02,\n",
            "          6.5129e-02,  3.3880e-01, -1.1937e-01, -5.4780e-02,  3.6465e-01,\n",
            "          2.2739e-02, -1.4237e-01,  7.4659e-02, -1.0673e-01,  2.0002e-01,\n",
            "         -1.1965e-01, -2.8842e-02, -4.7533e-01,  3.7402e-01,  1.0944e-01,\n",
            "         -2.6974e-01,  4.4372e-02,  5.2608e-02, -4.1959e-02, -8.2398e-02,\n",
            "          3.5727e+00, -2.9040e-02,  3.2614e-02,  1.5512e-01,  1.2221e-01,\n",
            "          1.0003e-01,  9.9599e-02, -4.9769e-03, -1.1049e-01,  6.5973e-02,\n",
            "          2.6047e-02,  3.5683e-02,  3.6941e-02,  2.0850e-01, -1.9138e-01,\n",
            "          8.0428e-02,  1.6420e-01, -1.5171e-01, -1.2848e-01, -7.5413e-02,\n",
            "          2.3845e-01,  1.1243e-01,  1.6437e-01, -1.5891e-02, -1.1823e+00,\n",
            "         -3.9776e-02,  1.5999e-01, -3.0544e-01,  2.4809e-01, -4.5131e-03,\n",
            "         -1.2085e-01,  1.8269e-03, -1.1104e-03,  9.1960e-02,  1.9021e-01,\n",
            "         -1.0746e-02,  1.3543e-01,  1.8933e-01,  1.9844e-01, -2.4609e-01,\n",
            "          2.2704e-01,  1.8756e-01,  1.9820e-02, -8.3310e-02,  3.9763e-02,\n",
            "          1.3475e-01, -2.4862e-01, -1.3573e-01, -1.0175e-01,  7.5262e-02,\n",
            "          3.3437e-02, -7.8380e-02,  7.5527e-02, -1.5418e-01, -4.9511e-02,\n",
            "         -1.3396e-01, -1.2878e-02,  8.3121e-02,  4.5249e-02, -1.6847e-01,\n",
            "          2.7127e-02, -2.7690e-02, -1.6939e-01, -2.5761e-02,  2.3491e-01,\n",
            "         -1.4882e-01,  7.7250e-02, -1.7147e-01, -4.1981e+00,  1.8235e-02,\n",
            "          5.0425e-02,  1.5842e-01,  2.5545e-01,  6.9077e-03,  2.6726e-01,\n",
            "          9.2382e-02,  2.0444e-01, -3.6579e-01,  7.3434e-02,  1.7368e-01,\n",
            "         -2.5656e-02,  1.0155e-01, -2.9859e-01,  1.7681e-01, -6.7990e-03,\n",
            "         -1.7406e-01, -1.9978e-02,  7.1345e-03, -2.0362e-02,  2.1270e-01,\n",
            "         -2.1537e-01,  3.0113e-02, -3.6798e-02, -1.4119e-01,  1.4477e-02,\n",
            "         -1.5803e-01,  3.9264e-01, -8.7364e-02, -8.4698e-02, -1.0178e-01,\n",
            "          2.0696e-01, -1.4076e-01, -1.6393e-01, -2.4259e+00,  9.0164e-02,\n",
            "         -1.0545e-01, -1.4428e-01,  5.0900e-02, -1.0954e-01,  2.0034e-01,\n",
            "         -1.4811e-01, -3.6446e-01,  5.7970e-02,  1.4776e-01, -4.8381e-03,\n",
            "         -1.1046e-01,  1.9530e-01,  3.5268e-01, -1.7863e-01, -9.3188e-02,\n",
            "         -1.0548e-01,  1.6100e-01,  2.2590e-01, -3.6949e-02, -8.0652e-02,\n",
            "         -1.2417e-01, -9.9830e-02,  8.9419e-02,  3.0115e-01, -4.0292e-01,\n",
            "          6.3017e-02, -2.3965e-01, -3.6706e-02, -1.9132e-01, -1.1254e-01,\n",
            "         -1.0118e-02,  8.3296e-02, -2.3813e-02, -1.9389e-01,  1.4846e-01,\n",
            "          1.3152e-01,  9.0962e-02, -7.2862e-02, -1.5728e-01,  2.8305e-01,\n",
            "          1.3410e-03,  1.8496e-01,  3.3793e-01,  6.7186e-02,  1.8811e-01,\n",
            "         -2.2263e-01, -1.0653e-01, -6.2588e-02,  1.3543e-02,  1.3539e-01,\n",
            "          1.1174e+00,  1.3951e-01,  9.8153e-02, -1.6124e-01,  2.1697e-01,\n",
            "          1.0880e-01,  8.4445e-02,  6.0807e-02,  2.6460e-01,  1.0800e-02,\n",
            "          1.0969e-01,  6.7569e-02, -4.6410e-02, -9.1961e-02,  2.5364e-01,\n",
            "         -3.2124e-01,  8.7444e-02, -3.4521e-02,  1.6120e-02, -1.1213e-01,\n",
            "         -3.7869e-02, -6.0515e-01, -1.1362e-01,  1.7341e-01, -2.2033e-02,\n",
            "          5.9073e-02,  9.1664e-04,  4.4855e-02, -1.9531e-01, -1.8482e-02,\n",
            "         -1.5434e-02,  1.3454e-01, -7.4706e-02, -3.9060e-02, -2.9883e-01,\n",
            "          1.9909e-02, -1.0703e-03, -1.7694e-02,  1.2347e-01,  1.8095e-01,\n",
            "          2.7663e-02,  2.8166e-01,  9.8733e-02, -4.3489e-02,  1.8613e-01,\n",
            "         -6.2743e-01,  1.3214e-01, -1.2334e-01, -1.1310e-01, -3.0477e-01,\n",
            "         -1.3528e-01,  2.1427e-02, -1.4990e-01, -2.3053e-01, -7.6550e-02,\n",
            "          3.6503e-01, -8.4248e-02,  1.3625e-01,  1.4415e-01,  2.7531e-01,\n",
            "          9.2659e-02,  6.5783e-02,  7.8776e-01, -7.6453e-03,  6.3825e-02,\n",
            "          2.5603e-01, -1.3428e-01,  1.9215e-01,  1.1506e-01, -2.1528e-02,\n",
            "         -3.1555e-01,  1.3217e-02, -4.7009e-02,  2.8015e-02,  5.1112e-02,\n",
            "         -3.4055e-01, -1.6770e-01, -2.6179e-03,  1.7905e-02,  2.7129e-02,\n",
            "         -1.2722e-01, -5.0489e-01, -1.4500e-02, -1.6000e-01, -1.1678e-01,\n",
            "          2.7223e-02,  1.1158e-01,  1.3185e-01,  2.7755e-01,  1.4216e-01,\n",
            "         -1.7605e-02,  3.1379e-01, -8.2785e-02,  3.3493e-01,  7.1293e-02,\n",
            "          1.0065e-01, -7.3071e-02,  9.4425e-02, -1.8008e-01, -8.9302e-02,\n",
            "          5.1244e-02,  1.1781e-02,  6.9097e-04,  1.5878e-01,  9.6841e-02,\n",
            "          3.2169e-02,  8.6110e-03,  1.0361e-01,  1.1567e-01,  1.6937e-01,\n",
            "         -1.1914e+00,  1.4822e-01,  2.8572e-01,  7.5464e-02,  8.1584e-02,\n",
            "          7.4819e-03, -1.5111e-01,  3.5056e-01,  1.4812e-01,  1.9560e-01,\n",
            "         -1.9797e-02,  2.1943e-01, -1.6503e-01,  1.2605e-02, -1.1448e-01,\n",
            "          3.8977e-02,  1.0257e-01,  1.2431e-01,  1.3797e-01,  1.7945e-01,\n",
            "          5.6076e-02,  7.6177e-02,  9.1883e-03, -2.6519e-03, -3.5716e-01,\n",
            "          5.9442e-02, -1.4653e-01,  2.3702e-01, -8.9915e-03, -5.3730e-02,\n",
            "          1.2348e-01, -2.9663e-01, -4.6666e-01, -1.4099e-01,  6.5454e-02,\n",
            "         -1.0160e-02,  4.1447e-02,  1.2959e-01,  1.4755e-01,  6.2910e-02,\n",
            "         -4.0963e-01,  2.3763e-01,  5.3742e-02, -1.5305e-01,  3.8188e-01,\n",
            "          1.1367e-01, -9.1228e-02,  1.9869e-01, -1.8616e-02, -1.5775e-01,\n",
            "         -1.3848e-01, -2.7079e-02, -8.4361e-02, -1.9715e-02,  1.5783e-01,\n",
            "         -6.9875e-02, -4.1190e-02, -7.3246e-02, -2.1516e-01,  2.6322e-02,\n",
            "          1.1042e-01, -1.3714e-01, -2.2542e-01,  2.6560e-01, -2.2820e-01,\n",
            "         -5.7218e-01, -2.0098e-01, -1.2566e-01, -3.4150e-02,  1.4867e-01,\n",
            "          1.7417e-01,  1.9225e-02, -1.5201e-01, -1.0276e-01, -2.4116e-02,\n",
            "          2.4561e-01,  2.5155e-02,  1.8822e-01,  1.4568e-01, -3.1045e-01,\n",
            "         -9.7277e-03,  1.3263e-01, -2.6272e-03,  1.6513e-01,  5.7662e-02,\n",
            "          7.5814e-02, -7.9262e-02, -7.1345e-03, -1.9329e-01, -6.8105e-02,\n",
            "         -2.2094e-01, -2.6495e-01,  1.1473e-01, -8.5436e-02,  6.3126e-02,\n",
            "          1.9983e-01,  3.1201e-01, -3.3396e-02, -2.2894e-01,  3.5266e-02,\n",
            "         -1.9004e-01,  2.5305e-01,  1.6835e-01,  7.3160e-02,  1.8570e-01,\n",
            "          1.1164e-01,  2.1383e-01,  7.9331e-02, -1.3096e-01, -5.2251e-02,\n",
            "         -6.1062e-02,  1.3912e-01,  9.1585e-02, -7.9250e-02, -1.6307e-01,\n",
            "         -2.2670e-01,  1.3385e-01, -1.7534e-01,  2.1269e+00,  3.2471e-01,\n",
            "         -2.6211e-03, -2.6499e-01,  8.0447e-02, -5.5818e-02,  1.0285e-01,\n",
            "          2.2205e-01, -8.8225e-02,  2.8026e-01, -3.7674e-02,  1.6147e-01,\n",
            "          6.7287e-02,  9.5320e-03,  3.2739e-01, -4.6458e-02,  2.9608e-02,\n",
            "         -1.2217e-01, -3.8833e-01, -1.8401e-01, -3.1209e-01,  9.4043e-02,\n",
            "          2.2778e-01, -7.1277e-02, -7.4049e-02,  1.2115e-01,  2.2647e-02,\n",
            "         -1.0845e-01,  2.1890e-02,  9.0043e-02, -4.1176e-02, -7.5949e-02,\n",
            "         -8.4193e-02,  1.6922e-01, -1.5809e-01,  3.0788e-02,  2.0228e-01,\n",
            "         -9.2700e-02, -1.2381e-01, -9.2184e-02,  1.3180e-01, -2.2731e-01,\n",
            "          1.2643e-01, -7.1765e-02,  1.8819e-01,  1.6790e-01, -1.4177e-02,\n",
            "         -9.1120e-02,  3.1611e-01,  1.1484e-01, -6.1316e-02, -6.9189e-02,\n",
            "         -9.0090e-02,  9.2345e-02, -3.5097e-02, -2.4078e-01,  4.5961e-02,\n",
            "         -2.4792e-01,  1.6229e-01,  1.2906e-01, -5.6944e-02,  1.2793e-01,\n",
            "          2.6016e-01,  1.1673e-01,  1.0791e-01,  1.1238e-01, -1.2967e-01,\n",
            "         -3.3522e-02, -5.9857e-02, -1.3557e-01, -1.2545e-01,  3.9175e-02,\n",
            "          3.1633e-01, -7.8773e-02,  2.6089e-01,  1.1353e-01,  4.0131e-01,\n",
            "          9.2380e-02, -6.0576e-02, -2.9233e+00, -5.1876e-02, -3.5782e-02,\n",
            "          5.5098e-02,  4.4209e-02,  4.9394e-01,  3.7819e-01, -2.7030e-01,\n",
            "          1.1889e-01, -1.7410e-01,  1.1357e-01,  2.3427e-01,  5.7641e-02,\n",
            "          2.0041e-01, -7.9178e-03,  1.3818e-01,  1.8767e-01, -1.2235e-01,\n",
            "         -8.8928e-02,  4.8166e-02, -9.4844e-02,  1.2722e-01, -6.6497e-02,\n",
            "         -2.0509e-01, -2.3247e-01,  1.6571e-01, -1.1115e-01,  5.1732e-02,\n",
            "          2.2200e-02,  2.0382e-01, -1.5163e-01,  2.8294e-01, -1.3474e-01,\n",
            "          3.2639e-02, -3.4362e-02, -1.9795e-01, -1.1394e-01, -1.6481e-01,\n",
            "         -2.9483e-02, -2.6767e-02,  4.4265e-02,  3.6267e-01,  1.3865e-01,\n",
            "         -5.7352e-02,  1.2206e-01, -8.2945e-02,  3.4013e-01, -1.9555e-01,\n",
            "          1.7477e-01, -2.9413e-01, -4.1646e-02,  6.8508e-02,  1.7337e-01,\n",
            "          9.1838e-03,  1.9749e-01,  8.7209e-02,  6.8549e-02,  1.8619e-02,\n",
            "          4.0999e-02, -8.5639e-02,  9.7062e-02,  5.4662e-02,  8.1088e-03,\n",
            "         -7.9338e-02,  2.3481e-01, -5.2332e-02, -7.3106e-02,  8.6984e-02,\n",
            "         -6.3082e-02,  5.9330e-03, -2.2763e-01, -1.2831e-01,  1.8726e-01,\n",
            "          8.0006e-02,  1.5455e-01, -3.4822e-02,  3.2261e-01,  3.1409e-01,\n",
            "          1.3222e-01, -1.1085e-02,  5.8345e-02, -2.7428e-01,  3.7859e-03,\n",
            "         -2.7500e-01, -5.6834e-02, -8.0004e+00, -1.7681e-01,  1.3058e-02,\n",
            "         -1.6325e-01,  2.5703e-01,  1.7243e-02,  2.4462e-02, -2.4326e-01,\n",
            "          5.1957e-02, -2.0985e-01,  1.2111e-01,  1.5801e-01, -1.2476e-01,\n",
            "         -1.4627e-01,  4.1183e-01,  2.9897e-01]], grad_fn=<SliceBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Some weights of DistilBertForSequenceClassification were not initialized from the model checkpoint at distilbert-base-uncased and are newly initialized: ['pre_classifier.bias', 'classifier.bias', 'classifier.weight', 'pre_classifier.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Sentiment Label: LABEL_0\n",
            "Confidence Score: 0.5293472409248352\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import pipeline, DistilBertModel, DistilBertTokenizer\n",
        "\n",
        "# Load pre-trained DistilBERT model and tokenizer\n",
        "model = DistilBertModel.from_pretrained('distilbert-base-uncased')\n",
        "tokenizer = DistilBertTokenizer.from_pretrained('distilbert-base-uncased')\n",
        "\n",
        "# Example texts for sentiment analysis\n",
        "texts = [\"That movie was absolutely awful\", \"I loved the performance in the play\", \"The weather today is fantastic\"]\n",
        "\n",
        "# Tokenize and encode the texts\n",
        "tokens = tokenizer(texts, return_tensors='pt', padding=True, truncation=True)\n",
        "\n",
        "# Get the model's output\n",
        "outputs = model(**tokens)\n",
        "\n",
        "# Access the output embeddings or other information from 'outputs'\n",
        "# For example, you can extract the embeddings for the [CLS] token\n",
        "cls_embeddings = outputs.last_hidden_state[:, 0, :]\n",
        "\n",
        "# Perform sentiment analysis using the pipeline\n",
        "sentiment_analysis = pipeline('sentiment-analysis', model='distilbert-base-uncased')\n",
        "results = sentiment_analysis(texts)\n",
        "\n",
        "# Extract sentiment labels and confidence scores for each text\n",
        "for i, result in enumerate(results):\n",
        "    sentiment_label = result['label']\n",
        "    confidence_score = result['score']\n",
        "\n",
        "    print(f\"\\nSentiment Analysis for Text {i + 1}:\")\n",
        "    print(\"Text:\", texts[i])\n",
        "    print(\"Sentiment Label:\", sentiment_label)\n",
        "    print(\"Confidence Score:\", confidence_score)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "C0hu-vTztBV8",
        "outputId": "66ab6e9c-db18-48e5-fb17-d8dd6a261bb7"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Some weights of DistilBertForSequenceClassification were not initialized from the model checkpoint at distilbert-base-uncased and are newly initialized: ['pre_classifier.bias', 'classifier.bias', 'classifier.weight', 'pre_classifier.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Sentiment Analysis for Text 1:\n",
            "Text: That movie was absolutely awful\n",
            "Sentiment Label: LABEL_1\n",
            "Confidence Score: 0.527485191822052\n",
            "\n",
            "Sentiment Analysis for Text 2:\n",
            "Text: I loved the performance in the play\n",
            "Sentiment Label: LABEL_1\n",
            "Confidence Score: 0.5284966230392456\n",
            "\n",
            "Sentiment Analysis for Text 3:\n",
            "Text: The weather today is fantastic\n",
            "Sentiment Label: LABEL_1\n",
            "Confidence Score: 0.5225977301597595\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "fzNagVpCtBbN"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [],
      "metadata": {
        "id": "fInShxpV3vlI"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install pandas\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cxY03Gn-3xMS",
        "outputId": "4e57a6ed-7f59-4c7e-88ef-92aa25944962"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (1.5.3)\n",
            "Requirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.10/dist-packages (from pandas) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas) (2023.3.post1)\n",
            "Requirement already satisfied: numpy>=1.21.0 in /usr/local/lib/python3.10/dist-packages (from pandas) (1.23.5)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.8.1->pandas) (1.16.0)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "K_5CtelH3zKN"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "VzDmhUYi7-3F"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "imdb = keras.datasets.imdb"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 177
        },
        "id": "va34GTpr7_CV",
        "outputId": "74b8eead-08a3-4bc8-e32c-b87c061aa9bd"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "error",
          "ename": "NameError",
          "evalue": "name 'keras' is not defined",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-67-cf8728ba76b7>\u001b[0m in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mimdb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mkeras\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdatasets\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mimdb\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m: name 'keras' is not defined"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "from transformers import pipeline, DistilBertModel, DistilBertTokenizer\n",
        "\n",
        "# Load pre-trained DistilBERT model and tokenizer\n",
        "model = DistilBertModel.from_pretrained('distilbert-base-uncased')\n",
        "tokenizer = DistilBertTokenizer.from_pretrained('distilbert-base-uncased')\n",
        "\n",
        "# Load the dataset from a CSV file\n",
        "dataset_path = 'sentiment_dataset.csv'\n",
        "df = pd.read_csv(dataset_path)\n",
        "#pd.read_csv('https://github.com/clairett/pytorch-sentiment-classification/raw/master/data/SST2/train.tsv'\n",
        "\n",
        "# Extract the 'text' column from the dataset\n",
        "texts = df['text'].tolist()\n",
        "\n",
        "# Tokenize and encode the texts\n",
        "tokens = tokenizer(texts, return_tensors='pt', padding=True, truncation=True)\n",
        "\n",
        "# Get the model's output\n",
        "outputs = model(**tokens)\n",
        "\n",
        "# Access the output embeddings or other information from 'outputs'\n",
        "# For example, you can extract the embeddings for the [CLS] token\n",
        "cls_embeddings = outputs.last_hidden_state[:, 0, :]\n",
        "\n",
        "# Perform sentiment analysis using the pipeline\n",
        "sentiment_analysis = pipeline('sentiment-analysis', model='distilbert-base-uncased')\n",
        "results = sentiment_analysis(texts)\n",
        "\n",
        "# Extract sentiment labels and confidence scores for each text\n",
        "for i, result in enumerate(results):\n",
        "    sentiment_label = result['label']\n",
        "    confidence_score = result['score']\n",
        "\n",
        "    print(f\"\\nSentiment Analysis for Text {i + 1}:\")\n",
        "    print(\"Text:\", texts[i])\n",
        "    print(\"Sentiment Label:\", sentiment_label)\n",
        "    print(\"Confidence Score:\", confidence_score)\n"
      ],
      "metadata": {
        "id": "YEgwmmsU33RS"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [],
      "metadata": {
        "id": "IOKCj_453voK"
      }
    },
    {
      "cell_type": "markdown",
      "source": [],
      "metadata": {
        "id": "a3RZzfRY3vs3"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "#NEW EXERCISE"
      ],
      "metadata": {
        "id": "L65ZTiOBtCor"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "import numpy as np"
      ],
      "metadata": {
        "id": "Tht4U-TLtWCC"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df = pd.read_csv('https://github.com/clairett/pytorch-sentiment-classification/raw/master/data/SST2/train.tsv', delimiter='\\t', header=None)"
      ],
      "metadata": {
        "id": "KLsF5IO2tFb9"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "OdfnEG5w4_1D"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "batch_1 = df[:1000]"
      ],
      "metadata": {
        "id": "9XHAisRrtZO3"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "batch_1[1].value_counts()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "sXrvoN_kti0g",
        "outputId": "75eb2bbd-29f4-4c1b-eddd-4474e097796f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1    521\n",
              "0    479\n",
              "Name: 1, dtype: int64"
            ]
          },
          "metadata": {},
          "execution_count": 36
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# For DistilBERT:\n",
        "from transformers import pipeline, DistilBertModel, DistilBertTokenizer\n",
        "\n"
      ],
      "metadata": {
        "id": "e7yOzYHTtkQM"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "tokenized = batch_1[0].apply((lambda x: tokenizer.encode(x, add_special_tokens=True)))"
      ],
      "metadata": {
        "id": "8ocWbT5nt-di"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(tokenized)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Ew6KPSVvuAN5",
        "outputId": "51d7d8b3-318e-44ce-c290-1ffebe9671d3"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "0      [101, 1037, 18385, 1010, 6057, 1998, 2633, 182...\n",
            "1      [101, 4593, 2128, 27241, 23931, 2013, 1996, 62...\n",
            "2      [101, 2027, 3653, 23545, 2037, 4378, 24185, 10...\n",
            "3      [101, 2023, 2003, 1037, 17453, 14726, 19379, 1...\n",
            "4      [101, 5655, 6262, 1005, 1055, 12075, 2571, 376...\n",
            "                             ...                        \n",
            "995    [101, 2612, 1997, 5599, 1996, 11680, 2272, 200...\n",
            "996    [101, 2023, 10722, 19068, 2080, 2323, 2031, 20...\n",
            "997    [101, 2412, 2156, 2028, 1997, 2216, 22092, 200...\n",
            "998    [101, 1996, 2143, 2003, 8052, 2005, 1996, 1592...\n",
            "999    [101, 2005, 2087, 1997, 2049, 8333, 1010, 1996...\n",
            "Name: 0, Length: 1000, dtype: object\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "max_len = 0\n",
        "for i in tokenized.values:\n",
        "    if len(i) > max_len:\n",
        "        max_len = len(i)\n",
        "\n",
        "padded = np.array([i + [0]*(max_len-len(i)) for i in tokenized.values])"
      ],
      "metadata": {
        "id": "i3dYsD73uMpM"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(padded)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5MjU1AU-uUz2",
        "outputId": "05c1d659-b11b-45d6-e2df-51a3b0bacc7c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[[  101  1037 18385 ...     0     0     0]\n",
            " [  101  4593  2128 ...     0     0     0]\n",
            " [  101  2027  3653 ...     0     0     0]\n",
            " ...\n",
            " [  101  2412  2156 ...     0     0     0]\n",
            " [  101  1996  2143 ...     0     0     0]\n",
            " [  101  2005  2087 ...     0     0     0]]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "np.array(padded).shape"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NmFW_S_KuXdF",
        "outputId": "7ae25e28-76c2-4beb-a9e4-a5e59f6816d5"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(1000, 59)"
            ]
          },
          "metadata": {},
          "execution_count": 46
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [],
      "metadata": {
        "id": "SFEOciotvrxM"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "#Question Answering"
      ],
      "metadata": {
        "id": "ZfC-sbjkvr3U"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import pipeline, DistilBertTokenizer, DistilBertForQuestionAnswering\n",
        "\n",
        "# Load pre-trained DistilBERT model and tokenizer\n",
        "model = DistilBertForQuestionAnswering.from_pretrained('distilbert-base-uncased-distilled-squad')\n",
        "tokenizer = DistilBertTokenizer.from_pretrained('distilbert-base-uncased-distilled-squad')\n",
        "\n",
        "# Example context and question\n",
        "context = \"DistilBERT is a lightweight version of BERT designed for speed and efficiency.\"\n",
        "question = \"What is DistilBERT designed for?\"\n",
        "\n",
        "# Tokenize and encode the context and question\n",
        "inputs = tokenizer(context, question, return_tensors='pt')\n",
        "\n",
        "# Get the model's output\n",
        "outputs = model(**inputs)\n",
        "\n",
        "# Extract start and end logits from the output\n",
        "start_logits = outputs.start_logits\n",
        "end_logits = outputs.end_logits\n",
        "\n",
        "# Use the tokenizer to convert logits to tokens\n",
        "start_index = start_logits.argmax(dim=-1).item()\n",
        "end_index = end_logits.argmax(dim=-1).item()\n",
        "tokens = tokenizer.convert_ids_to_tokens(inputs['input_ids'][0][start_index:end_index + 1])\n",
        "\n",
        "# Convert tokens back to text\n",
        "answer = tokenizer.convert_tokens_to_string(tokens)\n",
        "\n",
        "# Print the question-answering result\n",
        "print(\"Question:\", question)\n",
        "print(\"Answer:\", answer)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 212,
          "referenced_widgets": [
            "2adf07c3ed7e4db69675a94a53e9671a",
            "4983a3b9d57c46a1bea5e029e6bb397c",
            "02d1295785724218bad13fcd5657d80b",
            "7e9439093fd846dcb1a91181dea6547b",
            "a0604634bcac4fa389f8b55d8a480de1",
            "2efb175f8c644ef4a75345e8d1c36eda",
            "d64f030d434a48229c884c407b816462",
            "d5a5ff74fab241ec8527ea1043a435b3",
            "0de8950e790c4902aed5e4f2218e6bbf",
            "f5cb2645ed8040469cce3d5bde65475d",
            "4054f8ce21474ed5a2b425051c6109e4",
            "3df9e923b95740119b47f23d67a6510a",
            "218ec2bd10704946b81ee4cf676f8a4a",
            "75f4e300d50a476987e258ad7df60977",
            "791358bf4a9a46afbe7c6051ccc8528d",
            "211cde38356847258a32a0a3915a81bd",
            "88991280e20d4655849f644c3dc44bca",
            "2f20a9347c064533a11e51c057754121",
            "b47d638ccdb542a3987fbaff80dcdea9",
            "7932b1ac40ae4b60b2be9810f55f2844",
            "4c80492616d647c39ef966e998fd85e0",
            "a1296ed31718414bb16fcbfa432fbbee",
            "3d2523c23d104077aa8a7e70b70fd23e",
            "ca266a80ee06429f88029efed00cab38",
            "5b7be193d93f4b7cbc5deeed4a3fa23a",
            "5938660cc9a84373a3d0bda118ff5524",
            "f8b55da9f46841779ebbfe1fd15c0742",
            "c625a6c041f4466db3533b79f928908c",
            "f1c0b88c0871491db9cd45bbbb683111",
            "fbcd14631caf497ba4a311ccb434c338",
            "af9d41d330b2445893e3ce25c440fa82",
            "7538aa65c3c44924906f0b61e1b487f3",
            "e1fcc8d9e4064e8fac3803f5f22e4f55",
            "f20c7985bc834c00a5ed778a170efbda",
            "303f782f9cf04dcc80b851cb468297b7",
            "68ce75124cbf400abe6581656fa7c3fb",
            "23181efd60d248eb8e3c2680e1c5cc43",
            "926a1f7be5d74771afc361aad1da490a",
            "deab158c3a92492ab2d575809f5c5abd",
            "78cffeeb4aad49af83d9b3f83530324e",
            "a2b023d625c4442787d2636d8c6bc097",
            "e9edd12aa38b4595a5169294dbb7617f",
            "e4ac98859771482da96a330a02741718",
            "ee5b3988a31e4452bb313b06695ee4e4",
            "297232316a22434f961fa0be3e34f23c",
            "db728172fc994f7db4b5a109feace62a",
            "9157948e0de94670be3224774a58f737",
            "f9bb8fe36ff347faa6544d370337c9ea",
            "a38143786f6a4c0582f8016cea29c63e",
            "329c4f3cadec4ee7a2f8c37a3fe6e516",
            "75b37031eeb946d99d1aa1ba5e516486",
            "47801102d63b4e3f888e8fa63fdbd59a",
            "c9617a5645ab4b3cb62fd59c36cd9fec",
            "073c69aa138d4860980262bbbb9b8d26",
            "32a6df8150834c41b77b28163d84affd"
          ]
        },
        "id": "7E2Q3E2zvt29",
        "outputId": "86fa6b2b-cce8-4443-fed4-4a2ba69afad8"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "config.json:   0%|          | 0.00/451 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "2adf07c3ed7e4db69675a94a53e9671a"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "model.safetensors:   0%|          | 0.00/265M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "3df9e923b95740119b47f23d67a6510a"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "tokenizer_config.json:   0%|          | 0.00/28.0 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "3d2523c23d104077aa8a7e70b70fd23e"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "vocab.txt:   0%|          | 0.00/232k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "f20c7985bc834c00a5ed778a170efbda"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "tokenizer.json:   0%|          | 0.00/466k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "297232316a22434f961fa0be3e34f23c"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Question: What is DistilBERT designed for?\n",
            "Answer: speed and efficiency\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "In the above:\n",
        "In this example:\n",
        "\n",
        "We load a pre-trained DistilBERT model and tokenizer specifically fine-tuned for question-answering tasks (distilbert-base-uncased-distilled-squad).\n",
        "\n",
        "We provide a context (a piece of text) and a question related to that context.\n",
        "\n",
        "We tokenize and encode the context and question using the DistilBERT tokenizer.\n",
        "\n",
        "We pass the encoded inputs through the DistilBERT model to obtain start and end logits. These logits represent the probability distribution over tokens for the start and end positions of the answer.\n",
        "\n",
        "We use the tokenizer to convert logits to tokens and then convert the tokens back to text to form the final answer.\n",
        "\n",
        "The question and the obtained answer are printed.\n",
        "\n",
        "You can customize this example by providing your own context and question to perform question-answering on topics of interest to you."
      ],
      "metadata": {
        "id": "19oVrKMKwBwa"
      }
    },
    {
      "cell_type": "markdown",
      "source": [],
      "metadata": {
        "id": "ogeLeuHbwqcw"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "#Named Entity Extraction"
      ],
      "metadata": {
        "id": "_U1DUUsLwqnR"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import pipeline, DistilBertTokenizer, DistilBertForTokenClassification\n",
        "\n",
        "# Load pre-trained DistilBERT model and tokenizer for token classification\n",
        "model = DistilBertForTokenClassification.from_pretrained('distilbert-base-uncased')\n",
        "tokenizer = DistilBertTokenizer.from_pretrained('distilbert-base-uncased')\n",
        "\n",
        "# Example text for named entity recognition\n",
        "text = \"Apple Inc. was founded by Steve Jobs, Steve Wozniak, and Ronald Wayne.\"\n",
        "\n",
        "# Tokenize and encode the text\n",
        "tokens = tokenizer(text, return_tensors='pt')\n",
        "\n",
        "# Get the model's output\n",
        "outputs = model(**tokens)\n",
        "\n",
        "# Extract predicted labels (named entities) from the output\n",
        "predicted_labels = outputs.logits.argmax(dim=2).squeeze().numpy()\n",
        "\n",
        "# Map labels back to their corresponding tokens using the tokenizer\n",
        "predicted_tokens = tokenizer.convert_ids_to_tokens(tokens['input_ids'][0].numpy())\n",
        "predicted_entities = [tokenizer.convert_ids_to_tokens([token_id])[0] for token_id in predicted_labels]\n",
        "\n",
        "# Extract named entities from the predicted labels and tokens\n",
        "#named_entities = [(token, entity) for token, entity_id in zip(predicted_tokens, predicted_entities) if entity_id.startswith('B-')]\n",
        "named_entities = [(token, entity) for token, entity_id in zip(text[0].split(), predicted_entities[0]) if entity_id != '[PAD]']\n",
        "\n",
        "\n",
        "# Print the named entities\n",
        "print(\"Named Entities:\")\n",
        "for token, entity in named_entities:\n",
        "    print(f\"{token}: {entity}\")\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 492
        },
        "id": "BeEbd7Ihws-P",
        "outputId": "721593fe-35d2-480d-ba5b-69da795b61e6"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Some weights of DistilBertForTokenClassification were not initialized from the model checkpoint at distilbert-base-uncased and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ]
        },
        {
          "output_type": "error",
          "ename": "NameError",
          "evalue": "name 'entity' is not defined",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-61-9c6f8cf0ffd2>\u001b[0m in \u001b[0;36m<cell line: 25>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     23\u001b[0m \u001b[0;31m# Extract named entities from the predicted labels and tokens\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     24\u001b[0m \u001b[0;31m#named_entities = [(token, entity) for token, entity_id in zip(predicted_tokens, predicted_entities) if entity_id.startswith('B-')]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 25\u001b[0;31m \u001b[0mnamed_entities\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtoken\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mentity\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mtoken\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mentity_id\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mzip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtext\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msplit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpredicted_entities\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mentity_id\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0;34m'[PAD]'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     26\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     27\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-61-9c6f8cf0ffd2>\u001b[0m in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m     23\u001b[0m \u001b[0;31m# Extract named entities from the predicted labels and tokens\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     24\u001b[0m \u001b[0;31m#named_entities = [(token, entity) for token, entity_id in zip(predicted_tokens, predicted_entities) if entity_id.startswith('B-')]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 25\u001b[0;31m \u001b[0mnamed_entities\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtoken\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mentity\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mtoken\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mentity_id\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mzip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtext\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msplit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpredicted_entities\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mentity_id\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0;34m'[PAD]'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     26\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     27\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mNameError\u001b[0m: name 'entity' is not defined"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [],
      "metadata": {
        "id": "DHrL7D7zylCf"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "#NER"
      ],
      "metadata": {
        "id": "1VyP2pt0ylGj"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#pip install transformers\n"
      ],
      "metadata": {
        "id": "1QqXzXCzynBU"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "Apple    B-ORG\n",
        "Inc.     I-ORG\n",
        "was      O\n",
        "founded  O\n",
        "by       O\n",
        "Steve    B-PER\n",
        "Jobs,    I-PER\n",
        "Steve    B-PER\n",
        "Wozniak, I-PER\n",
        "and      O\n",
        "Ronald   B-PER\n",
        "Wayne.   I-PER\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 142
        },
        "id": "GBdni00xyw_1",
        "outputId": "d5e4b69c-933a-474b-ff3c-32e1ea52c068"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "error",
          "ename": "SyntaxError",
          "evalue": "invalid syntax (<ipython-input-56-4871fc2fa3ae>, line 1)",
          "traceback": [
            "\u001b[0;36m  File \u001b[0;32m\"<ipython-input-56-4871fc2fa3ae>\"\u001b[0;36m, line \u001b[0;32m1\u001b[0m\n\u001b[0;31m    Apple    B-ORG\u001b[0m\n\u001b[0m             ^\u001b[0m\n\u001b[0;31mSyntaxError\u001b[0m\u001b[0;31m:\u001b[0m invalid syntax\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [],
      "metadata": {
        "id": "o3ThR7q60Vf4"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "#TEXT GENERATION\n",
        "\n"
      ],
      "metadata": {
        "id": "wNCWQy180Viu"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "DistilBERT, as a distilled version of BERT, is primarily designed for tasks like question answering, text classification, and named entity recognition. It may not be the most suitable model for text generation, as its architecture is optimized for understanding context and relationships in input text rather than generating new content.\n",
        "\n",
        "However, for text generation tasks, models like GPT (Generative Pre-trained Transformer) are more commonly used. Since DistilBERT doesn't have a generative architecture, I'll provide you with an example using GPT-2 for text generation:"
      ],
      "metadata": {
        "id": "hfpXjt4x1EOU"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install transformers"
      ],
      "metadata": {
        "id": "smfJQ9pH1BU-"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import GPT2LMHeadModel, GPT2Tokenizer\n",
        "\n",
        "# Load pre-trained GPT-2 model and tokenizer for text generation\n",
        "model = GPT2LMHeadModel.from_pretrained('gpt2')\n",
        "tokenizer = GPT2Tokenizer.from_pretrained('gpt2')\n",
        "\n",
        "# Prompt for text generation\n",
        "prompt = \"Once upon a time in a land far, far away\"\n",
        "\n",
        "# Tokenize and encode the prompt\n",
        "input_ids = tokenizer.encode(prompt, return_tensors='pt')\n",
        "\n",
        "# Generate text using the model\n",
        "output = model.generate(input_ids, max_length=100, num_return_sequences=1, no_repeat_ngram_size=2, top_k=50, top_p=0.95, temperature=0.7)\n",
        "\n",
        "# Decode and print the generated text\n",
        "generated_text = tokenizer.decode(output[0], skip_special_tokens=True)\n",
        "print(\"Generated Text:\")\n",
        "print(generated_text)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 402,
          "referenced_widgets": [
            "0c44052698904deea17895d8c344afab",
            "43060b3bc1de41f0867498b356531175",
            "cee78aa0b2e1443691d37af57a8b69b3",
            "513c0fbe4be24c4880adc40e337ea67d",
            "e0096b48ff24408e86458635f32e352d",
            "d9ea534382814b0d9e11e97b2b2373a7",
            "87b78513bd674fa2ac44bcf776abc28d",
            "d90810023dbc493ca9698b357189ac9e",
            "00aa476a88f943a997a3bf3a29c24809",
            "3be01fe9bb2549ce8f3f2eca15e1a8aa",
            "40e05de6e45b45828e5b6d5099b9353a",
            "63f4fd0a10a64c69a4842b478f4f2227",
            "72ad3cfc11a140d7b04c458c710ced7c",
            "35e7c212ba054f47ae6cc84f3aa0a77c",
            "7b749c3536aa453aa1c4ac09422328fe",
            "65281acf40934b9aaffaeedf35210319",
            "06e82b0fa9244f86b035368c4d19f334",
            "65942244bed2418e8497f8fabbe71136",
            "6ac2966128684dd1b358b0a0355fed52",
            "a448704799a941c4a124824c7fccc78e",
            "ada5eb06591546dbae920a562e15419c",
            "a759fbbd7ea54b7b82e9d9f3b62aee7d",
            "1a19f14282924a31b152cd77952e9a8f",
            "3e8634c6200d43c4be1f5f95a94131f5",
            "f16f472e249c48dfab3e26d0e34d5ff9",
            "d5fbf04da152438c84afa4b9edebffe9",
            "632a03b6e2834c97828f6da4bdc58ceb",
            "2290c75f243449a3bb2c3753c76a56ef",
            "bcde98ad1c3c4e7aab1069e1188fda10",
            "7c5da0084ab4408cbdb66225dbd87c23",
            "e7df309fd349445ba7965a047f79de28",
            "92dfa3e5f2d44758a8f4e18b7c4eba7f",
            "2876538b77a04ed8be0e448bfbcb56af",
            "fe90c90391d241879e7f07f298a56565",
            "fde8ce15f9f94348bbc66efcd192a403",
            "983d32ede7a9476e85ef7cb169abe6b5",
            "06fe33a36232470f8f9c24e7210b4ae9",
            "172a6d15ec874c2086552e8808d71ae9",
            "3728454371b94e6d9a6ad452e3e49825",
            "b09143395771454a81dce318b793fe25",
            "1aacdb4eea2e465c8049e250dabce6d3",
            "5024b5a4cc7743269d88efa1766fccc9",
            "cc51a4e5936540d1b7f8bee913aab0c0",
            "9e66e23b275942739db0be9914d9f0f9",
            "bcbe7837620a4126aaeb51b0fbeda667",
            "e981a6d36c1b4e0baade56aa8640848e",
            "4fa4731c46344a418a39150f9cda5e96",
            "798d1482850643e18c16440d56af88c5",
            "a364b0a0cc0346b385d45f393a94d612",
            "73d5a52321174b74ab52c7846b8b4f07",
            "011bc5791c10401190b020066c39d7a3",
            "b4945e1893884a72a8c856e34c5fb8af",
            "b31047b1a5dc4ac69d100b28e2a8c10b",
            "9ffec65328c742e19a91fbf4403e646f",
            "d218810e7d80493f8c3d12599e28ce33",
            "27377480c46a47c085e9f659b7d31629",
            "1bbd6f79860c47ed8785e14b0eab9fc8",
            "469143e8f3a248c28a6cd7fcd3449fa5",
            "af29d8007eae43639dd17cf793a7aea3",
            "885945a372274fb1b569edddf5aa2cdb",
            "00b8b6c12aa34cb98b9bcfaaf0a1b60a",
            "f386a981d651442ead1c286176f21c6d",
            "07f19db2ad664cf986025bc442399cc5",
            "08f7cba4c2064d6a9c39e02c61dfcc36",
            "b11cccfce306417b9d05249447631685",
            "a901ccf6746b41afb3c3e5a64105e3d2"
          ]
        },
        "id": "D59wuX1h0ZMy",
        "outputId": "dc746621-ade4-486b-cd3e-7b35ac0dff2f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "config.json:   0%|          | 0.00/665 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "0c44052698904deea17895d8c344afab"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "model.safetensors:   0%|          | 0.00/548M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "63f4fd0a10a64c69a4842b478f4f2227"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "generation_config.json:   0%|          | 0.00/124 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "1a19f14282924a31b152cd77952e9a8f"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "vocab.json:   0%|          | 0.00/1.04M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "fe90c90391d241879e7f07f298a56565"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "merges.txt:   0%|          | 0.00/456k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "bcbe7837620a4126aaeb51b0fbeda667"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "tokenizer.json:   0%|          | 0.00/1.36M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "27377480c46a47c085e9f659b7d31629"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/transformers/generation/configuration_utils.py:381: UserWarning: `do_sample` is set to `False`. However, `temperature` is set to `0.7` -- this flag is only used in sample-based generation modes. You should set `do_sample=True` or unset `temperature`.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/transformers/generation/configuration_utils.py:386: UserWarning: `do_sample` is set to `False`. However, `top_p` is set to `0.95` -- this flag is only used in sample-based generation modes. You should set `do_sample=True` or unset `top_p`.\n",
            "  warnings.warn(\n",
            "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Generated Text:\n",
            "Once upon a time in a land far, far away, the world was a place of great beauty and great danger. The world of the gods was the land of darkness and darkness. And the darkness of this world, which was far from the light of day, was not the place where the sun and the moon met. It was in the midst of all the worlds, and it was there that the stars and all that were in them met, that they were all in one place.\n",
            "\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "In this example, I've used GPT-2 for text generation. You can customize the prompt variable to set the initial context for the generated text. Feel free to experiment with the parameters of the model.generate function to adjust the length, diversity, and other aspects of the generated text.\n",
        "\n",
        "Please note that GPT-2 is more suitable for text generation tasks, whereas DistilBERT is better suited for tasks like classification, question answering, and named entity recognition. If you specifically need text generation capabilities, GPT-2 or similar models would be more appropriate."
      ],
      "metadata": {
        "id": "LxTg-VJu1VF8"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "#MULTIPLE PROMPTS"
      ],
      "metadata": {
        "id": "PMAOCFVr1iCS"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# NEW EXERCISE\n",
        "from transformers import GPT2LMHeadModel, GPT2Tokenizer\n",
        "\n",
        "# Load pre-trained GPT-2 model and tokenizer for text generation\n",
        "model = GPT2LMHeadModel.from_pretrained('gpt2')\n",
        "tokenizer = GPT2Tokenizer.from_pretrained('gpt2')\n",
        "\n",
        "# List of prompts for text generation\n",
        "prompts = [\"Once upon a time in a land far, far away\",\n",
        "           \"In a galaxy not so distant, there was a hero\",\n",
        "           \"The mysterious door creaked open revealing\"]\n",
        "\n",
        "# Generate text for each prompt\n",
        "for prompt in prompts:\n",
        "    # Tokenize and encode the prompt\n",
        "    input_ids = tokenizer.encode(prompt, return_tensors='pt')\n",
        "\n",
        "    # Generate text using the model\n",
        "    output = model.generate(input_ids, max_length=100, num_return_sequences=1, no_repeat_ngram_size=2, top_k=50, top_p=0.95, temperature=0.7)\n",
        "\n",
        "    # Decode and print the generated text\n",
        "    generated_text = tokenizer.decode(output[0], skip_special_tokens=True)\n",
        "\n",
        "    print(f\"Prompt: {prompt}\")\n",
        "    print(\"Generated Text:\")\n",
        "    print(generated_text)\n",
        "    print(\"\\n\" + \"=\"*50 + \"\\n\")  # Separator between generated texts\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Q3nhG_gx1V1Q",
        "outputId": "7964f1f3-fb2e-461b-82cf-8441864845d6"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Prompt: Once upon a time in a land far, far away\n",
            "Generated Text:\n",
            "Once upon a time in a land far, far away, the world was a place of great beauty and great danger. The world of the gods was the land of darkness and darkness. And the darkness of this world, which was far from the light of day, was not the place where the sun and the moon met. It was in the midst of all the worlds, and it was there that the stars and all that were in them met, that they were all in one place.\n",
            "\n",
            "\n",
            "\n",
            "==================================================\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Prompt: In a galaxy not so distant, there was a hero\n",
            "Generated Text:\n",
            "In a galaxy not so distant, there was a hero who could save the galaxy.\n",
            "\n",
            "The hero was the hero of the Galactic Empire. He was known as the Emperor. The Emperor was an Emperor who had been born in the past, and had become a Jedi. His name was Darth Vader. Vader was born on the planet of Coruscant, where he was raised by his father, Darth Sidious. After his birth, Vader became a Sith Lord, a master of Sith sorcery.\n",
            "\n",
            "==================================================\n",
            "\n",
            "Prompt: The mysterious door creaked open revealing\n",
            "Generated Text:\n",
            "The mysterious door creaked open revealing a man in a black suit and a white shirt. He was wearing a red hooded sweatshirt and black pants.\n",
            "\n",
            "\"I'm a doctor,\" he said. \"I've been in the hospital for a year. I've never seen anything like this.\"\n",
            "...\n",
            ", a former nurse who worked at the University of California, Berkeley, and was a member of the medical staff at UC Berkeley's medical center. She was killed in an\n",
            "\n",
            "==================================================\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [],
      "metadata": {
        "id": "tPmtx0wi8wBi"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "#Fine Tuning LLM on Domain Specific Data"
      ],
      "metadata": {
        "id": "5Tump_GR8wE9"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import GPT2LMHeadModel, GPT2Tokenizer, GPT2Config\n",
        "from transformers import TextDataset, DataCollatorForLanguageModeling\n",
        "from transformers import Trainer, TrainingArguments\n",
        "\n",
        "# Load pre-trained GPT-2 model and tokenizer\n",
        "model = GPT2LMHeadModel.from_pretrained('gpt2')\n",
        "tokenizer = GPT2Tokenizer.from_pretrained('gpt2')\n",
        "\n",
        "# Load your domain-specific data\n",
        "train_dataset = TextDataset(\n",
        "    tokenizer=tokenizer,\n",
        "    file_path='/content/advertising.csv',\n",
        "   # file_path='path/to/your/train.txt',\n",
        "    block_size=128\n",
        ")\n",
        "\n",
        "\n",
        "\n",
        "# Define data collator\n",
        "data_collator = DataCollatorForLanguageModeling(\n",
        "    tokenizer=tokenizer,\n",
        "    mlm=False\n",
        ")\n",
        "\n",
        "# Define training arguments\n",
        "training_args = TrainingArguments(\n",
        "    output_dir='output_directory',\n",
        "    overwrite_output_dir=True,\n",
        "    num_train_epochs=3,\n",
        "    per_device_train_batch_size=8,\n",
        "    save_steps=10_000,\n",
        "    save_total_limit=2,\n",
        ")\n",
        "\n",
        "# Initialize Trainer\n",
        "trainer = Trainer(\n",
        "    model=model,\n",
        "    args=training_args,\n",
        "    data_collator=data_collator,\n",
        "    train_dataset=train_dataset,\n",
        ")\n",
        "\n",
        "# Fine-tune the model\n",
        "trainer.train()\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 620
        },
        "id": "3eIXRdSK8y6-",
        "outputId": "ec1f39d8-9836-4927-e164-22d613b7a39f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/transformers/data/datasets/language_modeling.py:53: FutureWarning: This dataset will be removed from the library soon, preprocessing should be handled with the 🤗 Datasets library. You can have a look at this example script for pointers: https://github.com/huggingface/transformers/blob/main/examples/pytorch/language-modeling/run_mlm.py\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "output_type": "error",
          "ename": "ImportError",
          "evalue": "Using the `Trainer` with `PyTorch` requires `accelerate>=0.20.1`: Please run `pip install transformers[torch]` or `pip install accelerate -U`",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mImportError\u001b[0m                               Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-68-69a0b5bfeacc>\u001b[0m in \u001b[0;36m<cell line: 26>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     24\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     25\u001b[0m \u001b[0;31m# Define training arguments\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 26\u001b[0;31m training_args = TrainingArguments(\n\u001b[0m\u001b[1;32m     27\u001b[0m     \u001b[0moutput_dir\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'output_directory'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     28\u001b[0m     \u001b[0moverwrite_output_dir\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/transformers/training_args.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, output_dir, overwrite_output_dir, do_train, do_eval, do_predict, evaluation_strategy, prediction_loss_only, per_device_train_batch_size, per_device_eval_batch_size, per_gpu_train_batch_size, per_gpu_eval_batch_size, gradient_accumulation_steps, eval_accumulation_steps, eval_delay, learning_rate, weight_decay, adam_beta1, adam_beta2, adam_epsilon, max_grad_norm, num_train_epochs, max_steps, lr_scheduler_type, warmup_ratio, warmup_steps, log_level, log_level_replica, log_on_each_node, logging_dir, logging_strategy, logging_first_step, logging_steps, logging_nan_inf_filter, save_strategy, save_steps, save_total_limit, save_safetensors, save_on_each_node, no_cuda, use_cpu, use_mps_device, seed, data_seed, jit_mode_eval, use_ipex, bf16, fp16, fp16_opt_level, half_precision_backend, bf16_full_eval, fp16_full_eval, tf32, local_rank, ddp_backend, tpu_num_cores, tpu_metrics_debug, debug, dataloader_drop_last, eval_steps, dataloader_num_workers, past_index, run_name, disable_tqdm, remove_unused_columns, label_names, load_best_model_at_end, metric_for_best_model, greater_is_better, ignore_data_skip, fsdp, fsdp_min_num_params, fsdp_config, fsdp_transformer_layer_cls_to_wrap, deepspeed, label_smoothing_factor, optim, optim_args, adafactor, group_by_length, length_column_name, report_to, ddp_find_unused_parameters, ddp_bucket_cap_mb, ddp_broadcast_buffers, dataloader_pin_memo...\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/transformers/training_args.py\u001b[0m in \u001b[0;36m__post_init__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1440\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mframework\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m\"pt\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1441\u001b[0m             \u001b[0;32mand\u001b[0m \u001b[0mis_torch_available\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1442\u001b[0;31m             \u001b[0;32mand\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtype\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0;34m\"cuda\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1443\u001b[0m             \u001b[0;32mand\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtype\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0;34m\"npu\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1444\u001b[0m             \u001b[0;32mand\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtype\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0;34m\"xpu\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/transformers/training_args.py\u001b[0m in \u001b[0;36mdevice\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1885\u001b[0m         \"\"\"\n\u001b[1;32m   1886\u001b[0m         \u001b[0mrequires_backends\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m\"torch\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1887\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_setup_devices\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1888\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1889\u001b[0m     \u001b[0;34m@\u001b[0m\u001b[0mproperty\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/transformers/utils/generic.py\u001b[0m in \u001b[0;36m__get__\u001b[0;34m(self, obj, objtype)\u001b[0m\n\u001b[1;32m     52\u001b[0m         \u001b[0mcached\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mattr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     53\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mcached\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 54\u001b[0;31m             \u001b[0mcached\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     55\u001b[0m             \u001b[0msetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mattr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcached\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     56\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mcached\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/transformers/training_args.py\u001b[0m in \u001b[0;36m_setup_devices\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1785\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mis_sagemaker_mp_enabled\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1786\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mis_accelerate_available\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmin_version\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"0.20.1\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1787\u001b[0;31m                 raise ImportError(\n\u001b[0m\u001b[1;32m   1788\u001b[0m                     \u001b[0;34m\"Using the `Trainer` with `PyTorch` requires `accelerate>=0.20.1`: Please run `pip install transformers[torch]` or `pip install accelerate -U`\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1789\u001b[0m                 )\n",
            "\u001b[0;31mImportError\u001b[0m: Using the `Trainer` with `PyTorch` requires `accelerate>=0.20.1`: Please run `pip install transformers[torch]` or `pip install accelerate -U`",
            "",
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0;32m\nNOTE: If your import is failing due to a missing package, you can\nmanually install dependencies using either !pip or !apt.\n\nTo view examples of installing some common dependencies, click the\n\"Open Examples\" button below.\n\u001b[0;31m---------------------------------------------------------------------------\u001b[0m\n"
          ],
          "errorDetails": {
            "actions": [
              {
                "action": "open_url",
                "actionText": "Open Examples",
                "url": "/notebooks/snippets/importing_libraries.ipynb"
              }
            ]
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "ImportError: Using the `Trainer` with `PyTorch` requires `accelerate>=0.20.1`: Please run `pip install transformers[torch]` or `pip install accelerate -U`"
      ],
      "metadata": {
        "id": "zGhweVti9WUr"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "pip install transformers[torch]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "m-eKTBEl9XaP",
        "outputId": "91b903f2-109a-4a15-e3a4-0af06de214ae"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: transformers[torch] in /usr/local/lib/python3.10/dist-packages (4.35.2)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from transformers[torch]) (3.13.1)\n",
            "Requirement already satisfied: huggingface-hub<1.0,>=0.16.4 in /usr/local/lib/python3.10/dist-packages (from transformers[torch]) (0.20.2)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from transformers[torch]) (1.23.5)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from transformers[torch]) (23.2)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from transformers[torch]) (6.0.1)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers[torch]) (2023.6.3)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from transformers[torch]) (2.31.0)\n",
            "Requirement already satisfied: tokenizers<0.19,>=0.14 in /usr/local/lib/python3.10/dist-packages (from transformers[torch]) (0.15.0)\n",
            "Requirement already satisfied: safetensors>=0.3.1 in /usr/local/lib/python3.10/dist-packages (from transformers[torch]) (0.4.1)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.10/dist-packages (from transformers[torch]) (4.66.1)\n",
            "Requirement already satisfied: torch!=1.12.0,>=1.10 in /usr/local/lib/python3.10/dist-packages (from transformers[torch]) (2.1.0+cu121)\n",
            "Collecting accelerate>=0.20.3 (from transformers[torch])\n",
            "  Downloading accelerate-0.26.0-py3-none-any.whl (270 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m270.7/270.7 kB\u001b[0m \u001b[31m4.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: psutil in /usr/local/lib/python3.10/dist-packages (from accelerate>=0.20.3->transformers[torch]) (5.9.5)\n",
            "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.16.4->transformers[torch]) (2023.6.0)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.16.4->transformers[torch]) (4.5.0)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch!=1.12.0,>=1.10->transformers[torch]) (1.12)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch!=1.12.0,>=1.10->transformers[torch]) (3.2.1)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch!=1.12.0,>=1.10->transformers[torch]) (3.1.2)\n",
            "Requirement already satisfied: triton==2.1.0 in /usr/local/lib/python3.10/dist-packages (from torch!=1.12.0,>=1.10->transformers[torch]) (2.1.0)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->transformers[torch]) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->transformers[torch]) (3.6)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->transformers[torch]) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->transformers[torch]) (2023.11.17)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch!=1.12.0,>=1.10->transformers[torch]) (2.1.3)\n",
            "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch!=1.12.0,>=1.10->transformers[torch]) (1.3.0)\n",
            "Installing collected packages: accelerate\n",
            "Successfully installed accelerate-0.26.0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install accelerate -U"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "LvVNLQmI9mfs",
        "outputId": "efc9b0f0-b145-43f8-ef86-0bf4ddfa2df5"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: accelerate in /usr/local/lib/python3.10/dist-packages (0.26.0)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from accelerate) (1.23.5)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from accelerate) (23.2)\n",
            "Requirement already satisfied: psutil in /usr/local/lib/python3.10/dist-packages (from accelerate) (5.9.5)\n",
            "Requirement already satisfied: pyyaml in /usr/local/lib/python3.10/dist-packages (from accelerate) (6.0.1)\n",
            "Requirement already satisfied: torch>=1.10.0 in /usr/local/lib/python3.10/dist-packages (from accelerate) (2.1.0+cu121)\n",
            "Requirement already satisfied: huggingface-hub in /usr/local/lib/python3.10/dist-packages (from accelerate) (0.20.2)\n",
            "Requirement already satisfied: safetensors>=0.3.1 in /usr/local/lib/python3.10/dist-packages (from accelerate) (0.4.1)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (3.13.1)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (4.5.0)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (1.12)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (3.2.1)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (3.1.2)\n",
            "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (2023.6.0)\n",
            "Requirement already satisfied: triton==2.1.0 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (2.1.0)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from huggingface-hub->accelerate) (2.31.0)\n",
            "Requirement already satisfied: tqdm>=4.42.1 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub->accelerate) (4.66.1)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch>=1.10.0->accelerate) (2.1.3)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub->accelerate) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub->accelerate) (3.6)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub->accelerate) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub->accelerate) (2023.11.17)\n",
            "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch>=1.10.0->accelerate) (1.3.0)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import GPT2LMHeadModel, GPT2Tokenizer, GPT2Config\n",
        "from transformers import TextDataset, DataCollatorForLanguageModeling\n",
        "from transformers import Trainer, TrainingArguments\n",
        "\n",
        "# Load pre-trained GPT-2 model and tokenizer\n",
        "model = GPT2LMHeadModel.from_pretrained('gpt2')\n",
        "tokenizer = GPT2Tokenizer.from_pretrained('gpt2')\n",
        "\n",
        "# Load your domain-specific data\n",
        "train_dataset = TextDataset(\n",
        "    tokenizer=tokenizer,\n",
        "    file_path='/content/advertising.csv',\n",
        "   # file_path='path/to/your/train.txt',\n",
        "    block_size=128\n",
        ")\n",
        "\n",
        "\n",
        "\n",
        "# Define data collator\n",
        "data_collator = DataCollatorForLanguageModeling(\n",
        "    tokenizer=tokenizer,\n",
        "    mlm=False\n",
        ")\n",
        "\n",
        "# Define training arguments\n",
        "training_args = TrainingArguments(\n",
        "    output_dir='output_directory',\n",
        "    overwrite_output_dir=True,\n",
        "    num_train_epochs=3,\n",
        "    per_device_train_batch_size=8,\n",
        "    save_steps=10_000,\n",
        "    save_total_limit=2,\n",
        ")\n",
        "\n",
        "# Initialize Trainer\n",
        "trainer = Trainer(\n",
        "    model=model,\n",
        "    args=training_args,\n",
        "    data_collator=data_collator,\n",
        "    train_dataset=train_dataset,\n",
        ")\n",
        "\n",
        "# Fine-tune the model\n",
        "trainer.train()\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 565
        },
        "id": "fL7RITUK9dup",
        "outputId": "a83d47d3-f369-48b9-a3ef-3a960fcc039b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "error",
          "ename": "ImportError",
          "evalue": "Using the `Trainer` with `PyTorch` requires `accelerate>=0.20.1`: Please run `pip install transformers[torch]` or `pip install accelerate -U`",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mImportError\u001b[0m                               Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-72-69a0b5bfeacc>\u001b[0m in \u001b[0;36m<cell line: 26>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     24\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     25\u001b[0m \u001b[0;31m# Define training arguments\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 26\u001b[0;31m training_args = TrainingArguments(\n\u001b[0m\u001b[1;32m     27\u001b[0m     \u001b[0moutput_dir\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'output_directory'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     28\u001b[0m     \u001b[0moverwrite_output_dir\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/transformers/training_args.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, output_dir, overwrite_output_dir, do_train, do_eval, do_predict, evaluation_strategy, prediction_loss_only, per_device_train_batch_size, per_device_eval_batch_size, per_gpu_train_batch_size, per_gpu_eval_batch_size, gradient_accumulation_steps, eval_accumulation_steps, eval_delay, learning_rate, weight_decay, adam_beta1, adam_beta2, adam_epsilon, max_grad_norm, num_train_epochs, max_steps, lr_scheduler_type, warmup_ratio, warmup_steps, log_level, log_level_replica, log_on_each_node, logging_dir, logging_strategy, logging_first_step, logging_steps, logging_nan_inf_filter, save_strategy, save_steps, save_total_limit, save_safetensors, save_on_each_node, no_cuda, use_cpu, use_mps_device, seed, data_seed, jit_mode_eval, use_ipex, bf16, fp16, fp16_opt_level, half_precision_backend, bf16_full_eval, fp16_full_eval, tf32, local_rank, ddp_backend, tpu_num_cores, tpu_metrics_debug, debug, dataloader_drop_last, eval_steps, dataloader_num_workers, past_index, run_name, disable_tqdm, remove_unused_columns, label_names, load_best_model_at_end, metric_for_best_model, greater_is_better, ignore_data_skip, fsdp, fsdp_min_num_params, fsdp_config, fsdp_transformer_layer_cls_to_wrap, deepspeed, label_smoothing_factor, optim, optim_args, adafactor, group_by_length, length_column_name, report_to, ddp_find_unused_parameters, ddp_bucket_cap_mb, ddp_broadcast_buffers, dataloader_pin_memo...\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/transformers/training_args.py\u001b[0m in \u001b[0;36m__post_init__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1440\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mframework\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m\"pt\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1441\u001b[0m             \u001b[0;32mand\u001b[0m \u001b[0mis_torch_available\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1442\u001b[0;31m             \u001b[0;32mand\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtype\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0;34m\"cuda\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1443\u001b[0m             \u001b[0;32mand\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtype\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0;34m\"npu\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1444\u001b[0m             \u001b[0;32mand\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtype\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0;34m\"xpu\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/transformers/training_args.py\u001b[0m in \u001b[0;36mdevice\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1885\u001b[0m         \"\"\"\n\u001b[1;32m   1886\u001b[0m         \u001b[0mrequires_backends\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m\"torch\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1887\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_setup_devices\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1888\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1889\u001b[0m     \u001b[0;34m@\u001b[0m\u001b[0mproperty\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/transformers/utils/generic.py\u001b[0m in \u001b[0;36m__get__\u001b[0;34m(self, obj, objtype)\u001b[0m\n\u001b[1;32m     52\u001b[0m         \u001b[0mcached\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mattr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     53\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mcached\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 54\u001b[0;31m             \u001b[0mcached\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     55\u001b[0m             \u001b[0msetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mattr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcached\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     56\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mcached\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/transformers/training_args.py\u001b[0m in \u001b[0;36m_setup_devices\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1785\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mis_sagemaker_mp_enabled\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1786\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mis_accelerate_available\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmin_version\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"0.20.1\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1787\u001b[0;31m                 raise ImportError(\n\u001b[0m\u001b[1;32m   1788\u001b[0m                     \u001b[0;34m\"Using the `Trainer` with `PyTorch` requires `accelerate>=0.20.1`: Please run `pip install transformers[torch]` or `pip install accelerate -U`\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1789\u001b[0m                 )\n",
            "\u001b[0;31mImportError\u001b[0m: Using the `Trainer` with `PyTorch` requires `accelerate>=0.20.1`: Please run `pip install transformers[torch]` or `pip install accelerate -U`",
            "",
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0;32m\nNOTE: If your import is failing due to a missing package, you can\nmanually install dependencies using either !pip or !apt.\n\nTo view examples of installing some common dependencies, click the\n\"Open Examples\" button below.\n\u001b[0;31m---------------------------------------------------------------------------\u001b[0m\n"
          ],
          "errorDetails": {
            "actions": [
              {
                "action": "open_url",
                "actionText": "Open Examples",
                "url": "/notebooks/snippets/importing_libraries.ipynb"
              }
            ]
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install tensorflow transformers\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "d_HcWjkE-Amn",
        "outputId": "d1a04e64-7804-49aa-e9b2-69427e75d2f0"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: tensorflow in /usr/local/lib/python3.10/dist-packages (2.15.0)\n",
            "Requirement already satisfied: transformers in /usr/local/lib/python3.10/dist-packages (4.35.2)\n",
            "Requirement already satisfied: absl-py>=1.0.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow) (1.4.0)\n",
            "Requirement already satisfied: astunparse>=1.6.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow) (1.6.3)\n",
            "Requirement already satisfied: flatbuffers>=23.5.26 in /usr/local/lib/python3.10/dist-packages (from tensorflow) (23.5.26)\n",
            "Requirement already satisfied: gast!=0.5.0,!=0.5.1,!=0.5.2,>=0.2.1 in /usr/local/lib/python3.10/dist-packages (from tensorflow) (0.5.4)\n",
            "Requirement already satisfied: google-pasta>=0.1.1 in /usr/local/lib/python3.10/dist-packages (from tensorflow) (0.2.0)\n",
            "Requirement already satisfied: h5py>=2.9.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow) (3.9.0)\n",
            "Requirement already satisfied: libclang>=13.0.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow) (16.0.6)\n",
            "Requirement already satisfied: ml-dtypes~=0.2.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow) (0.2.0)\n",
            "Requirement already satisfied: numpy<2.0.0,>=1.23.5 in /usr/local/lib/python3.10/dist-packages (from tensorflow) (1.23.5)\n",
            "Requirement already satisfied: opt-einsum>=2.3.2 in /usr/local/lib/python3.10/dist-packages (from tensorflow) (3.3.0)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.10/dist-packages (from tensorflow) (23.2)\n",
            "Requirement already satisfied: protobuf!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5,<5.0.0dev,>=3.20.3 in /usr/local/lib/python3.10/dist-packages (from tensorflow) (3.20.3)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.10/dist-packages (from tensorflow) (67.7.2)\n",
            "Requirement already satisfied: six>=1.12.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow) (1.16.0)\n",
            "Requirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow) (2.4.0)\n",
            "Requirement already satisfied: typing-extensions>=3.6.6 in /usr/local/lib/python3.10/dist-packages (from tensorflow) (4.5.0)\n",
            "Requirement already satisfied: wrapt<1.15,>=1.11.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow) (1.14.1)\n",
            "Requirement already satisfied: tensorflow-io-gcs-filesystem>=0.23.1 in /usr/local/lib/python3.10/dist-packages (from tensorflow) (0.35.0)\n",
            "Requirement already satisfied: grpcio<2.0,>=1.24.3 in /usr/local/lib/python3.10/dist-packages (from tensorflow) (1.60.0)\n",
            "Requirement already satisfied: tensorboard<2.16,>=2.15 in /usr/local/lib/python3.10/dist-packages (from tensorflow) (2.15.1)\n",
            "Requirement already satisfied: tensorflow-estimator<2.16,>=2.15.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow) (2.15.0)\n",
            "Requirement already satisfied: keras<2.16,>=2.15.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow) (2.15.0)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from transformers) (3.13.1)\n",
            "Requirement already satisfied: huggingface-hub<1.0,>=0.16.4 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.20.2)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (6.0.1)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (2023.6.3)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from transformers) (2.31.0)\n",
            "Requirement already satisfied: tokenizers<0.19,>=0.14 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.15.0)\n",
            "Requirement already satisfied: safetensors>=0.3.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.4.1)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.10/dist-packages (from transformers) (4.66.1)\n",
            "Requirement already satisfied: wheel<1.0,>=0.23.0 in /usr/local/lib/python3.10/dist-packages (from astunparse>=1.6.0->tensorflow) (0.42.0)\n",
            "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.16.4->transformers) (2023.6.0)\n",
            "Requirement already satisfied: google-auth<3,>=1.6.3 in /usr/local/lib/python3.10/dist-packages (from tensorboard<2.16,>=2.15->tensorflow) (2.17.3)\n",
            "Requirement already satisfied: google-auth-oauthlib<2,>=0.5 in /usr/local/lib/python3.10/dist-packages (from tensorboard<2.16,>=2.15->tensorflow) (1.2.0)\n",
            "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.10/dist-packages (from tensorboard<2.16,>=2.15->tensorflow) (3.5.1)\n",
            "Requirement already satisfied: tensorboard-data-server<0.8.0,>=0.7.0 in /usr/local/lib/python3.10/dist-packages (from tensorboard<2.16,>=2.15->tensorflow) (0.7.2)\n",
            "Requirement already satisfied: werkzeug>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from tensorboard<2.16,>=2.15->tensorflow) (3.0.1)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (3.6)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2023.11.17)\n",
            "Requirement already satisfied: cachetools<6.0,>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from google-auth<3,>=1.6.3->tensorboard<2.16,>=2.15->tensorflow) (5.3.2)\n",
            "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.10/dist-packages (from google-auth<3,>=1.6.3->tensorboard<2.16,>=2.15->tensorflow) (0.3.0)\n",
            "Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.10/dist-packages (from google-auth<3,>=1.6.3->tensorboard<2.16,>=2.15->tensorflow) (4.9)\n",
            "Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.10/dist-packages (from google-auth-oauthlib<2,>=0.5->tensorboard<2.16,>=2.15->tensorflow) (1.3.1)\n",
            "Requirement already satisfied: MarkupSafe>=2.1.1 in /usr/local/lib/python3.10/dist-packages (from werkzeug>=1.0.1->tensorboard<2.16,>=2.15->tensorflow) (2.1.3)\n",
            "Requirement already satisfied: pyasn1<0.6.0,>=0.4.6 in /usr/local/lib/python3.10/dist-packages (from pyasn1-modules>=0.2.1->google-auth<3,>=1.6.3->tensorboard<2.16,>=2.15->tensorflow) (0.5.1)\n",
            "Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.10/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<2,>=0.5->tensorboard<2.16,>=2.15->tensorflow) (3.2.2)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import TFGPT2LMHeadModel, GPT2Tokenizer, GPT2Config\n",
        "from transformers import TextDataset, DataCollatorForLanguageModeling\n",
        "from transformers import TFTrainer, TFTrainingArguments\n",
        "\n",
        "# Load pre-trained GPT-2 model and tokenizer\n",
        "model = TFGPT2LMHeadModel.from_pretrained('gpt2')\n",
        "tokenizer = GPT2Tokenizer.from_pretrained('gpt2')\n",
        "\n",
        "# Load your domain-specific data\n",
        "train_dataset = TextDataset(\n",
        "    tokenizer=tokenizer,\n",
        "    file_path='/content/advertising.csv',\n",
        "    block_size=128\n",
        ")\n",
        "\n",
        "# Define data collator\n",
        "data_collator = DataCollatorForLanguageModeling(\n",
        "    tokenizer=tokenizer,\n",
        "    mlm=False\n",
        ")\n",
        "\n",
        "# Define training arguments\n",
        "training_args = TFTrainingArguments(\n",
        "    output_dir='output_directory',\n",
        "    overwrite_output_dir=True,\n",
        "    num_train_epochs=3,\n",
        "    per_device_train_batch_size=8,\n",
        "    save_steps=10_000,\n",
        "    save_total_limit=2,\n",
        ")\n",
        "\n",
        "# Initialize Trainer\n",
        "trainer = TFTrainer(\n",
        "    model=model,\n",
        "    args=training_args,\n",
        "    data_collator=data_collator,\n",
        "    train_dataset=train_dataset,\n",
        ")\n",
        "\n",
        "# Fine-tune the model\n",
        "trainer.train()\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 335
        },
        "id": "LBcHlQR3-DMR",
        "outputId": "ab752150-ff75-40ee-892a-104be0226293"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "All PyTorch model weights were used when initializing TFGPT2LMHeadModel.\n",
            "\n",
            "All the weights of TFGPT2LMHeadModel were initialized from the PyTorch model.\n",
            "If your task is similar to the task the model of the checkpoint was trained on, you can already use TFGPT2LMHeadModel for predictions without further training.\n"
          ]
        },
        {
          "output_type": "error",
          "ename": "TypeError",
          "evalue": "TFTrainer.__init__() got an unexpected keyword argument 'data_collator'",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-74-a4d166f6e90a>\u001b[0m in \u001b[0;36m<cell line: 33>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     31\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     32\u001b[0m \u001b[0;31m# Initialize Trainer\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 33\u001b[0;31m trainer = TFTrainer(\n\u001b[0m\u001b[1;32m     34\u001b[0m     \u001b[0mmodel\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     35\u001b[0m     \u001b[0margs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtraining_args\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mTypeError\u001b[0m: TFTrainer.__init__() got an unexpected keyword argument 'data_collator'"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import TFGPT2LMHeadModel, GPT2Tokenizer, GPT2Config\n",
        "from transformers import TextDataset, DataCollatorForLanguageModeling\n",
        "from transformers import TFTrainer, TFTrainingArguments\n",
        "\n",
        "# Load pre-trained GPT-2 model and tokenizer\n",
        "model = TFGPT2LMHeadModel.from_pretrained('gpt2')\n",
        "tokenizer = GPT2Tokenizer.from_pretrained('gpt2')\n",
        "\n",
        "# Load your domain-specific data\n",
        "train_dataset = TextDataset(\n",
        "    tokenizer=tokenizer,\n",
        "      file_path='/content/advertising.csv',\n",
        "    block_size=128\n",
        ")\n",
        "\n",
        "# Define data collator\n",
        "data_collator = DataCollatorForLanguageModeling(\n",
        "    tokenizer=tokenizer,\n",
        "    mlm=False\n",
        ")\n",
        "\n",
        "# Define training arguments\n",
        "training_args = TFTrainingArguments(\n",
        "    output_dir='output_directory',\n",
        "    overwrite_output_dir=True,\n",
        "    num_train_epochs=3,\n",
        "    per_device_train_batch_size=8,\n",
        "    save_steps=10_000,\n",
        "    save_total_limit=2,\n",
        "    prediction_loss_only=True,  # This ensures only the prediction loss is computed (equivalent to mlm=False)\n",
        ")\n",
        "\n",
        "# Initialize Trainer\n",
        "trainer = TFTrainer(\n",
        "    model=model,\n",
        "    args=training_args,\n",
        "    train_dataset=train_dataset,\n",
        "    data_collator=data_collator,\n",
        ")\n",
        "\n",
        "# Fine-tune the model\n",
        "trainer.train()\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 335
        },
        "id": "5abOIith-ed6",
        "outputId": "1f2f2409-cf04-41e4-ce9a-d25f194703ca"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "All PyTorch model weights were used when initializing TFGPT2LMHeadModel.\n",
            "\n",
            "All the weights of TFGPT2LMHeadModel were initialized from the PyTorch model.\n",
            "If your task is similar to the task the model of the checkpoint was trained on, you can already use TFGPT2LMHeadModel for predictions without further training.\n"
          ]
        },
        {
          "output_type": "error",
          "ename": "TypeError",
          "evalue": "TFTrainer.__init__() got an unexpected keyword argument 'data_collator'",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-75-d2840edb0201>\u001b[0m in \u001b[0;36m<cell line: 34>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     32\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     33\u001b[0m \u001b[0;31m# Initialize Trainer\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 34\u001b[0;31m trainer = TFTrainer(\n\u001b[0m\u001b[1;32m     35\u001b[0m     \u001b[0mmodel\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     36\u001b[0m     \u001b[0margs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtraining_args\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mTypeError\u001b[0m: TFTrainer.__init__() got an unexpected keyword argument 'data_collator'"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "490fU5f--tvu"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [],
      "metadata": {
        "id": "T3tHJIWh-t_P"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import accelerate"
      ],
      "metadata": {
        "id": "GC6hQhO2_IgY"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install accelerate>=0.20.1"
      ],
      "metadata": {
        "id": "QLt74QXi-6mX"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [],
      "metadata": {
        "id": "UODQ-7YJ-uCK"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import GPT2LMHeadModel, GPT2Tokenizer, GPT2Config\n",
        "from transformers import TextDataset, DataCollatorForLanguageModeling\n",
        "from transformers import Trainer, TrainingArguments\n",
        "\n",
        "# Load pre-trained GPT-2 model and tokenizer\n",
        "model = GPT2LMHeadModel.from_pretrained('gpt2')\n",
        "tokenizer = GPT2Tokenizer.from_pretrained('gpt2')\n",
        "\n",
        "# Load your domain-specific data\n",
        "train_dataset = TextDataset(\n",
        "    tokenizer=tokenizer,\n",
        "     file_path='/content/advertising.csv',\n",
        "    block_size=128\n",
        ")\n",
        "\n",
        "# Define data collator\n",
        "data_collator = DataCollatorForLanguageModeling(\n",
        "    tokenizer=tokenizer,\n",
        "    mlm=False\n",
        ")\n",
        "\n",
        "# Define training arguments\n",
        "training_args = TrainingArguments(\n",
        "    output_dir='output_directory',\n",
        "    overwrite_output_dir=True,\n",
        "    num_train_epochs=3,\n",
        "    per_device_train_batch_size=8,\n",
        "    save_steps=10_000,\n",
        "    save_total_limit=2,\n",
        ")\n",
        "\n",
        "# Initialize Trainer\n",
        "trainer = Trainer(\n",
        "    model=model,\n",
        "    args=training_args,\n",
        "    data_collator=data_collator,\n",
        "    train_dataset=train_dataset,\n",
        ")\n",
        "\n",
        "# Fine-tune the model\n",
        "trainer.train()\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 620
        },
        "id": "RhA-zKsS-vgU",
        "outputId": "7e1277c8-2048-4cdc-baad-ecdfefabfcdc"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/transformers/data/datasets/language_modeling.py:53: FutureWarning: This dataset will be removed from the library soon, preprocessing should be handled with the 🤗 Datasets library. You can have a look at this example script for pointers: https://github.com/huggingface/transformers/blob/main/examples/pytorch/language-modeling/run_mlm.py\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "output_type": "error",
          "ename": "ImportError",
          "evalue": "Using the `Trainer` with `PyTorch` requires `accelerate>=0.20.1`: Please run `pip install transformers[torch]` or `pip install accelerate -U`",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mImportError\u001b[0m                               Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-84-a043037d0777>\u001b[0m in \u001b[0;36m<cell line: 23>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     21\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     22\u001b[0m \u001b[0;31m# Define training arguments\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 23\u001b[0;31m training_args = TrainingArguments(\n\u001b[0m\u001b[1;32m     24\u001b[0m     \u001b[0moutput_dir\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'output_directory'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     25\u001b[0m     \u001b[0moverwrite_output_dir\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/transformers/training_args.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, output_dir, overwrite_output_dir, do_train, do_eval, do_predict, evaluation_strategy, prediction_loss_only, per_device_train_batch_size, per_device_eval_batch_size, per_gpu_train_batch_size, per_gpu_eval_batch_size, gradient_accumulation_steps, eval_accumulation_steps, eval_delay, learning_rate, weight_decay, adam_beta1, adam_beta2, adam_epsilon, max_grad_norm, num_train_epochs, max_steps, lr_scheduler_type, warmup_ratio, warmup_steps, log_level, log_level_replica, log_on_each_node, logging_dir, logging_strategy, logging_first_step, logging_steps, logging_nan_inf_filter, save_strategy, save_steps, save_total_limit, save_safetensors, save_on_each_node, no_cuda, use_cpu, use_mps_device, seed, data_seed, jit_mode_eval, use_ipex, bf16, fp16, fp16_opt_level, half_precision_backend, bf16_full_eval, fp16_full_eval, tf32, local_rank, ddp_backend, tpu_num_cores, tpu_metrics_debug, debug, dataloader_drop_last, eval_steps, dataloader_num_workers, past_index, run_name, disable_tqdm, remove_unused_columns, label_names, load_best_model_at_end, metric_for_best_model, greater_is_better, ignore_data_skip, fsdp, fsdp_min_num_params, fsdp_config, fsdp_transformer_layer_cls_to_wrap, deepspeed, label_smoothing_factor, optim, optim_args, adafactor, group_by_length, length_column_name, report_to, ddp_find_unused_parameters, ddp_bucket_cap_mb, ddp_broadcast_buffers, dataloader_pin_memo...\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/transformers/training_args.py\u001b[0m in \u001b[0;36m__post_init__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1440\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mframework\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m\"pt\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1441\u001b[0m             \u001b[0;32mand\u001b[0m \u001b[0mis_torch_available\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1442\u001b[0;31m             \u001b[0;32mand\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtype\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0;34m\"cuda\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1443\u001b[0m             \u001b[0;32mand\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtype\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0;34m\"npu\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1444\u001b[0m             \u001b[0;32mand\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtype\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0;34m\"xpu\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/transformers/training_args.py\u001b[0m in \u001b[0;36mdevice\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1885\u001b[0m         \"\"\"\n\u001b[1;32m   1886\u001b[0m         \u001b[0mrequires_backends\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m\"torch\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1887\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_setup_devices\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1888\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1889\u001b[0m     \u001b[0;34m@\u001b[0m\u001b[0mproperty\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/transformers/utils/generic.py\u001b[0m in \u001b[0;36m__get__\u001b[0;34m(self, obj, objtype)\u001b[0m\n\u001b[1;32m     52\u001b[0m         \u001b[0mcached\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mattr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     53\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mcached\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 54\u001b[0;31m             \u001b[0mcached\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     55\u001b[0m             \u001b[0msetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mattr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcached\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     56\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mcached\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/transformers/training_args.py\u001b[0m in \u001b[0;36m_setup_devices\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1785\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mis_sagemaker_mp_enabled\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1786\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mis_accelerate_available\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmin_version\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"0.20.1\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1787\u001b[0;31m                 raise ImportError(\n\u001b[0m\u001b[1;32m   1788\u001b[0m                     \u001b[0;34m\"Using the `Trainer` with `PyTorch` requires `accelerate>=0.20.1`: Please run `pip install transformers[torch]` or `pip install accelerate -U`\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1789\u001b[0m                 )\n",
            "\u001b[0;31mImportError\u001b[0m: Using the `Trainer` with `PyTorch` requires `accelerate>=0.20.1`: Please run `pip install transformers[torch]` or `pip install accelerate -U`",
            "",
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0;32m\nNOTE: If your import is failing due to a missing package, you can\nmanually install dependencies using either !pip or !apt.\n\nTo view examples of installing some common dependencies, click the\n\"Open Examples\" button below.\n\u001b[0;31m---------------------------------------------------------------------------\u001b[0m\n"
          ],
          "errorDetails": {
            "actions": [
              {
                "action": "open_url",
                "actionText": "Open Examples",
                "url": "/notebooks/snippets/importing_libraries.ipynb"
              }
            ]
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "S9KUY1p2_rCe"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "AmL2lsAh_rMp"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "#Fine-Tune using Keras"
      ],
      "metadata": {
        "id": "LHyCJot1_ywD"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import tensorflow as tf\n",
        "from transformers import GPT2Tokenizer, TFGPT2LMHeadModel\n",
        "\n",
        "# Load pre-trained GPT-2 model and tokenizer\n",
        "model = TFGPT2LMHeadModel.from_pretrained('gpt2')\n",
        "tokenizer = GPT2Tokenizer.from_pretrained('gpt2')\n",
        "\n",
        "# Load your domain-specific data\n",
        "train_data_path = '/content/advertising.csv'\n",
        "with open(train_data_path, 'r', encoding='utf-8') as file:\n",
        "    train_texts = file.readlines()\n",
        "\n",
        "# Tokenize and encode the texts\n",
        "input_ids = tokenizer(train_texts, return_tensors='tf', padding=True, truncation=True)\n",
        "\n",
        "# Create TensorFlow Dataset\n",
        "train_dataset = tf.data.Dataset.from_tensor_slices((dict(input_ids), dict(input_ids)))\n",
        "\n",
        "# Define training parameters\n",
        "batch_size = 8\n",
        "num_epochs = 3\n",
        "learning_rate = 5e-5\n",
        "\n",
        "# Compile the model\n",
        "model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=learning_rate),\n",
        "              loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True))\n",
        "\n",
        "# Train the model\n",
        "model.fit(train_dataset.shuffle(1000).batch(batch_size),\n",
        "          epochs=num_epochs)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 585
        },
        "id": "R0z0ooQq_00x",
        "outputId": "c4f21cc1-81a1-48a3-c67e-4cef61538b24"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "All PyTorch model weights were used when initializing TFGPT2LMHeadModel.\n",
            "\n",
            "All the weights of TFGPT2LMHeadModel were initialized from the PyTorch model.\n",
            "If your task is similar to the task the model of the checkpoint was trained on, you can already use TFGPT2LMHeadModel for predictions without further training.\n"
          ]
        },
        {
          "output_type": "error",
          "ename": "ValueError",
          "evalue": "Asking to pad but the tokenizer does not have a padding token. Please select a token to use as `pad_token` `(tokenizer.pad_token = tokenizer.eos_token e.g.)` or add a new pad token via `tokenizer.add_special_tokens({'pad_token': '[PAD]'})`.",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-85-c2d81aa331df>\u001b[0m in \u001b[0;36m<cell line: 14>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     12\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     13\u001b[0m \u001b[0;31m# Tokenize and encode the texts\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 14\u001b[0;31m \u001b[0minput_ids\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtokenizer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_texts\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreturn_tensors\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'tf'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpadding\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtruncation\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     15\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     16\u001b[0m \u001b[0;31m# Create TensorFlow Dataset\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/transformers/tokenization_utils_base.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, text, text_pair, text_target, text_pair_target, add_special_tokens, padding, truncation, max_length, stride, is_split_into_words, pad_to_multiple_of, return_tensors, return_token_type_ids, return_attention_mask, return_overflowing_tokens, return_special_tokens_mask, return_offsets_mapping, return_length, verbose, **kwargs)\u001b[0m\n\u001b[1;32m   2796\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_in_target_context_manager\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2797\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_switch_to_input_mode\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2798\u001b[0;31m             \u001b[0mencodings\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call_one\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtext\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtext\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtext_pair\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtext_pair\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mall_kwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2799\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mtext_target\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2800\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_switch_to_target_mode\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/transformers/tokenization_utils_base.py\u001b[0m in \u001b[0;36m_call_one\u001b[0;34m(self, text, text_pair, add_special_tokens, padding, truncation, max_length, stride, is_split_into_words, pad_to_multiple_of, return_tensors, return_token_type_ids, return_attention_mask, return_overflowing_tokens, return_special_tokens_mask, return_offsets_mapping, return_length, verbose, **kwargs)\u001b[0m\n\u001b[1;32m   2882\u001b[0m                 )\n\u001b[1;32m   2883\u001b[0m             \u001b[0mbatch_text_or_text_pairs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mzip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtext\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtext_pair\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mtext_pair\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0mtext\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2884\u001b[0;31m             return self.batch_encode_plus(\n\u001b[0m\u001b[1;32m   2885\u001b[0m                 \u001b[0mbatch_text_or_text_pairs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mbatch_text_or_text_pairs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2886\u001b[0m                 \u001b[0madd_special_tokens\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0madd_special_tokens\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/transformers/tokenization_utils_base.py\u001b[0m in \u001b[0;36mbatch_encode_plus\u001b[0;34m(self, batch_text_or_text_pairs, add_special_tokens, padding, truncation, max_length, stride, is_split_into_words, pad_to_multiple_of, return_tensors, return_token_type_ids, return_attention_mask, return_overflowing_tokens, return_special_tokens_mask, return_offsets_mapping, return_length, verbose, **kwargs)\u001b[0m\n\u001b[1;32m   3064\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3065\u001b[0m         \u001b[0;31m# Backward compatibility for 'truncation_strategy', 'pad_to_max_length'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 3066\u001b[0;31m         padding_strategy, truncation_strategy, max_length, kwargs = self._get_padding_truncation_strategies(\n\u001b[0m\u001b[1;32m   3067\u001b[0m             \u001b[0mpadding\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpadding\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3068\u001b[0m             \u001b[0mtruncation\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtruncation\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/transformers/tokenization_utils_base.py\u001b[0m in \u001b[0;36m_get_padding_truncation_strategies\u001b[0;34m(self, padding, truncation, max_length, pad_to_multiple_of, verbose, **kwargs)\u001b[0m\n\u001b[1;32m   2701\u001b[0m         \u001b[0;31m# Test if we have a padding token\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2702\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mpadding_strategy\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0mPaddingStrategy\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mDO_NOT_PAD\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpad_token\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpad_token_id\u001b[0m \u001b[0;34m<\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2703\u001b[0;31m             raise ValueError(\n\u001b[0m\u001b[1;32m   2704\u001b[0m                 \u001b[0;34m\"Asking to pad but the tokenizer does not have a padding token. \"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2705\u001b[0m                 \u001b[0;34m\"Please select a token to use as `pad_token` `(tokenizer.pad_token = tokenizer.eos_token e.g.)` \"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mValueError\u001b[0m: Asking to pad but the tokenizer does not have a padding token. Please select a token to use as `pad_token` `(tokenizer.pad_token = tokenizer.eos_token e.g.)` or add a new pad token via `tokenizer.add_special_tokens({'pad_token': '[PAD]'})`."
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "Ox86OOWDAPiN"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import tensorflow as tf\n",
        "from transformers import GPT2Tokenizer, TFGPT2LMHeadModel\n",
        "\n",
        "# Load pre-trained GPT-2 model and tokenizer\n",
        "model = TFGPT2LMHeadModel.from_pretrained('gpt2')\n",
        "tokenizer = GPT2Tokenizer.from_pretrained('gpt2')\n",
        "\n",
        "# Add the EOS token as the pad token\n",
        "tokenizer.pad_token = tokenizer.eos_token\n",
        "\n",
        "# Load your domain-specific data\n",
        "train_data_path = '/content/advertising.csv'\n",
        "with open(train_data_path, 'r', encoding='utf-8') as file:\n",
        "    train_texts = file.readlines()\n",
        "\n",
        "# Tokenize and encode the texts\n",
        "input_ids = tokenizer(train_texts, return_tensors='tf', padding=True, truncation=True)\n",
        "\n",
        "# Create TensorFlow Dataset\n",
        "train_dataset = tf.data.Dataset.from_tensor_slices((dict(input_ids), dict(input_ids)))\n",
        "\n",
        "# Define training parameters\n",
        "batch_size = 8\n",
        "num_epochs = 3\n",
        "learning_rate = 5e-5\n",
        "\n",
        "# Compile the model\n",
        "model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=learning_rate),\n",
        "              loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True))\n",
        "\n",
        "# Train the model\n",
        "model.fit(train_dataset.shuffle(1000).batch(batch_size),\n",
        "          epochs=num_epochs)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 863
        },
        "id": "x8D8xj5OAPlT",
        "outputId": "39460ed2-4bbd-48e9-8aa0-73c26fbacff4"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "All PyTorch model weights were used when initializing TFGPT2LMHeadModel.\n",
            "\n",
            "All the weights of TFGPT2LMHeadModel were initialized from the PyTorch model.\n",
            "If your task is similar to the task the model of the checkpoint was trained on, you can already use TFGPT2LMHeadModel for predictions without further training.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/3\n"
          ]
        },
        {
          "output_type": "error",
          "ename": "TypeError",
          "evalue": "in user code:\n\n    File \"/usr/local/lib/python3.10/dist-packages/keras/src/engine/training.py\", line 1401, in train_function  *\n        return step_function(self, iterator)\n    File \"/usr/local/lib/python3.10/dist-packages/keras/src/engine/training.py\", line 1384, in step_function  **\n        outputs = model.distribute_strategy.run(run_step, args=(data,))\n    File \"/usr/local/lib/python3.10/dist-packages/keras/src/engine/training.py\", line 1373, in run_step  **\n        outputs = model.train_step(data)\n    File \"/usr/local/lib/python3.10/dist-packages/transformers/modeling_tf_utils.py\", line 1677, in train_step\n        self.optimizer.minimize(loss, self.trainable_variables, tape=tape)\n    File \"/usr/local/lib/python3.10/dist-packages/keras/src/optimizers/optimizer.py\", line 543, in minimize\n        grads_and_vars = self.compute_gradients(loss, var_list, tape)\n    File \"/usr/local/lib/python3.10/dist-packages/keras/src/optimizers/optimizer.py\", line 276, in compute_gradients\n        grads = tape.gradient(loss, var_list)\n\n    TypeError: Argument `target` should be a list or nested structure of Tensors, Variables or CompositeTensors to be differentiated, but received None.\n",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-86-5cdf87eae95c>\u001b[0m in \u001b[0;36m<cell line: 32>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     30\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     31\u001b[0m \u001b[0;31m# Train the model\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 32\u001b[0;31m model.fit(train_dataset.shuffle(1000).batch(batch_size),\n\u001b[0m\u001b[1;32m     33\u001b[0m           epochs=num_epochs)\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/keras/src/utils/traceback_utils.py\u001b[0m in \u001b[0;36merror_handler\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     68\u001b[0m             \u001b[0;31m# To get the full stack trace, call:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     69\u001b[0m             \u001b[0;31m# `tf.debugging.disable_traceback_filtering()`\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 70\u001b[0;31m             \u001b[0;32mraise\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwith_traceback\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfiltered_tb\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     71\u001b[0m         \u001b[0;32mfinally\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     72\u001b[0m             \u001b[0;32mdel\u001b[0m \u001b[0mfiltered_tb\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/keras/src/engine/training.py\u001b[0m in \u001b[0;36mtf__train_function\u001b[0;34m(iterator)\u001b[0m\n\u001b[1;32m     13\u001b[0m                 \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m                     \u001b[0mdo_return\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 15\u001b[0;31m                     \u001b[0mretval_\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mag__\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconverted_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mag__\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mld\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstep_function\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mag__\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mld\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mag__\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mld\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0miterator\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfscope\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     16\u001b[0m                 \u001b[0;32mexcept\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m                     \u001b[0mdo_return\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/transformers/modeling_tf_utils.py\u001b[0m in \u001b[0;36mtrain_step\u001b[0;34m(self, data)\u001b[0m\n\u001b[1;32m   1675\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1676\u001b[0m         \u001b[0;31m# Run backwards pass.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1677\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mminimize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mloss\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrainable_variables\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtape\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtape\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1678\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1679\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcompiled_metrics\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate_state\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_pred\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msample_weight\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mTypeError\u001b[0m: in user code:\n\n    File \"/usr/local/lib/python3.10/dist-packages/keras/src/engine/training.py\", line 1401, in train_function  *\n        return step_function(self, iterator)\n    File \"/usr/local/lib/python3.10/dist-packages/keras/src/engine/training.py\", line 1384, in step_function  **\n        outputs = model.distribute_strategy.run(run_step, args=(data,))\n    File \"/usr/local/lib/python3.10/dist-packages/keras/src/engine/training.py\", line 1373, in run_step  **\n        outputs = model.train_step(data)\n    File \"/usr/local/lib/python3.10/dist-packages/transformers/modeling_tf_utils.py\", line 1677, in train_step\n        self.optimizer.minimize(loss, self.trainable_variables, tape=tape)\n    File \"/usr/local/lib/python3.10/dist-packages/keras/src/optimizers/optimizer.py\", line 543, in minimize\n        grads_and_vars = self.compute_gradients(loss, var_list, tape)\n    File \"/usr/local/lib/python3.10/dist-packages/keras/src/optimizers/optimizer.py\", line 276, in compute_gradients\n        grads = tape.gradient(loss, var_list)\n\n    TypeError: Argument `target` should be a list or nested structure of Tensors, Variables or CompositeTensors to be differentiated, but received None.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "gv2JMXOWAlRE"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import tensorflow as tf\n",
        "from transformers import GPT2Tokenizer, TFGPT2LMHeadModel\n",
        "\n",
        "# Load pre-trained GPT-2 model and tokenizer\n",
        "model = TFGPT2LMHeadModel.from_pretrained('gpt2')\n",
        "tokenizer = GPT2Tokenizer.from_pretrained('gpt2')\n",
        "\n",
        "# Add the EOS token as the pad token\n",
        "tokenizer.pad_token = tokenizer.eos_token\n",
        "\n",
        "# Load your domain-specific data\n",
        "train_data_path = '/content/advertising.csv'\n",
        "with open(train_data_path, 'r', encoding='utf-8') as file:\n",
        "    train_texts = file.readlines()\n",
        "\n",
        "# Tokenize and encode the texts\n",
        "input_ids = tokenizer(train_texts, return_tensors='tf', padding=True, truncation=True)\n",
        "\n",
        "# Shift the input sequence to create target sequence\n",
        "labels = tf.roll(input_ids['input_ids'], shift=-1, axis=-1)\n",
        "\n",
        "# Create TensorFlow Dataset\n",
        "train_dataset = tf.data.Dataset.from_tensor_slices((dict(input_ids), labels))\n",
        "\n",
        "# Define training parameters\n",
        "batch_size = 8\n",
        "num_epochs = 3\n",
        "learning_rate = 5e-5\n",
        "\n",
        "# Compile the model\n",
        "model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=learning_rate),\n",
        "              loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True, reduction=tf.keras.losses.Reduction.NONE),\n",
        "              metrics=[tf.keras.metrics.SparseCategoricalAccuracy(name=\"accuracy\")])\n",
        "\n",
        "# Train the model\n",
        "model.fit(train_dataset.shuffle(1000).batch(batch_size),\n",
        "          epochs=num_epochs)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hFqo75c3Alaj",
        "outputId": "a423082e-389a-40a3-e0b7-e9414711555a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "All PyTorch model weights were used when initializing TFGPT2LMHeadModel.\n",
            "\n",
            "All the weights of TFGPT2LMHeadModel were initialized from the PyTorch model.\n",
            "If your task is similar to the task the model of the checkpoint was trained on, you can already use TFGPT2LMHeadModel for predictions without further training.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/3\n",
            "126/126 [==============================] - 60s 206ms/step - loss: 2.2341 - accuracy: 0.5788\n",
            "Epoch 2/3\n",
            "126/126 [==============================] - 26s 207ms/step - loss: 1.6445 - accuracy: 0.6345\n",
            "Epoch 3/3\n",
            "126/126 [==============================] - 25s 202ms/step - loss: 1.5473 - accuracy: 0.6497\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<keras.src.callbacks.History at 0x7b3bc1f3ee60>"
            ]
          },
          "metadata": {},
          "execution_count": 87
        }
      ]
    }
  ]
}