VectorDethcod.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "view-in-github"
   },
   "source": [
    "<a href=\"https://colab.research.google.com/github/khoda81/dethcod/blob/main/DETHCOD_demo.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "vINNjRbwnFOa",
    "outputId": "e84d83bf-0c13-433d-9a4e-346c37e9cc47"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: transformers in /usr/local/lib/python3.10/dist-packages (4.38.2)\n",
      "Collecting wandb\n",
      "  Downloading wandb-0.16.6-py3-none-any.whl (2.2 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.2/2.2 MB\u001b[0m \u001b[31m37.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting requests_cache\n",
      "  Downloading requests_cache-1.2.0-py3-none-any.whl (61 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m61.4/61.4 kB\u001b[0m \u001b[31m9.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting datasets\n",
      "  Downloading datasets-2.19.0-py3-none-any.whl (542 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m542.0/542.0 kB\u001b[0m \u001b[31m55.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (4.66.2)\n",
      "Collecting python-dotenv\n",
      "  Downloading python_dotenv-1.0.1-py3-none-any.whl (19 kB)\n",
      "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from transformers) (3.13.4)\n",
      "Requirement already satisfied: huggingface-hub<1.0,>=0.19.3 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.20.3)\n",
      "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (1.25.2)\n",
      "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from transformers) (24.0)\n",
      "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (6.0.1)\n",
      "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (2023.12.25)\n",
      "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from transformers) (2.31.0)\n",
      "Requirement already satisfied: tokenizers<0.19,>=0.14 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.15.2)\n",
      "Requirement already satisfied: safetensors>=0.4.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.4.3)\n",
      "Requirement already satisfied: Click!=8.0.0,>=7.1 in /usr/local/lib/python3.10/dist-packages (from wandb) (8.1.7)\n",
      "Collecting GitPython!=3.1.29,>=1.0.0 (from wandb)\n",
      "  Downloading GitPython-3.1.43-py3-none-any.whl (207 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m207.3/207.3 kB\u001b[0m \u001b[31m19.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: psutil>=5.0.0 in /usr/local/lib/python3.10/dist-packages (from wandb) (5.9.5)\n",
      "Collecting sentry-sdk>=1.0.0 (from wandb)\n",
      "  Downloading sentry_sdk-1.45.0-py2.py3-none-any.whl (267 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m267.1/267.1 kB\u001b[0m \u001b[31m31.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting docker-pycreds>=0.4.0 (from wandb)\n",
      "  Downloading docker_pycreds-0.4.0-py2.py3-none-any.whl (9.0 kB)\n",
      "Collecting setproctitle (from wandb)\n",
      "  Downloading setproctitle-1.3.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (30 kB)\n",
      "Requirement already satisfied: setuptools in /usr/local/lib/python3.10/dist-packages (from wandb) (67.7.2)\n",
      "Requirement already satisfied: appdirs>=1.4.3 in /usr/local/lib/python3.10/dist-packages (from wandb) (1.4.4)\n",
      "Requirement already satisfied: protobuf!=4.21.0,<5,>=3.19.0 in /usr/local/lib/python3.10/dist-packages (from wandb) (3.20.3)\n",
      "Requirement already satisfied: attrs>=21.2 in /usr/local/lib/python3.10/dist-packages (from requests_cache) (23.2.0)\n",
      "Collecting cattrs>=22.2 (from requests_cache)\n",
      "  Downloading cattrs-23.2.3-py3-none-any.whl (57 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m57.5/57.5 kB\u001b[0m \u001b[31m9.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: platformdirs>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests_cache) (4.2.0)\n",
      "Collecting url-normalize>=1.4 (from requests_cache)\n",
      "  Downloading url_normalize-1.4.3-py2.py3-none-any.whl (6.8 kB)\n",
      "Requirement already satisfied: urllib3>=1.25.5 in /usr/local/lib/python3.10/dist-packages (from requests_cache) (2.0.7)\n",
      "Requirement already satisfied: pyarrow>=12.0.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (14.0.2)\n",
      "Requirement already satisfied: pyarrow-hotfix in /usr/local/lib/python3.10/dist-packages (from datasets) (0.6)\n",
      "Collecting dill<0.3.9,>=0.3.0 (from datasets)\n",
      "  Downloading dill-0.3.8-py3-none-any.whl (116 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m116.3/116.3 kB\u001b[0m \u001b[31m16.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from datasets) (2.0.3)\n",
      "Collecting xxhash (from datasets)\n",
      "  Downloading xxhash-3.4.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (194 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m194.1/194.1 kB\u001b[0m \u001b[31m20.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting multiprocess (from datasets)\n",
      "  Downloading multiprocess-0.70.16-py310-none-any.whl (134 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m134.8/134.8 kB\u001b[0m \u001b[31m18.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: fsspec[http]<=2024.3.1,>=2023.1.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (2023.6.0)\n",
      "Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from datasets) (3.9.5)\n",
      "Collecting huggingface-hub<1.0,>=0.19.3 (from transformers)\n",
      "  Downloading huggingface_hub-0.22.2-py3-none-any.whl (388 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m388.9/388.9 kB\u001b[0m \u001b[31m40.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: exceptiongroup>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from cattrs>=22.2->requests_cache) (1.2.0)\n",
      "Requirement already satisfied: typing-extensions!=4.6.3,>=4.1.0 in /usr/local/lib/python3.10/dist-packages (from cattrs>=22.2->requests_cache) (4.11.0)\n",
      "Requirement already satisfied: six>=1.4.0 in /usr/local/lib/python3.10/dist-packages (from docker-pycreds>=0.4.0->wandb) (1.16.0)\n",
      "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.3.1)\n",
      "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.4.1)\n",
      "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (6.0.5)\n",
      "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.9.4)\n",
      "Requirement already satisfied: async-timeout<5.0,>=4.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (4.0.3)\n",
      "Collecting gitdb<5,>=4.0.1 (from GitPython!=3.1.29,>=1.0.0->wandb)\n",
      "  Downloading gitdb-4.0.11-py3-none-any.whl (62 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m62.7/62.7 kB\u001b[0m \u001b[31m9.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (3.3.2)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (3.7)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2024.2.2)\n",
      "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets) (2.8.2)\n",
      "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets) (2023.4)\n",
      "Requirement already satisfied: tzdata>=2022.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets) (2024.1)\n",
      "Collecting smmap<6,>=3.0.1 (from gitdb<5,>=4.0.1->GitPython!=3.1.29,>=1.0.0->wandb)\n",
      "  Downloading smmap-5.0.1-py3-none-any.whl (24 kB)\n",
      "Installing collected packages: xxhash, url-normalize, smmap, setproctitle, sentry-sdk, python-dotenv, docker-pycreds, dill, cattrs, requests_cache, multiprocess, huggingface-hub, gitdb, GitPython, wandb, datasets\n",
      "  Attempting uninstall: huggingface-hub\n",
      "    Found existing installation: huggingface-hub 0.20.3\n",
      "    Uninstalling huggingface-hub-0.20.3:\n",
      "      Successfully uninstalled huggingface-hub-0.20.3\n",
      "Successfully installed GitPython-3.1.43 cattrs-23.2.3 datasets-2.19.0 dill-0.3.8 docker-pycreds-0.4.0 gitdb-4.0.11 huggingface-hub-0.22.2 multiprocess-0.70.16 python-dotenv-1.0.1 requests_cache-1.2.0 sentry-sdk-1.45.0 setproctitle-1.3.3 smmap-5.0.1 url-normalize-1.4.3 wandb-0.16.6 xxhash-3.4.1\n"
     ]
    }
   ],
   "source": [
    "%pip install -r requirements.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "e-neGcFgTHdu",
    "outputId": "ed2deb1a-e533-4a0f-ca4f-209220e412aa"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[34m\u001b[1mwandb\u001b[0m: W&B API key is configured. Use \u001b[1m`wandb login --relogin`\u001b[0m to force relogin\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m If you're specifying your api key in code, ensure this code is not shared publicly.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Consider setting the WANDB_API_KEY environment variable, or running `wandb login` from the command line.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: Appending key for api.wandb.ai to your netrc file: C:\\Users\\20mah\\.netrc\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: Network error (ConnectTimeout), entering retry loop.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: Network error (ConnectTimeout), entering retry loop.\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import wandb\n",
    "\n",
    "\n",
    "# huggingface_hub.login(token=userdata.get('huggigface_token'))\n",
    "\n",
    "try:\n",
    "    from dotenv import load_dotenv\n",
    "    # Load environment variables from .env file\n",
    "    load_dotenv()\n",
    "\n",
    "except ImportError as e:\n",
    "    print(f\"Error importing dotenv: {e}\")\n",
    "\n",
    "\n",
    "# Check if running in Colab\n",
    "try:\n",
    "    from google.colab import userdata\n",
    "    # If running in Colab, use userdata.get to retrieve the token\n",
    "    wandb.login(key=userdata.get('wandb_token'))\n",
    "\n",
    "except ImportError:\n",
    "    # If not in Colab, load the token from the environment variable\n",
    "    wandb_token = os.getenv('WANDB_TOKEN')\n",
    "    if wandb_token:\n",
    "        wandb.login(key=wandb_token)\n",
    "    else:\n",
    "        print(\"W&B token not found in environment variable. Please set WANDB_TOKEN in your environment.\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "JQb9wuBJnFOc",
    "outputId": "34a4e916-a7f6-4add-f1b0-b2726150b843"
   },
   "outputs": [],
   "source": [
    "%run -m dethcod.dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%run -i -m dethcod.vector.train --batch-size 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from dethcod.vector.model import VectorCompressionModel\n",
    "\n",
    "model = VectorCompressionModel.from_pretrained(\"data/models/vector-t5-dethcod/\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import AutoTokenizer\n",
    "\n",
    "tokenizer = AutoTokenizer.from_pretrained(\"google-t5/t5-small\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import datasets\n",
    "\n",
    "data_files = [\"data/dataset/enwik8\"]\n",
    "dataset = datasets.load_dataset(\"text\", data_files=data_files)\n",
    "dataset = dataset[\"train\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "sample = random.choice(dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "5HPbtRovhsgL",
    "outputId": "3a54f97e-650e-4f94-e6fb-41abcf9c6737"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input Sequence: \n",
      "'* [[Combustion]], in which a substance reacts with an oxidizing element, such as [[oxygen]] gas.</s>'\n",
      "\n",
      "loss: 1.5137027502059937\n",
      "\n",
      "Generated Sequences: \n",
      "'* [[Combussion methylation, oxygen]], oxygenes, which offer a reaction, to</s>'\n",
      "'* Interactions in [[gas]], which have fueling fuel through oxygen, and fuel to fuel, the substance of that substance in a combustions.</s>'\n",
      "'* In exchange reaction reaction to elements, [[oxidation]] oxygenes, containing a substance, which gases in oxidation.</s>'\n",
      "'* Besides a composting substance, oxygening compounds to aceas, airing (reaction), and a [[combustion]]</s>'\n",
      "'* <unk>reactions in the compound-sicing process, like hydrogen, [[oxygen]], [[olinex]], in which anoxygens matter in which</s>'\n",
      "\"* The inoxygenation reaction comes into a component of Oxgenitine, such as ''oxygenoideline in air, such as fuel</s>\"\n",
      "'* In more gases, oxygens in oxidation like a reaction containing [[cinemit], in a substance, in substance in a form</s>'\n",
      "'* in [[conotation]], the reaction to [[gastine]], [[oxyglycine ice]], containing salt, reacts to being in such a manner, like in [[gas]].</s>'\n",
      "'* [[oxygenation]] - In [[aircraft]], in [[oxygens]] a reaction to which, in which Oxide reacts, acts like an incommission, like</s>'\n",
      "\"* ''Inoxygenium, a [[pressure]] reaction into [[aircraft], combustion]]s, as a reaction to a substance</s>\"\n",
      "'* [[Geonium to react shock gases, in which [[oxygen]] gases, use the gas of fuel, in addition to reacting with oxygens, compounding in which gases are absorbed.</s>'\n",
      "'* [[Commuting]], [[prostate gas]], the origin of [[nitrogen]] as it contains a substance, an oxygen-related substance:</s>'\n",
      "'* [[Recommission in [[oxygen]], a reaction that reacts to gases in other substances, like a [[oxygen]], as an intercommittotion, fuel, earth ([[gasoning]].</s>'\n",
      "\"* ''[[Goneoxylation]], an interoxygen from oxygen, which in its oxidation agent, thus fuels a reaction inoxygens, which acts a [[gas]], absorbing [[somerea]], [[acetone]], [[prostate]], [[carbonoxylin]], [oxygastine], [[carbon dioxide]]s</s>\"\n",
      "'* In [[oxidation]], [[combustion]], a compound reaction like oxygen, which then generates a substance.</s>'\n",
      "'* In addition, gases and [[reazolation]], as a fuel reaction containing compounds like gas based upon the</s>'\n"
     ]
    }
   ],
   "source": [
    "from transformers import GenerationConfig\n",
    "from transformers.modeling_outputs import BaseModelOutput\n",
    "\n",
    "input_ids = tokenizer.encode(\n",
    "    sample[\"text\"],\n",
    "    return_tensors=\"pt\",\n",
    "    padding=True,\n",
    "    truncation=True,\n",
    ").input_ids.to(model.device)\n",
    "\n",
    "print(\"Input Sequence: \")\n",
    "print(repr(tokenizer.decode(input_ids[0])))\n",
    "print()\n",
    "\n",
    "model_output = model.forward(input_ids)\n",
    "print(\"loss:\", model_output.loss.item())\n",
    "print()\n",
    "\n",
    "print(\"Compressing...\")\n",
    "compressed_form = model.compress(input_ids)\n",
    "\n",
    "print(\"Decompressing...\")\n",
    "last_hidden_state = model.unpooling_layer.forward(compressed_form)\n",
    "encoder_outputs = BaseModelOutput(last_hidden_state=last_hidden_state)\n",
    "generation_config = GenerationConfig(\n",
    "    do_sample=True,\n",
    "    num_return_sequences=16,\n",
    "    max_new_tokens=100,\n",
    ")\n",
    "\n",
    "generation_output = model.generate(\n",
    "    encoder_outputs=encoder_outputs,\n",
    "    generation_config=generation_config,\n",
    ")\n",
    "\n",
    "print(\"Generated Sequences: \")\n",
    "for seq in generation_output.tolist():\n",
    "    seq = [token for token in seq if token != 0]\n",
    "    print(repr(tokenizer.decode(seq)))\n"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "gpuType": "T4",
   "include_colab_link": true,
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.18"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "03a6bb59a8b84596b5092980729e9fc7": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "0ef902eca756472a8b4352dc19cfc25d": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "10924208b95049bda8d8a0df9c682bb2": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "3cfc9e627d02409792d0b4b6b39a8afa": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "54f9aaa4efaa4a77a1bc162f9ae994f5": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_725091c591c34c569b641a5f48130635",
      "max": 1128024,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_3cfc9e627d02409792d0b4b6b39a8afa",
      "value": 1128024
     }
    },
    "725091c591c34c569b641a5f48130635": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "7875e89599ea40b99edb43be81e67f21": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "a683eb3c780743a7afba4cb3bb5c9cc7": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "c91d4bfb8c4d434d9434dcb23cda86b7": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_a683eb3c780743a7afba4cb3bb5c9cc7",
      "placeholder": "​",
      "style": "IPY_MODEL_0ef902eca756472a8b4352dc19cfc25d",
      "value": " 1128024/1128024 [04:39&lt;00:00, 3025.60 examples/s]"
     }
    },
    "ce328cd910f24f78aadd4b7c1e8065b5": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_fc81b24e5e4a4402bd44d7dd4f9e706d",
       "IPY_MODEL_54f9aaa4efaa4a77a1bc162f9ae994f5",
       "IPY_MODEL_c91d4bfb8c4d434d9434dcb23cda86b7"
      ],
      "layout": "IPY_MODEL_03a6bb59a8b84596b5092980729e9fc7"
     }
    },
    "fc81b24e5e4a4402bd44d7dd4f9e706d": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_7875e89599ea40b99edb43be81e67f21",
      "placeholder": "​",
      "style": "IPY_MODEL_10924208b95049bda8d8a0df9c682bb2",
      "value": "Filter: 100%"
     }
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}