lab3.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "checkpoint = \"prajjwal1/bert-tiny\"\n",
    "tokenizer_checkpoint = \"bert-base-uncased\"\n",
    "dataset_name = \"imdb\"\n",
    "\n",
    "from transformers import AutoModel\n",
    "\n",
    "model = AutoModel.from_pretrained(checkpoint)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pathlib import Path\n",
    "import dill\n",
    "\n",
    "\n",
    "with open(f\"{Path.home()}/tutorial_5_best_model.pkl\", \"rb\") as f:\n",
    "    base_model = dill.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32mINFO    \u001b[0m \u001b[34mTokenizing dataset imdb with AutoTokenizer for bert-base-uncased.\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "from chop.tools import get_tokenized_dataset\n",
    "\n",
    "dataset, tokenizer = get_tokenized_dataset(\n",
    "    dataset=dataset_name,\n",
    "    checkpoint=tokenizer_checkpoint,\n",
    "    return_tokenizer=True,\n",
    ")\n",
    "import torch\n",
    "from chop.nn.quantized.modules.linear import (\n",
    "    LinearInteger,\n",
    "    LinearMinifloatDenorm,\n",
    "    LinearMinifloatIEEE,\n",
    "    LinearLog,\n",
    "    LinearBlockFP,\n",
    "    LinearBlockMinifloat,\n",
    "    LinearBlockLog,\n",
    "    LinearBinary,\n",
    "    LinearBinaryScaling,\n",
    "    LinearBinaryResidualSign,\n",
    ")\n",
    "\n",
    "search_space = {\n",
    "    \"linear_layer_choices\": [\n",
    "        torch.nn.Linear,\n",
    "        LinearInteger,\n",
    "    ],\n",
    "    'linear_layer_quantization': [8, 16, 32],\n",
    "    'linear_layer_quantization_fractional':  [2, 4, 8],\n",
    "}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from chop.tools.utils import deepsetattr\n",
    "from copy import deepcopy\n",
    "from transformers import AutoConfig, AutoModelForSequenceClassification\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from chop.tools.utils import deepsetattr\n",
    "from copy import deepcopy\n",
    "from transformers import AutoConfig, AutoModelForSequenceClassification\n",
    "\n",
    "def construct_model(trial):\n",
    "    # Fetch the model\n",
    "    my_config={}\n",
    "    trial_model = deepcopy(base_model)\n",
    "    for param in ['linear_layer_quantization','linear_layer_quantization_fractional']:\n",
    "        chosen_idx = trial.suggest_int(param, 0, len(search_space[param]) - 1)\n",
    "        my_config[param]=chosen_idx\n",
    "\n",
    "    # Quantize layers according to optuna suggestions\n",
    "    for name, layer in trial_model.named_modules():\n",
    "        if isinstance(layer, torch.nn.Linear):\n",
    "            new_layer_cls = trial.suggest_categorical(\n",
    "                f\"{name}_type\",\n",
    "                search_space[\"linear_layer_choices\"],\n",
    "            )\n",
    "\n",
    "            if new_layer_cls == torch.nn.Linear:\n",
    "                continue\n",
    "\n",
    "            kwargs = {\n",
    "                \"in_features\": layer.in_features,\n",
    "                \"out_features\": layer.out_features,\n",
    "            }\n",
    "\n",
    "            # If the chosen layer is integer, define the low precision config\n",
    "            if new_layer_cls == LinearInteger:\n",
    "                kwargs[\"config\"] = {\n",
    "                    \"data_in_width\": my_config['linear_layer_quantization'],\n",
    "                    \"data_in_frac_width\": my_config['linear_layer_quantization_fractional'],\n",
    "                    \"weight_width\": my_config['linear_layer_quantization'],\n",
    "                    \"weight_frac_width\": my_config['linear_layer_quantization_fractional'],\n",
    "                    \"bias_width\": my_config['linear_layer_quantization'],\n",
    "                    \"bias_frac_width\": my_config['linear_layer_quantization_fractional'],\n",
    "                }\n",
    "            # elif... (other precisions)\n",
    "\n",
    "            # Create the new layer (copy the weights)\n",
    "            new_layer = new_layer_cls(**kwargs)\n",
    "            new_layer.weight.data = layer.weight.data\n",
    "\n",
    "            # Replace the layer in the model\n",
    "            deepsetattr(trial_model, name, new_layer)\n",
    "\n",
    "    return trial_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-06 20:41:23,640] A new study created in memory with name: bert-tiny-nas-study\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearInteger'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 01:17, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.606000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.439800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.371200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.353200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.342600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.361000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 00:34]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-06 20:43:16,322] Trial 0 finished with value: 0.84784 and parameters: {'linear_layer_quantization': 2, 'linear_layer_quantization_fractional': 1, 'bert.encoder.layer.0.attention.self.query_type': <class 'chop.nn.quantized.modules.linear.LinearInteger'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearInteger'>, 'bert.encoder.layer.0.output.dense_type': <class 'chop.nn.quantized.modules.linear.LinearInteger'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'chop.nn.quantized.modules.linear.LinearInteger'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearInteger'>, 'bert.encoder.layer.1.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.pooler.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'classifier_type': <class 'torch.nn.modules.linear.Linear'>}. Best is trial 0 with value: 0.84784.\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearInteger'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 01:21, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.693100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.693100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.693100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.693200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.693200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.693200</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 00:47]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-06 20:45:26,600] Trial 1 finished with value: 0.5 and parameters: {'linear_layer_quantization': 1, 'linear_layer_quantization_fractional': 0, 'bert.encoder.layer.0.attention.self.query_type': <class 'chop.nn.quantized.modules.linear.LinearInteger'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'chop.nn.quantized.modules.linear.LinearInteger'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearInteger'>, 'bert.encoder.layer.0.output.dense_type': <class 'chop.nn.quantized.modules.linear.LinearInteger'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'chop.nn.quantized.modules.linear.LinearInteger'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.output.dense_type': <class 'chop.nn.quantized.modules.linear.LinearInteger'>, 'bert.pooler.dense_type': <class 'chop.nn.quantized.modules.linear.LinearInteger'>, 'classifier_type': <class 'torch.nn.modules.linear.Linear'>}. Best is trial 0 with value: 0.84784.\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearInteger'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 01:20, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.693100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.693100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.693100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.693100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.693100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.693100</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 00:38]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-06 20:47:26,961] Trial 2 finished with value: 0.5 and parameters: {'linear_layer_quantization': 2, 'linear_layer_quantization_fractional': 0, 'bert.encoder.layer.0.attention.self.query_type': <class 'chop.nn.quantized.modules.linear.LinearInteger'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearInteger'>, 'bert.encoder.layer.0.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearInteger'>, 'bert.encoder.layer.1.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.pooler.dense_type': <class 'chop.nn.quantized.modules.linear.LinearInteger'>, 'classifier_type': <class 'chop.nn.quantized.modules.linear.LinearInteger'>}. Best is trial 0 with value: 0.84784.\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearInteger'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 01:14, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.693100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.693100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.693100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.693100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.693100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.693100</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 00:30]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-06 20:49:13,570] Trial 3 finished with value: 0.5 and parameters: {'linear_layer_quantization': 1, 'linear_layer_quantization_fractional': 0, 'bert.encoder.layer.0.attention.self.query_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'chop.nn.quantized.modules.linear.LinearInteger'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearInteger'>, 'bert.encoder.layer.1.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.pooler.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'classifier_type': <class 'chop.nn.quantized.modules.linear.LinearInteger'>}. Best is trial 0 with value: 0.84784.\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearInteger'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 01:10, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.693100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.693100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.693100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.693200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.693100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.693100</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 00:30]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-06 20:50:55,667] Trial 4 finished with value: 0.5 and parameters: {'linear_layer_quantization': 2, 'linear_layer_quantization_fractional': 1, 'bert.encoder.layer.0.attention.self.query_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'chop.nn.quantized.modules.linear.LinearInteger'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.pooler.dense_type': <class 'chop.nn.quantized.modules.linear.LinearInteger'>, 'classifier_type': <class 'torch.nn.modules.linear.Linear'>}. Best is trial 0 with value: 0.84784.\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearInteger'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 01:18, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.697800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.695900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.697800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.696700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.695400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.696500</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 00:36]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-06 20:52:51,269] Trial 5 finished with value: 0.5 and parameters: {'linear_layer_quantization': 0, 'linear_layer_quantization_fractional': 0, 'bert.encoder.layer.0.attention.self.query_type': <class 'chop.nn.quantized.modules.linear.LinearInteger'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.output.dense_type': <class 'chop.nn.quantized.modules.linear.LinearInteger'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'chop.nn.quantized.modules.linear.LinearInteger'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.output.dense_type': <class 'chop.nn.quantized.modules.linear.LinearInteger'>, 'bert.pooler.dense_type': <class 'chop.nn.quantized.modules.linear.LinearInteger'>, 'classifier_type': <class 'torch.nn.modules.linear.Linear'>}. Best is trial 0 with value: 0.84784.\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearInteger'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 01:15, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.693100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.693100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.693100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.693100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.693100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.693100</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 00:33]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-06 20:54:41,806] Trial 6 finished with value: 0.5 and parameters: {'linear_layer_quantization': 0, 'linear_layer_quantization_fractional': 1, 'bert.encoder.layer.0.attention.self.query_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'chop.nn.quantized.modules.linear.LinearInteger'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearInteger'>, 'bert.encoder.layer.1.output.dense_type': <class 'chop.nn.quantized.modules.linear.LinearInteger'>, 'bert.pooler.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'classifier_type': <class 'chop.nn.quantized.modules.linear.LinearInteger'>}. Best is trial 0 with value: 0.84784.\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearInteger'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 01:11, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.693100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.693100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.693100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.693100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.693100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.693100</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 00:35]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-06 20:56:29,374] Trial 7 finished with value: 0.5 and parameters: {'linear_layer_quantization': 0, 'linear_layer_quantization_fractional': 2, 'bert.encoder.layer.0.attention.self.query_type': <class 'chop.nn.quantized.modules.linear.LinearInteger'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearInteger'>, 'bert.encoder.layer.1.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.pooler.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'classifier_type': <class 'chop.nn.quantized.modules.linear.LinearInteger'>}. Best is trial 0 with value: 0.84784.\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearInteger'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 01:11, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.693100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.693100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.693100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.693100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.693100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.693100</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 00:31]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-06 20:58:13,295] Trial 8 finished with value: 0.5 and parameters: {'linear_layer_quantization': 2, 'linear_layer_quantization_fractional': 2, 'bert.encoder.layer.0.attention.self.query_type': <class 'chop.nn.quantized.modules.linear.LinearInteger'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearInteger'>, 'bert.encoder.layer.0.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'chop.nn.quantized.modules.linear.LinearInteger'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.pooler.dense_type': <class 'chop.nn.quantized.modules.linear.LinearInteger'>, 'classifier_type': <class 'chop.nn.quantized.modules.linear.LinearInteger'>}. Best is trial 0 with value: 0.84784.\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearInteger'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 01:14, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.693100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.693100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.693100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.693100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.693100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.693100</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 00:34]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-06 21:00:03,633] Trial 9 finished with value: 0.5 and parameters: {'linear_layer_quantization': 1, 'linear_layer_quantization_fractional': 0, 'bert.encoder.layer.0.attention.self.query_type': <class 'chop.nn.quantized.modules.linear.LinearInteger'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearInteger'>, 'bert.encoder.layer.0.output.dense_type': <class 'chop.nn.quantized.modules.linear.LinearInteger'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.output.dense_type': <class 'chop.nn.quantized.modules.linear.LinearInteger'>, 'bert.pooler.dense_type': <class 'chop.nn.quantized.modules.linear.LinearInteger'>, 'classifier_type': <class 'chop.nn.quantized.modules.linear.LinearInteger'>}. Best is trial 0 with value: 0.84784.\n"
     ]
    }
   ],
   "source": [
    "from chop.tools import get_trainer\n",
    "import random\n",
    "\n",
    "\n",
    "def objective(trial):\n",
    "\n",
    "    # Define the model\n",
    "    model = construct_model(trial)\n",
    "\n",
    "    trainer = get_trainer(\n",
    "        model=model,\n",
    "        tokenized_dataset=dataset,\n",
    "        tokenizer=tokenizer,\n",
    "        evaluate_metric=\"accuracy\",\n",
    "        num_train_epochs=1,\n",
    "    )\n",
    "\n",
    "    trainer.train()\n",
    "    eval_results = trainer.evaluate()\n",
    "\n",
    "    trial.set_user_attr(\"model\", model)\n",
    "\n",
    "    return eval_results[\"eval_accuracy\"]\n",
    "\n",
    "\n",
    "from optuna.samplers import GridSampler, RandomSampler, TPESampler\n",
    "\n",
    "sampler = RandomSampler()\n",
    "\n",
    "import optuna\n",
    "\n",
    "study = optuna.create_study(\n",
    "    direction=\"maximize\",\n",
    "    study_name=\"bert-tiny-nas-study\",\n",
    "    sampler=sampler,\n",
    ")\n",
    "\n",
    "study.optimize(\n",
    "    objective,\n",
    "    n_trials=20,\n",
    "    timeout=60 * 60 * 24,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "task1 = [max([t.value for t in study.trials[:i+1]]) for i in range(len(study.trials))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0.84784,\n",
       " 0.84784,\n",
       " 0.84784,\n",
       " 0.84784,\n",
       " 0.84784,\n",
       " 0.84784,\n",
       " 0.84784,\n",
       " 0.84784,\n",
       " 0.84784,\n",
       " 0.84784]"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "task1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0.5,\n",
       " 0.5,\n",
       " 0.5,\n",
       " 0.5,\n",
       " 0.5,\n",
       " 0.8562,\n",
       " 0.8562,\n",
       " 0.8562,\n",
       " 0.8562,\n",
       " 0.8562,\n",
       " 0.8562,\n",
       " 0.8562,\n",
       " 0.87072,\n",
       " 0.87072,\n",
       " 0.87072,\n",
       " 0.87072,\n",
       " 0.87072,\n",
       " 0.87072,\n",
       " 0.87072,\n",
       " 0.87072]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "task1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "task1=[\n",
    " 0.5,\n",
    " 0.8562,\n",
    " 0.8562,\n",
    " 0.8562,\n",
    " 0.8562,\n",
    " 0.8562,\n",
    " 0.8562,\n",
    " 0.8562,\n",
    " 0.87072,\n",
    " 0.87072,\n",
    " 0.87072,\n",
    " 0.87072,\n",
    " 0.87072,\n",
    " 0.87072,\n",
    " 0.87072,\n",
    " 0.87072,\n",
    " 0.87072,\n",
    " 0.87072,\n",
    " 0.87072,\n",
    " 0.87072]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 270,
   "metadata": {},
   "outputs": [],
   "source": [
    "task1 = [max([t.value for t in study.trials[:i+1]]) for i in range(len(study.trials))]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "task1_pre"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkAAAAHHCAYAAABXx+fLAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/SrBM8AAAACXBIWXMAAA9hAAAPYQGoP6dpAABiEklEQVR4nO3deVhUZfsH8O8wwLDIIrIrAe4buGASLunP0NHMpXrdcgFNK3PJl8y0VNzJUiNLpUxQW4wWM0vDBZdKUUtxSyVRFDdW2REGZ87vD1+OTgzCwMAwM9/Pdc0lc+Y5z9xnDsjNs0oEQRBAREREZELM9B0AERERUX1jAkREREQmhwkQERERmRwmQERERGRymAARERGRyWECRERERCaHCRARERGZHCZAREREZHKYABEREZHJYQJERHqxefNmSCQSXLt2Td+hUDUtWrQIEolE32EQ6QQTIKJ6UP7L/q+//tL63OLiYixatAiHDh3SfWB1JDQ0FI0aNdJ3GHVKpVJh69atCAwMhJOTE+zs7NC6dWtMmDABx44dE8tduHABixYtqrNEb/369di8eXOd1E1kzJgAETVwxcXFWLx4sUElQNUxfvx43Lt3D97e3voOpUZmzpyJkJAQeHh4YNGiRVi5ciUGDRqEY8eOIS4uTix34cIFLF682CgSoPnz5+PevXv18l5Edc1c3wEQkfEoKiqCra1ttcpKpVJIpdI6jqjmVCoVFAoFrKysKryWnp6O9evXY8qUKfjss8/UXouMjERmZmaN3lMQBJSUlMDa2rpG59c1c3NzmJvz1wYZB7YAEelJeTfRrVu3MHz4cDRq1AguLi6YPXs2lEolAODatWtwcXEBACxevBgSiQQSiQSLFi0S67l06RL+85//wMnJCVZWVujWrRt27txZ4f3Onj2LPn36wNraGs2aNcOyZcsQExOjcRzOr7/+it69e8PW1hZ2dnYYPHgw/v77b43xX7lyBc8++yzs7OwwduzYal+/pjFAPj4+eO655/DHH3+ge/fusLKyQvPmzbF169YK5+fm5mLWrFnw8vKCTCZDy5YtsXLlSqhUKrVyq1atQo8ePdCkSRNYW1sjICAA33//fYX6JBIJpk+fjq+++godOnSATCZTa8l5VEpKCgRBQM+ePTXW4+rqKl7jiBEjAAD/93//J96/8ta88uvds2cPunXrBmtra3z66acAgJiYGPTr1w+urq6QyWRo3749NmzYoPZePj4++Pvvv3H48GGx7r59+2r9GWVnZ2P8+PGwt7eHo6MjQkJCcObMGUgkErXWpcrGAH355ZcICAiAtbU1nJycMHr0aNy4cUOtzOXLl/Hiiy/C3d0dVlZWaNasGUaPHo28vDyNnzFRXWMqT6RHSqUScrkcgYGBWLVqFfbv34/Vq1ejRYsWmDp1KlxcXLBhwwZMnToVzz//PF544QUAgL+/PwDg77//Rs+ePdG0aVPMnTsXtra2+PbbbzF8+HD88MMPeP755wEAt27dEn8Bz5s3D7a2tvj8888hk8kqxPTFF18gJCQEcrkcK1euRHFxMTZs2IBevXohMTERPj4+Ytn79+9DLpejV69eWLVqFWxsbGr9mSQnJ+M///kPXn75ZYSEhCA6OhqhoaEICAhAhw4dADzoFuzTpw9u3bqFV199FU888QSOHj2KefPm4c6dO4iMjBTr++ijjzB06FCMHTsWCoUC33zzDUaMGIFffvkFgwcPVnvvAwcO4Ntvv8X06dPh7Oysdq2PKu+2++677zBixIhKr/vpp5/GzJkzsXbtWrzzzjto164dAIj/AkBSUhLGjBmDV199FVOmTEGbNm0AABs2bECHDh0wdOhQmJub4+eff8brr78OlUqFadOmAXjQ2jRjxgw0atQI7777LgDAzc1Nq89IpVJhyJAhOHHiBKZOnYq2bdvip59+QkhISLXu1/Lly7FgwQKMHDkSkydPRmZmJj7++GM8/fTTSExMhKOjIxQKBeRyOUpLSzFjxgy4u7vj1q1b+OWXX5CbmwsHB4dqvReRTglEVOdiYmIEAMKff/4pHgsJCREACEuWLFEr26VLFyEgIEB8npmZKQAQwsPDK9T7zDPPCH5+fkJJSYl4TKVSCT169BBatWolHpsxY4YgkUiExMRE8Vh2drbg5OQkABBSUlIEQRCEgoICwdHRUZgyZYra+6SlpQkODg5qx8vjnzt3boW4QkJCBFtb22p9JuXvLQiC4O3tLQAQfvvtN/FYRkaGIJPJhDfffFM8tnTpUsHW1lb4559/1OqcO3euIJVKhdTUVPFYcXGxWhmFQiF07NhR6Nevn9pxAIKZmZnw999/PzbuchMmTBAACI0bNxaef/55YdWqVcLFixcrlPvuu+8EAMLBgwcrvFZ+vXFxcRVe+3fcgiAIcrlcaN68udqxDh06CH369KlQtrqf0Q8//CAAECIjI8UySqVS6NevnwBAiImJEY+Hh4cLj/7auHbtmiCVSoXly5ervce5c+cEc3Nz8XhiYqIAQPjuu+8qxEmkL+wCI9Kz1157Te157969cfXq1SrPu3v3Lg4cOICRI0eioKAAWVlZyMrKQnZ2NuRyOS5fvoxbt24BAOLi4hAUFITOnTuL5zs5OVXostq3bx9yc3MxZswYsb6srCxIpVIEBgbi4MGDFeKYOnVqDa66cu3bt0fv3r3F5y4uLmjTpo3aZ/Ldd9+hd+/eaNy4sVqcwcHBUCqV+O2338Syj46nycnJQV5eHnr37o1Tp05VeO8+ffqgffv21YozJiYGn3zyCXx9ffHjjz9i9uzZaNeuHZ555hnxc68OX19fyOXyCscfjTsvLw9ZWVno06cPrl69Wq1uo+p+RnFxcbCwsMCUKVPEc83MzMRWpsfZvn07VCoVRo4cqfYe7u7uaNWqlfj9Ut7Cs2fPHhQXF1dZL1F9YBcYkR5ZWVmJY3zKNW7cGDk5OVWem5ycDEEQsGDBAixYsEBjmYyMDDRt2hTXr19HUFBQhddbtmyp9vzy5csAgH79+mmsz97eXu25ubk5mjVrVmWs2njiiScqHPv3Z3L58mWcPXu2wmdXLiMjQ/z6l19+wbJly3D69GmUlpaKxzWNZfH19a12nOVJwrRp05CdnY0jR44gKioKv/76K0aPHo3ff/+9WvVU9p5HjhxBeHg4EhISKiQNeXl5VXYbVfczun79Ojw8PCp04/37e6Oy9xAEAa1atdL4uoWFBYAH1xgWFoY1a9bgq6++Qu/evTF06FCMGzeO3V+kN0yAiPSoNrOgygeyzp49W2MLAlC9X2Ka6vziiy/g7u5e4fV/zwCSyWQwM9NtQ3Jln4kgCOLXKpUK/fv3x5w5czSWbd26NQDg999/x9ChQ/H0009j/fr18PDwgIWFBWJiYvD1119XOK+ms6+aNGmCoUOHYujQoejbty8OHz6M69evV2uKv6b3vHLlCp555hm0bdsWa9asgZeXFywtLbF79258+OGHFQYxa1Ldz6g2VCoVJBIJfv31V4337dG1oFavXo3Q0FD89NNP2Lt3L2bOnImIiAgcO3ZM50k0UXUwASJq4Cpbebd58+YAHvyVHRwc/Ng6vL29kZycXOH4v4+1aNECAODq6lplnfrUokULFBYWVhnjDz/8ACsrK+zZs0dtwHdMTEydxdatWzccPnwYd+7cgbe3d41WTv75559RWlqKnTt3qrWIaeqCrKz+6n5G3t7eOHjwIIqLi9VagTR9v2h6D0EQ4OvrW62Eys/PD35+fpg/fz6OHj2Knj17IioqCsuWLavyXCJd4xggogau/JdSbm6u2nFXV1f07dsXn376Ke7cuVPhvEfXopHL5UhISMDp06fFY3fv3sVXX32ldo5cLoe9vT1WrFiBsrKyx9apTyNHjkRCQgL27NlT4bXc3Fzcv38fwIPWJIlEIi4rADxYWmDHjh21ev+0tDRcuHChwnGFQoH4+HiYmZmJrW/l6yL9+/49TnlryqOtXnl5eRoTN1tbW411V/czksvlKCsrw8aNG8XXVSoV1q1bV2WcL7zwAqRSKRYvXqwWa3ns2dnZAID8/Hzx/cr5+fnBzMxMrVuSqD6xBYiogbO2tkb79u0RGxuL1q1bw8nJCR07dkTHjh2xbt069OrVC35+fpgyZQqaN2+O9PR0JCQk4ObNmzhz5gwAYM6cOfjyyy/Rv39/zJgxQ5wG/8QTT+Du3btiK4K9vT02bNiA8ePHo2vXrhg9ejRcXFyQmpqKXbt2oWfPnvjkk0+qFXdZWZnGv+ydnJzw+uuv1+ozeeutt7Bz504899xz4hT5oqIinDt3Dt9//z2uXbsGZ2dnDB48GGvWrMHAgQPx0ksvISMjA+vWrUPLli1x9uzZGr//zZs30b17d/Tr1w/PPPMM3N3dkZGRgW3btuHMmTOYNWsWnJ2dAQCdO3eGVCrFypUrkZeXB5lMJq7vU5kBAwbA0tISQ4YMwauvvorCwkJs3LgRrq6uFZLdgIAAbNiwAcuWLUPLli3h6uqKfv36VfszGj58OLp3744333wTycnJaNu2LXbu3Im7d+8CqLyFCXjQArRs2TLMmzcP165dw/Dhw2FnZ4eUlBT8+OOPeOWVVzB79mwcOHAA06dPx4gRI9C6dWvcv38fX3zxBaRSKV588cUa3weiWtHjDDQik1HZNHhNU8X/PdVYEATh6NGjQkBAgGBpaVlhSvyVK1eECRMmCO7u7oKFhYXQtGlT4bnnnhO+//57tToSExOF3r17CzKZTGjWrJkQEREhrF27VgAgpKWlqZU9ePCgIJfLBQcHB8HKykpo0aKFEBoaKvz1119Vxl/+GgCNjxYtWqh9Jv+eBj948OAK9fXp06fCVO+CggJh3rx5QsuWLQVLS0vB2dlZ6NGjh7Bq1SpBoVCI5TZt2iS0atVKkMlkQtu2bYWYmBiNnzEAYdq0aRqv59/y8/OFjz76SJDL5UKzZs0ECwsLwc7OTggKChI2btwoqFQqtfIbN24UmjdvLkilUrUp8ZVdryAIws6dOwV/f3/ByspK8PHxEVauXClER0dX+MzS0tKEwYMHC3Z2dgIAtc+pup9RZmam8NJLLwl2dnaCg4ODEBoaKhw5ckQAIHzzzTdiOU2fmyA8mErfq1cvwdbWVrC1tRXatm0rTJs2TUhKShIEQRCuXr0qTJo0SWjRooVgZWUlODk5Cf/3f/8n7N+/v1qfN1FdkAjCv9otichkzJo1C59++ikKCwsb9LYUVP927NiB559/Hn/88YfGFa+JDB3HABGZiH9vYpmdnY0vvvgCvXr1YvJj4v79vaFUKvHxxx/D3t4eXbt21VNURHWLY4CITERQUBD69u2Ldu3aIT09HZs2bUJ+fn6lawiR6ZgxYwbu3buHoKAglJaWYvv27Th69ChWrFjRYDdmJaotdoERmYh33nkH33//PW7evAmJRIKuXbsiPDy8QU93p/rx9ddfY/Xq1UhOTkZJSQlatmyJqVOnYvr06foOjajOMAEiIiIik8MxQERERGRymAARERGRyeEgaA1UKhVu374NOzu7Gi1jT0RERPVPEAQUFBTA09Ozyn0KmQBpcPv2bXh5eek7DCIiIqqBGzduVLnJLhMgDezs7AA8+ADt7e31HA0RERFVR35+Pry8vMTf44/DBEiDR/dFYgJERERkWKozfIWDoImIiMjkMAEiIiIik8MEiIiIiEwOEyAiIiIyOUyAiIiIyOQwASIiIiKTwwSIiIiITA4TICIiIjI5TICIiIjI5DABIiIiIpPDBIiIiIhMDhMgIiIiMjncDJWIyEgJgoD0/FLcV6n0HQpRBXYyCzjYWOjt/ZkAEREZoWNXs7Fi90WcvZmn71CINHq9bwvMGdhWb+/PBIiIyIhcySzEe79ewr4L6QAAMwlgIeVoB2p4zM0k+n1/vb47ERHpRHZhKT6Kv4yvjqdCqRIgNZNg9JNemBXcGi52Mn2HR9TgMAEiIjJgJWVKRB9JwfqDV1BYeh8A8ExbV8wd1Bat3Oz0HB1Rw8UEiIjIAKlUAnacvoVVe5JwO68EANCxqT3eebYderRw1nN0RA0fEyAiIgNz9EoWVuy+iPO38gEAng5WeGtgGwzr1BRmeh5XQWQomAARERmI5IwCvPfrJey/mAEAaCQzx+v/1wKTevrCykKq5+iIDAsTICKiBi6zoBSR+//BN3/eEAc4jw18AjOfaQXnRhzgTFQTTICIiBqoewolNv1xFRsOXUGRQgkA6N/eDXMHtUULl0Z6jo7IsDEBIiJqYFQqAdsTHwxwTst/MMDZv5kD3n22HQKbN9FzdETGgQkQEVEDciQ5C8t3XcSFOw8GODd1tMacgW0wxN+TA5yJdIgJEBFRA/BPegEidl/EwaRMAICdlTmm/V9LhPbw4QBnojrABIiISI8yCkrw4b7LiP0zFSrhwfYA457yxsxnWsHJ1lLf4REZLSZARDrwcfxlbDh8BSpB0HcoZGAU91VQ/e/bZmAHd7w9qC18nW31GxSRCWACRKQDP56+heL/zdIh0lYnL0fMH9wOT/o46TsUIpOh9wRo3bp1+OCDD5CWloZOnTrh448/Rvfu3SstHxkZiQ0bNiA1NRXOzs74z3/+g4iICFhZWQEAFi1ahMWLF6ud06ZNG1y6dKlOr4NMW25xGQBgy6TuaM6/3kkL5lIJ3O2tIJFwgDNRfdJrAhQbG4uwsDBERUUhMDAQkZGRkMvlSEpKgqura4XyX3/9NebOnYvo6Gj06NED//zzD0JDQyGRSLBmzRqxXIcOHbB//37xubm53vM8MmIqlYDcYgUAoK27HdzsrfQcERERVcVMn2++Zs0aTJkyBRMnTkT79u0RFRUFGxsbREdHayx/9OhR9OzZEy+99BJ8fHwwYMAAjBkzBidOnFArZ25uDnd3d/Hh7MyNAanuFJTcF8dwONpY6DcYIiKqFr0lQAqFAidPnkRwcPDDYMzMEBwcjISEBI3n9OjRAydPnhQTnqtXr2L37t149tln1cpdvnwZnp6eaN68OcaOHYvU1NS6uxAyebn3HrT+2FhKITPndGUiIkOgt76hrKwsKJVKuLm5qR13c3OrdLzOSy+9hKysLPTq1QuCIOD+/ft47bXX8M4774hlAgMDsXnzZrRp0wZ37tzB4sWL0bt3b5w/fx52dnYa6y0tLUVpaan4PD8/XwdXSKYi53/jfxyt2fpDRGQo9NoFpq1Dhw5hxYoVWL9+PU6dOoXt27dj165dWLp0qVhm0KBBGDFiBPz9/SGXy7F7927k5ubi22+/rbTeiIgIODg4iA8vL6/6uBwyEjn/G//jaMM1W4iIDIXeWoCcnZ0hlUqRnp6udjw9PR3u7u4az1mwYAHGjx+PyZMnAwD8/PxQVFSEV155Be+++y7MzCrmc46OjmjdujWSk5MrjWXevHkICwsTn+fn5zMJomorHwDd2JYtQEREhkJvLUCWlpYICAhAfHy8eEylUiE+Ph5BQUEazykuLq6Q5EilD8ZcCJUsQFdYWIgrV67Aw8Oj0lhkMhns7e3VHkTVlVP0vy4wtgARERkMvc4PDwsLQ0hICLp164bu3bsjMjISRUVFmDhxIgBgwoQJaNq0KSIiIgAAQ4YMwZo1a9ClSxcEBgYiOTkZCxYswJAhQ8REaPbs2RgyZAi8vb1x+/ZthIeHQyqVYsyYMXq7TjJuYgsQZ4ARERkMvSZAo0aNQmZmJhYuXIi0tDR07twZcXFx4sDo1NRUtRaf+fPnQyKRYP78+bh16xZcXFwwZMgQLF++XCxz8+ZNjBkzBtnZ2XBxcUGvXr1w7NgxuLi41Pv1kWnIvfegBagxW4CIiAyGRKis78iE5efnw8HBAXl5eewOoyrN2JaIn8/cxvzB7TC5d3N9h0NEZLK0+f1tULPAiBqih11gbAEiIjIUTICIaimHs8CIiAwOEyCiWuIsMCIiw8MEiKiW8jgImojI4DABIqoFxX0VCkvvA+BWGEREhoQJEFEtlG+EKpEA9kyAiIgMBhMgolrI/d9GqA7WFpCaSfQcDRERVRcTIKJayCniFHgiIkPEBIioFnKKy2eAsfuLiMiQMAEiqoW8e2wBIiIyREyAiGpBbAHiAGgiIoPCBIioFspXgeYiiEREhoUJEFEt5BaVL4LIFiAiIkPCBIioFsQWIFu2ABERGRImQES1kHuPLUBERIaICRBRLeSWtwBZswWIiMiQMAEiqgWuA0REZJiYABHVkCAIYgtQY44BIiIyKEyAiGqoSKFEmVIAwDFARESGhgkQUQ2V7wNmaW4GawupnqMhIiJtMAEiqqG8R2aASSTcCZ6IyJAwASKqoRzOACMiMlhMgIhqiDPAiIgMFxMgohoSZ4BxHzAiIoPDBIiohnLK9wGzZQsQEZGhYQJEVEO597gTPBGRoWICRFRDucXcB4yIyFAxASKqIc4CIyIyXEyAiGqIs8CIiAwXEyCiGuI+YEREhosJEFENlW+FwTFARESGhwkQUQ0oVQLyS+4D4CwwIiJDxASIqAbK9wEDAAdrtgARERkaJkBENVA+A8xOZg4LKX+MiIgMDf/nJqqB8gHQjlwFmojIIOk9AVq3bh18fHxgZWWFwMBAnDhx4rHlIyMj0aZNG1hbW8PLywv//e9/UVJSUqs6ibQlboPB8T9ERAZJrwlQbGwswsLCEB4ejlOnTqFTp06Qy+XIyMjQWP7rr7/G3LlzER4ejosXL2LTpk2IjY3FO++8U+M6iWoi9175GkBMgIiIDJFeE6A1a9ZgypQpmDhxItq3b4+oqCjY2NggOjpaY/mjR4+iZ8+eeOmll+Dj44MBAwZgzJgxai082tZJVBMPd4JnFxgRkSHSWwKkUChw8uRJBAcHPwzGzAzBwcFISEjQeE6PHj1w8uRJMeG5evUqdu/ejWeffbbGdQJAaWkp8vPz1R5Ej/NwGwwmQEREhshcX2+clZUFpVIJNzc3teNubm64dOmSxnNeeuklZGVloVevXhAEAffv38drr70mdoHVpE4AiIiIwOLFi2t5RWRKHm6DwS4wIiJDpPdB0No4dOgQVqxYgfXr1+PUqVPYvn07du3ahaVLl9aq3nnz5iEvL0983LhxQ0cRk7FiFxgRkWHTWwuQs7MzpFIp0tPT1Y6np6fD3d1d4zkLFizA+PHjMXnyZACAn58fioqK8Morr+Ddd9+tUZ0AIJPJIJPJanlFZErEWWDcB4yIyCBp3QLUp08fbN26Fffu3avVG1taWiIgIADx8fHiMZVKhfj4eAQFBWk8p7i4GGZm6iFLpVIAgCAINaqTqCY4C4yIyLBpnQB16dIFs2fPhru7O6ZMmYJjx47V+M3DwsKwceNGbNmyBRcvXsTUqVNRVFSEiRMnAgAmTJiAefPmieWHDBmCDRs24JtvvkFKSgr27duHBQsWYMiQIWIiVFWdRLqQy0HQREQGTesusMjISKxatQo7d+7Eli1b8PTTT6Nly5aYNGkSxo8fX2EA8uOMGjUKmZmZWLhwIdLS0tC5c2fExcWJdaSmpqq1+MyfPx8SiQTz58/HrVu34OLigiFDhmD58uXVrpNIF3LEMUBsASIiMkQSQRCE2lSQkZGBzz77DMuXL4dSqcSzzz6LmTNnol+/frqKsd7l5+fDwcEBeXl5sLe313c41MCUlCnRdkEcAODsogGwt2IrEBFRQ6DN7+9azQI7ceIEwsPDsXr1ari6umLevHlwdnbGc889h9mzZ9emaqIGq7z1x9xMAjuZ3uYREBFRLWj9v3dGRga++OILxMTE4PLlyxgyZAi2bdsGuVwOiUQCAAgNDcXAgQOxatUqnQdMpG+54hpAFuL3PBERGRatE6BmzZqhRYsWmDRpEkJDQ+Hi4lKhjL+/P5588kmdBEjU0IirQHP8DxGRwdI6AYqPj0fv3r0fW8be3h4HDx6scVBEDZnYAsQZYEREBkvrMUDNmjXD5cuXKxy/fPkyrl27pouYiBo0tgARERk+rROg0NBQHD16tMLx48ePIzQ0VBcxETVo5S1A3AaDiMhwaZ0AJSYmomfPnhWOP/XUUzh9+rQuYiJq0HKK/rcGELfBICIyWFonQBKJBAUFBRWO5+XlQalU6iQooobs4TYYbAEiIjJUWidATz/9NCIiItSSHaVSiYiICPTq1UunwRE1RLlcBZqIyOBpPQts5cqVePrpp9GmTRtxNtjvv/+O/Px8HDhwQOcBEjU0OZwFRkRk8LRuAWrfvj3Onj2LkSNHIiMjAwUFBZgwYQIuXbqEjh071kWMRA0KZ4ERERm+Gq3j7+npiRUrVug6FiKDIM4Cs2ULEBGRoarxRkbFxcVITU2FQqFQO+7v71/roIgaKpVK4BggIiIjoHUClJmZiYkTJ+LXX3/V+DpngpExKyi9D5Xw4GsHjgEiIjJYWo8BmjVrFnJzc3H8+HFYW1sjLi4OW7ZsQatWrbBz5866iJGowShv/bG2kMLKQqrnaIiIqKa0bgE6cOAAfvrpJ3Tr1g1mZmbw9vZG//79YW9vj4iICAwePLgu4iRqEHK4CjQRkVHQugWoqKgIrq6uAIDGjRsjMzMTAODn54dTp07pNjqiBoYzwIiIjIPWCVCbNm2QlJQEAOjUqRM+/fRT3Lp1C1FRUfDw8NB5gEQNiTgAmjPAiIgMmtZdYG+88Qbu3LkDAAgPD8fAgQPx1VdfwdLSEps3b9Z1fEQNSvkUeLYAEREZNq0ToHHjxolfBwQE4Pr167h06RKeeOIJODs76zQ4ooaGY4CIiIyDVl1gZWVlaNGiBS5evCges7GxQdeuXZn8kEko7wJztGYLEBGRIdMqAbKwsEBJSUldxULU4In7gLEFiIjIoGk9CHratGlYuXIl7t+/XxfxEDVoXAWaiMg4aD0G6M8//0R8fDz27t0LPz8/2Nraqr2+fft2nQVH1NBwHzAiIuOgdQLk6OiIF198sS5iIWrwuA4QEZFx0DoBiomJqYs4iAyCOA2e+4ARERk0rccAEZkqxX0VCksfjH3jGCAiIsOmdQuQr68vJBJJpa9fvXq1VgERNVS59x50f0kkgD1bgIiIDJrWCdCsWbPUnpeVlSExMRFxcXF46623dBUXUYNT3v3lYG0BqVnlfwQQEVHDV6OtMDRZt24d/vrrr1oHRNRQiTPA2P1FRGTwdDYGaNCgQfjhhx90VR1Rg/NwBhi7v4iIDJ3OEqDvv/8eTk5OuqqOqMF5uA0GEyAiIkOndRdYly5d1AZBC4KAtLQ0ZGZmYv369ToNjqghyWEXGBGR0dA6ARo+fLjaczMzM7i4uKBv375o27atruIianC4CCIRkfHQOgEKDw/XeRDr1q3DBx98gLS0NHTq1Akff/wxunfvrrFs3759cfjw4QrHn332WezatQsAEBoaii1btqi9LpfLERcXp/PYyXTkiS1A7AIjIjJ0WidAu3fvhlQqhVwuVzu+Z88eqFQqDBo0SKv6YmNjERYWhqioKAQGBiIyMhJyuRxJSUlwdXWtUH779u1QKBTi8+zsbHTq1AkjRoxQKzdw4EC1VatlMplWcRH9m9gCZMsWICIiQ6f1IOi5c+dCqVRWOC4IAubOnat1AGvWrMGUKVMwceJEtG/fHlFRUbCxsUF0dLTG8k5OTnB3dxcf+/btg42NTYUESCaTqZVr3Lix1rERPSqH22AQERkNrROgy5cvo3379hWOt23bFsnJyVrVpVAocPLkSQQHBz8MyMwMwcHBSEhIqFYdmzZtwujRoyvsSn/o0CG4urqiTZs2mDp1KrKzsyuto7S0FPn5+WoPon8rnwXGQdBERIZP6wTIwcFB43YXycnJFZKQqmRlZUGpVMLNzU3tuJubG9LS0qo8/8SJEzh//jwmT56sdnzgwIHYunUr4uPjsXLlShw+fBiDBg3S2HIFABEREXBwcBAfXl5eWl0HmQaxBYhjgIiIDJ7WCdCwYcMwa9YsXLlyRTyWnJyMN998E0OHDtVpcFXZtGkT/Pz8KgyYHj16NIYOHQo/Pz8MHz4cv/zyC/78808cOnRIYz3z5s1DXl6e+Lhx40Y9RE+GRBCEhy1AHANERGTwtE6A3n//fdja2qJt27bw9fWFr68v2rVrhyZNmmDVqlVa1eXs7AypVIr09HS14+np6XB3d3/suUVFRfjmm2/w8ssvV/k+zZs3h7Ozc6VddDKZDPb29moPokcVK5QoUwoAOAuMiMgYaD0LzMHBAUePHsW+fftw5swZWFtbw9/fH08//bTWb25paYmAgADEx8eL6wupVCrEx8dj+vTpjz33u+++Q2lpKcaNG1fl+9y8eRPZ2dnw8PDQOkYi4OEMMEtzM1hbSPUcDRER1ZbWCRAASCQSDBgwAAMGDKh1AGFhYQgJCUG3bt3QvXt3REZGoqioCBMnTgQATJgwAU2bNkVERITaeZs2bcLw4cPRpEkTteOFhYVYvHgxXnzxRbi7u+PKlSuYM2cOWrZsWWHqPlF15T4yA+zRldCJiMgwaZ0AzZw5Ey1btsTMmTPVjn/yySdITk5GZGSkVvWNGjUKmZmZWLhwIdLS0tC5c2fExcWJA6NTU1NhZqbeU5eUlIQ//vgDe/furVCfVCrF2bNnsWXLFuTm5sLT0xMDBgzA0qVLuRYQ1VgOZ4ARERkViSAIgjYnNG3aFDt37kRAQIDa8VOnTmHo0KG4efOmTgPUh/z8fDg4OCAvL4/jgQgAsPPMbczclohAXyfEvhqk73CIiEgDbX5/az0IOjs7Gw4ODhWO29vbIysrS9vqiAxCHluAiIiMitYJUMuWLTXuqfXrr7+iefPmOgmKqKERd4K35QwwIiJjoPUYoLCwMEyfPh2ZmZno168fACA+Ph6rV6/WevwPkaHgTvBERMZF6wRo0qRJKC0txfLly7F06VIAgI+PDzZs2IAJEyboPECihiCX+4ARERmVGk2Dnzp1KqZOnYrMzExYW1ujUaNGAIC7d+/CyclJpwESNQScBUZEZFy0HgP0KBcXFzRq1Ah79+7FyJEj0bRpU13FRdSgcB8wIiLjUuME6Pr16wgPD4ePjw9GjBgBMzMzbN26VZexETUYedwHjIjIqGjVBaZQKLB9+3Z8/vnnOHLkCIKDg3Hz5k0kJibCz8+vrmIk0jtxFhhbgIiIjEK1W4BmzJgBT09PfPTRR3j++edx8+ZN/Pzzz5BIJJBKuTcSGS+lSkB+yYMEyMGaLUBERMag2i1AGzZswNtvv425c+fCzs6uLmMialDy7pWhfL10jgEiIjIO1W4B+uKLL3DixAl4eHhg1KhR+OWXX6BUKusyNqIGoXwGmJ3MHBbSWs0bICKiBqLa/5uPGTMG+/btw7lz59C2bVtMmzYN7u7uUKlUuHDhQl3GSKRX4hpAXAWaiMhoaP3nrK+vLxYvXoxr167hyy+/xIsvvohx48ahWbNmFXaIJzIGuVwDiIjI6NRoIUQAkEgkkMvlkMvluHv3LrZu3YqYmBhdxkbUIDxcA4gJEBGRsdDJgAYnJyfMmjULZ86c0UV1RA1KeQsQt8EgIjIeHNFJVIWH22AwASIiMhZMgIiqwC4wIiLjwwSIqAp5XAWaiMjoMAEiqkIO9wEjIjI61ZoFdvbs2WpX6O/vX+NgiBqi8i4wBw6CJiIyGtVKgDp37gyJRAJBECCRSB5blqtDk7HhOkBERManWl1gKSkpuHr1KlJSUvDDDz/A19cX69evR2JiIhITE7F+/Xq0aNECP/zwQ13HS1TvcpgAEREZnWq1AHl7e4tfjxgxAmvXrsWzzz4rHvP394eXlxcWLFiA4cOH6zxIIn0pKVOipEwFgFthEBEZE60HQZ87dw6+vr4Vjvv6+nJPMDI65fuAmZtJYCer8cLpRETUwGidALVr1w4RERFQKBTiMYVCgYiICLRr106nwRHpW3n3l6ONRZXj34iIyHBo/SdtVFQUhgwZgmbNmokzvs6ePQuJRIKff/5Z5wES6VN5AsQZYERExkXrBKh79+64evUqvvrqK1y6dAkAMGrUKLz00kuwtbXVeYBE+pQrLoLIAdBERMakRoMabG1t8corr+g6FqIG52EXGBMgIiJjUqOVoL/44gv06tULnp6euH79OgDgww8/xE8//aTT4Ij0LZfbYBARGSWtE6ANGzYgLCwMgwYNQk5OjrjwYePGjREZGanr+Ij0KpfbYBARGSWtE6CPP/4YGzduxLvvvgtz84c9aN26dcO5c+d0GhyRvj3cCZ4tQERExkTrBCglJQVdunSpcFwmk6GoqEgnQRE1FOUtQI7WbAEiIjImWidAvr6+OH36dIXjcXFxXAeIjE4OxwARERklrWeBhYWFYdq0aSgpKYEgCDhx4gS2bduGiIgIfP7553URI5He5HIWGBGRUdK6BWjy5MlYuXIl5s+fj+LiYrz00kvYsGEDPvroI4wePbpGQaxbtw4+Pj6wsrJCYGAgTpw4UWnZvn37QiKRVHgMHjxYLCMIAhYuXAgPDw9YW1sjODgYly9frlFsZNrEWWDcB4yIyKjUaBr82LFjcfnyZRQWFiItLQ03b97Eyy+/XKMAYmNjERYWhvDwcJw6dQqdOnWCXC5HRkaGxvLbt2/HnTt3xMf58+chlUoxYsQIscz777+PtWvXIioqCsePH4etrS3kcjlKSkpqFCOZJkEQkHuPCyESERkjrROgZcuWISUlBQBgY2MDV1fXWgWwZs0aTJkyBRMnTkT79u0RFRUFGxsbREdHayzv5OQEd3d38bFv3z7Y2NiICZAgCIiMjMT8+fMxbNgw+Pv7Y+vWrbh9+zZ27NhRq1jJtOSX3IdSJQDgVhhERMZG6wTou+++Q8uWLdGjRw+sX78eWVlZNX5zhUKBkydPIjg4+GFAZmYIDg5GQkJCterYtGkTRo8eLW7DkZKSgrS0NLU6HRwcEBgYWO06iYCH43+sLaSwspDqORoiItIlrROgM2fO4OzZs+jbty9WrVoFT09PDB48GF9//TWKi4u1qisrKwtKpRJubm5qx93c3JCWllbl+SdOnMD58+cxefJk8Vj5edrUWVpaivz8fLUHEWeAEREZrxqNAerQoQNWrFiBq1ev4uDBg/Dx8cGsWbPg7u6u6/gea9OmTfDz80P37t1rVU9ERAQcHBzEh5eXl44iJEPGGWBERMarRgnQo2xtbWFtbQ1LS0uUlZVpda6zszOkUinS09PVjqenp1eZTBUVFeGbb76pMPi6/Dxt6pw3bx7y8vLEx40bN7S6DjJOnAFGRGS8apQApaSkYPny5ejQoQO6deuGxMRELF68uFrdVo+ytLREQEAA4uPjxWMqlQrx8fEICgp67LnfffcdSktLMW7cOLXjvr6+cHd3V6szPz8fx48fr7ROmUwGe3t7tQcRd4InIjJeWi+E+NRTT+HPP/+Ev78/Jk6ciDFjxqBp06Y1DiAsLAwhISHo1q0bunfvjsjISBQVFWHixIkAgAkTJqBp06aIiIhQO2/Tpk0YPnw4mjRponZcIpFg1qxZWLZsGVq1agVfX18sWLAAnp6eGD58eI3jJNMj7gPGGWBEREZH6wTomWeeQXR0NNq3b6+TAEaNGoXMzEwsXLgQaWlp6Ny5M+Li4sRBzKmpqTAzU2+oSkpKwh9//IG9e/dqrHPOnDkoKirCK6+8gtzcXPTq1QtxcXGwsrLSScxkGsSd4NkCRERkdCSCIAg1OVGhUCAlJQUtWrRQ2xXeGOTn58PBwQF5eXnsDjNhM7clYueZ25g/uB0m926u73CIiKgK2vz+1noM0L179/Dyyy/DxsYGHTp0QGpqKgBgxowZeO+992oWMVEDlMMWICIio6V1AjR37lycOXMGhw4dUutSCg4ORmxsrE6DI9InzgIjIjJeWvdd7dixA7GxsXjqqacgkUjE4x06dMCVK1d0GhyRPpW3ADlYswWIiMjYaN0ClJmZqXH/r6KiIrWEiMjQ5XIlaCIio6V1AtStWzfs2rVLfF6e9Hz++edVrt1DZCgU91UoLL0PgGOAiIiMkdZdYCtWrMCgQYNw4cIF3L9/Hx999BEuXLiAo0eP4vDhw3URI1G9y7v3oPVHIgHsuQ4QEZHR0boFqFevXjh9+jTu378PPz8/7N27F66urkhISEBAQEBdxEhU73LF8T8WkJqxa5eIyNjUaAGfFi1aYOPGjbqOhajBeLgTPLu/iIiMUbUSoPz8fHFBofz8/MeW5cKBZAxyHmkBIiIi41OtBKhx48a4c+cOXF1d4ejoqHG2lyAIkEgkUCqVOg+SqL493AaDCRARkTGqVgJ04MABODk5AQAOHjxYpwERNQS57AIjIjJq1UqA+vTpo/FrImMl7gTPBIiIyCjVaBB0bm4uTpw4gYyMDKhUKrXXJkyYoJPAiPSJXWBERMZN6wTo559/xtixY1FYWAh7e3u18UASiYQJEBmF8kHQjrZsASIiMkZarwP05ptvYtKkSSgsLERubi5ycnLEx927d+siRqJ6J3aBcRYYEZFR0joBunXrFmbOnAkbG5u6iIeoQXjYBcYWICIiY6R1AiSXy/HXX3/VRSxEDUauOAiaLUBERMaoWmOAdu7cKX49ePBgvPXWW7hw4QL8/PxgYaH+C2Lo0KG6jZCongmC8HAaPMcAEREZpWolQMOHD69wbMmSJRWOcSFEMgbFCiUUygezGzkLjIjIOFUrAfr3VHciY1Y+A8xSagZrC6meoyEiorqg9RggImP36PgfTdu+EBGR4dM6AZo5cybWrl1b4fgnn3yCWbNm6SImIr3iNhhERMZP6wTohx9+QM+ePSsc79GjB77//nudBEWkT+IiiBz/Q0RktLROgLKzs+Hg4FDhuL29PbKysnQSFJE+cQ0gIiLjp3UC1LJlS8TFxVU4/uuvv6J58+Y6CYpIn3LEKfBsASIiMlZa7wUWFhaG6dOnIzMzE/369QMAxMfHY/Xq1YiMjNR1fET1rrwLzMGaLUBERMZK6wRo0qRJKC0txfLly7F06VIAgI+PDzZs2MCNUMkoPBwEzRYgIiJjpXUCBABTp07F1KlTkZmZCWtrazRq1AgAcPfuXTg5Oek0QKL6xjFARETGr1brALm4uKBRo0bYu3cvRo4ciaZNm+oqLiK9yeE+YERERq/GCdD169cRHh4OHx8fjBgxAmZmZti6dasuYyPSC7EFiPuAEREZLa26wBQKBbZv347PP/8cR44cQXBwMG7evInExET4+fnVVYxE9UpsAbJmCxARkbGqdgvQjBkz4OnpiY8++gjPP/88bt68iZ9//hkSiQRSKfdLIuOgVAnILynvAmMLEBGRsap2C9CGDRvw9ttvY+7cubCzs6vLmIj0Jv9eGQThwdccA0REZLyq3QL0xRdf4MSJE/Dw8MCoUaPwyy+/QKlU1mVsRPWufA0gO5k5LKTcK5iIyFhV+3/4MWPGYN++fTh37hzatm2LadOmwd3dHSqVChcuXKjLGInqjTj+h6tAExEZNa3/xPX19cXixYtx7do1fPnll3jxxRcxbtw4NGvWDDNnztQ6gHXr1sHHxwdWVlYIDAzEiRMnHls+NzcX06ZNg4eHB2QyGVq3bo3du3eLry9atAgSiUTt0bZtW63jItPENYCIiExDjRZCBACJRAK5XA65XI67d+9i69atiImJ0aqO2NhYhIWFISoqCoGBgYiMjIRcLkdSUhJcXV0rlFcoFOjfvz9cXV3x/fffo2nTprh+/TocHR3VynXo0AH79+8Xn5ub1/gyycSUtwA5cAYYEZFR00lm4OTkhFmzZmHWrFlanbdmzRpMmTIFEydOBABERUVh165diI6Oxty5cyuUj46Oxt27d3H06FFYWDz4BeXj41OhnLm5Odzd3bW+DiK2ABERmQa9jfJUKBQ4efIkgoODHwZjZobg4GAkJCRoPGfnzp0ICgrCtGnT4Obmho4dO2LFihUVBmNfvnwZnp6eaN68OcaOHYvU1NTHxlJaWor8/Hy1B5km7gNGRGQa9JYAZWVlQalUws3NTe24m5sb0tLSNJ5z9epVfP/991Aqldi9ezcWLFiA1atXY9myZWKZwMBAbN68GXFxcdiwYQNSUlLQu3dvFBQUVBpLREQEHBwcxIeXl5duLpIMTvksMK4BRERk3AxqcIxKpYKrqys+++wzSKVSBAQE4NatW/jggw8QHh4OABg0aJBY3t/fH4GBgfD29sa3336Ll19+WWO98+bNQ1hYmPg8Pz+fSZCJYgsQEZFp0FsC5OzsDKlUivT0dLXj6enplY7f8fDwgIWFhdrK0+3atUNaWhoUCgUsLSv+1e7o6IjWrVsjOTm50lhkMhlkMlkNr4SMCVuAiIhMQ40SoJKSEpw9exYZGRlQqVRqrw0dOrRadVhaWiIgIADx8fEYPnw4gActPPHx8Zg+fbrGc3r27Imvv/4aKpUKZmYPeu/++ecfeHh4aEx+AKCwsBBXrlzB+PHjq3l1ZMq4EzwRkWnQOgGKi4vDhAkTkJWVVeE1iUSi1erQYWFhCAkJQbdu3dC9e3dERkaiqKhInBU2YcIENG3aFBEREQCAqVOn4pNPPsEbb7yBGTNm4PLly1ixYoXa+kOzZ8/GkCFD4O3tjdu3byM8PBxSqRRjxozR9lLJBOVxFhgRkUnQOgGaMWMGRowYgYULF1YYwKytUaNGITMzEwsXLkRaWho6d+6MuLg4sd7U1FSxpQcAvLy8sGfPHvz3v/+Fv78/mjZtijfeeANvv/22WObmzZsYM2YMsrOz4eLigl69euHYsWNwcXGpVaxkGnLEMUBMgIiIjJlEEMq3fqwee3t7JCYmokWLFnUVk97l5+fDwcEBeXl5sLe313c4VE9KypRouyAOAHB20QDYW7EbjIjIkGjz+1vrafD/+c9/cOjQoZrGRtRglc8Ak5pJYCczqAmSRESkJa3/l//kk08wYsQI/P777/Dz8xNXZC5Xk/3AiBoCcQaYtQUkEomeoyEiorqkdQK0bds27N27F1ZWVjh06JDaLwqJRMIEiAzWwynw7PoiIjJ2WidA7777LhYvXoy5c+eqDVAmMnR5HABNRGQytM5gFAoFRo0axeSHjM7DNYCYABERGTuts5iQkBDExsbWRSxEepUjrgHELjAiImOndReYUqnE+++/jz179sDf37/CIOg1a9boLDii+pRbngDZsgWIiMjYaZ0AnTt3Dl26dAEAnD9/Xu01zpwhQ1beBeZgzRYgIiJjp3UCdPDgwbqIg0jvcjkImojIZHAkM9H/5HIMEBGRydC6Bej//u//HtvVdeDAgVoFRKQvD9cBYgsQEZGx0zoB6ty5s9rzsrIynD59GufPn0dISIiu4iKqd2IXmC1bgIiIjJ3WCdCHH36o8fiiRYtQWFhY64CI9EEQBOTe+986QNZsASIiMnY6GwM0btw4REdH66o6onqVX3IfSpUAgFthEBGZAp0lQAkJCbCystJVdUT1qnwbDGsLKawspHqOhoiI6prWXWAvvPCC2nNBEHDnzh389ddfWLBggc4CI6pPXAWaiMi0aJ0AOTg4qD03MzNDmzZtsGTJEgwYMEBngRHVJ84AIyIyLVonQDExMXURB5FecQYYEZFp0ToBelRhYSFUKpXaMXt7+1oFRKQPYgsQZ4AREZkErQdBp6SkYPDgwbC1tYWDgwMaN26Mxo0bw9HREY0bN66LGInqXHkLEGeAERGZBq1bgMaNGwdBEBAdHQ03NzdugEpG4eE2GGwBIiIyBVonQGfOnMHJkyfRpk2buoiHSC9y2AJERGRStO4Ce/LJJ3Hjxo26iIVIb3LYAkREZFK0bgH6/PPP8dprr+HWrVvo2LEjLCzU/2L29/fXWXBE9YVjgIiITIvWCVBmZiauXLmCiRMnisckEgkEQYBEIoFSqdRpgET1gesAERGZFq0ToEmTJqFLly7Ytm0bB0GT0SjfCoMrQRMRmQatE6Dr169j586daNmyZV3EQ1TvypQqFJTeB8AxQEREpkLrQdD9+vXDmTNn6iIWIr0oH/8jkQD21mwBIiIyBVq3AA0ZMgT//e9/ce7cOfj5+VUYBD106FCdBUdUH8rXAHKwtoDUjF26RESmQOsE6LXXXgMALFmypMJrHARNhkhcA4itP0REJkPrBOjfe38RGbpczgAjIjI5Wo8BIjI2uZwBRkRkcqrVArR27Vq88sorsLKywtq1ax9bdubMmToJjKi+cBVoIiLTU60E6MMPP8TYsWNhZWWFDz/8sNJyEomECRAZnIf7gDEBIiIyFdXqAktJSUGTJk3Eryt7XL16VesA1q1bBx8fH1hZWSEwMBAnTpx4bPnc3FxMmzYNHh4ekMlkaN26NXbv3l2rOsm0PdwJnl1gRESmQusxQCUlJZW+dufOHa3qio2NRVhYGMLDw3Hq1Cl06tQJcrkcGRkZGssrFAr0798f165dw/fff4+kpCRs3LgRTZs2rXGdRA+3wWACRERkKrROgLp27YrTp09XOP7DDz9ovRHqmjVrMGXKFEycOBHt27dHVFQUbGxsEB0drbF8dHQ07t69ix07dqBnz57w8fFBnz590KlTpxrXSZTLLjAiIpOjdQLUt29fPPXUU1i5ciUAoKioCKGhoRg/fjzeeeedatejUChw8uRJBAcHPwzGzAzBwcFISEjQeM7OnTsRFBSEadOmwc3NDR07dsSKFSvEtYdqUicAlJaWIj8/X+1BpuPhLDAmQEREpkLrdYDWr1+PwYMHY/Lkyfjll19w584dNGrUCCdOnEDHjh2rXU9WVhaUSiXc3NzUjru5ueHSpUsaz7l69SoOHDiAsWPHYvfu3UhOTsbrr7+OsrIyhIeH16hOAIiIiMDixYurHTsZF3aBERGZnhqtAzRo0CC88MILOHLkCFJTU7Fy5Uqtkp+aUqlUcHV1xWeffYaAgACMGjUK7777LqKiompV77x585CXlyc+bty4oaOIqaETBOFhC5AtW4CIiEyF1i1AV65cwUsvvYS0tDTs2bMHhw8fxtChQ/HGG29g+fLlFfYGq4yzszOkUinS09PVjqenp8Pd3V3jOR4eHrCwsIBUKhWPtWvXDmlpaVAoFDWqEwBkMhlkMlm14ibjUqxQQqF8sLo5t8IgIjIdWrcAde7cGb6+vjhz5gz69++PZcuW4eDBg9i+fTu6d+9e7XosLS0REBCA+Ph48ZhKpUJ8fDyCgoI0ntOzZ08kJyerbcfxzz//wMPDA5aWljWqk0xb7r0HrT+WUjPYWEqrKE1ERMZC6wRo/fr1+Oabb+Do6Cge69GjBxITE9G1a1et6goLC8PGjRuxZcsWXLx4EVOnTkVRUREmTpwIAJgwYQLmzZsnlp86dSru3r2LN954A//88w927dqFFStWYNq0adWuk+hROUUPx/9IJNwJnojIVGjdBTZ+/HiNx+3s7LBp0yat6ho1ahQyMzOxcOFCpKWloXPnzoiLixMHMaempsLM7GGO5uXlhT179uC///0v/P390bRpU7zxxht4++23q10n0aM4A4yIyDRJBEEQanLihQsXkJqaCoVC8bAyiQRDhgzRWXD6kp+fDwcHB+Tl5cHe3l7f4VAd+vnMbczYlohAXyfEvspuUiIiQ6bN72+tW4CuXr2K559/HufOnYNEIkF5/lTefVC+Jg+RIcjlRqhERCZJ6zFAb7zxBnx9fZGRkQEbGxv8/fff+O2339CtWzccOnSoDkIkqjsPN0LlDDAiIlOidQtQQkICDhw4AGdnZ5iZmcHMzAy9evVCREQEZs6cicTExLqIk6hOcBsMIiLTpHULkFKphJ2dHYAHa/ncvn0bAODt7Y2kpCTdRkdUx7gTPBGRadK6Bahjx444c+YMfH19ERgYiPfffx+Wlpb47LPP0Lx587qIkajO5HAMEBGRSdI6AZo/fz6KiooAAEuWLMFzzz2H3r17o0mTJoiNjdV5gER1iWOAiIhMk9YJkFwuF79u2bIlLl26hLt376Jx48ZcSI4MTq64ESpbgIiITInWCZAmTk5OuqiGqN6Vb4XBMUBERKal2gnQpEmTqlUuOjq6xsEQ1SelSkDePc4CIyIyRdVOgDZv3gxvb2906dIFNVw8mqhByb9XhvJvZY4BIiIyLdVOgKZOnYpt27YhJSUFEydOxLhx49j1RQatfAaYncwcFlKtV4QgIiIDVu3/9detW4c7d+5gzpw5+Pnnn+Hl5YWRI0diz549bBEigyTOALNl6w8RkanR6s9emUyGMWPGYN++fbhw4QI6dOiA119/HT4+PigsLKyrGInqhDgDzJrjf4iITE2N2/3NzMzEzVC5ASoZolyuAUREZLK0SoBKS0uxbds29O/fH61bt8a5c+fwySefIDU1FY0aNaqrGInqBFeBJiIyXdUeBP3666/jm2++gZeXFyZNmoRt27bB2dm5LmMjqlPlLUBcA4iIyPRUOwGKiorCE088gebNm+Pw4cM4fPiwxnLbt2/XWXBEdSmHq0ATEZmsaidAEyZM4FYXZFQ4BoiIyHRptRAikTHJvccxQEREpoqrv5HJyiliCxARkaliAkQmK5ezwIiITBYTIDJZOeIsMCZARESmhgkQmaSSMiXulT1YwJNbYRARmR4mQGSSymeASc0ksJNVey4AEREZCSZAZJLKZ4A5WltweQciIhPEBIhMEmeAERGZNiZAZJI4A4yIyLQxASKTlCOuAs0EiIjIFDEBIpP0cCd4doEREZkiJkBkkvLucQwQEZEpYwJEJimniDvBExGZMiZAZJK4CjQRkWljAkQmKZdjgIiITBoTIDJJ5YOg2QVGRGSaGkQCtG7dOvj4+MDKygqBgYE4ceJEpWU3b94MiUSi9rCyslIrExoaWqHMwIED6/oyyIDkFnMQNBGRKdP7JkixsbEICwtDVFQUAgMDERkZCblcjqSkJLi6umo8x97eHklJSeJzTVsZDBw4EDExMeJzmUym++DJIAmCgNx7HANERGTK9N4CtGbNGkyZMgUTJ05E+/btERUVBRsbG0RHR1d6jkQigbu7u/hwc3OrUEYmk6mVady4cV1eBhmQgtL7UKoEAGwBIiIyVXpNgBQKBU6ePIng4GDxmJmZGYKDg5GQkFDpeYWFhfD29oaXlxeGDRuGv//+u0KZQ4cOwdXVFW3atMHUqVORnZ1dJ9dAhif3f/uAWVtIYWUh1XM0RESkD3pNgLKysqBUKiu04Li5uSEtLU3jOW3atEF0dDR++uknfPnll1CpVOjRowdu3rwplhk4cCC2bt2K+Ph4rFy5EocPH8agQYOgVCo11llaWor8/Hy1BxkvrgJNRER6HwOkraCgIAQFBYnPe/TogXbt2uHTTz/F0qVLAQCjR48WX/fz84O/vz9atGiBQ4cO4ZlnnqlQZ0REBBYvXlz3wVODwBlgRESk1xYgZ2dnSKVSpKenqx1PT0+Hu7t7teqwsLBAly5dkJycXGmZ5s2bw9nZudIy8+bNQ15envi4ceNG9S+CDA63wSAiIr0mQJaWlggICEB8fLx4TKVSIT4+Xq2V53GUSiXOnTsHDw+PSsvcvHkT2dnZlZaRyWSwt7dXe5DxKt8GgzPAiIhMl95ngYWFhWHjxo3YsmULLl68iKlTp6KoqAgTJ04EAEyYMAHz5s0Tyy9ZsgR79+7F1atXcerUKYwbNw7Xr1/H5MmTATwYIP3WW2/h2LFjuHbtGuLj4zFs2DC0bNkScrlcL9dIDUsO1wAiIjJ5eh8DNGrUKGRmZmLhwoVIS0tD586dERcXJw6MTk1NhZnZwzwtJycHU6ZMQVpaGho3boyAgAAcPXoU7du3BwBIpVKcPXsWW7ZsQW5uLjw9PTFgwAAsXbqUawERgEe3wWALEBGRqZIIgiDoO4iGJj8/Hw4ODsjLy2N3mBGauS0RO8/cxvzB7TC5d3N9h0NERDqize9vvXeBEdU3zgIjIiImQGRy8sRtMDgGiIjIVDEBIpPDFiAiImICRCanfCsMtgAREZkuJkBkUsqUKhSU3gfAWWBERKaMCRCZlNz/rQEkkQD21mwBIiIyVUyAyKTk3Xsw/sfeygJSM4meoyEiIn1hAkQmpXwVaI7/ISIybUyAyKSU7wPGGWBERKaNCRCZlFy2ABEREZgAkYnJ4T5gREQEJkBkYh7uBM8EiIjIlDEBIpNSPgvMkV1gREQmjQkQmZQcrgJNRERgAkQmhvuAERERwASITMzDWWBMgIiITBkTIDIpD1uA2AVGRGTKmACRyRAEAbn3ymeBMQEiIjJlTIDIZNwrU0JxXwWAXWBERKaOCRCZjPI1gCylZrCxlOo5GiIi0icmQGQyHu4DZgGJhDvBExGZMiZAZDI4A4yIiMoxASKTwRlgRERUzlzfARDVF84AI9I/pVKJsrIyfYdBBkoqlcLc3FwnwxiYAJHJyC3iTvBE+lRYWIibN29CEAR9h0IGzMbGBh4eHrC0rN3/5UyAyGRwJ3gi/VEqlbh58yZsbGzg4uLCiQikNUEQoFAokJmZiZSUFLRq1QpmZjUfycMEiExGbnF5CxC7wIjqW1lZGQRBgIuLC6ytrfUdDhkoa2trWFhY4Pr161AoFLCysqpxXRwETSYjp5hdYET6xpYfqq3atPqo1aOTWogMQPkgaAe2ABERmTwmQGQyuA4QEVFF165dg0QiwenTp/UdSr1iAkQmI4djgIhIS6GhoRg+fLhW50gkEuzYsaNO4qmpvn37YtasWRpf8/Lywp07d9CxY8f6DUrPmACRSVCqBOTd4ywwIjItCoWiyjJSqRTu7u4wN9fvvCilUgmVSlVv78cEiExC/r0ylC89woUQiaim+vbti5kzZ2LOnDlwcnKCu7s7Fi1aJL7u4+MDAHj++echkUjE5wDw008/oWvXrrCyskLz5s2xePFi3L9/X3z90qVL6NWrF6ysrNC+fXvs37+/QmvSjRs3MHLkSDg6OsLJyQnDhg3DtWvXxNfLW6yWL18OT09PtGnTpspr+ncX2KFDhyCRSBAfH49u3brBxsYGPXr0QFJSktp5VV3PmjVr4OfnB1tbW3h5eeH1119HYWGh+PrmzZvh6OiInTt3on379pDJZEhNTa0yXl3hNHgyCeXdX3Yyc1hImfcT6ZsgCLhXptTLe1tbSGs1G23Lli0ICwvD8ePHkZCQgNDQUPTs2RP9+/fHn3/+CVdXV8TExGDgwIGQSqUAgN9//x0TJkzA2rVr0bt3b1y5cgWvvPIKACA8PBxKpRLDhw/HE088gePHj6OgoABvvvmm2vuWlZVBLpcjKCgIv//+O8zNzbFs2TIMHDgQZ8+eFRcGjI+Ph729Pfbt21fjawSAd999F6tXr4aLiwtee+01TJo0CUeOHKnW9QAPZmutXbsWvr6+uHr1Kl5//XXMmTMH69evF9+juLgYK1euxOeff44mTZrA1dW1VjFrgwkQmQTOACNqWO6VKdF+4R69vPeFJXLYWNb815+/v7/4S75Vq1b45JNPEB8fj/79+8PFxQUA4OjoCHd3d/GcxYsXY+7cuQgJCQEANG/eHEuXLsWcOXMQHh6Offv24cqVKzh06JB43vLly9G/f3+xjtjYWKhUKnz++ediAhcTEwNHR0ccOnQIAwYMAADY2tri888/r/VKycuXL0efPn0AAHPnzsXgwYNRUlICKyurKq8HgNqYIx8fHyxbtgyvvfaaWgJUVlaG9evXo1OnTrWKtSYaxJ/C69atg4+PD6ysrBAYGIgTJ05UWnbz5s2QSCRqj38vhCQIAhYuXAgPDw9YW1sjODgYly9fruvLoAYsl2sAEZGO+Pv7qz338PBARkbGY885c+YMlixZgkaNGomPKVOm4M6dOyguLkZSUhK8vLzUkqbu3btXqCM5ORl2dnZiHU5OTigpKcGVK1fEcn5+frVOfv59nR4eHgAgXmdV1wMA+/fvxzPPPIOmTZvCzs4O48ePR3Z2tvg6AFhaWlb4POuL3luAYmNjERYWhqioKAQGBiIyMhJyuRxJSUmVNoXZ29ur9UX+uynz/fffx9q1a7Flyxb4+vpiwYIFkMvluHDhQq1WjSTDlVPEjVCJGhJrCykuLJHr7b1rw8JC/f8RiURS5eDdwsJCLF68GC+88EKF16r7e6mwsBABAQH46quvKrxW3vIEPGgB0oVHr7P892z5dVZ1PdeuXcNzzz2HqVOnYvny5XBycsIff/yBl19+GQqFAjY2NgAerOysr8Ux9Z4ArVmzBlOmTMHEiRMBAFFRUdi1axeio6Mxd+5cjedIJBK1LPlRgiAgMjIS8+fPx7BhwwAAW7duhZubG3bs2IHRo0fXzYVUQ35JGfLvcRdkfbh+98FfHGwBImoYJBJJrbqhGjILCwsolerjm7p27YqkpCS0bNlS4zlt2rTBjRs3kJ6eDjc3NwDAn3/+WaGO2NhYuLq6wt7evm6Cr6aqrufkyZNQqVRYvXq1uHLzt99+W58hVkmv330KhQInT57EvHnzxGNmZmYIDg5GQkJCpecVFhbC29sbKpUKXbt2xYoVK9ChQwcAQEpKCtLS0hAcHCyWd3BwQGBgIBISEjQmQKWlpSgtLRWf5+fn6+LyKvjy2HW8H5dUdUGqM1wDiIjqmo+PD+Lj49GzZ0/IZDI0btwYCxcuxHPPPYcnnngC//nPf2BmZoYzZ87g/PnzWLZsGfr3748WLVogJCQE77//PgoKCjB//nwAD1tfxo4diw8++ADDhg3DkiVL0KxZM1y/fh3bt2/HnDlz0KxZs8fGlZmZWWGxw/KuLW1VdT0tW7ZEWVkZPv74YwwZMgRHjhxBVFRUjd6rruh1DFBWVhaUSqWY7ZZzc3NDWlqaxnPatGmD6Oho/PTTT/jyyy+hUqnQo0cP3Lx5EwDE87SpMyIiAg4ODuLDy8urtpemkbmZBDJzMz709HCytUT/9ppbDomIdGX16tXYt28fvLy80KVLFwCAXC7HL7/8gr179+LJJ5/EU089hQ8//BDe3t4AHqzFs2PHDhQWFuLJJ5/E5MmT8e677wJ42EVmY2OD3377DU888QReeOEFtGvXDi+//DJKSkqq1SL09ddfo0uXLmqPjRs31ugaq7qeTp06Yc2aNVi5ciU6duyIr776ChERETV6r7oiEYTy1VHq3+3bt9G0aVMcPXoUQUFB4vE5c+bg8OHDOH78eJV1lJWVoV27dhgzZgyWLl2Ko0ePomfPnrh9+7ZaZjty5EhIJBLExsZWqENTC5CXlxfy8vL03sxIRGQMSkpKkJKSAl9fX47FrKYjR46gV69eSE5ORosWLfQdToPxuO+l/Px8ODg4VOv3t167wJydnSGVSpGenq52PD09vdIxPv9mYWGBLl26IDk5GQDE89LT09USoPT0dHTu3FljHTKZDDKZrAZXQEREpBs//vgjGjVqhFatWiE5ORlvvPEGevbsyeSnjui1C8zS0hIBAQGIj48Xj6lUKsTHx6u1CD2OUqnEuXPnxGTH19cX7u7uanXm5+fj+PHj1a6TiIiovhUUFGDatGlo27YtQkND8eSTT+Knn37Sd1hGS+9D8MPCwhASEoJu3bqhe/fuiIyMRFFRkTgrbMKECWjatKnYd7hkyRI89dRTaNmyJXJzc/HBBx/g+vXrmDx5MoAHg8VmzZqFZcuWoVWrVuI0eE9PT603tCMiIqovEyZMwIQJE/QdhsnQewI0atQoZGZmYuHChUhLS0Pnzp0RFxcnDmJOTU0Vp9ABQE5ODqZMmYK0tDQ0btwYAQEBOHr0KNq3by+WmTNnDoqKivDKK68gNzcXvXr1QlxcHPudiYiICICeB0E3VNoMoiIioqpxEDTpiq4GQTeIrTCIiMg08G9uqi1dfQ8xASIiojpXviu6QqHQcyRk6Mr3Evv3liTa0vsYICIiMn7m5uawsbFBZmYmLCws1MZ2ElWHIAgoLi5GRkYGHB0dxaS6ppgAERFRnZNIJPDw8EBKSgquX7+u73DIgDk6OlZ7rcDHYQJERET1wtLSEq1atWI3GNWYhYVFrVt+yjEBIiKiemNmZsZZYNQgsBOWiIiITA4TICIiIjI5TICIiIjI5HAMkAbliyzl5+frORIiIiKqrvLf29VZLJEJkAYFBQUAAC8vLz1HQkRERNoqKCiAg4PDY8twLzANVCoVbt++DTs7O0gkEp3WnZ+fDy8vL9y4ccPo9xnjtRovU7peXqvxMqXrNZVrFQQBBQUF8PT0rHKxTbYAaWBmZoZmzZrV6XvY29sb9Tfho3itxsuUrpfXarxM6XpN4Vqravkpx0HQREREZHKYABEREZHJYQJUz2QyGcLDwyGTyfQdSp3jtRovU7peXqvxMqXrNaVrrS4OgiYiIiKTwxYgIiIiMjlMgIiIiMjkMAEiIiIik8MEiIiIiEwOE6A6sG7dOvj4+MDKygqBgYE4ceLEY8t/9913aNu2LaysrODn54fdu3fXU6Q1FxERgSeffBJ2dnZwdXXF8OHDkZSU9NhzNm/eDIlEovawsrKqp4hrbtGiRRXibtu27WPPMcR7Ws7Hx6fC9UokEkybNk1jeUO6r7/99huGDBkCT09PSCQS7NixQ+11QRCwcOFCeHh4wNraGsHBwbh8+XKV9Wr7M18fHnetZWVlePvtt+Hn5wdbW1t4enpiwoQJuH379mPrrMnPQn2p6t6GhoZWiH3gwIFV1mto9xaAxp9fiUSCDz74oNI6G/K9rStMgHQsNjYWYWFhCA8Px6lTp9CpUyfI5XJkZGRoLH/06FGMGTMGL7/8MhITEzF8+HAMHz4c58+fr+fItXP48GFMmzYNx44dw759+1BWVoYBAwagqKjosefZ29vjzp074uP69ev1FHHtdOjQQS3uP/74o9KyhnpPy/35559q17pv3z4AwIgRIyo9x1Dua1FRETp16oR169ZpfP3999/H2rVrERUVhePHj8PW1hZyuRwlJSWV1qntz3x9edy1FhcX49SpU1iwYAFOnTqF7du3IykpCUOHDq2yXm1+FupTVfcWAAYOHKgW+7Zt2x5bpyHeWwBq13jnzh1ER0dDIpHgxRdffGy9DfXe1hmBdKp79+7CtGnTxOdKpVLw9PQUIiIiNJYfOXKkMHjwYLVjgYGBwquvvlqncepaRkaGAEA4fPhwpWViYmIEBweH+gtKR8LDw4VOnTpVu7yx3NNyb7zxhtCiRQtBpVJpfN1Q7ysA4ccffxSfq1Qqwd3dXfjggw/EY7m5uYJMJhO2bdtWaT3a/szrw7+vVZMTJ04IAITr169XWkbbnwV90XS9ISEhwrBhw7Sqx1ju7bBhw4R+/fo9toyh3FtdYguQDikUCpw8eRLBwcHiMTMzMwQHByMhIUHjOQkJCWrlAUAul1davqHKy8sDADg5OT22XGFhIby9veHl5YVhw4bh77//ro/wau3y5cvw9PRE8+bNMXbsWKSmplZa1ljuKfDge/rLL7/EpEmTHrsxsKHe10elpKQgLS1N7d45ODggMDCw0ntXk5/5hiovLw8SiQSOjo6PLafNz0JDc+jQIbi6uqJNmzaYOnUqsrOzKy1rLPc2PT0du3btwssvv1xlWUO+tzXBBEiHsrKyoFQq4ebmpnbczc0NaWlpGs9JS0vTqnxDpFKpMGvWLPTs2RMdO3astFybNm0QHR2Nn376CV9++SVUKhV69OiBmzdv1mO02gsMDMTmzZsRFxeHDRs2ICUlBb1790ZBQYHG8sZwT8vt2LEDubm5CA0NrbSMod7Xfyu/P9rcu5r8zDdEJSUlePvttzFmzJjHbpSp7c9CQzJw4EBs3boV8fHxWLlyJQ4fPoxBgwZBqVRqLG8s93bLli2ws7PDCy+88Nhyhnxva4q7wVOtTZs2DefPn6+yvzgoKAhBQUHi8x49eqBdu3b49NNPsXTp0roOs8YGDRokfu3v74/AwEB4e3vj22+/rdZfVYZs06ZNGDRoEDw9PSstY6j3lR4oKyvDyJEjIQgCNmzY8NiyhvyzMHr0aPFrPz8/+Pv7o0WLFjh06BCeeeYZPUZWt6KjozF27NgqJyYY8r2tKbYA6ZCzszOkUinS09PVjqenp8Pd3V3jOe7u7lqVb2imT5+OX375BQcPHkSzZs20OtfCwgJdunRBcnJyHUVXNxwdHdG6detK4zb0e1ru+vXr2L9/PyZPnqzVeYZ6X8vvjzb3riY/8w1JefJz/fp17Nu377GtP5pU9bPQkDVv3hzOzs6Vxm7o9xYAfv/9dyQlJWn9MwwY9r2tLiZAOmRpaYmAgADEx8eLx1QqFeLj49X+Qn5UUFCQWnkA2LdvX6XlGwpBEDB9+nT8+OOPOHDgAHx9fbWuQ6lU4ty5c/Dw8KiDCOtOYWEhrly5UmnchnpP/y0mJgaurq4YPHiwVucZ6n319fWFu7u72r3Lz8/H8ePHK713NfmZbyjKk5/Lly9j//79aNKkidZ1VPWz0JDdvHkT2dnZlcZuyPe23KZNmxAQEIBOnTppfa4h39tq0/cobGPzzTffCDKZTNi8ebNw4cIF4ZVXXhEcHR2FtLQ0QRAEYfz48cLcuXPF8keOHBHMzc2FVatWCRcvXhTCw8MFCwsL4dy5c/q6hGqZOnWq4ODgIBw6dEi4c+eO+CguLhbL/PtaFy9eLOzZs0e4cuWKcPLkSWH06NGClZWV8Pfff+vjEqrtzTffFA4dOiSkpKQIR44cEYKDgwVnZ2chIyNDEATjuaePUiqVwhNPPCG8/fbbFV4z5PtaUFAgJCYmComJiQIAYc2aNUJiYqI48+m9994THB0dhZ9++kk4e/asMGzYMMHX11e4d++eWEe/fv2Ejz/+WHxe1c+8vjzuWhUKhTB06FChWbNmwunTp9V+hktLS8U6/n2tVf0s6NPjrregoECYPXu2kJCQIKSkpAj79+8XunbtKrRq1UooKSkR6zCGe1suLy9PsLGxETZs2KCxDkO6t3WFCVAd+Pjjj4UnnnhCsLS0FLp37y4cO3ZMfK1Pnz5CSEiIWvlvv/1WaN26tWBpaSl06NBB2LVrVz1HrD0AGh8xMTFimX9f66xZs8TPxc3NTXj22WeFU6dO1X/wWho1apTg4eEhWFpaCk2bNhVGjRolJCcni68byz191J49ewQAQlJSUoXXDPm+Hjx4UOP3bfn1qFQqYcGCBYKbm5sgk8mEZ555psJn4O3tLYSHh6sde9zPvL487lpTUlIq/Rk+ePCgWMe/r7WqnwV9etz1FhcXCwMGDBBcXFwECwsLwdvbW5gyZUqFRMYY7m25Tz/9VLC2thZyc3M11mFI97auSARBEOq0iYmIiIiogeEYICIiIjI5TICIiIjI5DABIiIiIpPDBIiIiIhMDhMgIiIiMjlMgIiIiMjkMAEiIiIik8MEiIganGvXrkEikeD06dP6DkV06dIlPPXUU7CyskLnzp11UueiRYu0rksikWDHjh06eX8iU8YEiIgqCA0NhUQiwXvvvad2fMeOHZBIJHqKSr/Cw8Nha2uLpKSkCnu9AQ8Sk8c9Fi1aVOGc2bNna6yLiOqeub4DIKKGycrKCitXrsSrr76Kxo0b6zscnVAoFLC0tKzRuVeuXMHgwYPh7e2t8fU7d+6IX8fGxmLhwoVISkoSjzVq1Ej8WhAEKJVKNGrUSO04EdUftgARkUbBwcFwd3dHREREpWU0deFERkbCx8dHfB4aGorhw4djxYoVcHNzg6OjI5YsWYL79+/jrbfegpOTE5o1a4aYmJgK9V+6dAk9evSAlZUVOnbsiMOHD6u9fv78eQwaNAiNGjWCm5sbxo8fj6ysLPH1vn37Yvr06Zg1axacnZ0hl8s1XodKpcKSJUvQrFkzyGQydO7cGXFxceLrEokEJ0+exJIlSyptzXF3dxcfDg4OkEgk4vNLly7Bzs4Ov/76KwICAiCTyfDHH39U+Pz+/PNP9O/fH87OznBwcECfPn1w6tSpSj9/hUKB6dOnw8PDA1ZWVvD29n7s/SKih5gAEZFGUqkUK1aswMcff4ybN2/Wqq4DBw7g9u3b+O2337BmzRqEh4fjueeeQ+PGjXH8+HG89tprePXVVyu8z1tvvYU333wTiYmJCAoKwpAhQ5CdnQ0AyM3NRb9+/dClSxf89ddfiIuLQ3p6OkaOHKlWx5YtW2BpaYkjR44gKipKY3wfffQRVq9ejVWrVuHs2bOQy+UYOnQoLl++DOBB606HDh3w5ptv4s6dO5g9e3aNPoe5c+fivffew8WLF+Hv71/h9YKCAoSEhOCPP/7AsWPH0KpVKzz77LMoKCjQWN/atWuxc+dOfPvtt0hKSsJXX32llnwS0WPoeTNWImqAQkJChGHDhgmCIAhPPfWUMGnSJEEQBOHHH38UHv1vIzw8XOjUqZPauR9++KHg7e2tVpe3t7egVCrFY23atBF69+4tPr9//75ga2srbNu2TRAEQdyt/L333hPLlJWVCc2aNRNWrlwpCIIgLF26VBgwYIDae9+4cUNtF/s+ffoIXbp0qfJ6PT09heXLl6sde/LJJ4XXX39dfN6pU6cKO4VXJiYmRnBwcBCfl+/evWPHDrVymj6/RymVSsHOzk74+eefxWMAhB9//FEQBEGYMWOG0K9fP0GlUlUrLiJ6iC1ARPRYK1euxJYtW3Dx4sUa19GhQweYmT3878bNzQ1+fn7ic6lUiiZNmiAjI0PtvKCgIPFrc3NzdOvWTYzjzJkzOHjwoDiOplGjRmjbti2AB+N1ygUEBDw2tvz8fNy+fRs9e/ZUO96zZ89aXbMm3bp1e+zr6enpmDJlClq1agUHBwfY29ujsLAQqampGsuHhobi9OnTaNOmDWbOnIm9e/fqNF4iY8ZB0ET0WE8//TTkcjnmzZuH0NBQtdfMzMwgCILasbKysgp1WFhYqD2XSCQaj6lUqmrHVVhYiCFDhmDlypUVXvPw8BC/trW1rXadda2qWEJCQpCdnY2PPvoI3t7ekMlkCAoKgkKh0Fi+a9euSElJwa+//or9+/dj5MiRCA4Oxvfff18X4RMZFbYAEVGV3nvvPfz8889ISEhQO+7i4oK0tDS1JEiXa/ccO3ZM/Pr+/fs4efIk2rVrB+DBL/+///4bPj4+aNmypdpDm6TH3t4enp6eOHLkiNrxI0eOoH379rq5kGo6cuQIZs6ciWeffRYdOnSATCZTG9Stib29PUaNGoWNGzciNjYWP/zwA+7evVtPERMZLiZARFQlPz8/jB07FmvXrlU73rdvX2RmZuL999/HlStXsG7dOvz66686e99169bhxx9/xKVLlzBt2jTk5ORg0qRJAIBp06bh7t27GDNmDP78809cuXIFe/bswcSJE6FUKrV6n7feegsrV65EbGwskpKSMHfuXJw+fRpvvPGGzq6lOlq1aoUvvvgCFy9exPHjxzF27FhYW1tXWn7NmjXYtm0bLl26hH/++Qffffcd3N3d4ejoWH9BExkoJkBEVC1Lliyp0EXVrl07rF+/HuvWrUOnTp1w4sSJGs+Q0uS9997De++9h06dOuGPP/7Azp074ezsDABiq41SqcSAAQPg5+eHWbNmwdHRUW28UXXMnDkTYWFhePPNN+Hn54e4uDjs3LkTrVq10tm1VMemTZuQk5ODrl27Yvz48Zg5cyZcXV0rLW9nZ4f3338f3bp1w5NPPolr165h9+7dWl8/kSmSCP/uwCciIiIycvwzgYiIiEwOEyAiIiIyOUyAiIiIyOQwASIiIiKTwwSIiIiITA4TICIiIjI5TICIiIjI5DABIiIiIpPDBIiIiIhMDhMgIiIiMjlMgIiIiMjkMAEiIiIik/P/fvGxgkabjywAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# 横轴：试验次数\n",
    "x_task1_len= range(len(task1))\n",
    "\n",
    "# 绘制曲线\n",
    "plt.plot(x_task1_len, task1, label=\"IntegerLinear \")\n",
    "\n",
    "# 图形设置\n",
    "plt.xlabel(\"Number of Trials\")\n",
    "plt.ylabel(\"Maximum Achieved Accuracy\")\n",
    "plt.title(\"IntegerLinear Strategies\")\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 273,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pathlib import Path\n",
    "import dill\n",
    "\n",
    "with open(f\"{Path.home()}/tutorial_5_best_model.pkl\", \"rb\") as f:\n",
    "    base_model = dill.load(f)\n",
    "from chop.nn.quantized.modules.linear import (\n",
    "    LinearInteger,\n",
    "    LinearMinifloatDenorm,\n",
    "    LinearMinifloatIEEE,\n",
    "    LinearLog,\n",
    "    LinearBlockFP,\n",
    "    LinearBlockMinifloat,\n",
    "    LinearBlockLog,\n",
    "    LinearBinary,\n",
    "    LinearBinaryScaling,\n",
    "    LinearBinaryResidualSign,\n",
    ")\n",
    "search_space = {\n",
    "    \"linear_layer_choices\": [\n",
    "        torch.nn.Linear,\n",
    "        LinearInteger,\n",
    "        LinearMinifloatDenorm,\n",
    "        LinearMinifloatIEEE,\n",
    "        LinearLog,\n",
    "        LinearBlockFP,\n",
    "        # LinearBlockMinifloat,\n",
    "        LinearBlockLog,\n",
    "        LinearBinary,\n",
    "        LinearBinaryScaling,\n",
    "        LinearBinaryResidualSign,\n",
    "    ],\n",
    "    'linear_layer_quantization': [8, 16, 32],\n",
    "    'linear_layer_quantization_fractional':  [2, 4, 8],\n",
    "    'LinearMinifloatDenorm': [8, 16, 32],\n",
    "    'LinearMinifloatDenorm_ew': [2, 4, 8],\n",
    "    'LinearMinifloatDenorm_eb': [2, 4, 8],\n",
    "    'LinearMinifloatIEEE': [8, 16, 32],\n",
    "    'LinearMinifloatIEEE_ew': [2, 4, 8],\n",
    "    'LinearMinifloatIEEE_eb': [2, 4, 8],\n",
    "    'LinearLog': [8, 16, 32],\n",
    "    'LinearLog_ew': [2, 4, 8],\n",
    "    'LinearLog_eb': [2, 4, 8],\n",
    "    'LinearBlockFP': [8, 16, 32],\n",
    "    'LinearBlockFP_block_size':[[4], [8], [16]] ,\n",
    "    'LinearBlockMinifloat_ebw': [2, 4, 8],\n",
    "    'LinearBlockMinifloat_ew': [2, 4, 8],\n",
    "    'LinearBlockMinifloat_block_size': [[4], [8], [16]],\n",
    "    'LinearBlockMinifloat_w': [2, 4, 8],\n",
    "    'LinearBlockLog': [8, 16, 32],\n",
    "    'LinearBlockLog_ebw': [2, 4, 8],\n",
    "    'LinearBlockLog_block_size': [[4], [8], [16]] ,\n",
    "    'LinearBinary_stochastic': [2, 4, 8],\n",
    "    'LinearBinary_bipolar': [2, 4, 8],\n",
    "    'LinearBinaryScaling_stochastic': [2, 4, 8],\n",
    "    'LinearBinaryScaling_bipolar': [2, 4, 8],\n",
    "    \"LinearBinaryScaling_binary_training\": [2, 4, 8],\n",
    "    'LinearBinaryResidualSign_stochastic': [2, 4, 8],\n",
    "    'LinearBinaryResidualSign_bipolar': [2, 4, 8],\n",
    "    \"LinearBinaryResidualSign_binary_training\": [2, 4, 8],\n",
    "}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 275,
   "metadata": {},
   "outputs": [],
   "source": [
    "def construct_model(trial):\n",
    "    # Fetch the model\n",
    "    my_config = {}\n",
    "    trial_model = deepcopy(base_model)\n",
    "    \n",
    "    # Select quantization and fractional width configurations\n",
    "    for param in ['linear_layer_quantization', \n",
    "                  'linear_layer_quantization_fractional',\n",
    "                  'LinearMinifloatDenorm',\n",
    "                  'LinearMinifloatDenorm_ew',\n",
    "                  'LinearMinifloatDenorm_eb',\n",
    "                  'LinearMinifloatIEEE',\n",
    "                  'LinearMinifloatIEEE_ew',\n",
    "                  'LinearMinifloatIEEE_eb',\n",
    "                    'LinearLog',\n",
    "                    'LinearLog_ew',\n",
    "                    'LinearLog_eb',\n",
    "                    'LinearBlockFP',\n",
    "                    'LinearBlockFP_block_size',\n",
    "                    'LinearBlockMinifloat_ebw',\n",
    "                    'LinearBlockMinifloat_ew',\n",
    "                    'LinearBlockMinifloat_block_size',\n",
    "                    'LinearBlockMinifloat_w',\n",
    "                    'LinearBlockLog',\n",
    "                    'LinearBlockLog_ebw',\n",
    "                    'LinearBlockLog_block_size',\n",
    "                    'LinearBinary_stochastic',\n",
    "                    'LinearBinary_bipolar',\n",
    "                    'LinearBinaryScaling_stochastic',\n",
    "                    'LinearBinaryScaling_bipolar',\n",
    "                    'LinearBinaryScaling_binary_training',\n",
    "                    'LinearBinaryResidualSign_stochastic',\n",
    "                    'LinearBinaryResidualSign_bipolar',\n",
    "                    'LinearBinaryResidualSign_binary_training'\n",
    "                  ]:\n",
    "        chosen_idx = trial.suggest_int(param, 0, len(search_space[param]) - 1)\n",
    "        my_config[param] = search_space[param][chosen_idx]\n",
    "\n",
    "    # Quantize layers according to Optuna suggestions\n",
    "    for name, layer in trial_model.named_modules():\n",
    "        if isinstance(layer, torch.nn.Linear):\n",
    "            new_layer_cls = trial.suggest_categorical(\n",
    "                f\"{name}_type\",\n",
    "                search_space[\"linear_layer_choices\"],\n",
    "            )\n",
    "\n",
    "            if new_layer_cls == torch.nn.Linear:\n",
    "                continue\n",
    "\n",
    "            kwargs = {\n",
    "                \"in_features\": layer.in_features,\n",
    "                \"out_features\": layer.out_features,\n",
    "            }\n",
    "\n",
    "            try:\n",
    "                # Configure parameters for specific layer types\n",
    "                if new_layer_cls == LinearInteger:\n",
    "                    kwargs[\"config\"] = {\n",
    "                        \"data_in_width\": my_config['linear_layer_quantization'],\n",
    "                        \"data_in_frac_width\": my_config['linear_layer_quantization_fractional'],\n",
    "                        \"weight_width\": my_config['linear_layer_quantization'],\n",
    "                        \"weight_frac_width\": my_config['linear_layer_quantization_fractional'],\n",
    "                        \"bias_width\": my_config['linear_layer_quantization'],\n",
    "                        \"bias_frac_width\": my_config['linear_layer_quantization_fractional'],\n",
    "                    }\n",
    "                elif new_layer_cls == LinearMinifloatDenorm:\n",
    "                    kwargs[\"config\"] = {\n",
    "                        \"data_in_width\": my_config['LinearMinifloatDenorm'],\n",
    "                        \"data_in_exponent_width\": my_config['LinearMinifloatDenorm_ew'],\n",
    "                        \"data_in_exponent_bias\": my_config['LinearMinifloatDenorm_eb'],\n",
    "                        \"weight_width\": my_config['LinearMinifloatDenorm'],\n",
    "                        \"weight_exponent_width\": my_config['LinearMinifloatDenorm_ew'],\n",
    "                        \"weight_exponent_bias\": my_config['LinearMinifloatDenorm_eb'],\n",
    "                        \"bias_width\": my_config['LinearMinifloatDenorm'],\n",
    "                        \"bias_exponent_width\": my_config['LinearMinifloatDenorm_ew'],\n",
    "                        \"bias_exponent_bias\": my_config['LinearMinifloatDenorm_eb'],\n",
    "                    }\n",
    "                elif new_layer_cls == LinearMinifloatIEEE:\n",
    "                    kwargs[\"config\"] = {\n",
    "                        \"data_in_width\": my_config['LinearMinifloatIEEE'],\n",
    "                        \"data_in_exponent_width\": my_config['LinearMinifloatIEEE_ew'],\n",
    "                        \"data_in_exponent_bias\": my_config['LinearMinifloatIEEE_eb'],\n",
    "                        \"weight_width\": my_config['LinearMinifloatIEEE'],\n",
    "                        \"weight_exponent_width\": my_config['LinearMinifloatIEEE_ew'],\n",
    "                        \"weight_exponent_bias\": my_config['LinearMinifloatIEEE_eb'],\n",
    "                        \"bias_width\": my_config['LinearMinifloatIEEE'],\n",
    "                        \"bias_exponent_width\": my_config['LinearMinifloatIEEE_ew'],\n",
    "                        \"bias_exponent_bias\": my_config['LinearMinifloatIEEE_eb'],\n",
    "                    }\n",
    "                elif new_layer_cls == LinearLog:\n",
    "                    kwargs[\"config\"] = {\n",
    "                        \"data_in_width\": my_config['LinearLog'],\n",
    "                        \"data_in_exponent_width\": my_config['LinearLog_ew'],\n",
    "                        \"data_in_exponent_bias\": my_config['LinearLog_eb'],\n",
    "                        \"weight_width\": my_config['LinearLog'],\n",
    "                        \"weight_exponent_width\": my_config['LinearLog_ew'],\n",
    "                        \"weight_exponent_bias\": my_config['LinearLog_eb'],\n",
    "                        \"bias_width\": my_config['LinearLog'],\n",
    "                        \"bias_exponent_width\": my_config['LinearLog_ew'],\n",
    "                        \"bias_exponent_bias\": my_config['LinearLog_eb'],\n",
    "                    }\n",
    "                elif new_layer_cls == LinearBlockFP:\n",
    "                    kwargs[\"config\"] = {\n",
    "                        \"weight_width\": my_config['LinearBlockFP'],\n",
    "                        \"weight_exponent_width\": my_config['LinearBlockFP'],\n",
    "                        \"weight_block_size\": my_config['LinearBlockFP_block_size'],\n",
    "                        \"weight_exponent_bias\": my_config['LinearBlockFP'],\n",
    "                        \"data_in_width\": my_config['LinearBlockFP'],\n",
    "                        \"data_in_exponent_width\": my_config['LinearBlockFP'],\n",
    "                        \"data_in_block_size\": my_config['LinearBlockFP_block_size'],\n",
    "                        \"data_in_exponent_bias\": my_config['LinearBlockFP'],\n",
    "                        \"bias_width\": my_config['LinearBlockFP'],\n",
    "                        \"bias_exponent_width\": my_config['LinearBlockFP'],\n",
    "                        \"bias_block_size\": my_config['LinearBlockFP_block_size'],\n",
    "                        \"bias_exponent_bias\": my_config['LinearBlockFP'],\n",
    "                    }\n",
    "                # elif new_layer_cls == LinearBlockMinifloat:\n",
    "                #     kwargs[\"config\"] = {\n",
    "                #         \"weight_exponent_bias_width\" : my_config['LinearBlockMinifloat_ebw'],\n",
    "                #         \"weight_exponent_width\": my_config['LinearBlockMinifloat_ew'],\n",
    "                #         \"weight_block_size\": my_config['LinearBlockMinifloat_block_size'],\n",
    "                #         \"weight_width\": my_config['LinearBlockMinifloat_w'],\n",
    "                #         \"data_in_width\":my_config[\"LinearBlockMinifloat_w\"],\n",
    "                #         \"data_in_exponent_bias_width\":my_config[\"LinearBlockMinifloat_ew\"],\n",
    "                #         \"data_in_exponent_width\":my_config[\"LinearBlockMinifloat_ebw\"],\n",
    "                #         \"data_in_block_size\":my_config[\"LinearBlockMinifloat_block_size\"],\n",
    "                #         \"bias_width\":my_config[\"LinearBlockMinifloat_w\"],\n",
    "                #         \"bias_exponent_width\":my_config[\"LinearBlockMinifloat_ew\"],\n",
    "                #         \"bias_exponent_bias_width\":my_config[\"LinearBlockMinifloat_ebw\"],\n",
    "                #         \"bias_block_size\":my_config[\"LinearBlockMinifloat_block_size\"],\n",
    "                #     }\n",
    "                elif new_layer_cls == LinearBlockLog:\n",
    "                    kwargs[\"config\"] = {\n",
    "                        \"data_in_width\": my_config['LinearBlockLog'],\n",
    "                        \"data_in_exponent_bias_width\": my_config['LinearBlockLog_ebw'],\n",
    "                        \"data_in_block_size\": my_config['LinearBlockLog_block_size'],\n",
    "                        \"weight_width\": my_config['LinearBlockLog'],\n",
    "                        \"weight_exponent_bias_width\": my_config['LinearBlockLog_ebw'],\n",
    "                        \"weight_block_size\": my_config['LinearBlockLog_block_size'],\n",
    "                        \"bias_width\": my_config['LinearBlockLog'],\n",
    "                        \"bias_exponent_bias_width\": my_config['LinearBlockLog_ebw'],\n",
    "                        \"bias_block_size\": my_config['LinearBlockLog_block_size'],\n",
    "                    }\n",
    "\n",
    "                elif new_layer_cls == LinearBinary:\n",
    "                    kwargs[\"config\"] = {\n",
    "                        'weight_stochastic':my_config['LinearBinary_stochastic'],\n",
    "                        'weight_bipolar':my_config['LinearBinary_bipolar'],\n",
    "                    }\n",
    "\n",
    "                    \n",
    "                elif new_layer_cls == LinearBinaryScaling:\n",
    "                    kwargs[\"config\"] = {\n",
    "                        'weight_stochastic':my_config['LinearBinaryScaling_stochastic'],\n",
    "                        'weight_bipolar':my_config['LinearBinaryScaling_bipolar'],\n",
    "                        \"data_in_stochastic\": my_config['LinearBinaryScaling_stochastic'],\n",
    "                        \"data_in_bipolar\": my_config['LinearBinaryScaling_bipolar'],\n",
    "                        \"bias_stochastic\": my_config['LinearBinaryScaling_stochastic'],\n",
    "                        \"bias_bipolar\": my_config['LinearBinaryScaling_bipolar'],\n",
    "                        \"binary_training\": my_config['LinearBinaryScaling_binary_training'],\n",
    "                    }\n",
    "                elif new_layer_cls == LinearBinaryResidualSign:\n",
    "                    kwargs[\"config\"] = {\n",
    "                        'weight_stochastic':my_config['LinearBinaryResidualSign_stochastic'],\n",
    "                        'weight_bipolar':my_config['LinearBinaryResidualSign_bipolar'],\n",
    "                        \"data_in_stochastic\": my_config['LinearBinaryResidualSign_stochastic'],\n",
    "                        \"data_in_bipolar\": my_config['LinearBinaryResidualSign_bipolar'],\n",
    "                        \"bias_stochastic\": my_config['LinearBinaryResidualSign_stochastic'],\n",
    "                        \"bias_bipolar\": my_config['LinearBinaryResidualSign_bipolar'],\n",
    "                        \"binary_training\": my_config['LinearBinaryResidualSign_binary_training'],\n",
    "                    }\n",
    "                # Create the new layer (copy the weights if applicable)\n",
    "                new_layer = new_layer_cls(**kwargs)\n",
    "                new_layer.weight.data = layer.weight.data.clone()\n",
    "\n",
    "                # Replace the layer in the model\n",
    "                deepsetattr(trial_model, name, new_layer)\n",
    "            except TypeError as e:\n",
    "                print(f\"Error creating layer {name} with class {new_layer_cls}: {e}\")\n",
    "                raise\n",
    "\n",
    "    return trial_model\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from chop.tools import get_trainer\n",
    "import random\n",
    "\n",
    "\n",
    "def objective(trial):\n",
    "\n",
    "    # Define the model\n",
    "    model = construct_model(trial)\n",
    "\n",
    "    trainer = get_trainer(\n",
    "        model=model,\n",
    "        tokenized_dataset=dataset,\n",
    "        tokenizer=tokenizer,\n",
    "        evaluate_metric=\"accuracy\",\n",
    "        num_train_epochs=1,\n",
    "    )\n",
    "\n",
    "    trainer.train()\n",
    "    eval_results = trainer.evaluate()\n",
    "\n",
    "    trial.set_user_attr(\"model\", model)\n",
    "\n",
    "    return eval_results[\"eval_accuracy\"]\n",
    "\n",
    "\n",
    "from optuna.samplers import GridSampler, RandomSampler, TPESampler\n",
    "\n",
    "sampler = RandomSampler()\n",
    "\n",
    "import optuna\n",
    "\n",
    "study = optuna.create_study(\n",
    "    direction=\"maximize\",\n",
    "    study_name=\"bert-tiny-nas-study\",\n",
    "    sampler=sampler,\n",
    ")\n",
    "\n",
    "study.optimize(\n",
    "    objective,\n",
    "    n_trials=10,\n",
    "    timeout=60 * 60 * 24,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 277,
   "metadata": {},
   "outputs": [],
   "source": [
    "task2 = [max([t.value for t in study.trials[:i+1]]) for i in range(len(study.trials))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "task2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# 横轴：试验次数\n",
    "x_task2_len= range(len(task2))\n",
    "\n",
    "# 绘制曲线\n",
    "plt.plot(x_task2_len, task2, label=\" all supported precisions for the Linear layer \")\n",
    "\n",
    "# 图形设置\n",
    "plt.xlabel(\"Number of Trials\")\n",
    "plt.ylabel(\"Maximum Achieved Accuracy\")\n",
    "plt.title(\" all supported precisions for the Linear layer Strategies\")\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## -------------------------------------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32mINFO    \u001b[0m \u001b[34mTokenizing dataset imdb with AutoTokenizer for bert-base-uncased.\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "dataset, tokenizer = get_tokenized_dataset(\n",
    "    dataset=dataset_name,\n",
    "    checkpoint=tokenizer_checkpoint,\n",
    "    return_tokenizer=True,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-06 12:05:11,350] A new study created in memory with name: bert-tiny-nas-study\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2025-02-06 12:05:17,250] [INFO] [real_accelerator.py:158:get_accelerator] Setting ds_accelerator to cuda (auto detect)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 02:24, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.340800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.301800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.331500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.344600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.334200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.366500</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 01:36]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-06 12:09:22,562] Trial 0 finished with value: 0.87256 and parameters: {'linear_layer_quantization': 1, 'linear_layer_quantization_fractional': 1, 'LinearMinifloatDenorm': 2, 'LinearMinifloatDenorm_ew': 1, 'LinearMinifloatDenorm_eb': 0, 'LinearMinifloatIEEE': 0, 'LinearMinifloatIEEE_ew': 1, 'LinearMinifloatIEEE_eb': 2, 'LinearLog': 1, 'LinearLog_ew': 1, 'LinearLog_eb': 0, 'LinearBlockFP': 1, 'LinearBlockFP_block_size': 1, 'LinearBlockMinifloat_ebw': 0, 'LinearBlockMinifloat_ew': 2, 'LinearBlockMinifloat_block_size': 1, 'LinearBlockMinifloat_w': 1, 'LinearBlockLog': 2, 'LinearBlockLog_ebw': 2, 'LinearBlockLog_block_size': 1, 'LinearBinary_stochastic': 0, 'LinearBinary_bipolar': 2, 'LinearBinaryScaling_stochastic': 0, 'LinearBinaryScaling_bipolar': 1, 'LinearBinaryScaling_binary_training': 1, 'LinearBinaryResidualSign_stochastic': 1, 'LinearBinaryResidualSign_bipolar': 1, 'LinearBinaryResidualSign_binary_training': 0, 'bert.encoder.layer.0.attention.self.query_type': <class 'chop.nn.quantized.modules.linear.LinearMinifloatDenorm'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearMinifloatDenorm'>, 'bert.encoder.layer.0.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'chop.nn.quantized.modules.linear.LinearMinifloatDenorm'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.output.dense_type': <class 'chop.nn.quantized.modules.linear.LinearMinifloatDenorm'>, 'bert.pooler.dense_type': <class 'chop.nn.quantized.modules.linear.LinearMinifloatDenorm'>, 'classifier_type': <class 'chop.nn.quantized.modules.linear.LinearMinifloatDenorm'>}. Best is trial 0 with value: 0.87256.\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearMinifloatDenorm'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 02:21, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.340700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.307400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.336300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.343300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.335100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.359800</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 01:34]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-06 12:13:19,388] Trial 1 finished with value: 0.87268 and parameters: {'linear_layer_quantization': 1, 'linear_layer_quantization_fractional': 0, 'LinearMinifloatDenorm': 2, 'LinearMinifloatDenorm_ew': 2, 'LinearMinifloatDenorm_eb': 2, 'LinearMinifloatIEEE': 2, 'LinearMinifloatIEEE_ew': 0, 'LinearMinifloatIEEE_eb': 0, 'LinearLog': 1, 'LinearLog_ew': 0, 'LinearLog_eb': 2, 'LinearBlockFP': 1, 'LinearBlockFP_block_size': 0, 'LinearBlockMinifloat_ebw': 2, 'LinearBlockMinifloat_ew': 1, 'LinearBlockMinifloat_block_size': 0, 'LinearBlockMinifloat_w': 2, 'LinearBlockLog': 0, 'LinearBlockLog_ebw': 0, 'LinearBlockLog_block_size': 0, 'LinearBinary_stochastic': 2, 'LinearBinary_bipolar': 2, 'LinearBinaryScaling_stochastic': 2, 'LinearBinaryScaling_bipolar': 0, 'LinearBinaryScaling_binary_training': 2, 'LinearBinaryResidualSign_stochastic': 2, 'LinearBinaryResidualSign_bipolar': 0, 'LinearBinaryResidualSign_binary_training': 0, 'bert.encoder.layer.0.attention.self.query_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'chop.nn.quantized.modules.linear.LinearMinifloatDenorm'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearMinifloatDenorm'>, 'bert.encoder.layer.0.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'chop.nn.quantized.modules.linear.LinearMinifloatDenorm'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'chop.nn.quantized.modules.linear.LinearMinifloatDenorm'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearMinifloatDenorm'>, 'bert.encoder.layer.1.output.dense_type': <class 'chop.nn.quantized.modules.linear.LinearMinifloatDenorm'>, 'bert.pooler.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'classifier_type': <class 'torch.nn.modules.linear.Linear'>}. Best is trial 1 with value: 0.87268.\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearMinifloatDenorm'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 02:36, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.663100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.620900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.597900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.580400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.564000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.561900</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 01:46]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-06 12:17:43,549] Trial 2 finished with value: 0.84592 and parameters: {'linear_layer_quantization': 0, 'linear_layer_quantization_fractional': 1, 'LinearMinifloatDenorm': 1, 'LinearMinifloatDenorm_ew': 0, 'LinearMinifloatDenorm_eb': 2, 'LinearMinifloatIEEE': 2, 'LinearMinifloatIEEE_ew': 1, 'LinearMinifloatIEEE_eb': 0, 'LinearLog': 1, 'LinearLog_ew': 0, 'LinearLog_eb': 1, 'LinearBlockFP': 1, 'LinearBlockFP_block_size': 2, 'LinearBlockMinifloat_ebw': 2, 'LinearBlockMinifloat_ew': 1, 'LinearBlockMinifloat_block_size': 0, 'LinearBlockMinifloat_w': 0, 'LinearBlockLog': 2, 'LinearBlockLog_ebw': 0, 'LinearBlockLog_block_size': 0, 'LinearBinary_stochastic': 1, 'LinearBinary_bipolar': 2, 'LinearBinaryScaling_stochastic': 0, 'LinearBinaryScaling_bipolar': 2, 'LinearBinaryScaling_binary_training': 1, 'LinearBinaryResidualSign_stochastic': 0, 'LinearBinaryResidualSign_bipolar': 1, 'LinearBinaryResidualSign_binary_training': 2, 'bert.encoder.layer.0.attention.self.query_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearMinifloatDenorm'>, 'bert.encoder.layer.0.output.dense_type': <class 'chop.nn.quantized.modules.linear.LinearMinifloatDenorm'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'chop.nn.quantized.modules.linear.LinearMinifloatDenorm'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearMinifloatDenorm'>, 'bert.encoder.layer.1.output.dense_type': <class 'chop.nn.quantized.modules.linear.LinearMinifloatDenorm'>, 'bert.pooler.dense_type': <class 'chop.nn.quantized.modules.linear.LinearMinifloatDenorm'>, 'classifier_type': <class 'torch.nn.modules.linear.Linear'>}. Best is trial 1 with value: 0.87268.\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearMinifloatDenorm'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 02:28, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.341400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.313600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.345300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.337400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.335100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.359000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 01:24]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-06 12:21:37,682] Trial 3 finished with value: 0.87264 and parameters: {'linear_layer_quantization': 0, 'linear_layer_quantization_fractional': 2, 'LinearMinifloatDenorm': 2, 'LinearMinifloatDenorm_ew': 1, 'LinearMinifloatDenorm_eb': 0, 'LinearMinifloatIEEE': 2, 'LinearMinifloatIEEE_ew': 1, 'LinearMinifloatIEEE_eb': 2, 'LinearLog': 1, 'LinearLog_ew': 1, 'LinearLog_eb': 1, 'LinearBlockFP': 0, 'LinearBlockFP_block_size': 2, 'LinearBlockMinifloat_ebw': 0, 'LinearBlockMinifloat_ew': 2, 'LinearBlockMinifloat_block_size': 2, 'LinearBlockMinifloat_w': 1, 'LinearBlockLog': 1, 'LinearBlockLog_ebw': 2, 'LinearBlockLog_block_size': 0, 'LinearBinary_stochastic': 2, 'LinearBinary_bipolar': 2, 'LinearBinaryScaling_stochastic': 0, 'LinearBinaryScaling_bipolar': 1, 'LinearBinaryScaling_binary_training': 2, 'LinearBinaryResidualSign_stochastic': 1, 'LinearBinaryResidualSign_bipolar': 0, 'LinearBinaryResidualSign_binary_training': 2, 'bert.encoder.layer.0.attention.self.query_type': <class 'chop.nn.quantized.modules.linear.LinearMinifloatDenorm'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'chop.nn.quantized.modules.linear.LinearMinifloatDenorm'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.output.dense_type': <class 'chop.nn.quantized.modules.linear.LinearMinifloatDenorm'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'chop.nn.quantized.modules.linear.LinearMinifloatDenorm'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.pooler.dense_type': <class 'chop.nn.quantized.modules.linear.LinearMinifloatDenorm'>, 'classifier_type': <class 'chop.nn.quantized.modules.linear.LinearMinifloatDenorm'>}. Best is trial 1 with value: 0.87268.\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearMinifloatDenorm'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 01:31, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.339300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.306300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.335700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.346400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.336400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.362500</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 00:54]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-06 12:24:04,789] Trial 4 finished with value: 0.87284 and parameters: {'linear_layer_quantization': 2, 'linear_layer_quantization_fractional': 0, 'LinearMinifloatDenorm': 1, 'LinearMinifloatDenorm_ew': 2, 'LinearMinifloatDenorm_eb': 1, 'LinearMinifloatIEEE': 1, 'LinearMinifloatIEEE_ew': 1, 'LinearMinifloatIEEE_eb': 0, 'LinearLog': 1, 'LinearLog_ew': 2, 'LinearLog_eb': 2, 'LinearBlockFP': 1, 'LinearBlockFP_block_size': 1, 'LinearBlockMinifloat_ebw': 2, 'LinearBlockMinifloat_ew': 1, 'LinearBlockMinifloat_block_size': 0, 'LinearBlockMinifloat_w': 0, 'LinearBlockLog': 1, 'LinearBlockLog_ebw': 1, 'LinearBlockLog_block_size': 0, 'LinearBinary_stochastic': 1, 'LinearBinary_bipolar': 0, 'LinearBinaryScaling_stochastic': 2, 'LinearBinaryScaling_bipolar': 0, 'LinearBinaryScaling_binary_training': 0, 'LinearBinaryResidualSign_stochastic': 1, 'LinearBinaryResidualSign_bipolar': 1, 'LinearBinaryResidualSign_binary_training': 0, 'bert.encoder.layer.0.attention.self.query_type': <class 'chop.nn.quantized.modules.linear.LinearMinifloatDenorm'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearMinifloatDenorm'>, 'bert.encoder.layer.0.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.pooler.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'classifier_type': <class 'chop.nn.quantized.modules.linear.LinearMinifloatDenorm'>}. Best is trial 4 with value: 0.87284.\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearMinifloatDenorm'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 01:47, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.339400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.310100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.343300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.345500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.337600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.363500</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 01:07]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-06 12:27:00,492] Trial 5 finished with value: 0.87284 and parameters: {'linear_layer_quantization': 0, 'linear_layer_quantization_fractional': 0, 'LinearMinifloatDenorm': 2, 'LinearMinifloatDenorm_ew': 2, 'LinearMinifloatDenorm_eb': 2, 'LinearMinifloatIEEE': 0, 'LinearMinifloatIEEE_ew': 2, 'LinearMinifloatIEEE_eb': 1, 'LinearLog': 1, 'LinearLog_ew': 1, 'LinearLog_eb': 2, 'LinearBlockFP': 1, 'LinearBlockFP_block_size': 0, 'LinearBlockMinifloat_ebw': 0, 'LinearBlockMinifloat_ew': 2, 'LinearBlockMinifloat_block_size': 1, 'LinearBlockMinifloat_w': 0, 'LinearBlockLog': 0, 'LinearBlockLog_ebw': 1, 'LinearBlockLog_block_size': 2, 'LinearBinary_stochastic': 0, 'LinearBinary_bipolar': 2, 'LinearBinaryScaling_stochastic': 1, 'LinearBinaryScaling_bipolar': 0, 'LinearBinaryScaling_binary_training': 1, 'LinearBinaryResidualSign_stochastic': 2, 'LinearBinaryResidualSign_bipolar': 1, 'LinearBinaryResidualSign_binary_training': 1, 'bert.encoder.layer.0.attention.self.query_type': <class 'chop.nn.quantized.modules.linear.LinearMinifloatDenorm'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearMinifloatDenorm'>, 'bert.encoder.layer.0.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'chop.nn.quantized.modules.linear.LinearMinifloatDenorm'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearMinifloatDenorm'>, 'bert.encoder.layer.1.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.pooler.dense_type': <class 'chop.nn.quantized.modules.linear.LinearMinifloatDenorm'>, 'classifier_type': <class 'torch.nn.modules.linear.Linear'>}. Best is trial 4 with value: 0.87284.\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearMinifloatDenorm'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 02:13, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.337200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.306800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.331000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.346900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.334900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.366000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 01:28]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-06 12:30:43,815] Trial 6 finished with value: 0.87296 and parameters: {'linear_layer_quantization': 0, 'linear_layer_quantization_fractional': 1, 'LinearMinifloatDenorm': 1, 'LinearMinifloatDenorm_ew': 2, 'LinearMinifloatDenorm_eb': 0, 'LinearMinifloatIEEE': 2, 'LinearMinifloatIEEE_ew': 1, 'LinearMinifloatIEEE_eb': 2, 'LinearLog': 2, 'LinearLog_ew': 2, 'LinearLog_eb': 0, 'LinearBlockFP': 1, 'LinearBlockFP_block_size': 1, 'LinearBlockMinifloat_ebw': 0, 'LinearBlockMinifloat_ew': 1, 'LinearBlockMinifloat_block_size': 2, 'LinearBlockMinifloat_w': 0, 'LinearBlockLog': 0, 'LinearBlockLog_ebw': 2, 'LinearBlockLog_block_size': 0, 'LinearBinary_stochastic': 1, 'LinearBinary_bipolar': 2, 'LinearBinaryScaling_stochastic': 1, 'LinearBinaryScaling_bipolar': 0, 'LinearBinaryScaling_binary_training': 2, 'LinearBinaryResidualSign_stochastic': 2, 'LinearBinaryResidualSign_bipolar': 1, 'LinearBinaryResidualSign_binary_training': 2, 'bert.encoder.layer.0.attention.self.query_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearMinifloatDenorm'>, 'bert.encoder.layer.0.output.dense_type': <class 'chop.nn.quantized.modules.linear.LinearMinifloatDenorm'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'chop.nn.quantized.modules.linear.LinearMinifloatDenorm'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'chop.nn.quantized.modules.linear.LinearMinifloatDenorm'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearMinifloatDenorm'>, 'bert.encoder.layer.1.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.pooler.dense_type': <class 'chop.nn.quantized.modules.linear.LinearMinifloatDenorm'>, 'classifier_type': <class 'chop.nn.quantized.modules.linear.LinearMinifloatDenorm'>}. Best is trial 6 with value: 0.87296.\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearMinifloatDenorm'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 01:21, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.342900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.333500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.358800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.345600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.326600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.369900</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 00:50]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-06 12:32:57,109] Trial 7 finished with value: 0.8698 and parameters: {'linear_layer_quantization': 0, 'linear_layer_quantization_fractional': 0, 'LinearMinifloatDenorm': 2, 'LinearMinifloatDenorm_ew': 0, 'LinearMinifloatDenorm_eb': 1, 'LinearMinifloatIEEE': 1, 'LinearMinifloatIEEE_ew': 0, 'LinearMinifloatIEEE_eb': 1, 'LinearLog': 1, 'LinearLog_ew': 2, 'LinearLog_eb': 1, 'LinearBlockFP': 0, 'LinearBlockFP_block_size': 0, 'LinearBlockMinifloat_ebw': 2, 'LinearBlockMinifloat_ew': 2, 'LinearBlockMinifloat_block_size': 2, 'LinearBlockMinifloat_w': 2, 'LinearBlockLog': 2, 'LinearBlockLog_ebw': 0, 'LinearBlockLog_block_size': 0, 'LinearBinary_stochastic': 1, 'LinearBinary_bipolar': 1, 'LinearBinaryScaling_stochastic': 1, 'LinearBinaryScaling_bipolar': 0, 'LinearBinaryScaling_binary_training': 1, 'LinearBinaryResidualSign_stochastic': 1, 'LinearBinaryResidualSign_bipolar': 1, 'LinearBinaryResidualSign_binary_training': 1, 'bert.encoder.layer.0.attention.self.query_type': <class 'chop.nn.quantized.modules.linear.LinearMinifloatDenorm'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'chop.nn.quantized.modules.linear.LinearMinifloatDenorm'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.pooler.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'classifier_type': <class 'torch.nn.modules.linear.Linear'>}. Best is trial 6 with value: 0.87296.\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearMinifloatDenorm'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 02:12, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.340800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.307100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.329200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.346700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.333400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.367700</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 01:26]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-06 12:36:37,513] Trial 8 finished with value: 0.8738 and parameters: {'linear_layer_quantization': 2, 'linear_layer_quantization_fractional': 2, 'LinearMinifloatDenorm': 1, 'LinearMinifloatDenorm_ew': 0, 'LinearMinifloatDenorm_eb': 0, 'LinearMinifloatIEEE': 2, 'LinearMinifloatIEEE_ew': 2, 'LinearMinifloatIEEE_eb': 1, 'LinearLog': 1, 'LinearLog_ew': 2, 'LinearLog_eb': 0, 'LinearBlockFP': 2, 'LinearBlockFP_block_size': 0, 'LinearBlockMinifloat_ebw': 2, 'LinearBlockMinifloat_ew': 1, 'LinearBlockMinifloat_block_size': 0, 'LinearBlockMinifloat_w': 0, 'LinearBlockLog': 0, 'LinearBlockLog_ebw': 0, 'LinearBlockLog_block_size': 1, 'LinearBinary_stochastic': 0, 'LinearBinary_bipolar': 1, 'LinearBinaryScaling_stochastic': 0, 'LinearBinaryScaling_bipolar': 2, 'LinearBinaryScaling_binary_training': 1, 'LinearBinaryResidualSign_stochastic': 0, 'LinearBinaryResidualSign_bipolar': 0, 'LinearBinaryResidualSign_binary_training': 1, 'bert.encoder.layer.0.attention.self.query_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearMinifloatDenorm'>, 'bert.encoder.layer.0.output.dense_type': <class 'chop.nn.quantized.modules.linear.LinearMinifloatDenorm'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'chop.nn.quantized.modules.linear.LinearMinifloatDenorm'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearMinifloatDenorm'>, 'bert.encoder.layer.1.output.dense_type': <class 'chop.nn.quantized.modules.linear.LinearMinifloatDenorm'>, 'bert.pooler.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'classifier_type': <class 'torch.nn.modules.linear.Linear'>}. Best is trial 8 with value: 0.8738.\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearMinifloatDenorm'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 01:39, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.341600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.306600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.328900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.338600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.328700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.365600</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 01:02]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-06 12:39:20,587] Trial 9 finished with value: 0.87188 and parameters: {'linear_layer_quantization': 1, 'linear_layer_quantization_fractional': 0, 'LinearMinifloatDenorm': 2, 'LinearMinifloatDenorm_ew': 0, 'LinearMinifloatDenorm_eb': 0, 'LinearMinifloatIEEE': 0, 'LinearMinifloatIEEE_ew': 2, 'LinearMinifloatIEEE_eb': 1, 'LinearLog': 1, 'LinearLog_ew': 2, 'LinearLog_eb': 1, 'LinearBlockFP': 1, 'LinearBlockFP_block_size': 1, 'LinearBlockMinifloat_ebw': 2, 'LinearBlockMinifloat_ew': 1, 'LinearBlockMinifloat_block_size': 1, 'LinearBlockMinifloat_w': 1, 'LinearBlockLog': 2, 'LinearBlockLog_ebw': 0, 'LinearBlockLog_block_size': 2, 'LinearBinary_stochastic': 2, 'LinearBinary_bipolar': 2, 'LinearBinaryScaling_stochastic': 0, 'LinearBinaryScaling_bipolar': 2, 'LinearBinaryScaling_binary_training': 2, 'LinearBinaryResidualSign_stochastic': 0, 'LinearBinaryResidualSign_bipolar': 1, 'LinearBinaryResidualSign_binary_training': 0, 'bert.encoder.layer.0.attention.self.query_type': <class 'chop.nn.quantized.modules.linear.LinearMinifloatDenorm'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'chop.nn.quantized.modules.linear.LinearMinifloatDenorm'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'chop.nn.quantized.modules.linear.LinearMinifloatDenorm'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.pooler.dense_type': <class 'chop.nn.quantized.modules.linear.LinearMinifloatDenorm'>, 'classifier_type': <class 'torch.nn.modules.linear.Linear'>}. Best is trial 8 with value: 0.8738.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.87256,\n",
       " 0.87268,\n",
       " 0.87268,\n",
       " 0.87268,\n",
       " 0.87284,\n",
       " 0.87284,\n",
       " 0.87296,\n",
       " 0.87296,\n",
       " 0.8738,\n",
       " 0.8738]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "from pathlib import Path\n",
    "import dill\n",
    "import torch\n",
    "\n",
    "with open(f\"{Path.home()}/tutorial_5_best_model.pkl\", \"rb\") as f:\n",
    "    base_model = dill.load(f)\n",
    "from chop.nn.quantized.modules.linear import (\n",
    "    LinearInteger,\n",
    "    LinearMinifloatDenorm,\n",
    "    LinearMinifloatIEEE,\n",
    "    LinearLog,\n",
    "    LinearBlockFP,\n",
    "    LinearBlockMinifloat,\n",
    "    LinearBlockLog,\n",
    "    LinearBinary,\n",
    "    LinearBinaryScaling,\n",
    "    LinearBinaryResidualSign,\n",
    ")\n",
    "search_space = {\n",
    "    \"linear_layer_choices\": [\n",
    "        torch.nn.Linear,\n",
    "        # LinearInteger,\n",
    "        LinearMinifloatDenorm,\n",
    "        # LinearMinifloatIEEE,\n",
    "        # LinearLog,\n",
    "        # LinearBlockFP,\n",
    "        # # LinearBlockMinifloat,\n",
    "        # LinearBlockLog,\n",
    "        # LinearBinary,\n",
    "        # LinearBinaryScaling,\n",
    "        # LinearBinaryResidualSign,\n",
    "    ],\n",
    "    'linear_layer_quantization': [8, 16, 32],\n",
    "    'linear_layer_quantization_fractional':  [2, 4, 8],\n",
    "    'LinearMinifloatDenorm': [8, 16, 32],\n",
    "    'LinearMinifloatDenorm_ew': [2, 4, 8],\n",
    "    'LinearMinifloatDenorm_eb': [2, 4, 8],\n",
    "    'LinearMinifloatIEEE': [8, 16, 32],\n",
    "    'LinearMinifloatIEEE_ew': [2, 4, 8],\n",
    "    'LinearMinifloatIEEE_eb': [2, 4, 8],\n",
    "    'LinearLog': [8, 16, 32],\n",
    "    'LinearLog_ew': [2, 4, 8],\n",
    "    'LinearLog_eb': [2, 4, 8],\n",
    "    'LinearBlockFP': [8, 16, 32],\n",
    "    'LinearBlockFP_block_size':[[4], [8], [16]] ,\n",
    "    'LinearBlockMinifloat_ebw': [2, 4, 8],\n",
    "    'LinearBlockMinifloat_ew': [2, 4, 8],\n",
    "    'LinearBlockMinifloat_block_size': [[4], [8], [16]],\n",
    "    'LinearBlockMinifloat_w': [2, 4, 8],\n",
    "    'LinearBlockLog': [8, 16, 32],\n",
    "    'LinearBlockLog_ebw': [2, 4, 8],\n",
    "    'LinearBlockLog_block_size': [[4], [8], [16]] ,\n",
    "    'LinearBinary_stochastic': [2, 4, 8],\n",
    "    'LinearBinary_bipolar': [2, 4, 8],\n",
    "    'LinearBinaryScaling_stochastic': [2, 4, 8],\n",
    "    'LinearBinaryScaling_bipolar': [2, 4, 8],\n",
    "    \"LinearBinaryScaling_binary_training\": [2, 4, 8],\n",
    "    'LinearBinaryResidualSign_stochastic': [2, 4, 8],\n",
    "    'LinearBinaryResidualSign_bipolar': [2, 4, 8],\n",
    "    \"LinearBinaryResidualSign_binary_training\": [2, 4, 8],\n",
    "}\n",
    "\n",
    "def construct_model(trial):\n",
    "    # Fetch the model\n",
    "    my_config = {}\n",
    "    trial_model = deepcopy(base_model)\n",
    "    \n",
    "    # Select quantization and fractional width configurations\n",
    "    for param in ['linear_layer_quantization', \n",
    "                  'linear_layer_quantization_fractional',\n",
    "                  'LinearMinifloatDenorm',\n",
    "                  'LinearMinifloatDenorm_ew',\n",
    "                  'LinearMinifloatDenorm_eb',\n",
    "                  'LinearMinifloatIEEE',\n",
    "                  'LinearMinifloatIEEE_ew',\n",
    "                  'LinearMinifloatIEEE_eb',\n",
    "                    'LinearLog',\n",
    "                    'LinearLog_ew',\n",
    "                    'LinearLog_eb',\n",
    "                    'LinearBlockFP',\n",
    "                    'LinearBlockFP_block_size',\n",
    "                    'LinearBlockMinifloat_ebw',\n",
    "                    'LinearBlockMinifloat_ew',\n",
    "                    'LinearBlockMinifloat_block_size',\n",
    "                    'LinearBlockMinifloat_w',\n",
    "                    'LinearBlockLog',\n",
    "                    'LinearBlockLog_ebw',\n",
    "                    'LinearBlockLog_block_size',\n",
    "                    'LinearBinary_stochastic',\n",
    "                    'LinearBinary_bipolar',\n",
    "                    'LinearBinaryScaling_stochastic',\n",
    "                    'LinearBinaryScaling_bipolar',\n",
    "                    'LinearBinaryScaling_binary_training',\n",
    "                    'LinearBinaryResidualSign_stochastic',\n",
    "                    'LinearBinaryResidualSign_bipolar',\n",
    "                    'LinearBinaryResidualSign_binary_training'\n",
    "                  ]:\n",
    "        chosen_idx = trial.suggest_int(param, 0, len(search_space[param]) - 1)\n",
    "        my_config[param] = search_space[param][chosen_idx]\n",
    "\n",
    "    # Quantize layers according to Optuna suggestions\n",
    "    for name, layer in trial_model.named_modules():\n",
    "        if isinstance(layer, torch.nn.Linear):\n",
    "            new_layer_cls = trial.suggest_categorical(\n",
    "                f\"{name}_type\",\n",
    "                search_space[\"linear_layer_choices\"],\n",
    "            )\n",
    "\n",
    "            if new_layer_cls == torch.nn.Linear:\n",
    "                continue\n",
    "\n",
    "            kwargs = {\n",
    "                \"in_features\": layer.in_features,\n",
    "                \"out_features\": layer.out_features,\n",
    "            }\n",
    "\n",
    "            try:\n",
    "                # Configure parameters for specific layer types\n",
    "                # if new_layer_cls == LinearInteger:\n",
    "                #     kwargs[\"config\"] = {\n",
    "                #         \"data_in_width\": my_config['linear_layer_quantization'],\n",
    "                #         \"data_in_frac_width\": my_config['linear_layer_quantization_fractional'],\n",
    "                #         \"weight_width\": my_config['linear_layer_quantization'],\n",
    "                #         \"weight_frac_width\": my_config['linear_layer_quantization_fractional'],\n",
    "                #         \"bias_width\": my_config['linear_layer_quantization'],\n",
    "                #         \"bias_frac_width\": my_config['linear_layer_quantization_fractional'],\n",
    "                #     }\n",
    "                if new_layer_cls == LinearMinifloatDenorm:\n",
    "                    kwargs[\"config\"] = {\n",
    "                        \"data_in_width\": my_config['LinearMinifloatDenorm'],\n",
    "                        \"data_in_exponent_width\": my_config['LinearMinifloatDenorm_ew'],\n",
    "                        \"data_in_exponent_bias\": my_config['LinearMinifloatDenorm_eb'],\n",
    "                        \"weight_width\": my_config['LinearMinifloatDenorm'],\n",
    "                        \"weight_exponent_width\": my_config['LinearMinifloatDenorm_ew'],\n",
    "                        \"weight_exponent_bias\": my_config['LinearMinifloatDenorm_eb'],\n",
    "                        \"bias_width\": my_config['LinearMinifloatDenorm'],\n",
    "                        \"bias_exponent_width\": my_config['LinearMinifloatDenorm_ew'],\n",
    "                        \"bias_exponent_bias\": my_config['LinearMinifloatDenorm_eb'],\n",
    "                    }\n",
    "                # elif new_layer_cls == LinearMinifloatIEEE:\n",
    "                #     kwargs[\"config\"] = {\n",
    "                #         \"data_in_width\": my_config['LinearMinifloatIEEE'],\n",
    "                #         \"data_in_exponent_width\": my_config['LinearMinifloatIEEE_ew'],\n",
    "                #         \"data_in_exponent_bias\": my_config['LinearMinifloatIEEE_eb'],\n",
    "                #         \"weight_width\": my_config['LinearMinifloatIEEE'],\n",
    "                #         \"weight_exponent_width\": my_config['LinearMinifloatIEEE_ew'],\n",
    "                #         \"weight_exponent_bias\": my_config['LinearMinifloatIEEE_eb'],\n",
    "                #         \"bias_width\": my_config['LinearMinifloatIEEE'],\n",
    "                #         \"bias_exponent_width\": my_config['LinearMinifloatIEEE_ew'],\n",
    "                #         \"bias_exponent_bias\": my_config['LinearMinifloatIEEE_eb'],\n",
    "                #     }\n",
    "                # elif new_layer_cls == LinearLog:\n",
    "                #     kwargs[\"config\"] = {\n",
    "                #         \"data_in_width\": my_config['LinearLog'],\n",
    "                #         \"data_in_exponent_width\": my_config['LinearLog_ew'],\n",
    "                #         \"data_in_exponent_bias\": my_config['LinearLog_eb'],\n",
    "                #         \"weight_width\": my_config['LinearLog'],\n",
    "                #         \"weight_exponent_width\": my_config['LinearLog_ew'],\n",
    "                #         \"weight_exponent_bias\": my_config['LinearLog_eb'],\n",
    "                #         \"bias_width\": my_config['LinearLog'],\n",
    "                #         \"bias_exponent_width\": my_config['LinearLog_ew'],\n",
    "                #         \"bias_exponent_bias\": my_config['LinearLog_eb'],\n",
    "                #     }\n",
    "                # elif new_layer_cls == LinearBlockFP:\n",
    "                #     kwargs[\"config\"] = {\n",
    "                #         \"weight_width\": my_config['LinearBlockFP'],\n",
    "                #         \"weight_exponent_width\": my_config['LinearBlockFP'],\n",
    "                #         \"weight_block_size\": my_config['LinearBlockFP_block_size'],\n",
    "                #         \"weight_exponent_bias\": my_config['LinearBlockFP'],\n",
    "                #         \"data_in_width\": my_config['LinearBlockFP'],\n",
    "                #         \"data_in_exponent_width\": my_config['LinearBlockFP'],\n",
    "                #         \"data_in_block_size\": my_config['LinearBlockFP_block_size'],\n",
    "                #         \"data_in_exponent_bias\": my_config['LinearBlockFP'],\n",
    "                #         \"bias_width\": my_config['LinearBlockFP'],\n",
    "                #         \"bias_exponent_width\": my_config['LinearBlockFP'],\n",
    "                #         \"bias_block_size\": my_config['LinearBlockFP_block_size'],\n",
    "                #         \"bias_exponent_bias\": my_config['LinearBlockFP'],\n",
    "                #     }\n",
    "                # # elif new_layer_cls == LinearBlockMinifloat:\n",
    "                # #     kwargs[\"config\"] = {\n",
    "                # #         \"weight_exponent_bias_width\" : my_config['LinearBlockMinifloat_ebw'],\n",
    "                # #         \"weight_exponent_width\": my_config['LinearBlockMinifloat_ew'],\n",
    "                # #         \"weight_block_size\": my_config['LinearBlockMinifloat_block_size'],\n",
    "                # #         \"weight_width\": my_config['LinearBlockMinifloat_w'],\n",
    "                # #         \"data_in_width\":my_config[\"LinearBlockMinifloat_w\"],\n",
    "                # #         \"data_in_exponent_bias_width\":my_config[\"LinearBlockMinifloat_ew\"],\n",
    "                # #         \"data_in_exponent_width\":my_config[\"LinearBlockMinifloat_ebw\"],\n",
    "                # #         \"data_in_block_size\":my_config[\"LinearBlockMinifloat_block_size\"],\n",
    "                # #         \"bias_width\":my_config[\"LinearBlockMinifloat_w\"],\n",
    "                # #         \"bias_exponent_width\":my_config[\"LinearBlockMinifloat_ew\"],\n",
    "                # #         \"bias_exponent_bias_width\":my_config[\"LinearBlockMinifloat_ebw\"],\n",
    "                # #         \"bias_block_size\":my_config[\"LinearBlockMinifloat_block_size\"],\n",
    "                # #     }\n",
    "                # elif new_layer_cls == LinearBlockLog:\n",
    "                #     kwargs[\"config\"] = {\n",
    "                #         \"data_in_width\": my_config['LinearBlockLog'],\n",
    "                #         \"data_in_exponent_bias_width\": my_config['LinearBlockLog_ebw'],\n",
    "                #         \"data_in_block_size\": my_config['LinearBlockLog_block_size'],\n",
    "                #         \"weight_width\": my_config['LinearBlockLog'],\n",
    "                #         \"weight_exponent_bias_width\": my_config['LinearBlockLog_ebw'],\n",
    "                #         \"weight_block_size\": my_config['LinearBlockLog_block_size'],\n",
    "                #         \"bias_width\": my_config['LinearBlockLog'],\n",
    "                #         \"bias_exponent_bias_width\": my_config['LinearBlockLog_ebw'],\n",
    "                #         \"bias_block_size\": my_config['LinearBlockLog_block_size'],\n",
    "                #     }\n",
    "\n",
    "                # elif new_layer_cls == LinearBinary:\n",
    "                #     kwargs[\"config\"] = {\n",
    "                #         'weight_stochastic':my_config['LinearBinary_stochastic'],\n",
    "                #         'weight_bipolar':my_config['LinearBinary_bipolar'],\n",
    "                #     }\n",
    "\n",
    "                    \n",
    "                # elif new_layer_cls == LinearBinaryScaling:\n",
    "                #     kwargs[\"config\"] = {\n",
    "                #         'weight_stochastic':my_config['LinearBinaryScaling_stochastic'],\n",
    "                #         'weight_bipolar':my_config['LinearBinaryScaling_bipolar'],\n",
    "                #         \"data_in_stochastic\": my_config['LinearBinaryScaling_stochastic'],\n",
    "                #         \"data_in_bipolar\": my_config['LinearBinaryScaling_bipolar'],\n",
    "                #         \"bias_stochastic\": my_config['LinearBinaryScaling_stochastic'],\n",
    "                #         \"bias_bipolar\": my_config['LinearBinaryScaling_bipolar'],\n",
    "                #         \"binary_training\": my_config['LinearBinaryScaling_binary_training'],\n",
    "                #     }\n",
    "                # elif new_layer_cls == LinearBinaryResidualSign:\n",
    "                #     kwargs[\"config\"] = {\n",
    "                #         'weight_stochastic':my_config['LinearBinaryResidualSign_stochastic'],\n",
    "                #         'weight_bipolar':my_config['LinearBinaryResidualSign_bipolar'],\n",
    "                #         \"data_in_stochastic\": my_config['LinearBinaryResidualSign_stochastic'],\n",
    "                #         \"data_in_bipolar\": my_config['LinearBinaryResidualSign_bipolar'],\n",
    "                #         \"bias_stochastic\": my_config['LinearBinaryResidualSign_stochastic'],\n",
    "                #         \"bias_bipolar\": my_config['LinearBinaryResidualSign_bipolar'],\n",
    "                #         \"binary_training\": my_config['LinearBinaryResidualSign_binary_training'],\n",
    "                #     }\n",
    "                # Create the new layer (copy the weights if applicable)\n",
    "                new_layer = new_layer_cls(**kwargs)\n",
    "                new_layer.weight.data = layer.weight.data.clone()\n",
    "\n",
    "                # Replace the layer in the model\n",
    "                deepsetattr(trial_model, name, new_layer)\n",
    "            except TypeError as e:\n",
    "                print(f\"Error creating layer {name} with class {new_layer_cls}: {e}\")\n",
    "                raise\n",
    "\n",
    "    return trial_model\n",
    "from chop.tools import get_trainer\n",
    "import random\n",
    "\n",
    "\n",
    "def objective(trial):\n",
    "\n",
    "    # Define the model\n",
    "    model = construct_model(trial)\n",
    "\n",
    "    trainer = get_trainer(\n",
    "        model=model,\n",
    "        tokenized_dataset=dataset,\n",
    "        tokenizer=tokenizer,\n",
    "        evaluate_metric=\"accuracy\",\n",
    "        num_train_epochs=1,\n",
    "    )\n",
    "\n",
    "    trainer.train()\n",
    "    eval_results = trainer.evaluate()\n",
    "\n",
    "    trial.set_user_attr(\"model\", model)\n",
    "\n",
    "    return eval_results[\"eval_accuracy\"]\n",
    "\n",
    "\n",
    "from optuna.samplers import GridSampler, RandomSampler, TPESampler\n",
    "\n",
    "sampler = RandomSampler()\n",
    "\n",
    "import optuna\n",
    "\n",
    "study = optuna.create_study(\n",
    "    direction=\"maximize\",\n",
    "    study_name=\"bert-tiny-nas-study\",\n",
    "    sampler=sampler,\n",
    ")\n",
    "\n",
    "study.optimize(\n",
    "    objective,\n",
    "    n_trials=20,\n",
    "    timeout=60 * 60 * 24,\n",
    ")\n",
    "\n",
    "task3 = [max([t.value for t in study.trials[:i+1]]) for i in range(len(study.trials))]\n",
    "task3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "task3 =[0.87256,\n",
    " 0.87268,\n",
    " 0.87268,\n",
    " 0.87268,\n",
    " 0.87284,\n",
    " 0.87284,\n",
    " 0.87296,\n",
    " 0.87296,\n",
    " 0.8738,\n",
    " 0.8738]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-06 12:39:21,569] A new study created in memory with name: bert-tiny-nas-study\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearMinifloatIEEE'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 01:57, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.340100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.307700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.333000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.343500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.335300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.368200</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 01:32]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-06 12:42:53,402] Trial 0 finished with value: 0.87332 and parameters: {'linear_layer_quantization': 2, 'linear_layer_quantization_fractional': 0, 'LinearMinifloatDenorm': 1, 'LinearMinifloatDenorm_ew': 1, 'LinearMinifloatDenorm_eb': 1, 'LinearMinifloatIEEE': 2, 'LinearMinifloatIEEE_ew': 2, 'LinearMinifloatIEEE_eb': 2, 'LinearLog': 2, 'LinearLog_ew': 0, 'LinearLog_eb': 0, 'LinearBlockFP': 2, 'LinearBlockFP_block_size': 0, 'LinearBlockMinifloat_ebw': 2, 'LinearBlockMinifloat_ew': 0, 'LinearBlockMinifloat_block_size': 1, 'LinearBlockMinifloat_w': 0, 'LinearBlockLog': 0, 'LinearBlockLog_ebw': 0, 'LinearBlockLog_block_size': 1, 'LinearBinary_stochastic': 1, 'LinearBinary_bipolar': 0, 'LinearBinaryScaling_stochastic': 0, 'LinearBinaryScaling_bipolar': 0, 'LinearBinaryScaling_binary_training': 0, 'LinearBinaryResidualSign_stochastic': 2, 'LinearBinaryResidualSign_bipolar': 0, 'LinearBinaryResidualSign_binary_training': 2, 'bert.encoder.layer.0.attention.self.query_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.output.dense_type': <class 'chop.nn.quantized.modules.linear.LinearMinifloatIEEE'>, 'bert.pooler.dense_type': <class 'chop.nn.quantized.modules.linear.LinearMinifloatIEEE'>, 'classifier_type': <class 'torch.nn.modules.linear.Linear'>}. Best is trial 0 with value: 0.87332.\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearMinifloatIEEE'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 03:05, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.337500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.311000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.330500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.344200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.335300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.361700</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 02:05]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-06 12:48:05,475] Trial 1 finished with value: 0.87348 and parameters: {'linear_layer_quantization': 0, 'linear_layer_quantization_fractional': 0, 'LinearMinifloatDenorm': 0, 'LinearMinifloatDenorm_ew': 1, 'LinearMinifloatDenorm_eb': 0, 'LinearMinifloatIEEE': 1, 'LinearMinifloatIEEE_ew': 1, 'LinearMinifloatIEEE_eb': 0, 'LinearLog': 0, 'LinearLog_ew': 1, 'LinearLog_eb': 0, 'LinearBlockFP': 0, 'LinearBlockFP_block_size': 2, 'LinearBlockMinifloat_ebw': 2, 'LinearBlockMinifloat_ew': 2, 'LinearBlockMinifloat_block_size': 1, 'LinearBlockMinifloat_w': 2, 'LinearBlockLog': 1, 'LinearBlockLog_ebw': 2, 'LinearBlockLog_block_size': 0, 'LinearBinary_stochastic': 1, 'LinearBinary_bipolar': 2, 'LinearBinaryScaling_stochastic': 1, 'LinearBinaryScaling_bipolar': 2, 'LinearBinaryScaling_binary_training': 0, 'LinearBinaryResidualSign_stochastic': 2, 'LinearBinaryResidualSign_bipolar': 0, 'LinearBinaryResidualSign_binary_training': 0, 'bert.encoder.layer.0.attention.self.query_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.output.dense_type': <class 'chop.nn.quantized.modules.linear.LinearMinifloatIEEE'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'chop.nn.quantized.modules.linear.LinearMinifloatIEEE'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'chop.nn.quantized.modules.linear.LinearMinifloatIEEE'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearMinifloatIEEE'>, 'bert.encoder.layer.1.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.pooler.dense_type': <class 'chop.nn.quantized.modules.linear.LinearMinifloatIEEE'>, 'classifier_type': <class 'chop.nn.quantized.modules.linear.LinearMinifloatIEEE'>}. Best is trial 1 with value: 0.87348.\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearMinifloatIEEE'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 02:01, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.339000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.303800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.330200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.344700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.331700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.361400</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 01:24]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-06 12:51:30,843] Trial 2 finished with value: 0.87268 and parameters: {'linear_layer_quantization': 2, 'linear_layer_quantization_fractional': 1, 'LinearMinifloatDenorm': 2, 'LinearMinifloatDenorm_ew': 0, 'LinearMinifloatDenorm_eb': 0, 'LinearMinifloatIEEE': 2, 'LinearMinifloatIEEE_ew': 1, 'LinearMinifloatIEEE_eb': 1, 'LinearLog': 2, 'LinearLog_ew': 2, 'LinearLog_eb': 2, 'LinearBlockFP': 0, 'LinearBlockFP_block_size': 2, 'LinearBlockMinifloat_ebw': 0, 'LinearBlockMinifloat_ew': 2, 'LinearBlockMinifloat_block_size': 2, 'LinearBlockMinifloat_w': 0, 'LinearBlockLog': 0, 'LinearBlockLog_ebw': 1, 'LinearBlockLog_block_size': 0, 'LinearBinary_stochastic': 0, 'LinearBinary_bipolar': 1, 'LinearBinaryScaling_stochastic': 0, 'LinearBinaryScaling_bipolar': 2, 'LinearBinaryScaling_binary_training': 0, 'LinearBinaryResidualSign_stochastic': 0, 'LinearBinaryResidualSign_bipolar': 0, 'LinearBinaryResidualSign_binary_training': 2, 'bert.encoder.layer.0.attention.self.query_type': <class 'chop.nn.quantized.modules.linear.LinearMinifloatIEEE'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'chop.nn.quantized.modules.linear.LinearMinifloatIEEE'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'chop.nn.quantized.modules.linear.LinearMinifloatIEEE'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.pooler.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'classifier_type': <class 'chop.nn.quantized.modules.linear.LinearMinifloatIEEE'>}. Best is trial 1 with value: 0.87348.\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearMinifloatIEEE'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 02:15, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.343300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.311200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.339600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.341800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.334400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.371300</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 01:36]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-06 12:55:24,031] Trial 3 finished with value: 0.8738 and parameters: {'linear_layer_quantization': 2, 'linear_layer_quantization_fractional': 0, 'LinearMinifloatDenorm': 2, 'LinearMinifloatDenorm_ew': 0, 'LinearMinifloatDenorm_eb': 2, 'LinearMinifloatIEEE': 1, 'LinearMinifloatIEEE_ew': 1, 'LinearMinifloatIEEE_eb': 0, 'LinearLog': 0, 'LinearLog_ew': 2, 'LinearLog_eb': 0, 'LinearBlockFP': 2, 'LinearBlockFP_block_size': 2, 'LinearBlockMinifloat_ebw': 2, 'LinearBlockMinifloat_ew': 0, 'LinearBlockMinifloat_block_size': 1, 'LinearBlockMinifloat_w': 0, 'LinearBlockLog': 0, 'LinearBlockLog_ebw': 1, 'LinearBlockLog_block_size': 0, 'LinearBinary_stochastic': 1, 'LinearBinary_bipolar': 1, 'LinearBinaryScaling_stochastic': 2, 'LinearBinaryScaling_bipolar': 0, 'LinearBinaryScaling_binary_training': 2, 'LinearBinaryResidualSign_stochastic': 2, 'LinearBinaryResidualSign_bipolar': 1, 'LinearBinaryResidualSign_binary_training': 1, 'bert.encoder.layer.0.attention.self.query_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearMinifloatIEEE'>, 'bert.encoder.layer.0.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'chop.nn.quantized.modules.linear.LinearMinifloatIEEE'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'chop.nn.quantized.modules.linear.LinearMinifloatIEEE'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearMinifloatIEEE'>, 'bert.encoder.layer.1.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.pooler.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'classifier_type': <class 'chop.nn.quantized.modules.linear.LinearMinifloatIEEE'>}. Best is trial 3 with value: 0.8738.\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearMinifloatIEEE'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 02:14, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.339800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.305600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.333800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.335300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.331600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.360700</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 01:37]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-06 12:59:17,328] Trial 4 finished with value: 0.87252 and parameters: {'linear_layer_quantization': 0, 'linear_layer_quantization_fractional': 2, 'LinearMinifloatDenorm': 1, 'LinearMinifloatDenorm_ew': 2, 'LinearMinifloatDenorm_eb': 2, 'LinearMinifloatIEEE': 2, 'LinearMinifloatIEEE_ew': 1, 'LinearMinifloatIEEE_eb': 2, 'LinearLog': 0, 'LinearLog_ew': 0, 'LinearLog_eb': 2, 'LinearBlockFP': 1, 'LinearBlockFP_block_size': 2, 'LinearBlockMinifloat_ebw': 2, 'LinearBlockMinifloat_ew': 1, 'LinearBlockMinifloat_block_size': 0, 'LinearBlockMinifloat_w': 1, 'LinearBlockLog': 2, 'LinearBlockLog_ebw': 0, 'LinearBlockLog_block_size': 1, 'LinearBinary_stochastic': 2, 'LinearBinary_bipolar': 2, 'LinearBinaryScaling_stochastic': 0, 'LinearBinaryScaling_bipolar': 0, 'LinearBinaryScaling_binary_training': 2, 'LinearBinaryResidualSign_stochastic': 2, 'LinearBinaryResidualSign_bipolar': 2, 'LinearBinaryResidualSign_binary_training': 1, 'bert.encoder.layer.0.attention.self.query_type': <class 'chop.nn.quantized.modules.linear.LinearMinifloatIEEE'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearMinifloatIEEE'>, 'bert.encoder.layer.0.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'chop.nn.quantized.modules.linear.LinearMinifloatIEEE'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearMinifloatIEEE'>, 'bert.encoder.layer.1.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.pooler.dense_type': <class 'chop.nn.quantized.modules.linear.LinearMinifloatIEEE'>, 'classifier_type': <class 'torch.nn.modules.linear.Linear'>}. Best is trial 3 with value: 0.8738.\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearMinifloatIEEE'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 03:13, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.343000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.337200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.357800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.356600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.350100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.390700</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 02:27]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-06 13:04:59,440] Trial 5 finished with value: 0.86972 and parameters: {'linear_layer_quantization': 1, 'linear_layer_quantization_fractional': 0, 'LinearMinifloatDenorm': 1, 'LinearMinifloatDenorm_ew': 1, 'LinearMinifloatDenorm_eb': 2, 'LinearMinifloatIEEE': 1, 'LinearMinifloatIEEE_ew': 0, 'LinearMinifloatIEEE_eb': 1, 'LinearLog': 2, 'LinearLog_ew': 0, 'LinearLog_eb': 0, 'LinearBlockFP': 2, 'LinearBlockFP_block_size': 1, 'LinearBlockMinifloat_ebw': 2, 'LinearBlockMinifloat_ew': 2, 'LinearBlockMinifloat_block_size': 1, 'LinearBlockMinifloat_w': 2, 'LinearBlockLog': 1, 'LinearBlockLog_ebw': 0, 'LinearBlockLog_block_size': 2, 'LinearBinary_stochastic': 2, 'LinearBinary_bipolar': 2, 'LinearBinaryScaling_stochastic': 2, 'LinearBinaryScaling_bipolar': 0, 'LinearBinaryScaling_binary_training': 0, 'LinearBinaryResidualSign_stochastic': 2, 'LinearBinaryResidualSign_bipolar': 2, 'LinearBinaryResidualSign_binary_training': 2, 'bert.encoder.layer.0.attention.self.query_type': <class 'chop.nn.quantized.modules.linear.LinearMinifloatIEEE'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearMinifloatIEEE'>, 'bert.encoder.layer.0.output.dense_type': <class 'chop.nn.quantized.modules.linear.LinearMinifloatIEEE'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'chop.nn.quantized.modules.linear.LinearMinifloatIEEE'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.output.dense_type': <class 'chop.nn.quantized.modules.linear.LinearMinifloatIEEE'>, 'bert.pooler.dense_type': <class 'chop.nn.quantized.modules.linear.LinearMinifloatIEEE'>, 'classifier_type': <class 'torch.nn.modules.linear.Linear'>}. Best is trial 3 with value: 0.8738.\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearMinifloatIEEE'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 02:52, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.341300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.310300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.335200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.338700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.330800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.361300</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 02:08]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-06 13:10:01,945] Trial 6 finished with value: 0.8732 and parameters: {'linear_layer_quantization': 0, 'linear_layer_quantization_fractional': 1, 'LinearMinifloatDenorm': 2, 'LinearMinifloatDenorm_ew': 2, 'LinearMinifloatDenorm_eb': 0, 'LinearMinifloatIEEE': 2, 'LinearMinifloatIEEE_ew': 2, 'LinearMinifloatIEEE_eb': 0, 'LinearLog': 1, 'LinearLog_ew': 0, 'LinearLog_eb': 1, 'LinearBlockFP': 2, 'LinearBlockFP_block_size': 0, 'LinearBlockMinifloat_ebw': 1, 'LinearBlockMinifloat_ew': 2, 'LinearBlockMinifloat_block_size': 2, 'LinearBlockMinifloat_w': 2, 'LinearBlockLog': 1, 'LinearBlockLog_ebw': 1, 'LinearBlockLog_block_size': 0, 'LinearBinary_stochastic': 0, 'LinearBinary_bipolar': 0, 'LinearBinaryScaling_stochastic': 2, 'LinearBinaryScaling_bipolar': 0, 'LinearBinaryScaling_binary_training': 0, 'LinearBinaryResidualSign_stochastic': 1, 'LinearBinaryResidualSign_bipolar': 0, 'LinearBinaryResidualSign_binary_training': 2, 'bert.encoder.layer.0.attention.self.query_type': <class 'chop.nn.quantized.modules.linear.LinearMinifloatIEEE'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'chop.nn.quantized.modules.linear.LinearMinifloatIEEE'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'chop.nn.quantized.modules.linear.LinearMinifloatIEEE'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.output.dense_type': <class 'chop.nn.quantized.modules.linear.LinearMinifloatIEEE'>, 'bert.pooler.dense_type': <class 'chop.nn.quantized.modules.linear.LinearMinifloatIEEE'>, 'classifier_type': <class 'chop.nn.quantized.modules.linear.LinearMinifloatIEEE'>}. Best is trial 3 with value: 0.8738.\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearMinifloatIEEE'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 03:05, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.339300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.301300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.333500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.347700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.334500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.367200</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 02:28]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-06 13:15:37,374] Trial 7 finished with value: 0.8738 and parameters: {'linear_layer_quantization': 1, 'linear_layer_quantization_fractional': 2, 'LinearMinifloatDenorm': 0, 'LinearMinifloatDenorm_ew': 0, 'LinearMinifloatDenorm_eb': 0, 'LinearMinifloatIEEE': 0, 'LinearMinifloatIEEE_ew': 1, 'LinearMinifloatIEEE_eb': 1, 'LinearLog': 2, 'LinearLog_ew': 1, 'LinearLog_eb': 2, 'LinearBlockFP': 0, 'LinearBlockFP_block_size': 2, 'LinearBlockMinifloat_ebw': 1, 'LinearBlockMinifloat_ew': 1, 'LinearBlockMinifloat_block_size': 2, 'LinearBlockMinifloat_w': 0, 'LinearBlockLog': 2, 'LinearBlockLog_ebw': 0, 'LinearBlockLog_block_size': 0, 'LinearBinary_stochastic': 1, 'LinearBinary_bipolar': 1, 'LinearBinaryScaling_stochastic': 1, 'LinearBinaryScaling_bipolar': 2, 'LinearBinaryScaling_binary_training': 0, 'LinearBinaryResidualSign_stochastic': 0, 'LinearBinaryResidualSign_bipolar': 1, 'LinearBinaryResidualSign_binary_training': 2, 'bert.encoder.layer.0.attention.self.query_type': <class 'chop.nn.quantized.modules.linear.LinearMinifloatIEEE'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'chop.nn.quantized.modules.linear.LinearMinifloatIEEE'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearMinifloatIEEE'>, 'bert.encoder.layer.0.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'chop.nn.quantized.modules.linear.LinearMinifloatIEEE'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearMinifloatIEEE'>, 'bert.encoder.layer.1.output.dense_type': <class 'chop.nn.quantized.modules.linear.LinearMinifloatIEEE'>, 'bert.pooler.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'classifier_type': <class 'chop.nn.quantized.modules.linear.LinearMinifloatIEEE'>}. Best is trial 3 with value: 0.8738.\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearMinifloatIEEE'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 02:26, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.344100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.343100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.346200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.347600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.336100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.376500</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 01:46]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-06 13:19:51,699] Trial 8 finished with value: 0.87076 and parameters: {'linear_layer_quantization': 1, 'linear_layer_quantization_fractional': 1, 'LinearMinifloatDenorm': 2, 'LinearMinifloatDenorm_ew': 0, 'LinearMinifloatDenorm_eb': 0, 'LinearMinifloatIEEE': 0, 'LinearMinifloatIEEE_ew': 0, 'LinearMinifloatIEEE_eb': 1, 'LinearLog': 2, 'LinearLog_ew': 1, 'LinearLog_eb': 2, 'LinearBlockFP': 1, 'LinearBlockFP_block_size': 0, 'LinearBlockMinifloat_ebw': 2, 'LinearBlockMinifloat_ew': 1, 'LinearBlockMinifloat_block_size': 0, 'LinearBlockMinifloat_w': 2, 'LinearBlockLog': 1, 'LinearBlockLog_ebw': 0, 'LinearBlockLog_block_size': 0, 'LinearBinary_stochastic': 2, 'LinearBinary_bipolar': 0, 'LinearBinaryScaling_stochastic': 1, 'LinearBinaryScaling_bipolar': 0, 'LinearBinaryScaling_binary_training': 1, 'LinearBinaryResidualSign_stochastic': 0, 'LinearBinaryResidualSign_bipolar': 0, 'LinearBinaryResidualSign_binary_training': 2, 'bert.encoder.layer.0.attention.self.query_type': <class 'chop.nn.quantized.modules.linear.LinearMinifloatIEEE'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'chop.nn.quantized.modules.linear.LinearMinifloatIEEE'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearMinifloatIEEE'>, 'bert.encoder.layer.0.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'chop.nn.quantized.modules.linear.LinearMinifloatIEEE'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'chop.nn.quantized.modules.linear.LinearMinifloatIEEE'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.pooler.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'classifier_type': <class 'chop.nn.quantized.modules.linear.LinearMinifloatIEEE'>}. Best is trial 3 with value: 0.8738.\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearMinifloatIEEE'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 01:58, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.338600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.302800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.334800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.347100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.336100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.366600</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 01:21]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-06 13:23:13,316] Trial 9 finished with value: 0.87352 and parameters: {'linear_layer_quantization': 0, 'linear_layer_quantization_fractional': 1, 'LinearMinifloatDenorm': 0, 'LinearMinifloatDenorm_ew': 2, 'LinearMinifloatDenorm_eb': 2, 'LinearMinifloatIEEE': 0, 'LinearMinifloatIEEE_ew': 1, 'LinearMinifloatIEEE_eb': 2, 'LinearLog': 1, 'LinearLog_ew': 2, 'LinearLog_eb': 0, 'LinearBlockFP': 2, 'LinearBlockFP_block_size': 0, 'LinearBlockMinifloat_ebw': 1, 'LinearBlockMinifloat_ew': 0, 'LinearBlockMinifloat_block_size': 1, 'LinearBlockMinifloat_w': 2, 'LinearBlockLog': 0, 'LinearBlockLog_ebw': 2, 'LinearBlockLog_block_size': 0, 'LinearBinary_stochastic': 1, 'LinearBinary_bipolar': 0, 'LinearBinaryScaling_stochastic': 1, 'LinearBinaryScaling_bipolar': 2, 'LinearBinaryScaling_binary_training': 2, 'LinearBinaryResidualSign_stochastic': 2, 'LinearBinaryResidualSign_bipolar': 2, 'LinearBinaryResidualSign_binary_training': 0, 'bert.encoder.layer.0.attention.self.query_type': <class 'chop.nn.quantized.modules.linear.LinearMinifloatIEEE'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'chop.nn.quantized.modules.linear.LinearMinifloatIEEE'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearMinifloatIEEE'>, 'bert.encoder.layer.1.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.pooler.dense_type': <class 'chop.nn.quantized.modules.linear.LinearMinifloatIEEE'>, 'classifier_type': <class 'torch.nn.modules.linear.Linear'>}. Best is trial 3 with value: 0.8738.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.87332,\n",
       " 0.87348,\n",
       " 0.87348,\n",
       " 0.8738,\n",
       " 0.8738,\n",
       " 0.8738,\n",
       " 0.8738,\n",
       " 0.8738,\n",
       " 0.8738,\n",
       " 0.8738]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "from pathlib import Path\n",
    "import dill\n",
    "\n",
    "with open(f\"{Path.home()}/tutorial_5_best_model.pkl\", \"rb\") as f:\n",
    "    base_model = dill.load(f)\n",
    "from chop.nn.quantized.modules.linear import (\n",
    "    LinearInteger,\n",
    "    LinearMinifloatDenorm,\n",
    "    LinearMinifloatIEEE,\n",
    "    LinearLog,\n",
    "    LinearBlockFP,\n",
    "    LinearBlockMinifloat,\n",
    "    LinearBlockLog,\n",
    "    LinearBinary,\n",
    "    LinearBinaryScaling,\n",
    "    LinearBinaryResidualSign,\n",
    ")\n",
    "search_space = {\n",
    "    \"linear_layer_choices\": [\n",
    "        torch.nn.Linear,\n",
    "        # LinearInteger,\n",
    "        # LinearMinifloatDenorm,\n",
    "        LinearMinifloatIEEE,\n",
    "        # LinearLog,\n",
    "        # LinearBlockFP,\n",
    "        # # LinearBlockMinifloat,\n",
    "        # LinearBlockLog,\n",
    "        # LinearBinary,\n",
    "        # LinearBinaryScaling,\n",
    "        # LinearBinaryResidualSign,\n",
    "    ],\n",
    "    'linear_layer_quantization': [8, 16, 32],\n",
    "    'linear_layer_quantization_fractional':  [2, 4, 8],\n",
    "    'LinearMinifloatDenorm': [8, 16, 32],\n",
    "    'LinearMinifloatDenorm_ew': [2, 4, 8],\n",
    "    'LinearMinifloatDenorm_eb': [2, 4, 8],\n",
    "    'LinearMinifloatIEEE': [8, 16, 32],\n",
    "    'LinearMinifloatIEEE_ew': [2, 4, 8],\n",
    "    'LinearMinifloatIEEE_eb': [2, 4, 8],\n",
    "    'LinearLog': [8, 16, 32],\n",
    "    'LinearLog_ew': [2, 4, 8],\n",
    "    'LinearLog_eb': [2, 4, 8],\n",
    "    'LinearBlockFP': [8, 16, 32],\n",
    "    'LinearBlockFP_block_size':[[4], [8], [16]] ,\n",
    "    'LinearBlockMinifloat_ebw': [2, 4, 8],\n",
    "    'LinearBlockMinifloat_ew': [2, 4, 8],\n",
    "    'LinearBlockMinifloat_block_size': [[4], [8], [16]],\n",
    "    'LinearBlockMinifloat_w': [2, 4, 8],\n",
    "    'LinearBlockLog': [8, 16, 32],\n",
    "    'LinearBlockLog_ebw': [2, 4, 8],\n",
    "    'LinearBlockLog_block_size': [[4], [8], [16]] ,\n",
    "    'LinearBinary_stochastic': [2, 4, 8],\n",
    "    'LinearBinary_bipolar': [2, 4, 8],\n",
    "    'LinearBinaryScaling_stochastic': [2, 4, 8],\n",
    "    'LinearBinaryScaling_bipolar': [2, 4, 8],\n",
    "    \"LinearBinaryScaling_binary_training\": [2, 4, 8],\n",
    "    'LinearBinaryResidualSign_stochastic': [2, 4, 8],\n",
    "    'LinearBinaryResidualSign_bipolar': [2, 4, 8],\n",
    "    \"LinearBinaryResidualSign_binary_training\": [2, 4, 8],\n",
    "}\n",
    "\n",
    "def construct_model(trial):\n",
    "    # Fetch the model\n",
    "    my_config = {}\n",
    "    trial_model = deepcopy(base_model)\n",
    "    \n",
    "    # Select quantization and fractional width configurations\n",
    "    for param in ['linear_layer_quantization', \n",
    "                  'linear_layer_quantization_fractional',\n",
    "                  'LinearMinifloatDenorm',\n",
    "                  'LinearMinifloatDenorm_ew',\n",
    "                  'LinearMinifloatDenorm_eb',\n",
    "                  'LinearMinifloatIEEE',\n",
    "                  'LinearMinifloatIEEE_ew',\n",
    "                  'LinearMinifloatIEEE_eb',\n",
    "                    'LinearLog',\n",
    "                    'LinearLog_ew',\n",
    "                    'LinearLog_eb',\n",
    "                    'LinearBlockFP',\n",
    "                    'LinearBlockFP_block_size',\n",
    "                    'LinearBlockMinifloat_ebw',\n",
    "                    'LinearBlockMinifloat_ew',\n",
    "                    'LinearBlockMinifloat_block_size',\n",
    "                    'LinearBlockMinifloat_w',\n",
    "                    'LinearBlockLog',\n",
    "                    'LinearBlockLog_ebw',\n",
    "                    'LinearBlockLog_block_size',\n",
    "                    'LinearBinary_stochastic',\n",
    "                    'LinearBinary_bipolar',\n",
    "                    'LinearBinaryScaling_stochastic',\n",
    "                    'LinearBinaryScaling_bipolar',\n",
    "                    'LinearBinaryScaling_binary_training',\n",
    "                    'LinearBinaryResidualSign_stochastic',\n",
    "                    'LinearBinaryResidualSign_bipolar',\n",
    "                    'LinearBinaryResidualSign_binary_training'\n",
    "                  ]:\n",
    "        chosen_idx = trial.suggest_int(param, 0, len(search_space[param]) - 1)\n",
    "        my_config[param] = search_space[param][chosen_idx]\n",
    "\n",
    "    # Quantize layers according to Optuna suggestions\n",
    "    for name, layer in trial_model.named_modules():\n",
    "        if isinstance(layer, torch.nn.Linear):\n",
    "            new_layer_cls = trial.suggest_categorical(\n",
    "                f\"{name}_type\",\n",
    "                search_space[\"linear_layer_choices\"],\n",
    "            )\n",
    "\n",
    "            if new_layer_cls == torch.nn.Linear:\n",
    "                continue\n",
    "\n",
    "            kwargs = {\n",
    "                \"in_features\": layer.in_features,\n",
    "                \"out_features\": layer.out_features,\n",
    "            }\n",
    "\n",
    "            try:\n",
    "                # Configure parameters for specific layer types\n",
    "                # if new_layer_cls == LinearInteger:\n",
    "                #     kwargs[\"config\"] = {\n",
    "                #         \"data_in_width\": my_config['linear_layer_quantization'],\n",
    "                #         \"data_in_frac_width\": my_config['linear_layer_quantization_fractional'],\n",
    "                #         \"weight_width\": my_config['linear_layer_quantization'],\n",
    "                #         \"weight_frac_width\": my_config['linear_layer_quantization_fractional'],\n",
    "                #         \"bias_width\": my_config['linear_layer_quantization'],\n",
    "                #         \"bias_frac_width\": my_config['linear_layer_quantization_fractional'],\n",
    "                #     }\n",
    "                # elif new_layer_cls == LinearMinifloatDenorm:\n",
    "                #     kwargs[\"config\"] = {\n",
    "                #         \"data_in_width\": my_config['LinearMinifloatDenorm'],\n",
    "                #         \"data_in_exponent_width\": my_config['LinearMinifloatDenorm_ew'],\n",
    "                #         \"data_in_exponent_bias\": my_config['LinearMinifloatDenorm_eb'],\n",
    "                #         \"weight_width\": my_config['LinearMinifloatDenorm'],\n",
    "                #         \"weight_exponent_width\": my_config['LinearMinifloatDenorm_ew'],\n",
    "                #         \"weight_exponent_bias\": my_config['LinearMinifloatDenorm_eb'],\n",
    "                #         \"bias_width\": my_config['LinearMinifloatDenorm'],\n",
    "                #         \"bias_exponent_width\": my_config['LinearMinifloatDenorm_ew'],\n",
    "                #         \"bias_exponent_bias\": my_config['LinearMinifloatDenorm_eb'],\n",
    "                #     }\n",
    "                if new_layer_cls == LinearMinifloatIEEE:\n",
    "                    kwargs[\"config\"] = {\n",
    "                        \"data_in_width\": my_config['LinearMinifloatIEEE'],\n",
    "                        \"data_in_exponent_width\": my_config['LinearMinifloatIEEE_ew'],\n",
    "                        \"data_in_exponent_bias\": my_config['LinearMinifloatIEEE_eb'],\n",
    "                        \"weight_width\": my_config['LinearMinifloatIEEE'],\n",
    "                        \"weight_exponent_width\": my_config['LinearMinifloatIEEE_ew'],\n",
    "                        \"weight_exponent_bias\": my_config['LinearMinifloatIEEE_eb'],\n",
    "                        \"bias_width\": my_config['LinearMinifloatIEEE'],\n",
    "                        \"bias_exponent_width\": my_config['LinearMinifloatIEEE_ew'],\n",
    "                        \"bias_exponent_bias\": my_config['LinearMinifloatIEEE_eb'],\n",
    "                    }\n",
    "                # elif new_layer_cls == LinearLog:\n",
    "                #     kwargs[\"config\"] = {\n",
    "                #         \"data_in_width\": my_config['LinearLog'],\n",
    "                #         \"data_in_exponent_width\": my_config['LinearLog_ew'],\n",
    "                #         \"data_in_exponent_bias\": my_config['LinearLog_eb'],\n",
    "                #         \"weight_width\": my_config['LinearLog'],\n",
    "                #         \"weight_exponent_width\": my_config['LinearLog_ew'],\n",
    "                #         \"weight_exponent_bias\": my_config['LinearLog_eb'],\n",
    "                #         \"bias_width\": my_config['LinearLog'],\n",
    "                #         \"bias_exponent_width\": my_config['LinearLog_ew'],\n",
    "                #         \"bias_exponent_bias\": my_config['LinearLog_eb'],\n",
    "                #     }\n",
    "                # elif new_layer_cls == LinearBlockFP:\n",
    "                #     kwargs[\"config\"] = {\n",
    "                #         \"weight_width\": my_config['LinearBlockFP'],\n",
    "                #         \"weight_exponent_width\": my_config['LinearBlockFP'],\n",
    "                #         \"weight_block_size\": my_config['LinearBlockFP_block_size'],\n",
    "                #         \"weight_exponent_bias\": my_config['LinearBlockFP'],\n",
    "                #         \"data_in_width\": my_config['LinearBlockFP'],\n",
    "                #         \"data_in_exponent_width\": my_config['LinearBlockFP'],\n",
    "                #         \"data_in_block_size\": my_config['LinearBlockFP_block_size'],\n",
    "                #         \"data_in_exponent_bias\": my_config['LinearBlockFP'],\n",
    "                #         \"bias_width\": my_config['LinearBlockFP'],\n",
    "                #         \"bias_exponent_width\": my_config['LinearBlockFP'],\n",
    "                #         \"bias_block_size\": my_config['LinearBlockFP_block_size'],\n",
    "                #         \"bias_exponent_bias\": my_config['LinearBlockFP'],\n",
    "                #     }\n",
    "                # # elif new_layer_cls == LinearBlockMinifloat:\n",
    "                # #     kwargs[\"config\"] = {\n",
    "                # #         \"weight_exponent_bias_width\" : my_config['LinearBlockMinifloat_ebw'],\n",
    "                # #         \"weight_exponent_width\": my_config['LinearBlockMinifloat_ew'],\n",
    "                # #         \"weight_block_size\": my_config['LinearBlockMinifloat_block_size'],\n",
    "                # #         \"weight_width\": my_config['LinearBlockMinifloat_w'],\n",
    "                # #         \"data_in_width\":my_config[\"LinearBlockMinifloat_w\"],\n",
    "                # #         \"data_in_exponent_bias_width\":my_config[\"LinearBlockMinifloat_ew\"],\n",
    "                # #         \"data_in_exponent_width\":my_config[\"LinearBlockMinifloat_ebw\"],\n",
    "                # #         \"data_in_block_size\":my_config[\"LinearBlockMinifloat_block_size\"],\n",
    "                # #         \"bias_width\":my_config[\"LinearBlockMinifloat_w\"],\n",
    "                # #         \"bias_exponent_width\":my_config[\"LinearBlockMinifloat_ew\"],\n",
    "                # #         \"bias_exponent_bias_width\":my_config[\"LinearBlockMinifloat_ebw\"],\n",
    "                # #         \"bias_block_size\":my_config[\"LinearBlockMinifloat_block_size\"],\n",
    "                # #     }\n",
    "                # elif new_layer_cls == LinearBlockLog:\n",
    "                #     kwargs[\"config\"] = {\n",
    "                #         \"data_in_width\": my_config['LinearBlockLog'],\n",
    "                #         \"data_in_exponent_bias_width\": my_config['LinearBlockLog_ebw'],\n",
    "                #         \"data_in_block_size\": my_config['LinearBlockLog_block_size'],\n",
    "                #         \"weight_width\": my_config['LinearBlockLog'],\n",
    "                #         \"weight_exponent_bias_width\": my_config['LinearBlockLog_ebw'],\n",
    "                #         \"weight_block_size\": my_config['LinearBlockLog_block_size'],\n",
    "                #         \"bias_width\": my_config['LinearBlockLog'],\n",
    "                #         \"bias_exponent_bias_width\": my_config['LinearBlockLog_ebw'],\n",
    "                #         \"bias_block_size\": my_config['LinearBlockLog_block_size'],\n",
    "                #     }\n",
    "\n",
    "                # elif new_layer_cls == LinearBinary:\n",
    "                #     kwargs[\"config\"] = {\n",
    "                #         'weight_stochastic':my_config['LinearBinary_stochastic'],\n",
    "                #         'weight_bipolar':my_config['LinearBinary_bipolar'],\n",
    "                #     }\n",
    "\n",
    "                    \n",
    "                # elif new_layer_cls == LinearBinaryScaling:\n",
    "                #     kwargs[\"config\"] = {\n",
    "                #         'weight_stochastic':my_config['LinearBinaryScaling_stochastic'],\n",
    "                #         'weight_bipolar':my_config['LinearBinaryScaling_bipolar'],\n",
    "                #         \"data_in_stochastic\": my_config['LinearBinaryScaling_stochastic'],\n",
    "                #         \"data_in_bipolar\": my_config['LinearBinaryScaling_bipolar'],\n",
    "                #         \"bias_stochastic\": my_config['LinearBinaryScaling_stochastic'],\n",
    "                #         \"bias_bipolar\": my_config['LinearBinaryScaling_bipolar'],\n",
    "                #         \"binary_training\": my_config['LinearBinaryScaling_binary_training'],\n",
    "                #     }\n",
    "                # elif new_layer_cls == LinearBinaryResidualSign:\n",
    "                #     kwargs[\"config\"] = {\n",
    "                #         'weight_stochastic':my_config['LinearBinaryResidualSign_stochastic'],\n",
    "                #         'weight_bipolar':my_config['LinearBinaryResidualSign_bipolar'],\n",
    "                #         \"data_in_stochastic\": my_config['LinearBinaryResidualSign_stochastic'],\n",
    "                #         \"data_in_bipolar\": my_config['LinearBinaryResidualSign_bipolar'],\n",
    "                #         \"bias_stochastic\": my_config['LinearBinaryResidualSign_stochastic'],\n",
    "                #         \"bias_bipolar\": my_config['LinearBinaryResidualSign_bipolar'],\n",
    "                #         \"binary_training\": my_config['LinearBinaryResidualSign_binary_training'],\n",
    "                    # }\n",
    "                # Create the new layer (copy the weights if applicable)\n",
    "                new_layer = new_layer_cls(**kwargs)\n",
    "                new_layer.weight.data = layer.weight.data.clone()\n",
    "\n",
    "                # Replace the layer in the model\n",
    "                deepsetattr(trial_model, name, new_layer)\n",
    "            except TypeError as e:\n",
    "                print(f\"Error creating layer {name} with class {new_layer_cls}: {e}\")\n",
    "                raise\n",
    "\n",
    "    return trial_model\n",
    "from chop.tools import get_trainer\n",
    "import random\n",
    "\n",
    "\n",
    "def objective(trial):\n",
    "\n",
    "    # Define the model\n",
    "    model = construct_model(trial)\n",
    "\n",
    "    trainer = get_trainer(\n",
    "        model=model,\n",
    "        tokenized_dataset=dataset,\n",
    "        tokenizer=tokenizer,\n",
    "        evaluate_metric=\"accuracy\",\n",
    "        num_train_epochs=1,\n",
    "    )\n",
    "\n",
    "    trainer.train()\n",
    "    eval_results = trainer.evaluate()\n",
    "\n",
    "    trial.set_user_attr(\"model\", model)\n",
    "\n",
    "    return eval_results[\"eval_accuracy\"]\n",
    "\n",
    "\n",
    "from optuna.samplers import GridSampler, RandomSampler, TPESampler\n",
    "\n",
    "sampler = RandomSampler()\n",
    "\n",
    "import optuna\n",
    "\n",
    "study = optuna.create_study(\n",
    "    direction=\"maximize\",\n",
    "    study_name=\"bert-tiny-nas-study\",\n",
    "    sampler=sampler,\n",
    ")\n",
    "\n",
    "study.optimize(\n",
    "    objective,\n",
    "    n_trials=20,\n",
    "    timeout=60 * 60 * 24,\n",
    ")\n",
    "\n",
    "task4 = [max([t.value for t in study.trials[:i+1]]) for i in range(len(study.trials))]\n",
    "task4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "task4 =[0.87332,\n",
    " 0.87348,\n",
    " 0.87348,\n",
    " 0.8738,\n",
    " 0.8738,\n",
    " 0.8738,\n",
    " 0.8738,\n",
    " 0.8738,\n",
    " 0.8738,\n",
    " 0.8738]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-06 13:23:14,100] A new study created in memory with name: bert-tiny-nas-study\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearLog'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 01:16, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.377100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.320300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.329700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.341500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.335000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.353600</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 00:38]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-06 13:25:10,774] Trial 0 finished with value: 0.87404 and parameters: {'linear_layer_quantization': 2, 'linear_layer_quantization_fractional': 1, 'LinearMinifloatDenorm': 1, 'LinearMinifloatDenorm_ew': 1, 'LinearMinifloatDenorm_eb': 0, 'LinearMinifloatIEEE': 1, 'LinearMinifloatIEEE_ew': 0, 'LinearMinifloatIEEE_eb': 2, 'LinearLog': 2, 'LinearLog_ew': 0, 'LinearLog_eb': 1, 'LinearBlockFP': 1, 'LinearBlockFP_block_size': 0, 'LinearBlockMinifloat_ebw': 0, 'LinearBlockMinifloat_ew': 0, 'LinearBlockMinifloat_block_size': 2, 'LinearBlockMinifloat_w': 1, 'LinearBlockLog': 2, 'LinearBlockLog_ebw': 0, 'LinearBlockLog_block_size': 1, 'LinearBinary_stochastic': 0, 'LinearBinary_bipolar': 0, 'LinearBinaryScaling_stochastic': 0, 'LinearBinaryScaling_bipolar': 2, 'LinearBinaryScaling_binary_training': 1, 'LinearBinaryResidualSign_stochastic': 0, 'LinearBinaryResidualSign_bipolar': 1, 'LinearBinaryResidualSign_binary_training': 0, 'bert.encoder.layer.0.attention.self.query_type': <class 'chop.nn.quantized.modules.linear.LinearLog'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearLog'>, 'bert.encoder.layer.0.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'chop.nn.quantized.modules.linear.LinearLog'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.output.dense_type': <class 'chop.nn.quantized.modules.linear.LinearLog'>, 'bert.pooler.dense_type': <class 'chop.nn.quantized.modules.linear.LinearLog'>, 'classifier_type': <class 'torch.nn.modules.linear.Linear'>}. Best is trial 0 with value: 0.87404.\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearLog'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 01:14, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.342000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.309100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.343600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.348000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.326200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.361300</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 00:38]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-06 13:27:03,745] Trial 1 finished with value: 0.8734 and parameters: {'linear_layer_quantization': 0, 'linear_layer_quantization_fractional': 0, 'LinearMinifloatDenorm': 0, 'LinearMinifloatDenorm_ew': 1, 'LinearMinifloatDenorm_eb': 2, 'LinearMinifloatIEEE': 1, 'LinearMinifloatIEEE_ew': 0, 'LinearMinifloatIEEE_eb': 1, 'LinearLog': 1, 'LinearLog_ew': 1, 'LinearLog_eb': 2, 'LinearBlockFP': 2, 'LinearBlockFP_block_size': 1, 'LinearBlockMinifloat_ebw': 2, 'LinearBlockMinifloat_ew': 1, 'LinearBlockMinifloat_block_size': 2, 'LinearBlockMinifloat_w': 0, 'LinearBlockLog': 2, 'LinearBlockLog_ebw': 1, 'LinearBlockLog_block_size': 1, 'LinearBinary_stochastic': 1, 'LinearBinary_bipolar': 2, 'LinearBinaryScaling_stochastic': 1, 'LinearBinaryScaling_bipolar': 0, 'LinearBinaryScaling_binary_training': 0, 'LinearBinaryResidualSign_stochastic': 0, 'LinearBinaryResidualSign_bipolar': 2, 'LinearBinaryResidualSign_binary_training': 1, 'bert.encoder.layer.0.attention.self.query_type': <class 'chop.nn.quantized.modules.linear.LinearLog'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearLog'>, 'bert.encoder.layer.0.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'chop.nn.quantized.modules.linear.LinearLog'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.pooler.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'classifier_type': <class 'torch.nn.modules.linear.Linear'>}. Best is trial 0 with value: 0.87404.\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearLog'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 01:20, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.335500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.310300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.318800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.330100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.316500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.351000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 00:39]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-06 13:29:04,118] Trial 2 finished with value: 0.87156 and parameters: {'linear_layer_quantization': 0, 'linear_layer_quantization_fractional': 1, 'LinearMinifloatDenorm': 1, 'LinearMinifloatDenorm_ew': 1, 'LinearMinifloatDenorm_eb': 2, 'LinearMinifloatIEEE': 0, 'LinearMinifloatIEEE_ew': 1, 'LinearMinifloatIEEE_eb': 0, 'LinearLog': 0, 'LinearLog_ew': 2, 'LinearLog_eb': 2, 'LinearBlockFP': 2, 'LinearBlockFP_block_size': 0, 'LinearBlockMinifloat_ebw': 1, 'LinearBlockMinifloat_ew': 1, 'LinearBlockMinifloat_block_size': 1, 'LinearBlockMinifloat_w': 0, 'LinearBlockLog': 1, 'LinearBlockLog_ebw': 1, 'LinearBlockLog_block_size': 0, 'LinearBinary_stochastic': 0, 'LinearBinary_bipolar': 1, 'LinearBinaryScaling_stochastic': 2, 'LinearBinaryScaling_bipolar': 2, 'LinearBinaryScaling_binary_training': 0, 'LinearBinaryResidualSign_stochastic': 2, 'LinearBinaryResidualSign_bipolar': 2, 'LinearBinaryResidualSign_binary_training': 0, 'bert.encoder.layer.0.attention.self.query_type': <class 'chop.nn.quantized.modules.linear.LinearLog'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'chop.nn.quantized.modules.linear.LinearLog'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearLog'>, 'bert.encoder.layer.0.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearLog'>, 'bert.encoder.layer.1.output.dense_type': <class 'chop.nn.quantized.modules.linear.LinearLog'>, 'bert.pooler.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'classifier_type': <class 'chop.nn.quantized.modules.linear.LinearLog'>}. Best is trial 0 with value: 0.87404.\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearLog'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 01:17, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.390700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.322300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.344200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.336000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.318000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.358500</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 00:36]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-06 13:30:59,073] Trial 3 finished with value: 0.85916 and parameters: {'linear_layer_quantization': 0, 'linear_layer_quantization_fractional': 2, 'LinearMinifloatDenorm': 1, 'LinearMinifloatDenorm_ew': 0, 'LinearMinifloatDenorm_eb': 1, 'LinearMinifloatIEEE': 2, 'LinearMinifloatIEEE_ew': 1, 'LinearMinifloatIEEE_eb': 0, 'LinearLog': 0, 'LinearLog_ew': 1, 'LinearLog_eb': 0, 'LinearBlockFP': 1, 'LinearBlockFP_block_size': 1, 'LinearBlockMinifloat_ebw': 0, 'LinearBlockMinifloat_ew': 2, 'LinearBlockMinifloat_block_size': 1, 'LinearBlockMinifloat_w': 2, 'LinearBlockLog': 1, 'LinearBlockLog_ebw': 2, 'LinearBlockLog_block_size': 0, 'LinearBinary_stochastic': 2, 'LinearBinary_bipolar': 2, 'LinearBinaryScaling_stochastic': 1, 'LinearBinaryScaling_bipolar': 2, 'LinearBinaryScaling_binary_training': 0, 'LinearBinaryResidualSign_stochastic': 2, 'LinearBinaryResidualSign_bipolar': 0, 'LinearBinaryResidualSign_binary_training': 1, 'bert.encoder.layer.0.attention.self.query_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'chop.nn.quantized.modules.linear.LinearLog'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.output.dense_type': <class 'chop.nn.quantized.modules.linear.LinearLog'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'chop.nn.quantized.modules.linear.LinearLog'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearLog'>, 'bert.encoder.layer.1.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.pooler.dense_type': <class 'chop.nn.quantized.modules.linear.LinearLog'>, 'classifier_type': <class 'torch.nn.modules.linear.Linear'>}. Best is trial 0 with value: 0.87404.\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearLog'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 01:22, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.336100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.304000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.322800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.336300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.316600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.358800</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 00:47]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-06 13:33:09,852] Trial 4 finished with value: 0.87272 and parameters: {'linear_layer_quantization': 2, 'linear_layer_quantization_fractional': 2, 'LinearMinifloatDenorm': 0, 'LinearMinifloatDenorm_ew': 1, 'LinearMinifloatDenorm_eb': 2, 'LinearMinifloatIEEE': 2, 'LinearMinifloatIEEE_ew': 0, 'LinearMinifloatIEEE_eb': 0, 'LinearLog': 2, 'LinearLog_ew': 2, 'LinearLog_eb': 2, 'LinearBlockFP': 1, 'LinearBlockFP_block_size': 2, 'LinearBlockMinifloat_ebw': 0, 'LinearBlockMinifloat_ew': 1, 'LinearBlockMinifloat_block_size': 2, 'LinearBlockMinifloat_w': 1, 'LinearBlockLog': 1, 'LinearBlockLog_ebw': 1, 'LinearBlockLog_block_size': 1, 'LinearBinary_stochastic': 2, 'LinearBinary_bipolar': 2, 'LinearBinaryScaling_stochastic': 0, 'LinearBinaryScaling_bipolar': 1, 'LinearBinaryScaling_binary_training': 1, 'LinearBinaryResidualSign_stochastic': 0, 'LinearBinaryResidualSign_bipolar': 2, 'LinearBinaryResidualSign_binary_training': 0, 'bert.encoder.layer.0.attention.self.query_type': <class 'chop.nn.quantized.modules.linear.LinearLog'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'chop.nn.quantized.modules.linear.LinearLog'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearLog'>, 'bert.encoder.layer.0.output.dense_type': <class 'chop.nn.quantized.modules.linear.LinearLog'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.pooler.dense_type': <class 'chop.nn.quantized.modules.linear.LinearLog'>, 'classifier_type': <class 'chop.nn.quantized.modules.linear.LinearLog'>}. Best is trial 0 with value: 0.87404.\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearLog'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 01:26, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.399500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.323300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.325900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.326300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.316100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.363000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 00:52]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-06 13:35:30,064] Trial 5 finished with value: 0.87196 and parameters: {'linear_layer_quantization': 0, 'linear_layer_quantization_fractional': 0, 'LinearMinifloatDenorm': 1, 'LinearMinifloatDenorm_ew': 1, 'LinearMinifloatDenorm_eb': 2, 'LinearMinifloatIEEE': 2, 'LinearMinifloatIEEE_ew': 1, 'LinearMinifloatIEEE_eb': 1, 'LinearLog': 1, 'LinearLog_ew': 1, 'LinearLog_eb': 1, 'LinearBlockFP': 1, 'LinearBlockFP_block_size': 2, 'LinearBlockMinifloat_ebw': 0, 'LinearBlockMinifloat_ew': 0, 'LinearBlockMinifloat_block_size': 0, 'LinearBlockMinifloat_w': 0, 'LinearBlockLog': 1, 'LinearBlockLog_ebw': 2, 'LinearBlockLog_block_size': 1, 'LinearBinary_stochastic': 2, 'LinearBinary_bipolar': 0, 'LinearBinaryScaling_stochastic': 2, 'LinearBinaryScaling_bipolar': 2, 'LinearBinaryScaling_binary_training': 0, 'LinearBinaryResidualSign_stochastic': 1, 'LinearBinaryResidualSign_bipolar': 1, 'LinearBinaryResidualSign_binary_training': 1, 'bert.encoder.layer.0.attention.self.query_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.output.dense_type': <class 'chop.nn.quantized.modules.linear.LinearLog'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'chop.nn.quantized.modules.linear.LinearLog'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'chop.nn.quantized.modules.linear.LinearLog'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.output.dense_type': <class 'chop.nn.quantized.modules.linear.LinearLog'>, 'bert.pooler.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'classifier_type': <class 'chop.nn.quantized.modules.linear.LinearLog'>}. Best is trial 0 with value: 0.87404.\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearLog'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 01:25, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.403300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.326400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.313300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.334600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.326700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.351200</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 00:38]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-06 13:37:35,226] Trial 6 finished with value: 0.87328 and parameters: {'linear_layer_quantization': 2, 'linear_layer_quantization_fractional': 2, 'LinearMinifloatDenorm': 1, 'LinearMinifloatDenorm_ew': 0, 'LinearMinifloatDenorm_eb': 1, 'LinearMinifloatIEEE': 0, 'LinearMinifloatIEEE_ew': 0, 'LinearMinifloatIEEE_eb': 2, 'LinearLog': 0, 'LinearLog_ew': 2, 'LinearLog_eb': 1, 'LinearBlockFP': 1, 'LinearBlockFP_block_size': 2, 'LinearBlockMinifloat_ebw': 1, 'LinearBlockMinifloat_ew': 1, 'LinearBlockMinifloat_block_size': 1, 'LinearBlockMinifloat_w': 0, 'LinearBlockLog': 2, 'LinearBlockLog_ebw': 2, 'LinearBlockLog_block_size': 1, 'LinearBinary_stochastic': 0, 'LinearBinary_bipolar': 1, 'LinearBinaryScaling_stochastic': 1, 'LinearBinaryScaling_bipolar': 0, 'LinearBinaryScaling_binary_training': 1, 'LinearBinaryResidualSign_stochastic': 1, 'LinearBinaryResidualSign_bipolar': 1, 'LinearBinaryResidualSign_binary_training': 0, 'bert.encoder.layer.0.attention.self.query_type': <class 'chop.nn.quantized.modules.linear.LinearLog'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearLog'>, 'bert.encoder.layer.0.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'chop.nn.quantized.modules.linear.LinearLog'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearLog'>, 'bert.encoder.layer.1.output.dense_type': <class 'chop.nn.quantized.modules.linear.LinearLog'>, 'bert.pooler.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'classifier_type': <class 'chop.nn.quantized.modules.linear.LinearLog'>}. Best is trial 0 with value: 0.87404.\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearLog'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 01:12, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.400300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.323400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.331500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.327300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.310100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.356200</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 00:38]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-06 13:39:27,234] Trial 7 finished with value: 0.87388 and parameters: {'linear_layer_quantization': 0, 'linear_layer_quantization_fractional': 0, 'LinearMinifloatDenorm': 0, 'LinearMinifloatDenorm_ew': 0, 'LinearMinifloatDenorm_eb': 1, 'LinearMinifloatIEEE': 0, 'LinearMinifloatIEEE_ew': 2, 'LinearMinifloatIEEE_eb': 0, 'LinearLog': 2, 'LinearLog_ew': 2, 'LinearLog_eb': 1, 'LinearBlockFP': 0, 'LinearBlockFP_block_size': 0, 'LinearBlockMinifloat_ebw': 2, 'LinearBlockMinifloat_ew': 0, 'LinearBlockMinifloat_block_size': 2, 'LinearBlockMinifloat_w': 0, 'LinearBlockLog': 2, 'LinearBlockLog_ebw': 2, 'LinearBlockLog_block_size': 2, 'LinearBinary_stochastic': 2, 'LinearBinary_bipolar': 0, 'LinearBinaryScaling_stochastic': 1, 'LinearBinaryScaling_bipolar': 0, 'LinearBinaryScaling_binary_training': 1, 'LinearBinaryResidualSign_stochastic': 1, 'LinearBinaryResidualSign_bipolar': 2, 'LinearBinaryResidualSign_binary_training': 1, 'bert.encoder.layer.0.attention.self.query_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'chop.nn.quantized.modules.linear.LinearLog'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.pooler.dense_type': <class 'chop.nn.quantized.modules.linear.LinearLog'>, 'classifier_type': <class 'chop.nn.quantized.modules.linear.LinearLog'>}. Best is trial 0 with value: 0.87404.\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearLog'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 01:21, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.340200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.307000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.325200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.329800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.316000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.349800</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 00:43]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-06 13:41:33,579] Trial 8 finished with value: 0.87288 and parameters: {'linear_layer_quantization': 0, 'linear_layer_quantization_fractional': 1, 'LinearMinifloatDenorm': 1, 'LinearMinifloatDenorm_ew': 2, 'LinearMinifloatDenorm_eb': 1, 'LinearMinifloatIEEE': 1, 'LinearMinifloatIEEE_ew': 2, 'LinearMinifloatIEEE_eb': 2, 'LinearLog': 0, 'LinearLog_ew': 1, 'LinearLog_eb': 2, 'LinearBlockFP': 2, 'LinearBlockFP_block_size': 1, 'LinearBlockMinifloat_ebw': 0, 'LinearBlockMinifloat_ew': 1, 'LinearBlockMinifloat_block_size': 1, 'LinearBlockMinifloat_w': 2, 'LinearBlockLog': 1, 'LinearBlockLog_ebw': 1, 'LinearBlockLog_block_size': 0, 'LinearBinary_stochastic': 0, 'LinearBinary_bipolar': 1, 'LinearBinaryScaling_stochastic': 0, 'LinearBinaryScaling_bipolar': 2, 'LinearBinaryScaling_binary_training': 1, 'LinearBinaryResidualSign_stochastic': 0, 'LinearBinaryResidualSign_bipolar': 2, 'LinearBinaryResidualSign_binary_training': 2, 'bert.encoder.layer.0.attention.self.query_type': <class 'chop.nn.quantized.modules.linear.LinearLog'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'chop.nn.quantized.modules.linear.LinearLog'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearLog'>, 'bert.encoder.layer.0.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'chop.nn.quantized.modules.linear.LinearLog'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.output.dense_type': <class 'chop.nn.quantized.modules.linear.LinearLog'>, 'bert.pooler.dense_type': <class 'chop.nn.quantized.modules.linear.LinearLog'>, 'classifier_type': <class 'chop.nn.quantized.modules.linear.LinearLog'>}. Best is trial 0 with value: 0.87404.\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearLog'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 01:18, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.466900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.308800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.332500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.335100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.315700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.358600</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 00:37]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-06 13:43:30,494] Trial 9 finished with value: 0.87224 and parameters: {'linear_layer_quantization': 1, 'linear_layer_quantization_fractional': 0, 'LinearMinifloatDenorm': 2, 'LinearMinifloatDenorm_ew': 2, 'LinearMinifloatDenorm_eb': 2, 'LinearMinifloatIEEE': 0, 'LinearMinifloatIEEE_ew': 2, 'LinearMinifloatIEEE_eb': 1, 'LinearLog': 0, 'LinearLog_ew': 1, 'LinearLog_eb': 1, 'LinearBlockFP': 0, 'LinearBlockFP_block_size': 0, 'LinearBlockMinifloat_ebw': 2, 'LinearBlockMinifloat_ew': 0, 'LinearBlockMinifloat_block_size': 0, 'LinearBlockMinifloat_w': 0, 'LinearBlockLog': 1, 'LinearBlockLog_ebw': 2, 'LinearBlockLog_block_size': 1, 'LinearBinary_stochastic': 0, 'LinearBinary_bipolar': 1, 'LinearBinaryScaling_stochastic': 0, 'LinearBinaryScaling_bipolar': 0, 'LinearBinaryScaling_binary_training': 2, 'LinearBinaryResidualSign_stochastic': 1, 'LinearBinaryResidualSign_bipolar': 0, 'LinearBinaryResidualSign_binary_training': 2, 'bert.encoder.layer.0.attention.self.query_type': <class 'chop.nn.quantized.modules.linear.LinearLog'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'chop.nn.quantized.modules.linear.LinearLog'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'chop.nn.quantized.modules.linear.LinearLog'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearLog'>, 'bert.encoder.layer.1.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.pooler.dense_type': <class 'chop.nn.quantized.modules.linear.LinearLog'>, 'classifier_type': <class 'chop.nn.quantized.modules.linear.LinearLog'>}. Best is trial 0 with value: 0.87404.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.87404,\n",
       " 0.87404,\n",
       " 0.87404,\n",
       " 0.87404,\n",
       " 0.87404,\n",
       " 0.87404,\n",
       " 0.87404,\n",
       " 0.87404,\n",
       " 0.87404,\n",
       " 0.87404]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "from pathlib import Path\n",
    "import dill\n",
    "\n",
    "with open(f\"{Path.home()}/tutorial_5_best_model.pkl\", \"rb\") as f:\n",
    "    base_model = dill.load(f)\n",
    "from chop.nn.quantized.modules.linear import (\n",
    "    LinearInteger,\n",
    "    LinearMinifloatDenorm,\n",
    "    LinearMinifloatIEEE,\n",
    "    LinearLog,\n",
    "    LinearBlockFP,\n",
    "    LinearBlockMinifloat,\n",
    "    LinearBlockLog,\n",
    "    LinearBinary,\n",
    "    LinearBinaryScaling,\n",
    "    LinearBinaryResidualSign,\n",
    ")\n",
    "search_space = {\n",
    "    \"linear_layer_choices\": [\n",
    "        torch.nn.Linear,\n",
    "        # LinearInteger,\n",
    "        # LinearMinifloatDenorm,\n",
    "        # LinearMinifloatIEEE,\n",
    "        LinearLog,\n",
    "        # LinearBlockFP,\n",
    "        # # LinearBlockMinifloat,\n",
    "        # LinearBlockLog,\n",
    "        # LinearBinary,\n",
    "        # LinearBinaryScaling,\n",
    "        # LinearBinaryResidualSign,\n",
    "    ],\n",
    "    'linear_layer_quantization': [8, 16, 32],\n",
    "    'linear_layer_quantization_fractional':  [2, 4, 8],\n",
    "    'LinearMinifloatDenorm': [8, 16, 32],\n",
    "    'LinearMinifloatDenorm_ew': [2, 4, 8],\n",
    "    'LinearMinifloatDenorm_eb': [2, 4, 8],\n",
    "    'LinearMinifloatIEEE': [8, 16, 32],\n",
    "    'LinearMinifloatIEEE_ew': [2, 4, 8],\n",
    "    'LinearMinifloatIEEE_eb': [2, 4, 8],\n",
    "    'LinearLog': [8, 16, 32],\n",
    "    'LinearLog_ew': [2, 4, 8],\n",
    "    'LinearLog_eb': [2, 4, 8],\n",
    "    'LinearBlockFP': [8, 16, 32],\n",
    "    'LinearBlockFP_block_size':[[4], [8], [16]] ,\n",
    "    'LinearBlockMinifloat_ebw': [2, 4, 8],\n",
    "    'LinearBlockMinifloat_ew': [2, 4, 8],\n",
    "    'LinearBlockMinifloat_block_size': [[4], [8], [16]],\n",
    "    'LinearBlockMinifloat_w': [2, 4, 8],\n",
    "    'LinearBlockLog': [8, 16, 32],\n",
    "    'LinearBlockLog_ebw': [2, 4, 8],\n",
    "    'LinearBlockLog_block_size': [[4], [8], [16]] ,\n",
    "    'LinearBinary_stochastic': [2, 4, 8],\n",
    "    'LinearBinary_bipolar': [2, 4, 8],\n",
    "    'LinearBinaryScaling_stochastic': [2, 4, 8],\n",
    "    'LinearBinaryScaling_bipolar': [2, 4, 8],\n",
    "    \"LinearBinaryScaling_binary_training\": [2, 4, 8],\n",
    "    'LinearBinaryResidualSign_stochastic': [2, 4, 8],\n",
    "    'LinearBinaryResidualSign_bipolar': [2, 4, 8],\n",
    "    \"LinearBinaryResidualSign_binary_training\": [2, 4, 8],\n",
    "}\n",
    "\n",
    "def construct_model(trial):\n",
    "    # Fetch the model\n",
    "    my_config = {}\n",
    "    trial_model = deepcopy(base_model)\n",
    "    \n",
    "    # Select quantization and fractional width configurations\n",
    "    for param in ['linear_layer_quantization', \n",
    "                  'linear_layer_quantization_fractional',\n",
    "                  'LinearMinifloatDenorm',\n",
    "                  'LinearMinifloatDenorm_ew',\n",
    "                  'LinearMinifloatDenorm_eb',\n",
    "                  'LinearMinifloatIEEE',\n",
    "                  'LinearMinifloatIEEE_ew',\n",
    "                  'LinearMinifloatIEEE_eb',\n",
    "                    'LinearLog',\n",
    "                    'LinearLog_ew',\n",
    "                    'LinearLog_eb',\n",
    "                    'LinearBlockFP',\n",
    "                    'LinearBlockFP_block_size',\n",
    "                    'LinearBlockMinifloat_ebw',\n",
    "                    'LinearBlockMinifloat_ew',\n",
    "                    'LinearBlockMinifloat_block_size',\n",
    "                    'LinearBlockMinifloat_w',\n",
    "                    'LinearBlockLog',\n",
    "                    'LinearBlockLog_ebw',\n",
    "                    'LinearBlockLog_block_size',\n",
    "                    'LinearBinary_stochastic',\n",
    "                    'LinearBinary_bipolar',\n",
    "                    'LinearBinaryScaling_stochastic',\n",
    "                    'LinearBinaryScaling_bipolar',\n",
    "                    'LinearBinaryScaling_binary_training',\n",
    "                    'LinearBinaryResidualSign_stochastic',\n",
    "                    'LinearBinaryResidualSign_bipolar',\n",
    "                    'LinearBinaryResidualSign_binary_training'\n",
    "                  ]:\n",
    "        chosen_idx = trial.suggest_int(param, 0, len(search_space[param]) - 1)\n",
    "        my_config[param] = search_space[param][chosen_idx]\n",
    "\n",
    "    # Quantize layers according to Optuna suggestions\n",
    "    for name, layer in trial_model.named_modules():\n",
    "        if isinstance(layer, torch.nn.Linear):\n",
    "            new_layer_cls = trial.suggest_categorical(\n",
    "                f\"{name}_type\",\n",
    "                search_space[\"linear_layer_choices\"],\n",
    "            )\n",
    "\n",
    "            if new_layer_cls == torch.nn.Linear:\n",
    "                continue\n",
    "\n",
    "            kwargs = {\n",
    "                \"in_features\": layer.in_features,\n",
    "                \"out_features\": layer.out_features,\n",
    "            }\n",
    "\n",
    "            try:\n",
    "                # Configure parameters for specific layer types\n",
    "                # if new_layer_cls == LinearInteger:\n",
    "                #     kwargs[\"config\"] = {\n",
    "                #         \"data_in_width\": my_config['linear_layer_quantization'],\n",
    "                #         \"data_in_frac_width\": my_config['linear_layer_quantization_fractional'],\n",
    "                #         \"weight_width\": my_config['linear_layer_quantization'],\n",
    "                #         \"weight_frac_width\": my_config['linear_layer_quantization_fractional'],\n",
    "                #         \"bias_width\": my_config['linear_layer_quantization'],\n",
    "                #         \"bias_frac_width\": my_config['linear_layer_quantization_fractional'],\n",
    "                #     }\n",
    "                # elif new_layer_cls == LinearMinifloatDenorm:\n",
    "                #     kwargs[\"config\"] = {\n",
    "                #         \"data_in_width\": my_config['LinearMinifloatDenorm'],\n",
    "                #         \"data_in_exponent_width\": my_config['LinearMinifloatDenorm_ew'],\n",
    "                #         \"data_in_exponent_bias\": my_config['LinearMinifloatDenorm_eb'],\n",
    "                #         \"weight_width\": my_config['LinearMinifloatDenorm'],\n",
    "                #         \"weight_exponent_width\": my_config['LinearMinifloatDenorm_ew'],\n",
    "                #         \"weight_exponent_bias\": my_config['LinearMinifloatDenorm_eb'],\n",
    "                #         \"bias_width\": my_config['LinearMinifloatDenorm'],\n",
    "                #         \"bias_exponent_width\": my_config['LinearMinifloatDenorm_ew'],\n",
    "                #         \"bias_exponent_bias\": my_config['LinearMinifloatDenorm_eb'],\n",
    "                #     }\n",
    "                # elif new_layer_cls == LinearMinifloatIEEE:\n",
    "                #     kwargs[\"config\"] = {\n",
    "                #         \"data_in_width\": my_config['LinearMinifloatIEEE'],\n",
    "                #         \"data_in_exponent_width\": my_config['LinearMinifloatIEEE_ew'],\n",
    "                #         \"data_in_exponent_bias\": my_config['LinearMinifloatIEEE_eb'],\n",
    "                #         \"weight_width\": my_config['LinearMinifloatIEEE'],\n",
    "                #         \"weight_exponent_width\": my_config['LinearMinifloatIEEE_ew'],\n",
    "                #         \"weight_exponent_bias\": my_config['LinearMinifloatIEEE_eb'],\n",
    "                #         \"bias_width\": my_config['LinearMinifloatIEEE'],\n",
    "                #         \"bias_exponent_width\": my_config['LinearMinifloatIEEE_ew'],\n",
    "                #         \"bias_exponent_bias\": my_config['LinearMinifloatIEEE_eb'],\n",
    "                #     }\n",
    "                if new_layer_cls == LinearLog:\n",
    "                    kwargs[\"config\"] = {\n",
    "                        \"data_in_width\": my_config['LinearLog'],\n",
    "                        \"data_in_exponent_width\": my_config['LinearLog_ew'],\n",
    "                        \"data_in_exponent_bias\": my_config['LinearLog_eb'],\n",
    "                        \"weight_width\": my_config['LinearLog'],\n",
    "                        \"weight_exponent_width\": my_config['LinearLog_ew'],\n",
    "                        \"weight_exponent_bias\": my_config['LinearLog_eb'],\n",
    "                        \"bias_width\": my_config['LinearLog'],\n",
    "                        \"bias_exponent_width\": my_config['LinearLog_ew'],\n",
    "                        \"bias_exponent_bias\": my_config['LinearLog_eb'],\n",
    "                    }\n",
    "                # elif new_layer_cls == LinearBlockFP:\n",
    "                #     kwargs[\"config\"] = {\n",
    "                #         \"weight_width\": my_config['LinearBlockFP'],\n",
    "                #         \"weight_exponent_width\": my_config['LinearBlockFP'],\n",
    "                #         \"weight_block_size\": my_config['LinearBlockFP_block_size'],\n",
    "                #         \"weight_exponent_bias\": my_config['LinearBlockFP'],\n",
    "                #         \"data_in_width\": my_config['LinearBlockFP'],\n",
    "                #         \"data_in_exponent_width\": my_config['LinearBlockFP'],\n",
    "                #         \"data_in_block_size\": my_config['LinearBlockFP_block_size'],\n",
    "                #         \"data_in_exponent_bias\": my_config['LinearBlockFP'],\n",
    "                #         \"bias_width\": my_config['LinearBlockFP'],\n",
    "                #         \"bias_exponent_width\": my_config['LinearBlockFP'],\n",
    "                #         \"bias_block_size\": my_config['LinearBlockFP_block_size'],\n",
    "                #         \"bias_exponent_bias\": my_config['LinearBlockFP'],\n",
    "                #     }\n",
    "                # # elif new_layer_cls == LinearBlockMinifloat:\n",
    "                # #     kwargs[\"config\"] = {\n",
    "                # #         \"weight_exponent_bias_width\" : my_config['LinearBlockMinifloat_ebw'],\n",
    "                # #         \"weight_exponent_width\": my_config['LinearBlockMinifloat_ew'],\n",
    "                # #         \"weight_block_size\": my_config['LinearBlockMinifloat_block_size'],\n",
    "                # #         \"weight_width\": my_config['LinearBlockMinifloat_w'],\n",
    "                # #         \"data_in_width\":my_config[\"LinearBlockMinifloat_w\"],\n",
    "                # #         \"data_in_exponent_bias_width\":my_config[\"LinearBlockMinifloat_ew\"],\n",
    "                # #         \"data_in_exponent_width\":my_config[\"LinearBlockMinifloat_ebw\"],\n",
    "                # #         \"data_in_block_size\":my_config[\"LinearBlockMinifloat_block_size\"],\n",
    "                # #         \"bias_width\":my_config[\"LinearBlockMinifloat_w\"],\n",
    "                # #         \"bias_exponent_width\":my_config[\"LinearBlockMinifloat_ew\"],\n",
    "                # #         \"bias_exponent_bias_width\":my_config[\"LinearBlockMinifloat_ebw\"],\n",
    "                # #         \"bias_block_size\":my_config[\"LinearBlockMinifloat_block_size\"],\n",
    "                # #     }\n",
    "                # elif new_layer_cls == LinearBlockLog:\n",
    "                #     kwargs[\"config\"] = {\n",
    "                #         \"data_in_width\": my_config['LinearBlockLog'],\n",
    "                #         \"data_in_exponent_bias_width\": my_config['LinearBlockLog_ebw'],\n",
    "                #         \"data_in_block_size\": my_config['LinearBlockLog_block_size'],\n",
    "                #         \"weight_width\": my_config['LinearBlockLog'],\n",
    "                #         \"weight_exponent_bias_width\": my_config['LinearBlockLog_ebw'],\n",
    "                #         \"weight_block_size\": my_config['LinearBlockLog_block_size'],\n",
    "                #         \"bias_width\": my_config['LinearBlockLog'],\n",
    "                #         \"bias_exponent_bias_width\": my_config['LinearBlockLog_ebw'],\n",
    "                #         \"bias_block_size\": my_config['LinearBlockLog_block_size'],\n",
    "                #     }\n",
    "\n",
    "                # elif new_layer_cls == LinearBinary:\n",
    "                #     kwargs[\"config\"] = {\n",
    "                #         'weight_stochastic':my_config['LinearBinary_stochastic'],\n",
    "                #         'weight_bipolar':my_config['LinearBinary_bipolar'],\n",
    "                #     }\n",
    "\n",
    "                    \n",
    "                # elif new_layer_cls == LinearBinaryScaling:\n",
    "                #     kwargs[\"config\"] = {\n",
    "                #         'weight_stochastic':my_config['LinearBinaryScaling_stochastic'],\n",
    "                #         'weight_bipolar':my_config['LinearBinaryScaling_bipolar'],\n",
    "                #         \"data_in_stochastic\": my_config['LinearBinaryScaling_stochastic'],\n",
    "                #         \"data_in_bipolar\": my_config['LinearBinaryScaling_bipolar'],\n",
    "                #         \"bias_stochastic\": my_config['LinearBinaryScaling_stochastic'],\n",
    "                #         \"bias_bipolar\": my_config['LinearBinaryScaling_bipolar'],\n",
    "                #         \"binary_training\": my_config['LinearBinaryScaling_binary_training'],\n",
    "                #     }\n",
    "                # elif new_layer_cls == LinearBinaryResidualSign:\n",
    "                #     kwargs[\"config\"] = {\n",
    "                #         'weight_stochastic':my_config['LinearBinaryResidualSign_stochastic'],\n",
    "                #         'weight_bipolar':my_config['LinearBinaryResidualSign_bipolar'],\n",
    "                #         \"data_in_stochastic\": my_config['LinearBinaryResidualSign_stochastic'],\n",
    "                #         \"data_in_bipolar\": my_config['LinearBinaryResidualSign_bipolar'],\n",
    "                #         \"bias_stochastic\": my_config['LinearBinaryResidualSign_stochastic'],\n",
    "                #         \"bias_bipolar\": my_config['LinearBinaryResidualSign_bipolar'],\n",
    "                #         \"binary_training\": my_config['LinearBinaryResidualSign_binary_training'],\n",
    "                #     }\n",
    "                # Create the new layer (copy the weights if applicable)\n",
    "                new_layer = new_layer_cls(**kwargs)\n",
    "                new_layer.weight.data = layer.weight.data.clone()\n",
    "\n",
    "                # Replace the layer in the model\n",
    "                deepsetattr(trial_model, name, new_layer)\n",
    "            except TypeError as e:\n",
    "                print(f\"Error creating layer {name} with class {new_layer_cls}: {e}\")\n",
    "                raise\n",
    "\n",
    "    return trial_model\n",
    "from chop.tools import get_trainer\n",
    "import random\n",
    "\n",
    "\n",
    "def objective(trial):\n",
    "\n",
    "    # Define the model\n",
    "    model = construct_model(trial)\n",
    "\n",
    "    trainer = get_trainer(\n",
    "        model=model,\n",
    "        tokenized_dataset=dataset,\n",
    "        tokenizer=tokenizer,\n",
    "        evaluate_metric=\"accuracy\",\n",
    "        num_train_epochs=1,\n",
    "    )\n",
    "\n",
    "    trainer.train()\n",
    "    eval_results = trainer.evaluate()\n",
    "\n",
    "    trial.set_user_attr(\"model\", model)\n",
    "\n",
    "    return eval_results[\"eval_accuracy\"]\n",
    "\n",
    "\n",
    "from optuna.samplers import GridSampler, RandomSampler, TPESampler\n",
    "\n",
    "sampler = RandomSampler()\n",
    "\n",
    "import optuna\n",
    "\n",
    "study = optuna.create_study(\n",
    "    direction=\"maximize\",\n",
    "    study_name=\"bert-tiny-nas-study\",\n",
    "    sampler=sampler,\n",
    ")\n",
    "\n",
    "study.optimize(\n",
    "    objective,\n",
    "    n_trials=20,\n",
    "    timeout=60 * 60 * 24,\n",
    ")\n",
    "\n",
    "task5 = [max([t.value for t in study.trials[:i+1]]) for i in range(len(study.trials))]\n",
    "task5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "task5 =[0.87404,\n",
    " 0.87404,\n",
    " 0.87404,\n",
    " 0.87404,\n",
    " 0.87404,\n",
    " 0.87404,\n",
    " 0.87404,\n",
    " 0.87404,\n",
    " 0.87404,\n",
    " 0.87404]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-06 13:43:31,260] A new study created in memory with name: bert-tiny-nas-study\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearBlockFP'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 01:45, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.339800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.310700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.342500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.338900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.332300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.364200</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 01:07]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-06 13:46:25,485] Trial 0 finished with value: 0.87308 and parameters: {'linear_layer_quantization': 1, 'linear_layer_quantization_fractional': 2, 'LinearMinifloatDenorm': 1, 'LinearMinifloatDenorm_ew': 1, 'LinearMinifloatDenorm_eb': 2, 'LinearMinifloatIEEE': 1, 'LinearMinifloatIEEE_ew': 1, 'LinearMinifloatIEEE_eb': 2, 'LinearLog': 0, 'LinearLog_ew': 0, 'LinearLog_eb': 0, 'LinearBlockFP': 1, 'LinearBlockFP_block_size': 1, 'LinearBlockMinifloat_ebw': 2, 'LinearBlockMinifloat_ew': 2, 'LinearBlockMinifloat_block_size': 2, 'LinearBlockMinifloat_w': 0, 'LinearBlockLog': 1, 'LinearBlockLog_ebw': 0, 'LinearBlockLog_block_size': 2, 'LinearBinary_stochastic': 0, 'LinearBinary_bipolar': 1, 'LinearBinaryScaling_stochastic': 2, 'LinearBinaryScaling_bipolar': 1, 'LinearBinaryScaling_binary_training': 1, 'LinearBinaryResidualSign_stochastic': 1, 'LinearBinaryResidualSign_bipolar': 0, 'LinearBinaryResidualSign_binary_training': 2, 'bert.encoder.layer.0.attention.self.query_type': <class 'chop.nn.quantized.modules.linear.LinearBlockFP'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'chop.nn.quantized.modules.linear.LinearBlockFP'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.pooler.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'classifier_type': <class 'torch.nn.modules.linear.Linear'>}. Best is trial 0 with value: 0.87308.\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearBlockFP'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 04:09, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.337700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.304800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.336600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.340500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.328600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.353500</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 03:23]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-06 13:54:00,094] Trial 1 finished with value: 0.87244 and parameters: {'linear_layer_quantization': 2, 'linear_layer_quantization_fractional': 2, 'LinearMinifloatDenorm': 0, 'LinearMinifloatDenorm_ew': 0, 'LinearMinifloatDenorm_eb': 1, 'LinearMinifloatIEEE': 0, 'LinearMinifloatIEEE_ew': 0, 'LinearMinifloatIEEE_eb': 1, 'LinearLog': 1, 'LinearLog_ew': 1, 'LinearLog_eb': 0, 'LinearBlockFP': 1, 'LinearBlockFP_block_size': 2, 'LinearBlockMinifloat_ebw': 1, 'LinearBlockMinifloat_ew': 2, 'LinearBlockMinifloat_block_size': 1, 'LinearBlockMinifloat_w': 2, 'LinearBlockLog': 0, 'LinearBlockLog_ebw': 1, 'LinearBlockLog_block_size': 0, 'LinearBinary_stochastic': 1, 'LinearBinary_bipolar': 0, 'LinearBinaryScaling_stochastic': 0, 'LinearBinaryScaling_bipolar': 2, 'LinearBinaryScaling_binary_training': 2, 'LinearBinaryResidualSign_stochastic': 0, 'LinearBinaryResidualSign_bipolar': 1, 'LinearBinaryResidualSign_binary_training': 0, 'bert.encoder.layer.0.attention.self.query_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'chop.nn.quantized.modules.linear.LinearBlockFP'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBlockFP'>, 'bert.encoder.layer.0.output.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBlockFP'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'chop.nn.quantized.modules.linear.LinearBlockFP'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBlockFP'>, 'bert.encoder.layer.1.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.pooler.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBlockFP'>, 'classifier_type': <class 'chop.nn.quantized.modules.linear.LinearBlockFP'>}. Best is trial 0 with value: 0.87308.\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearBlockFP'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 03:43, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.340400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.310900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.332400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.343600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.331600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.362500</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 02:29]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-06 14:00:14,801] Trial 2 finished with value: 0.87284 and parameters: {'linear_layer_quantization': 1, 'linear_layer_quantization_fractional': 2, 'LinearMinifloatDenorm': 1, 'LinearMinifloatDenorm_ew': 1, 'LinearMinifloatDenorm_eb': 1, 'LinearMinifloatIEEE': 1, 'LinearMinifloatIEEE_ew': 2, 'LinearMinifloatIEEE_eb': 1, 'LinearLog': 2, 'LinearLog_ew': 1, 'LinearLog_eb': 1, 'LinearBlockFP': 1, 'LinearBlockFP_block_size': 0, 'LinearBlockMinifloat_ebw': 0, 'LinearBlockMinifloat_ew': 1, 'LinearBlockMinifloat_block_size': 0, 'LinearBlockMinifloat_w': 2, 'LinearBlockLog': 2, 'LinearBlockLog_ebw': 0, 'LinearBlockLog_block_size': 0, 'LinearBinary_stochastic': 2, 'LinearBinary_bipolar': 1, 'LinearBinaryScaling_stochastic': 2, 'LinearBinaryScaling_bipolar': 2, 'LinearBinaryScaling_binary_training': 0, 'LinearBinaryResidualSign_stochastic': 2, 'LinearBinaryResidualSign_bipolar': 0, 'LinearBinaryResidualSign_binary_training': 1, 'bert.encoder.layer.0.attention.self.query_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'chop.nn.quantized.modules.linear.LinearBlockFP'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBlockFP'>, 'bert.encoder.layer.0.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'chop.nn.quantized.modules.linear.LinearBlockFP'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBlockFP'>, 'bert.encoder.layer.1.output.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBlockFP'>, 'bert.pooler.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'classifier_type': <class 'chop.nn.quantized.modules.linear.LinearBlockFP'>}. Best is trial 0 with value: 0.87308.\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearBlockFP'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 02:53, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.340800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.308800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.339300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.338900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.329400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.359000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 02:10]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-06 14:05:18,160] Trial 3 finished with value: 0.87244 and parameters: {'linear_layer_quantization': 1, 'linear_layer_quantization_fractional': 2, 'LinearMinifloatDenorm': 1, 'LinearMinifloatDenorm_ew': 0, 'LinearMinifloatDenorm_eb': 1, 'LinearMinifloatIEEE': 2, 'LinearMinifloatIEEE_ew': 0, 'LinearMinifloatIEEE_eb': 0, 'LinearLog': 2, 'LinearLog_ew': 1, 'LinearLog_eb': 0, 'LinearBlockFP': 0, 'LinearBlockFP_block_size': 0, 'LinearBlockMinifloat_ebw': 2, 'LinearBlockMinifloat_ew': 2, 'LinearBlockMinifloat_block_size': 0, 'LinearBlockMinifloat_w': 1, 'LinearBlockLog': 0, 'LinearBlockLog_ebw': 1, 'LinearBlockLog_block_size': 1, 'LinearBinary_stochastic': 0, 'LinearBinary_bipolar': 0, 'LinearBinaryScaling_stochastic': 2, 'LinearBinaryScaling_bipolar': 1, 'LinearBinaryScaling_binary_training': 0, 'LinearBinaryResidualSign_stochastic': 1, 'LinearBinaryResidualSign_bipolar': 1, 'LinearBinaryResidualSign_binary_training': 1, 'bert.encoder.layer.0.attention.self.query_type': <class 'chop.nn.quantized.modules.linear.LinearBlockFP'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBlockFP'>, 'bert.encoder.layer.0.output.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBlockFP'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBlockFP'>, 'bert.encoder.layer.1.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.pooler.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBlockFP'>, 'classifier_type': <class 'torch.nn.modules.linear.Linear'>}. Best is trial 0 with value: 0.87308.\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearBlockFP'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 02:47, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.340300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.309700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.346200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.347800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.334800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.360800</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 02:07]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-06 14:10:14,610] Trial 4 finished with value: 0.87264 and parameters: {'linear_layer_quantization': 2, 'linear_layer_quantization_fractional': 0, 'LinearMinifloatDenorm': 2, 'LinearMinifloatDenorm_ew': 0, 'LinearMinifloatDenorm_eb': 2, 'LinearMinifloatIEEE': 2, 'LinearMinifloatIEEE_ew': 1, 'LinearMinifloatIEEE_eb': 2, 'LinearLog': 2, 'LinearLog_ew': 2, 'LinearLog_eb': 1, 'LinearBlockFP': 2, 'LinearBlockFP_block_size': 0, 'LinearBlockMinifloat_ebw': 1, 'LinearBlockMinifloat_ew': 0, 'LinearBlockMinifloat_block_size': 0, 'LinearBlockMinifloat_w': 2, 'LinearBlockLog': 2, 'LinearBlockLog_ebw': 0, 'LinearBlockLog_block_size': 1, 'LinearBinary_stochastic': 2, 'LinearBinary_bipolar': 2, 'LinearBinaryScaling_stochastic': 1, 'LinearBinaryScaling_bipolar': 0, 'LinearBinaryScaling_binary_training': 1, 'LinearBinaryResidualSign_stochastic': 0, 'LinearBinaryResidualSign_bipolar': 1, 'LinearBinaryResidualSign_binary_training': 0, 'bert.encoder.layer.0.attention.self.query_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'chop.nn.quantized.modules.linear.LinearBlockFP'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'chop.nn.quantized.modules.linear.LinearBlockFP'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.output.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBlockFP'>, 'bert.pooler.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBlockFP'>, 'classifier_type': <class 'chop.nn.quantized.modules.linear.LinearBlockFP'>}. Best is trial 0 with value: 0.87308.\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearBlockFP'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 03:08, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.340100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.307100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.338000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.341900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.334700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.359700</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 02:24]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-06 14:15:47,972] Trial 5 finished with value: 0.87264 and parameters: {'linear_layer_quantization': 2, 'linear_layer_quantization_fractional': 2, 'LinearMinifloatDenorm': 2, 'LinearMinifloatDenorm_ew': 1, 'LinearMinifloatDenorm_eb': 2, 'LinearMinifloatIEEE': 1, 'LinearMinifloatIEEE_ew': 2, 'LinearMinifloatIEEE_eb': 2, 'LinearLog': 1, 'LinearLog_ew': 2, 'LinearLog_eb': 1, 'LinearBlockFP': 0, 'LinearBlockFP_block_size': 0, 'LinearBlockMinifloat_ebw': 1, 'LinearBlockMinifloat_ew': 2, 'LinearBlockMinifloat_block_size': 1, 'LinearBlockMinifloat_w': 0, 'LinearBlockLog': 0, 'LinearBlockLog_ebw': 1, 'LinearBlockLog_block_size': 0, 'LinearBinary_stochastic': 2, 'LinearBinary_bipolar': 1, 'LinearBinaryScaling_stochastic': 2, 'LinearBinaryScaling_bipolar': 2, 'LinearBinaryScaling_binary_training': 0, 'LinearBinaryResidualSign_stochastic': 2, 'LinearBinaryResidualSign_bipolar': 0, 'LinearBinaryResidualSign_binary_training': 0, 'bert.encoder.layer.0.attention.self.query_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'chop.nn.quantized.modules.linear.LinearBlockFP'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBlockFP'>, 'bert.encoder.layer.0.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBlockFP'>, 'bert.encoder.layer.1.output.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBlockFP'>, 'bert.pooler.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBlockFP'>, 'classifier_type': <class 'chop.nn.quantized.modules.linear.LinearBlockFP'>}. Best is trial 0 with value: 0.87308.\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearBlockFP'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 03:43, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.337700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.305100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.341800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.341300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.326600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.357800</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 03:49]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-06 14:23:21,641] Trial 6 finished with value: 0.8724 and parameters: {'linear_layer_quantization': 0, 'linear_layer_quantization_fractional': 2, 'LinearMinifloatDenorm': 2, 'LinearMinifloatDenorm_ew': 0, 'LinearMinifloatDenorm_eb': 1, 'LinearMinifloatIEEE': 1, 'LinearMinifloatIEEE_ew': 0, 'LinearMinifloatIEEE_eb': 1, 'LinearLog': 2, 'LinearLog_ew': 1, 'LinearLog_eb': 1, 'LinearBlockFP': 1, 'LinearBlockFP_block_size': 2, 'LinearBlockMinifloat_ebw': 1, 'LinearBlockMinifloat_ew': 0, 'LinearBlockMinifloat_block_size': 1, 'LinearBlockMinifloat_w': 0, 'LinearBlockLog': 1, 'LinearBlockLog_ebw': 1, 'LinearBlockLog_block_size': 0, 'LinearBinary_stochastic': 0, 'LinearBinary_bipolar': 2, 'LinearBinaryScaling_stochastic': 1, 'LinearBinaryScaling_bipolar': 2, 'LinearBinaryScaling_binary_training': 2, 'LinearBinaryResidualSign_stochastic': 0, 'LinearBinaryResidualSign_bipolar': 2, 'LinearBinaryResidualSign_binary_training': 1, 'bert.encoder.layer.0.attention.self.query_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'chop.nn.quantized.modules.linear.LinearBlockFP'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBlockFP'>, 'bert.encoder.layer.0.output.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBlockFP'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'chop.nn.quantized.modules.linear.LinearBlockFP'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.pooler.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBlockFP'>, 'classifier_type': <class 'chop.nn.quantized.modules.linear.LinearBlockFP'>}. Best is trial 0 with value: 0.87308.\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearBlockFP'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 02:41, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.336600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.300600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.325800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.344000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.332900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.362700</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 01:34]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-06 14:27:38,410] Trial 7 finished with value: 0.8742 and parameters: {'linear_layer_quantization': 1, 'linear_layer_quantization_fractional': 2, 'LinearMinifloatDenorm': 2, 'LinearMinifloatDenorm_ew': 0, 'LinearMinifloatDenorm_eb': 2, 'LinearMinifloatIEEE': 1, 'LinearMinifloatIEEE_ew': 0, 'LinearMinifloatIEEE_eb': 1, 'LinearLog': 1, 'LinearLog_ew': 1, 'LinearLog_eb': 2, 'LinearBlockFP': 2, 'LinearBlockFP_block_size': 1, 'LinearBlockMinifloat_ebw': 2, 'LinearBlockMinifloat_ew': 2, 'LinearBlockMinifloat_block_size': 0, 'LinearBlockMinifloat_w': 2, 'LinearBlockLog': 2, 'LinearBlockLog_ebw': 1, 'LinearBlockLog_block_size': 0, 'LinearBinary_stochastic': 2, 'LinearBinary_bipolar': 0, 'LinearBinaryScaling_stochastic': 0, 'LinearBinaryScaling_bipolar': 1, 'LinearBinaryScaling_binary_training': 0, 'LinearBinaryResidualSign_stochastic': 2, 'LinearBinaryResidualSign_bipolar': 1, 'LinearBinaryResidualSign_binary_training': 1, 'bert.encoder.layer.0.attention.self.query_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'chop.nn.quantized.modules.linear.LinearBlockFP'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.output.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBlockFP'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.pooler.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'classifier_type': <class 'chop.nn.quantized.modules.linear.LinearBlockFP'>}. Best is trial 7 with value: 0.8742.\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearBlockFP'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 05:23, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.340500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.304800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.332300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.339600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.331300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.361700</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 04:27]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-06 14:37:29,661] Trial 8 finished with value: 0.87324 and parameters: {'linear_layer_quantization': 0, 'linear_layer_quantization_fractional': 2, 'LinearMinifloatDenorm': 2, 'LinearMinifloatDenorm_ew': 2, 'LinearMinifloatDenorm_eb': 0, 'LinearMinifloatIEEE': 2, 'LinearMinifloatIEEE_ew': 0, 'LinearMinifloatIEEE_eb': 1, 'LinearLog': 0, 'LinearLog_ew': 1, 'LinearLog_eb': 1, 'LinearBlockFP': 0, 'LinearBlockFP_block_size': 2, 'LinearBlockMinifloat_ebw': 0, 'LinearBlockMinifloat_ew': 0, 'LinearBlockMinifloat_block_size': 0, 'LinearBlockMinifloat_w': 1, 'LinearBlockLog': 1, 'LinearBlockLog_ebw': 0, 'LinearBlockLog_block_size': 1, 'LinearBinary_stochastic': 0, 'LinearBinary_bipolar': 2, 'LinearBinaryScaling_stochastic': 2, 'LinearBinaryScaling_bipolar': 0, 'LinearBinaryScaling_binary_training': 2, 'LinearBinaryResidualSign_stochastic': 0, 'LinearBinaryResidualSign_bipolar': 2, 'LinearBinaryResidualSign_binary_training': 1, 'bert.encoder.layer.0.attention.self.query_type': <class 'chop.nn.quantized.modules.linear.LinearBlockFP'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'chop.nn.quantized.modules.linear.LinearBlockFP'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.output.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBlockFP'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'chop.nn.quantized.modules.linear.LinearBlockFP'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'chop.nn.quantized.modules.linear.LinearBlockFP'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.output.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBlockFP'>, 'bert.pooler.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBlockFP'>, 'classifier_type': <class 'chop.nn.quantized.modules.linear.LinearBlockFP'>}. Best is trial 7 with value: 0.8742.\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearBlockFP'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 04:35, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.339100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.304100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.333700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.348300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.332300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.360400</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 03:20]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-06 14:45:26,872] Trial 9 finished with value: 0.87304 and parameters: {'linear_layer_quantization': 1, 'linear_layer_quantization_fractional': 1, 'LinearMinifloatDenorm': 1, 'LinearMinifloatDenorm_ew': 1, 'LinearMinifloatDenorm_eb': 2, 'LinearMinifloatIEEE': 1, 'LinearMinifloatIEEE_ew': 2, 'LinearMinifloatIEEE_eb': 1, 'LinearLog': 2, 'LinearLog_ew': 0, 'LinearLog_eb': 1, 'LinearBlockFP': 1, 'LinearBlockFP_block_size': 2, 'LinearBlockMinifloat_ebw': 1, 'LinearBlockMinifloat_ew': 1, 'LinearBlockMinifloat_block_size': 2, 'LinearBlockMinifloat_w': 0, 'LinearBlockLog': 2, 'LinearBlockLog_ebw': 0, 'LinearBlockLog_block_size': 2, 'LinearBinary_stochastic': 0, 'LinearBinary_bipolar': 2, 'LinearBinaryScaling_stochastic': 0, 'LinearBinaryScaling_bipolar': 1, 'LinearBinaryScaling_binary_training': 0, 'LinearBinaryResidualSign_stochastic': 2, 'LinearBinaryResidualSign_bipolar': 0, 'LinearBinaryResidualSign_binary_training': 1, 'bert.encoder.layer.0.attention.self.query_type': <class 'chop.nn.quantized.modules.linear.LinearBlockFP'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'chop.nn.quantized.modules.linear.LinearBlockFP'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'chop.nn.quantized.modules.linear.LinearBlockFP'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBlockFP'>, 'bert.encoder.layer.1.output.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBlockFP'>, 'bert.pooler.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBlockFP'>, 'classifier_type': <class 'chop.nn.quantized.modules.linear.LinearBlockFP'>}. Best is trial 7 with value: 0.8742.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.87308,\n",
       " 0.87308,\n",
       " 0.87308,\n",
       " 0.87308,\n",
       " 0.87308,\n",
       " 0.87308,\n",
       " 0.87308,\n",
       " 0.8742,\n",
       " 0.8742,\n",
       " 0.8742]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from pathlib import Path\n",
    "import dill\n",
    "\n",
    "with open(f\"{Path.home()}/tutorial_5_best_model.pkl\", \"rb\") as f:\n",
    "    base_model = dill.load(f)\n",
    "from chop.nn.quantized.modules.linear import (\n",
    "    LinearInteger,\n",
    "    LinearMinifloatDenorm,\n",
    "    LinearMinifloatIEEE,\n",
    "    LinearLog,\n",
    "    LinearBlockFP,\n",
    "    LinearBlockMinifloat,\n",
    "    LinearBlockLog,\n",
    "    LinearBinary,\n",
    "    LinearBinaryScaling,\n",
    "    LinearBinaryResidualSign,\n",
    ")\n",
    "search_space = {\n",
    "    \"linear_layer_choices\": [\n",
    "        torch.nn.Linear,\n",
    "        # LinearInteger,\n",
    "        # LinearMinifloatDenorm,\n",
    "        # LinearMinifloatIEEE,\n",
    "        # LinearLog,\n",
    "        LinearBlockFP,\n",
    "        # LinearBlockMinifloat,\n",
    "        # LinearBlockLog,\n",
    "        # LinearBinary,\n",
    "        # LinearBinaryScaling,\n",
    "        # LinearBinaryResidualSign,\n",
    "    ],\n",
    "    'linear_layer_quantization': [8, 16, 32],\n",
    "    'linear_layer_quantization_fractional':  [2, 4, 8],\n",
    "    'LinearMinifloatDenorm': [8, 16, 32],\n",
    "    'LinearMinifloatDenorm_ew': [2, 4, 8],\n",
    "    'LinearMinifloatDenorm_eb': [2, 4, 8],\n",
    "    'LinearMinifloatIEEE': [8, 16, 32],\n",
    "    'LinearMinifloatIEEE_ew': [2, 4, 8],\n",
    "    'LinearMinifloatIEEE_eb': [2, 4, 8],\n",
    "    'LinearLog': [8, 16, 32],\n",
    "    'LinearLog_ew': [2, 4, 8],\n",
    "    'LinearLog_eb': [2, 4, 8],\n",
    "    'LinearBlockFP': [8, 16, 32],\n",
    "    'LinearBlockFP_block_size':[[4], [8], [16]] ,\n",
    "    'LinearBlockMinifloat_ebw': [2, 4, 8],\n",
    "    'LinearBlockMinifloat_ew': [2, 4, 8],\n",
    "    'LinearBlockMinifloat_block_size': [[4], [8], [16]],\n",
    "    'LinearBlockMinifloat_w': [2, 4, 8],\n",
    "    'LinearBlockLog': [8, 16, 32],\n",
    "    'LinearBlockLog_ebw': [2, 4, 8],\n",
    "    'LinearBlockLog_block_size': [[4], [8], [16]] ,\n",
    "    'LinearBinary_stochastic': [2, 4, 8],\n",
    "    'LinearBinary_bipolar': [2, 4, 8],\n",
    "    'LinearBinaryScaling_stochastic': [2, 4, 8],\n",
    "    'LinearBinaryScaling_bipolar': [2, 4, 8],\n",
    "    \"LinearBinaryScaling_binary_training\": [2, 4, 8],\n",
    "    'LinearBinaryResidualSign_stochastic': [2, 4, 8],\n",
    "    'LinearBinaryResidualSign_bipolar': [2, 4, 8],\n",
    "    \"LinearBinaryResidualSign_binary_training\": [2, 4, 8],\n",
    "}\n",
    "\n",
    "def construct_model(trial):\n",
    "    # Fetch the model\n",
    "    my_config = {}\n",
    "    trial_model = deepcopy(base_model)\n",
    "    \n",
    "    # Select quantization and fractional width configurations\n",
    "    for param in ['linear_layer_quantization', \n",
    "                  'linear_layer_quantization_fractional',\n",
    "                  'LinearMinifloatDenorm',\n",
    "                  'LinearMinifloatDenorm_ew',\n",
    "                  'LinearMinifloatDenorm_eb',\n",
    "                  'LinearMinifloatIEEE',\n",
    "                  'LinearMinifloatIEEE_ew',\n",
    "                  'LinearMinifloatIEEE_eb',\n",
    "                    'LinearLog',\n",
    "                    'LinearLog_ew',\n",
    "                    'LinearLog_eb',\n",
    "                    'LinearBlockFP',\n",
    "                    'LinearBlockFP_block_size',\n",
    "                    'LinearBlockMinifloat_ebw',\n",
    "                    'LinearBlockMinifloat_ew',\n",
    "                    'LinearBlockMinifloat_block_size',\n",
    "                    'LinearBlockMinifloat_w',\n",
    "                    'LinearBlockLog',\n",
    "                    'LinearBlockLog_ebw',\n",
    "                    'LinearBlockLog_block_size',\n",
    "                    'LinearBinary_stochastic',\n",
    "                    'LinearBinary_bipolar',\n",
    "                    'LinearBinaryScaling_stochastic',\n",
    "                    'LinearBinaryScaling_bipolar',\n",
    "                    'LinearBinaryScaling_binary_training',\n",
    "                    'LinearBinaryResidualSign_stochastic',\n",
    "                    'LinearBinaryResidualSign_bipolar',\n",
    "                    'LinearBinaryResidualSign_binary_training'\n",
    "                  ]:\n",
    "        chosen_idx = trial.suggest_int(param, 0, len(search_space[param]) - 1)\n",
    "        my_config[param] = search_space[param][chosen_idx]\n",
    "\n",
    "    # Quantize layers according to Optuna suggestions\n",
    "    for name, layer in trial_model.named_modules():\n",
    "        if isinstance(layer, torch.nn.Linear):\n",
    "            new_layer_cls = trial.suggest_categorical(\n",
    "                f\"{name}_type\",\n",
    "                search_space[\"linear_layer_choices\"],\n",
    "            )\n",
    "\n",
    "            if new_layer_cls == torch.nn.Linear:\n",
    "                continue\n",
    "\n",
    "            kwargs = {\n",
    "                \"in_features\": layer.in_features,\n",
    "                \"out_features\": layer.out_features,\n",
    "            }\n",
    "\n",
    "            try:\n",
    "                # Configure parameters for specific layer types\n",
    "                # if new_layer_cls == LinearInteger:\n",
    "                #     kwargs[\"config\"] = {\n",
    "                #         \"data_in_width\": my_config['linear_layer_quantization'],\n",
    "                #         \"data_in_frac_width\": my_config['linear_layer_quantization_fractional'],\n",
    "                #         \"weight_width\": my_config['linear_layer_quantization'],\n",
    "                #         \"weight_frac_width\": my_config['linear_layer_quantization_fractional'],\n",
    "                #         \"bias_width\": my_config['linear_layer_quantization'],\n",
    "                #         \"bias_frac_width\": my_config['linear_layer_quantization_fractional'],\n",
    "                #     }\n",
    "                # elif new_layer_cls == LinearMinifloatDenorm:\n",
    "                #     kwargs[\"config\"] = {\n",
    "                #         \"data_in_width\": my_config['LinearMinifloatDenorm'],\n",
    "                #         \"data_in_exponent_width\": my_config['LinearMinifloatDenorm_ew'],\n",
    "                #         \"data_in_exponent_bias\": my_config['LinearMinifloatDenorm_eb'],\n",
    "                #         \"weight_width\": my_config['LinearMinifloatDenorm'],\n",
    "                #         \"weight_exponent_width\": my_config['LinearMinifloatDenorm_ew'],\n",
    "                #         \"weight_exponent_bias\": my_config['LinearMinifloatDenorm_eb'],\n",
    "                #         \"bias_width\": my_config['LinearMinifloatDenorm'],\n",
    "                #         \"bias_exponent_width\": my_config['LinearMinifloatDenorm_ew'],\n",
    "                #         \"bias_exponent_bias\": my_config['LinearMinifloatDenorm_eb'],\n",
    "                #     }\n",
    "                # elif new_layer_cls == LinearMinifloatIEEE:\n",
    "                #     kwargs[\"config\"] = {\n",
    "                #         \"data_in_width\": my_config['LinearMinifloatIEEE'],\n",
    "                #         \"data_in_exponent_width\": my_config['LinearMinifloatIEEE_ew'],\n",
    "                #         \"data_in_exponent_bias\": my_config['LinearMinifloatIEEE_eb'],\n",
    "                #         \"weight_width\": my_config['LinearMinifloatIEEE'],\n",
    "                #         \"weight_exponent_width\": my_config['LinearMinifloatIEEE_ew'],\n",
    "                #         \"weight_exponent_bias\": my_config['LinearMinifloatIEEE_eb'],\n",
    "                #         \"bias_width\": my_config['LinearMinifloatIEEE'],\n",
    "                #         \"bias_exponent_width\": my_config['LinearMinifloatIEEE_ew'],\n",
    "                #         \"bias_exponent_bias\": my_config['LinearMinifloatIEEE_eb'],\n",
    "                #     }\n",
    "                # elif new_layer_cls == LinearLog:\n",
    "                #     kwargs[\"config\"] = {\n",
    "                #         \"data_in_width\": my_config['LinearLog'],\n",
    "                #         \"data_in_exponent_width\": my_config['LinearLog_ew'],\n",
    "                #         \"data_in_exponent_bias\": my_config['LinearLog_eb'],\n",
    "                #         \"weight_width\": my_config['LinearLog'],\n",
    "                #         \"weight_exponent_width\": my_config['LinearLog_ew'],\n",
    "                #         \"weight_exponent_bias\": my_config['LinearLog_eb'],\n",
    "                #         \"bias_width\": my_config['LinearLog'],\n",
    "                #         \"bias_exponent_width\": my_config['LinearLog_ew'],\n",
    "                #         \"bias_exponent_bias\": my_config['LinearLog_eb'],\n",
    "                #     }\n",
    "                if new_layer_cls == LinearBlockFP:\n",
    "                    kwargs[\"config\"] = {\n",
    "                        \"weight_width\": my_config['LinearBlockFP'],\n",
    "                        \"weight_exponent_width\": my_config['LinearBlockFP'],\n",
    "                        \"weight_block_size\": my_config['LinearBlockFP_block_size'],\n",
    "                        \"weight_exponent_bias\": my_config['LinearBlockFP'],\n",
    "                        \"data_in_width\": my_config['LinearBlockFP'],\n",
    "                        \"data_in_exponent_width\": my_config['LinearBlockFP'],\n",
    "                        \"data_in_block_size\": my_config['LinearBlockFP_block_size'],\n",
    "                        \"data_in_exponent_bias\": my_config['LinearBlockFP'],\n",
    "                        \"bias_width\": my_config['LinearBlockFP'],\n",
    "                        \"bias_exponent_width\": my_config['LinearBlockFP'],\n",
    "                        \"bias_block_size\": my_config['LinearBlockFP_block_size'],\n",
    "                        \"bias_exponent_bias\": my_config['LinearBlockFP'],\n",
    "                    }\n",
    "                # # elif new_layer_cls == LinearBlockMinifloat:\n",
    "                # #     kwargs[\"config\"] = {\n",
    "                # #         \"weight_exponent_bias_width\" : my_config['LinearBlockMinifloat_ebw'],\n",
    "                # #         \"weight_exponent_width\": my_config['LinearBlockMinifloat_ew'],\n",
    "                # #         \"weight_block_size\": my_config['LinearBlockMinifloat_block_size'],\n",
    "                # #         \"weight_width\": my_config['LinearBlockMinifloat_w'],\n",
    "                # #         \"data_in_width\":my_config[\"LinearBlockMinifloat_w\"],\n",
    "                # #         \"data_in_exponent_bias_width\":my_config[\"LinearBlockMinifloat_ew\"],\n",
    "                # #         \"data_in_exponent_width\":my_config[\"LinearBlockMinifloat_ebw\"],\n",
    "                # #         \"data_in_block_size\":my_config[\"LinearBlockMinifloat_block_size\"],\n",
    "                # #         \"bias_width\":my_config[\"LinearBlockMinifloat_w\"],\n",
    "                # #         \"bias_exponent_width\":my_config[\"LinearBlockMinifloat_ew\"],\n",
    "                # #         \"bias_exponent_bias_width\":my_config[\"LinearBlockMinifloat_ebw\"],\n",
    "                # #         \"bias_block_size\":my_config[\"LinearBlockMinifloat_block_size\"],\n",
    "                # #     }\n",
    "                # elif new_layer_cls == LinearBlockLog:\n",
    "                #     kwargs[\"config\"] = {\n",
    "                #         \"data_in_width\": my_config['LinearBlockLog'],\n",
    "                #         \"data_in_exponent_bias_width\": my_config['LinearBlockLog_ebw'],\n",
    "                #         \"data_in_block_size\": my_config['LinearBlockLog_block_size'],\n",
    "                #         \"weight_width\": my_config['LinearBlockLog'],\n",
    "                #         \"weight_exponent_bias_width\": my_config['LinearBlockLog_ebw'],\n",
    "                #         \"weight_block_size\": my_config['LinearBlockLog_block_size'],\n",
    "                #         \"bias_width\": my_config['LinearBlockLog'],\n",
    "                #         \"bias_exponent_bias_width\": my_config['LinearBlockLog_ebw'],\n",
    "                #         \"bias_block_size\": my_config['LinearBlockLog_block_size'],\n",
    "                #     }\n",
    "\n",
    "                # elif new_layer_cls == LinearBinary:\n",
    "                #     kwargs[\"config\"] = {\n",
    "                #         'weight_stochastic':my_config['LinearBinary_stochastic'],\n",
    "                #         'weight_bipolar':my_config['LinearBinary_bipolar'],\n",
    "                #     }\n",
    "\n",
    "                    \n",
    "                # elif new_layer_cls == LinearBinaryScaling:\n",
    "                #     kwargs[\"config\"] = {\n",
    "                #         'weight_stochastic':my_config['LinearBinaryScaling_stochastic'],\n",
    "                #         'weight_bipolar':my_config['LinearBinaryScaling_bipolar'],\n",
    "                #         \"data_in_stochastic\": my_config['LinearBinaryScaling_stochastic'],\n",
    "                #         \"data_in_bipolar\": my_config['LinearBinaryScaling_bipolar'],\n",
    "                #         \"bias_stochastic\": my_config['LinearBinaryScaling_stochastic'],\n",
    "                #         \"bias_bipolar\": my_config['LinearBinaryScaling_bipolar'],\n",
    "                #         \"binary_training\": my_config['LinearBinaryScaling_binary_training'],\n",
    "                #     }\n",
    "                # elif new_layer_cls == LinearBinaryResidualSign:\n",
    "                #     kwargs[\"config\"] = {\n",
    "                #         'weight_stochastic':my_config['LinearBinaryResidualSign_stochastic'],\n",
    "                #         'weight_bipolar':my_config['LinearBinaryResidualSign_bipolar'],\n",
    "                #         \"data_in_stochastic\": my_config['LinearBinaryResidualSign_stochastic'],\n",
    "                #         \"data_in_bipolar\": my_config['LinearBinaryResidualSign_bipolar'],\n",
    "                #         \"bias_stochastic\": my_config['LinearBinaryResidualSign_stochastic'],\n",
    "                #         \"bias_bipolar\": my_config['LinearBinaryResidualSign_bipolar'],\n",
    "                #         \"binary_training\": my_config['LinearBinaryResidualSign_binary_training'],\n",
    "                #     }\n",
    "                # Create the new layer (copy the weights if applicable)\n",
    "                new_layer = new_layer_cls(**kwargs)\n",
    "                new_layer.weight.data = layer.weight.data.clone()\n",
    "\n",
    "                # Replace the layer in the model\n",
    "                deepsetattr(trial_model, name, new_layer)\n",
    "            except TypeError as e:\n",
    "                print(f\"Error creating layer {name} with class {new_layer_cls}: {e}\")\n",
    "                raise\n",
    "\n",
    "    return trial_model\n",
    "from chop.tools import get_trainer\n",
    "import random\n",
    "\n",
    "\n",
    "def objective(trial):\n",
    "\n",
    "    # Define the model\n",
    "    model = construct_model(trial)\n",
    "\n",
    "    trainer = get_trainer(\n",
    "        model=model,\n",
    "        tokenized_dataset=dataset,\n",
    "        tokenizer=tokenizer,\n",
    "        evaluate_metric=\"accuracy\",\n",
    "        num_train_epochs=1,\n",
    "    )\n",
    "\n",
    "    trainer.train()\n",
    "    eval_results = trainer.evaluate()\n",
    "\n",
    "    trial.set_user_attr(\"model\", model)\n",
    "\n",
    "    return eval_results[\"eval_accuracy\"]\n",
    "\n",
    "\n",
    "from optuna.samplers import GridSampler, RandomSampler, TPESampler\n",
    "\n",
    "sampler = RandomSampler()\n",
    "\n",
    "import optuna\n",
    "\n",
    "study = optuna.create_study(\n",
    "    direction=\"maximize\",\n",
    "    study_name=\"bert-tiny-nas-study\",\n",
    "    sampler=sampler,\n",
    ")\n",
    "\n",
    "study.optimize(\n",
    "    objective,\n",
    "    n_trials=20,\n",
    "    timeout=60 * 60 * 24,\n",
    ")\n",
    "\n",
    "task6 = [max([t.value for t in study.trials[:i+1]]) for i in range(len(study.trials))]\n",
    "task6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "task6 =[0.87308,\n",
    " 0.87308,\n",
    " 0.87308,\n",
    " 0.87308,\n",
    " 0.87308,\n",
    " 0.87308,\n",
    " 0.87308,\n",
    " 0.8742,\n",
    " 0.8742,\n",
    " 0.8742]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-06 14:45:27,626] A new study created in memory with name: bert-tiny-nas-study\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearBlockLog'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 03:40, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.336200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.295300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.321300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.332500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.310700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.349800</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 03:25]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-06 14:52:35,136] Trial 0 finished with value: 0.87304 and parameters: {'linear_layer_quantization': 1, 'linear_layer_quantization_fractional': 2, 'LinearMinifloatDenorm': 1, 'LinearMinifloatDenorm_ew': 1, 'LinearMinifloatDenorm_eb': 1, 'LinearMinifloatIEEE': 2, 'LinearMinifloatIEEE_ew': 0, 'LinearMinifloatIEEE_eb': 0, 'LinearLog': 2, 'LinearLog_ew': 2, 'LinearLog_eb': 1, 'LinearBlockFP': 0, 'LinearBlockFP_block_size': 2, 'LinearBlockMinifloat_ebw': 2, 'LinearBlockMinifloat_ew': 2, 'LinearBlockMinifloat_block_size': 0, 'LinearBlockMinifloat_w': 0, 'LinearBlockLog': 2, 'LinearBlockLog_ebw': 2, 'LinearBlockLog_block_size': 1, 'LinearBinary_stochastic': 1, 'LinearBinary_bipolar': 0, 'LinearBinaryScaling_stochastic': 1, 'LinearBinaryScaling_bipolar': 1, 'LinearBinaryScaling_binary_training': 2, 'LinearBinaryResidualSign_stochastic': 2, 'LinearBinaryResidualSign_bipolar': 2, 'LinearBinaryResidualSign_binary_training': 1, 'bert.encoder.layer.0.attention.self.query_type': <class 'chop.nn.quantized.modules.linear.LinearBlockLog'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'chop.nn.quantized.modules.linear.LinearBlockLog'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBlockLog'>, 'bert.encoder.layer.0.output.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBlockLog'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBlockLog'>, 'bert.encoder.layer.1.output.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBlockLog'>, 'bert.pooler.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBlockLog'>, 'classifier_type': <class 'chop.nn.quantized.modules.linear.LinearBlockLog'>}. Best is trial 0 with value: 0.87304.\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearBlockLog'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 03:32, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.340300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.306000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.325600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.338600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.324600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.357200</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 02:50]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-06 14:58:58,970] Trial 1 finished with value: 0.874 and parameters: {'linear_layer_quantization': 2, 'linear_layer_quantization_fractional': 2, 'LinearMinifloatDenorm': 1, 'LinearMinifloatDenorm_ew': 1, 'LinearMinifloatDenorm_eb': 0, 'LinearMinifloatIEEE': 0, 'LinearMinifloatIEEE_ew': 0, 'LinearMinifloatIEEE_eb': 2, 'LinearLog': 1, 'LinearLog_ew': 2, 'LinearLog_eb': 0, 'LinearBlockFP': 2, 'LinearBlockFP_block_size': 2, 'LinearBlockMinifloat_ebw': 0, 'LinearBlockMinifloat_ew': 1, 'LinearBlockMinifloat_block_size': 2, 'LinearBlockMinifloat_w': 1, 'LinearBlockLog': 0, 'LinearBlockLog_ebw': 2, 'LinearBlockLog_block_size': 2, 'LinearBinary_stochastic': 2, 'LinearBinary_bipolar': 2, 'LinearBinaryScaling_stochastic': 1, 'LinearBinaryScaling_bipolar': 2, 'LinearBinaryScaling_binary_training': 1, 'LinearBinaryResidualSign_stochastic': 0, 'LinearBinaryResidualSign_bipolar': 0, 'LinearBinaryResidualSign_binary_training': 0, 'bert.encoder.layer.0.attention.self.query_type': <class 'chop.nn.quantized.modules.linear.LinearBlockLog'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBlockLog'>, 'bert.encoder.layer.0.output.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBlockLog'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'chop.nn.quantized.modules.linear.LinearBlockLog'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.output.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBlockLog'>, 'bert.pooler.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'classifier_type': <class 'torch.nn.modules.linear.Linear'>}. Best is trial 1 with value: 0.874.\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearBlockLog'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 04:23, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.330500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.302700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.327300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.328500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.317200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.349500</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 03:59]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-06 15:07:22,708] Trial 2 finished with value: 0.87188 and parameters: {'linear_layer_quantization': 1, 'linear_layer_quantization_fractional': 1, 'LinearMinifloatDenorm': 0, 'LinearMinifloatDenorm_ew': 1, 'LinearMinifloatDenorm_eb': 1, 'LinearMinifloatIEEE': 1, 'LinearMinifloatIEEE_ew': 1, 'LinearMinifloatIEEE_eb': 1, 'LinearLog': 0, 'LinearLog_ew': 1, 'LinearLog_eb': 1, 'LinearBlockFP': 2, 'LinearBlockFP_block_size': 0, 'LinearBlockMinifloat_ebw': 2, 'LinearBlockMinifloat_ew': 0, 'LinearBlockMinifloat_block_size': 1, 'LinearBlockMinifloat_w': 1, 'LinearBlockLog': 1, 'LinearBlockLog_ebw': 1, 'LinearBlockLog_block_size': 1, 'LinearBinary_stochastic': 2, 'LinearBinary_bipolar': 0, 'LinearBinaryScaling_stochastic': 2, 'LinearBinaryScaling_bipolar': 2, 'LinearBinaryScaling_binary_training': 0, 'LinearBinaryResidualSign_stochastic': 1, 'LinearBinaryResidualSign_bipolar': 0, 'LinearBinaryResidualSign_binary_training': 2, 'bert.encoder.layer.0.attention.self.query_type': <class 'chop.nn.quantized.modules.linear.LinearBlockLog'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'chop.nn.quantized.modules.linear.LinearBlockLog'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBlockLog'>, 'bert.encoder.layer.0.output.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBlockLog'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'chop.nn.quantized.modules.linear.LinearBlockLog'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'chop.nn.quantized.modules.linear.LinearBlockLog'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBlockLog'>, 'bert.encoder.layer.1.output.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBlockLog'>, 'bert.pooler.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBlockLog'>, 'classifier_type': <class 'chop.nn.quantized.modules.linear.LinearBlockLog'>}. Best is trial 1 with value: 0.874.\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearBlockLog'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 02:56, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.338200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.297100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.328700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.332000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.315900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.341900</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 01:45]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-06 15:12:05,035] Trial 3 finished with value: 0.8712 and parameters: {'linear_layer_quantization': 0, 'linear_layer_quantization_fractional': 1, 'LinearMinifloatDenorm': 1, 'LinearMinifloatDenorm_ew': 2, 'LinearMinifloatDenorm_eb': 2, 'LinearMinifloatIEEE': 1, 'LinearMinifloatIEEE_ew': 1, 'LinearMinifloatIEEE_eb': 1, 'LinearLog': 1, 'LinearLog_ew': 0, 'LinearLog_eb': 1, 'LinearBlockFP': 2, 'LinearBlockFP_block_size': 0, 'LinearBlockMinifloat_ebw': 1, 'LinearBlockMinifloat_ew': 1, 'LinearBlockMinifloat_block_size': 0, 'LinearBlockMinifloat_w': 1, 'LinearBlockLog': 2, 'LinearBlockLog_ebw': 1, 'LinearBlockLog_block_size': 0, 'LinearBinary_stochastic': 1, 'LinearBinary_bipolar': 2, 'LinearBinaryScaling_stochastic': 2, 'LinearBinaryScaling_bipolar': 1, 'LinearBinaryScaling_binary_training': 0, 'LinearBinaryResidualSign_stochastic': 1, 'LinearBinaryResidualSign_bipolar': 0, 'LinearBinaryResidualSign_binary_training': 1, 'bert.encoder.layer.0.attention.self.query_type': <class 'chop.nn.quantized.modules.linear.LinearBlockLog'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'chop.nn.quantized.modules.linear.LinearBlockLog'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'chop.nn.quantized.modules.linear.LinearBlockLog'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBlockLog'>, 'bert.encoder.layer.1.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.pooler.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'classifier_type': <class 'chop.nn.quantized.modules.linear.LinearBlockLog'>}. Best is trial 1 with value: 0.874.\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearBlockLog'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 02:57, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.393000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.335400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.358100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.357800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.329200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.376900</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 02:15]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-06 15:17:19,186] Trial 4 finished with value: 0.86776 and parameters: {'linear_layer_quantization': 0, 'linear_layer_quantization_fractional': 0, 'LinearMinifloatDenorm': 2, 'LinearMinifloatDenorm_ew': 0, 'LinearMinifloatDenorm_eb': 0, 'LinearMinifloatIEEE': 1, 'LinearMinifloatIEEE_ew': 0, 'LinearMinifloatIEEE_eb': 2, 'LinearLog': 0, 'LinearLog_ew': 2, 'LinearLog_eb': 2, 'LinearBlockFP': 2, 'LinearBlockFP_block_size': 1, 'LinearBlockMinifloat_ebw': 1, 'LinearBlockMinifloat_ew': 2, 'LinearBlockMinifloat_block_size': 2, 'LinearBlockMinifloat_w': 2, 'LinearBlockLog': 0, 'LinearBlockLog_ebw': 0, 'LinearBlockLog_block_size': 1, 'LinearBinary_stochastic': 0, 'LinearBinary_bipolar': 1, 'LinearBinaryScaling_stochastic': 0, 'LinearBinaryScaling_bipolar': 0, 'LinearBinaryScaling_binary_training': 1, 'LinearBinaryResidualSign_stochastic': 1, 'LinearBinaryResidualSign_bipolar': 0, 'LinearBinaryResidualSign_binary_training': 0, 'bert.encoder.layer.0.attention.self.query_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'chop.nn.quantized.modules.linear.LinearBlockLog'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBlockLog'>, 'bert.encoder.layer.0.output.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBlockLog'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'chop.nn.quantized.modules.linear.LinearBlockLog'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.output.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBlockLog'>, 'bert.pooler.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'classifier_type': <class 'torch.nn.modules.linear.Linear'>}. Best is trial 1 with value: 0.874.\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearBlockLog'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 02:22, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.338000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.314700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.334600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.336800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.329900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.363900</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 01:41]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-06 15:21:24,225] Trial 5 finished with value: 0.87088 and parameters: {'linear_layer_quantization': 0, 'linear_layer_quantization_fractional': 0, 'LinearMinifloatDenorm': 0, 'LinearMinifloatDenorm_ew': 1, 'LinearMinifloatDenorm_eb': 2, 'LinearMinifloatIEEE': 2, 'LinearMinifloatIEEE_ew': 0, 'LinearMinifloatIEEE_eb': 1, 'LinearLog': 0, 'LinearLog_ew': 1, 'LinearLog_eb': 0, 'LinearBlockFP': 0, 'LinearBlockFP_block_size': 1, 'LinearBlockMinifloat_ebw': 2, 'LinearBlockMinifloat_ew': 2, 'LinearBlockMinifloat_block_size': 2, 'LinearBlockMinifloat_w': 2, 'LinearBlockLog': 0, 'LinearBlockLog_ebw': 1, 'LinearBlockLog_block_size': 2, 'LinearBinary_stochastic': 0, 'LinearBinary_bipolar': 2, 'LinearBinaryScaling_stochastic': 2, 'LinearBinaryScaling_bipolar': 2, 'LinearBinaryScaling_binary_training': 0, 'LinearBinaryResidualSign_stochastic': 1, 'LinearBinaryResidualSign_bipolar': 0, 'LinearBinaryResidualSign_binary_training': 0, 'bert.encoder.layer.0.attention.self.query_type': <class 'chop.nn.quantized.modules.linear.LinearBlockLog'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'chop.nn.quantized.modules.linear.LinearBlockLog'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBlockLog'>, 'bert.encoder.layer.1.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.pooler.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'classifier_type': <class 'torch.nn.modules.linear.Linear'>}. Best is trial 1 with value: 0.874.\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearBlockLog'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 02:09, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.333100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.302100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.319700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.339000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.317000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.350800</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 01:29]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-06 15:25:04,493] Trial 6 finished with value: 0.87344 and parameters: {'linear_layer_quantization': 2, 'linear_layer_quantization_fractional': 2, 'LinearMinifloatDenorm': 2, 'LinearMinifloatDenorm_ew': 1, 'LinearMinifloatDenorm_eb': 1, 'LinearMinifloatIEEE': 2, 'LinearMinifloatIEEE_ew': 2, 'LinearMinifloatIEEE_eb': 2, 'LinearLog': 2, 'LinearLog_ew': 2, 'LinearLog_eb': 1, 'LinearBlockFP': 2, 'LinearBlockFP_block_size': 2, 'LinearBlockMinifloat_ebw': 2, 'LinearBlockMinifloat_ew': 2, 'LinearBlockMinifloat_block_size': 1, 'LinearBlockMinifloat_w': 2, 'LinearBlockLog': 2, 'LinearBlockLog_ebw': 2, 'LinearBlockLog_block_size': 0, 'LinearBinary_stochastic': 2, 'LinearBinary_bipolar': 2, 'LinearBinaryScaling_stochastic': 1, 'LinearBinaryScaling_bipolar': 1, 'LinearBinaryScaling_binary_training': 1, 'LinearBinaryResidualSign_stochastic': 2, 'LinearBinaryResidualSign_bipolar': 2, 'LinearBinaryResidualSign_binary_training': 0, 'bert.encoder.layer.0.attention.self.query_type': <class 'chop.nn.quantized.modules.linear.LinearBlockLog'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'chop.nn.quantized.modules.linear.LinearBlockLog'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'chop.nn.quantized.modules.linear.LinearBlockLog'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.pooler.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'classifier_type': <class 'chop.nn.quantized.modules.linear.LinearBlockLog'>}. Best is trial 1 with value: 0.874.\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearBlockLog'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 02:22, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.334900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.300800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.315300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.332500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.315300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.346200</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 01:41]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-06 15:29:09,537] Trial 7 finished with value: 0.87176 and parameters: {'linear_layer_quantization': 1, 'linear_layer_quantization_fractional': 1, 'LinearMinifloatDenorm': 0, 'LinearMinifloatDenorm_ew': 1, 'LinearMinifloatDenorm_eb': 1, 'LinearMinifloatIEEE': 0, 'LinearMinifloatIEEE_ew': 2, 'LinearMinifloatIEEE_eb': 2, 'LinearLog': 1, 'LinearLog_ew': 1, 'LinearLog_eb': 0, 'LinearBlockFP': 0, 'LinearBlockFP_block_size': 1, 'LinearBlockMinifloat_ebw': 2, 'LinearBlockMinifloat_ew': 2, 'LinearBlockMinifloat_block_size': 1, 'LinearBlockMinifloat_w': 2, 'LinearBlockLog': 0, 'LinearBlockLog_ebw': 1, 'LinearBlockLog_block_size': 1, 'LinearBinary_stochastic': 0, 'LinearBinary_bipolar': 1, 'LinearBinaryScaling_stochastic': 0, 'LinearBinaryScaling_bipolar': 1, 'LinearBinaryScaling_binary_training': 1, 'LinearBinaryResidualSign_stochastic': 0, 'LinearBinaryResidualSign_bipolar': 0, 'LinearBinaryResidualSign_binary_training': 2, 'bert.encoder.layer.0.attention.self.query_type': <class 'chop.nn.quantized.modules.linear.LinearBlockLog'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.output.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBlockLog'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBlockLog'>, 'bert.encoder.layer.1.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.pooler.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'classifier_type': <class 'chop.nn.quantized.modules.linear.LinearBlockLog'>}. Best is trial 1 with value: 0.874.\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearBlockLog'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 02:43, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.335900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.299500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.317300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.326600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.313800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.350100</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 02:03]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-06 15:33:57,721] Trial 8 finished with value: 0.87028 and parameters: {'linear_layer_quantization': 0, 'linear_layer_quantization_fractional': 2, 'LinearMinifloatDenorm': 0, 'LinearMinifloatDenorm_ew': 0, 'LinearMinifloatDenorm_eb': 1, 'LinearMinifloatIEEE': 0, 'LinearMinifloatIEEE_ew': 0, 'LinearMinifloatIEEE_eb': 2, 'LinearLog': 1, 'LinearLog_ew': 1, 'LinearLog_eb': 2, 'LinearBlockFP': 1, 'LinearBlockFP_block_size': 2, 'LinearBlockMinifloat_ebw': 1, 'LinearBlockMinifloat_ew': 0, 'LinearBlockMinifloat_block_size': 2, 'LinearBlockMinifloat_w': 2, 'LinearBlockLog': 1, 'LinearBlockLog_ebw': 2, 'LinearBlockLog_block_size': 0, 'LinearBinary_stochastic': 2, 'LinearBinary_bipolar': 0, 'LinearBinaryScaling_stochastic': 2, 'LinearBinaryScaling_bipolar': 0, 'LinearBinaryScaling_binary_training': 2, 'LinearBinaryResidualSign_stochastic': 2, 'LinearBinaryResidualSign_bipolar': 1, 'LinearBinaryResidualSign_binary_training': 1, 'bert.encoder.layer.0.attention.self.query_type': <class 'chop.nn.quantized.modules.linear.LinearBlockLog'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'chop.nn.quantized.modules.linear.LinearBlockLog'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'chop.nn.quantized.modules.linear.LinearBlockLog'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'chop.nn.quantized.modules.linear.LinearBlockLog'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBlockLog'>, 'bert.encoder.layer.1.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.pooler.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'classifier_type': <class 'chop.nn.quantized.modules.linear.LinearBlockLog'>}. Best is trial 1 with value: 0.874.\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearBlockLog'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 03:30, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.334300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.297100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.321500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.332800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.313800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.352200</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 03:26]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-06 15:40:55,491] Trial 9 finished with value: 0.87132 and parameters: {'linear_layer_quantization': 1, 'linear_layer_quantization_fractional': 1, 'LinearMinifloatDenorm': 2, 'LinearMinifloatDenorm_ew': 1, 'LinearMinifloatDenorm_eb': 0, 'LinearMinifloatIEEE': 1, 'LinearMinifloatIEEE_ew': 0, 'LinearMinifloatIEEE_eb': 2, 'LinearLog': 2, 'LinearLog_ew': 2, 'LinearLog_eb': 1, 'LinearBlockFP': 1, 'LinearBlockFP_block_size': 0, 'LinearBlockMinifloat_ebw': 1, 'LinearBlockMinifloat_ew': 0, 'LinearBlockMinifloat_block_size': 0, 'LinearBlockMinifloat_w': 0, 'LinearBlockLog': 1, 'LinearBlockLog_ebw': 2, 'LinearBlockLog_block_size': 2, 'LinearBinary_stochastic': 1, 'LinearBinary_bipolar': 0, 'LinearBinaryScaling_stochastic': 0, 'LinearBinaryScaling_bipolar': 1, 'LinearBinaryScaling_binary_training': 1, 'LinearBinaryResidualSign_stochastic': 1, 'LinearBinaryResidualSign_bipolar': 2, 'LinearBinaryResidualSign_binary_training': 1, 'bert.encoder.layer.0.attention.self.query_type': <class 'chop.nn.quantized.modules.linear.LinearBlockLog'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.output.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBlockLog'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'chop.nn.quantized.modules.linear.LinearBlockLog'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBlockLog'>, 'bert.encoder.layer.1.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.pooler.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBlockLog'>, 'classifier_type': <class 'chop.nn.quantized.modules.linear.LinearBlockLog'>}. Best is trial 1 with value: 0.874.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.87304, 0.874, 0.874, 0.874, 0.874, 0.874, 0.874, 0.874, 0.874, 0.874]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from pathlib import Path\n",
    "import dill\n",
    "\n",
    "with open(f\"{Path.home()}/tutorial_5_best_model.pkl\", \"rb\") as f:\n",
    "    base_model = dill.load(f)\n",
    "from chop.nn.quantized.modules.linear import (\n",
    "    LinearInteger,\n",
    "    LinearMinifloatDenorm,\n",
    "    LinearMinifloatIEEE,\n",
    "    LinearLog,\n",
    "    LinearBlockFP,\n",
    "    LinearBlockMinifloat,\n",
    "    LinearBlockLog,\n",
    "    LinearBinary,\n",
    "    LinearBinaryScaling,\n",
    "    LinearBinaryResidualSign,\n",
    ")\n",
    "search_space = {\n",
    "    \"linear_layer_choices\": [\n",
    "        torch.nn.Linear,\n",
    "        # LinearInteger,\n",
    "        # LinearMinifloatDenorm,\n",
    "        # LinearMinifloatIEEE,\n",
    "        # LinearLog,\n",
    "        # LinearBlockFP,\n",
    "        # LinearBlockMinifloat,\n",
    "        LinearBlockLog,\n",
    "        # LinearBinary,\n",
    "        # LinearBinaryScaling,\n",
    "        # LinearBinaryResidualSign,\n",
    "    ],\n",
    "    'linear_layer_quantization': [8, 16, 32],\n",
    "    'linear_layer_quantization_fractional':  [2, 4, 8],\n",
    "    'LinearMinifloatDenorm': [8, 16, 32],\n",
    "    'LinearMinifloatDenorm_ew': [2, 4, 8],\n",
    "    'LinearMinifloatDenorm_eb': [2, 4, 8],\n",
    "    'LinearMinifloatIEEE': [8, 16, 32],\n",
    "    'LinearMinifloatIEEE_ew': [2, 4, 8],\n",
    "    'LinearMinifloatIEEE_eb': [2, 4, 8],\n",
    "    'LinearLog': [8, 16, 32],\n",
    "    'LinearLog_ew': [2, 4, 8],\n",
    "    'LinearLog_eb': [2, 4, 8],\n",
    "    'LinearBlockFP': [8, 16, 32],\n",
    "    'LinearBlockFP_block_size':[[4], [8], [16]] ,\n",
    "    'LinearBlockMinifloat_ebw': [2, 4, 8],\n",
    "    'LinearBlockMinifloat_ew': [2, 4, 8],\n",
    "    'LinearBlockMinifloat_block_size': [[4], [8], [16]],\n",
    "    'LinearBlockMinifloat_w': [2, 4, 8],\n",
    "    'LinearBlockLog': [8, 16, 32],\n",
    "    'LinearBlockLog_ebw': [2, 4, 8],\n",
    "    'LinearBlockLog_block_size': [[4], [8], [16]] ,\n",
    "    'LinearBinary_stochastic': [2, 4, 8],\n",
    "    'LinearBinary_bipolar': [2, 4, 8],\n",
    "    'LinearBinaryScaling_stochastic': [2, 4, 8],\n",
    "    'LinearBinaryScaling_bipolar': [2, 4, 8],\n",
    "    \"LinearBinaryScaling_binary_training\": [2, 4, 8],\n",
    "    'LinearBinaryResidualSign_stochastic': [2, 4, 8],\n",
    "    'LinearBinaryResidualSign_bipolar': [2, 4, 8],\n",
    "    \"LinearBinaryResidualSign_binary_training\": [2, 4, 8],\n",
    "}\n",
    "\n",
    "def construct_model(trial):\n",
    "    # Fetch the model\n",
    "    my_config = {}\n",
    "    trial_model = deepcopy(base_model)\n",
    "    \n",
    "    # Select quantization and fractional width configurations\n",
    "    for param in ['linear_layer_quantization', \n",
    "                  'linear_layer_quantization_fractional',\n",
    "                  'LinearMinifloatDenorm',\n",
    "                  'LinearMinifloatDenorm_ew',\n",
    "                  'LinearMinifloatDenorm_eb',\n",
    "                  'LinearMinifloatIEEE',\n",
    "                  'LinearMinifloatIEEE_ew',\n",
    "                  'LinearMinifloatIEEE_eb',\n",
    "                    'LinearLog',\n",
    "                    'LinearLog_ew',\n",
    "                    'LinearLog_eb',\n",
    "                    'LinearBlockFP',\n",
    "                    'LinearBlockFP_block_size',\n",
    "                    'LinearBlockMinifloat_ebw',\n",
    "                    'LinearBlockMinifloat_ew',\n",
    "                    'LinearBlockMinifloat_block_size',\n",
    "                    'LinearBlockMinifloat_w',\n",
    "                    'LinearBlockLog',\n",
    "                    'LinearBlockLog_ebw',\n",
    "                    'LinearBlockLog_block_size',\n",
    "                    'LinearBinary_stochastic',\n",
    "                    'LinearBinary_bipolar',\n",
    "                    'LinearBinaryScaling_stochastic',\n",
    "                    'LinearBinaryScaling_bipolar',\n",
    "                    'LinearBinaryScaling_binary_training',\n",
    "                    'LinearBinaryResidualSign_stochastic',\n",
    "                    'LinearBinaryResidualSign_bipolar',\n",
    "                    'LinearBinaryResidualSign_binary_training'\n",
    "                  ]:\n",
    "        chosen_idx = trial.suggest_int(param, 0, len(search_space[param]) - 1)\n",
    "        my_config[param] = search_space[param][chosen_idx]\n",
    "\n",
    "    # Quantize layers according to Optuna suggestions\n",
    "    for name, layer in trial_model.named_modules():\n",
    "        if isinstance(layer, torch.nn.Linear):\n",
    "            new_layer_cls = trial.suggest_categorical(\n",
    "                f\"{name}_type\",\n",
    "                search_space[\"linear_layer_choices\"],\n",
    "            )\n",
    "\n",
    "            if new_layer_cls == torch.nn.Linear:\n",
    "                continue\n",
    "\n",
    "            kwargs = {\n",
    "                \"in_features\": layer.in_features,\n",
    "                \"out_features\": layer.out_features,\n",
    "            }\n",
    "\n",
    "            try:\n",
    "                # Configure parameters for specific layer types\n",
    "                # if new_layer_cls == LinearInteger:\n",
    "                #     kwargs[\"config\"] = {\n",
    "                #         \"data_in_width\": my_config['linear_layer_quantization'],\n",
    "                #         \"data_in_frac_width\": my_config['linear_layer_quantization_fractional'],\n",
    "                #         \"weight_width\": my_config['linear_layer_quantization'],\n",
    "                #         \"weight_frac_width\": my_config['linear_layer_quantization_fractional'],\n",
    "                #         \"bias_width\": my_config['linear_layer_quantization'],\n",
    "                #         \"bias_frac_width\": my_config['linear_layer_quantization_fractional'],\n",
    "                #     }\n",
    "                # elif new_layer_cls == LinearMinifloatDenorm:\n",
    "                #     kwargs[\"config\"] = {\n",
    "                #         \"data_in_width\": my_config['LinearMinifloatDenorm'],\n",
    "                #         \"data_in_exponent_width\": my_config['LinearMinifloatDenorm_ew'],\n",
    "                #         \"data_in_exponent_bias\": my_config['LinearMinifloatDenorm_eb'],\n",
    "                #         \"weight_width\": my_config['LinearMinifloatDenorm'],\n",
    "                #         \"weight_exponent_width\": my_config['LinearMinifloatDenorm_ew'],\n",
    "                #         \"weight_exponent_bias\": my_config['LinearMinifloatDenorm_eb'],\n",
    "                #         \"bias_width\": my_config['LinearMinifloatDenorm'],\n",
    "                #         \"bias_exponent_width\": my_config['LinearMinifloatDenorm_ew'],\n",
    "                #         \"bias_exponent_bias\": my_config['LinearMinifloatDenorm_eb'],\n",
    "                #     }\n",
    "                # elif new_layer_cls == LinearMinifloatIEEE:\n",
    "                #     kwargs[\"config\"] = {\n",
    "                #         \"data_in_width\": my_config['LinearMinifloatIEEE'],\n",
    "                #         \"data_in_exponent_width\": my_config['LinearMinifloatIEEE_ew'],\n",
    "                #         \"data_in_exponent_bias\": my_config['LinearMinifloatIEEE_eb'],\n",
    "                #         \"weight_width\": my_config['LinearMinifloatIEEE'],\n",
    "                #         \"weight_exponent_width\": my_config['LinearMinifloatIEEE_ew'],\n",
    "                #         \"weight_exponent_bias\": my_config['LinearMinifloatIEEE_eb'],\n",
    "                #         \"bias_width\": my_config['LinearMinifloatIEEE'],\n",
    "                #         \"bias_exponent_width\": my_config['LinearMinifloatIEEE_ew'],\n",
    "                #         \"bias_exponent_bias\": my_config['LinearMinifloatIEEE_eb'],\n",
    "                #     }\n",
    "                # elif new_layer_cls == LinearLog:\n",
    "                #     kwargs[\"config\"] = {\n",
    "                #         \"data_in_width\": my_config['LinearLog'],\n",
    "                #         \"data_in_exponent_width\": my_config['LinearLog_ew'],\n",
    "                #         \"data_in_exponent_bias\": my_config['LinearLog_eb'],\n",
    "                #         \"weight_width\": my_config['LinearLog'],\n",
    "                #         \"weight_exponent_width\": my_config['LinearLog_ew'],\n",
    "                #         \"weight_exponent_bias\": my_config['LinearLog_eb'],\n",
    "                #         \"bias_width\": my_config['LinearLog'],\n",
    "                #         \"bias_exponent_width\": my_config['LinearLog_ew'],\n",
    "                #         \"bias_exponent_bias\": my_config['LinearLog_eb'],\n",
    "                #     }\n",
    "                # elif new_layer_cls == LinearBlockFP:\n",
    "                #     kwargs[\"config\"] = {\n",
    "                #         \"weight_width\": my_config['LinearBlockFP'],\n",
    "                #         \"weight_exponent_width\": my_config['LinearBlockFP'],\n",
    "                #         \"weight_block_size\": my_config['LinearBlockFP_block_size'],\n",
    "                #         \"weight_exponent_bias\": my_config['LinearBlockFP'],\n",
    "                #         \"data_in_width\": my_config['LinearBlockFP'],\n",
    "                #         \"data_in_exponent_width\": my_config['LinearBlockFP'],\n",
    "                #         \"data_in_block_size\": my_config['LinearBlockFP_block_size'],\n",
    "                #         \"data_in_exponent_bias\": my_config['LinearBlockFP'],\n",
    "                #         \"bias_width\": my_config['LinearBlockFP'],\n",
    "                #         \"bias_exponent_width\": my_config['LinearBlockFP'],\n",
    "                #         \"bias_block_size\": my_config['LinearBlockFP_block_size'],\n",
    "                #         \"bias_exponent_bias\": my_config['LinearBlockFP'],\n",
    "                #     }\n",
    "                # # elif new_layer_cls == LinearBlockMinifloat:\n",
    "                # #     kwargs[\"config\"] = {\n",
    "                # #         \"weight_exponent_bias_width\" : my_config['LinearBlockMinifloat_ebw'],\n",
    "                # #         \"weight_exponent_width\": my_config['LinearBlockMinifloat_ew'],\n",
    "                # #         \"weight_block_size\": my_config['LinearBlockMinifloat_block_size'],\n",
    "                # #         \"weight_width\": my_config['LinearBlockMinifloat_w'],\n",
    "                # #         \"data_in_width\":my_config[\"LinearBlockMinifloat_w\"],\n",
    "                # #         \"data_in_exponent_bias_width\":my_config[\"LinearBlockMinifloat_ew\"],\n",
    "                # #         \"data_in_exponent_width\":my_config[\"LinearBlockMinifloat_ebw\"],\n",
    "                # #         \"data_in_block_size\":my_config[\"LinearBlockMinifloat_block_size\"],\n",
    "                # #         \"bias_width\":my_config[\"LinearBlockMinifloat_w\"],\n",
    "                # #         \"bias_exponent_width\":my_config[\"LinearBlockMinifloat_ew\"],\n",
    "                # #         \"bias_exponent_bias_width\":my_config[\"LinearBlockMinifloat_ebw\"],\n",
    "                # #         \"bias_block_size\":my_config[\"LinearBlockMinifloat_block_size\"],\n",
    "                # #     }\n",
    "                if new_layer_cls == LinearBlockLog:\n",
    "                    kwargs[\"config\"] = {\n",
    "                        \"data_in_width\": my_config['LinearBlockLog'],\n",
    "                        \"data_in_exponent_bias_width\": my_config['LinearBlockLog_ebw'],\n",
    "                        \"data_in_block_size\": my_config['LinearBlockLog_block_size'],\n",
    "                        \"weight_width\": my_config['LinearBlockLog'],\n",
    "                        \"weight_exponent_bias_width\": my_config['LinearBlockLog_ebw'],\n",
    "                        \"weight_block_size\": my_config['LinearBlockLog_block_size'],\n",
    "                        \"bias_width\": my_config['LinearBlockLog'],\n",
    "                        \"bias_exponent_bias_width\": my_config['LinearBlockLog_ebw'],\n",
    "                        \"bias_block_size\": my_config['LinearBlockLog_block_size'],\n",
    "                    }\n",
    "\n",
    "                # elif new_layer_cls == LinearBinary:\n",
    "                #     kwargs[\"config\"] = {\n",
    "                #         'weight_stochastic':my_config['LinearBinary_stochastic'],\n",
    "                #         'weight_bipolar':my_config['LinearBinary_bipolar'],\n",
    "                #     }\n",
    "\n",
    "                    \n",
    "                # elif new_layer_cls == LinearBinaryScaling:\n",
    "                #     kwargs[\"config\"] = {\n",
    "                #         'weight_stochastic':my_config['LinearBinaryScaling_stochastic'],\n",
    "                #         'weight_bipolar':my_config['LinearBinaryScaling_bipolar'],\n",
    "                #         \"data_in_stochastic\": my_config['LinearBinaryScaling_stochastic'],\n",
    "                #         \"data_in_bipolar\": my_config['LinearBinaryScaling_bipolar'],\n",
    "                #         \"bias_stochastic\": my_config['LinearBinaryScaling_stochastic'],\n",
    "                #         \"bias_bipolar\": my_config['LinearBinaryScaling_bipolar'],\n",
    "                #         \"binary_training\": my_config['LinearBinaryScaling_binary_training'],\n",
    "                #     }\n",
    "                # elif new_layer_cls == LinearBinaryResidualSign:\n",
    "                #     kwargs[\"config\"] = {\n",
    "                #         'weight_stochastic':my_config['LinearBinaryResidualSign_stochastic'],\n",
    "                #         'weight_bipolar':my_config['LinearBinaryResidualSign_bipolar'],\n",
    "                #         \"data_in_stochastic\": my_config['LinearBinaryResidualSign_stochastic'],\n",
    "                #         \"data_in_bipolar\": my_config['LinearBinaryResidualSign_bipolar'],\n",
    "                #         \"bias_stochastic\": my_config['LinearBinaryResidualSign_stochastic'],\n",
    "                #         \"bias_bipolar\": my_config['LinearBinaryResidualSign_bipolar'],\n",
    "                #         \"binary_training\": my_config['LinearBinaryResidualSign_binary_training'],\n",
    "                #     }\n",
    "                # Create the new layer (copy the weights if applicable)\n",
    "                new_layer = new_layer_cls(**kwargs)\n",
    "                new_layer.weight.data = layer.weight.data.clone()\n",
    "\n",
    "                # Replace the layer in the model\n",
    "                deepsetattr(trial_model, name, new_layer)\n",
    "            except TypeError as e:\n",
    "                print(f\"Error creating layer {name} with class {new_layer_cls}: {e}\")\n",
    "                raise\n",
    "\n",
    "    return trial_model\n",
    "from chop.tools import get_trainer\n",
    "import random\n",
    "\n",
    "\n",
    "def objective(trial):\n",
    "\n",
    "    # Define the model\n",
    "    model = construct_model(trial)\n",
    "\n",
    "    trainer = get_trainer(\n",
    "        model=model,\n",
    "        tokenized_dataset=dataset,\n",
    "        tokenizer=tokenizer,\n",
    "        evaluate_metric=\"accuracy\",\n",
    "        num_train_epochs=1,\n",
    "    )\n",
    "\n",
    "    trainer.train()\n",
    "    eval_results = trainer.evaluate()\n",
    "\n",
    "    trial.set_user_attr(\"model\", model)\n",
    "\n",
    "    return eval_results[\"eval_accuracy\"]\n",
    "\n",
    "\n",
    "from optuna.samplers import GridSampler, RandomSampler, TPESampler\n",
    "\n",
    "sampler = RandomSampler()\n",
    "\n",
    "import optuna\n",
    "\n",
    "study = optuna.create_study(\n",
    "    direction=\"maximize\",\n",
    "    study_name=\"bert-tiny-nas-study\",\n",
    "    sampler=sampler,\n",
    ")\n",
    "\n",
    "study.optimize(\n",
    "    objective,\n",
    "    n_trials=20,\n",
    "    timeout=60 * 60 * 24,\n",
    ")\n",
    "\n",
    "task7 = [max([t.value for t in study.trials[:i+1]]) for i in range(len(study.trials))]\n",
    "task7"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "task7 =[0.87304, 0.874, 0.874, 0.874, 0.874, 0.874, 0.874, 0.874, 0.874, 0.874]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-06 15:40:56,430] A new study created in memory with name: bert-tiny-nas-study\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearBinary'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 01:06, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.720300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.695500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.693200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.692900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.693300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.692600</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 00:29]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-06 15:42:34,167] Trial 0 finished with value: 0.514 and parameters: {'linear_layer_quantization': 1, 'linear_layer_quantization_fractional': 1, 'LinearMinifloatDenorm': 2, 'LinearMinifloatDenorm_ew': 1, 'LinearMinifloatDenorm_eb': 2, 'LinearMinifloatIEEE': 1, 'LinearMinifloatIEEE_ew': 2, 'LinearMinifloatIEEE_eb': 1, 'LinearLog': 1, 'LinearLog_ew': 1, 'LinearLog_eb': 2, 'LinearBlockFP': 0, 'LinearBlockFP_block_size': 1, 'LinearBlockMinifloat_ebw': 2, 'LinearBlockMinifloat_ew': 1, 'LinearBlockMinifloat_block_size': 1, 'LinearBlockMinifloat_w': 1, 'LinearBlockLog': 0, 'LinearBlockLog_ebw': 1, 'LinearBlockLog_block_size': 2, 'LinearBinary_stochastic': 1, 'LinearBinary_bipolar': 1, 'LinearBinaryScaling_stochastic': 2, 'LinearBinaryScaling_bipolar': 0, 'LinearBinaryScaling_binary_training': 1, 'LinearBinaryResidualSign_stochastic': 0, 'LinearBinaryResidualSign_bipolar': 2, 'LinearBinaryResidualSign_binary_training': 2, 'bert.encoder.layer.0.attention.self.query_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'chop.nn.quantized.modules.linear.LinearBinary'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinary'>, 'bert.encoder.layer.0.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.pooler.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'classifier_type': <class 'torch.nn.modules.linear.Linear'>}. Best is trial 0 with value: 0.514.\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearBinary'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 01:13, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.721300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.716300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.710600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.709700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.717700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.714700</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 00:34]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-06 15:44:23,565] Trial 1 finished with value: 0.48804 and parameters: {'linear_layer_quantization': 2, 'linear_layer_quantization_fractional': 1, 'LinearMinifloatDenorm': 2, 'LinearMinifloatDenorm_ew': 2, 'LinearMinifloatDenorm_eb': 0, 'LinearMinifloatIEEE': 0, 'LinearMinifloatIEEE_ew': 1, 'LinearMinifloatIEEE_eb': 2, 'LinearLog': 2, 'LinearLog_ew': 1, 'LinearLog_eb': 2, 'LinearBlockFP': 2, 'LinearBlockFP_block_size': 0, 'LinearBlockMinifloat_ebw': 0, 'LinearBlockMinifloat_ew': 0, 'LinearBlockMinifloat_block_size': 1, 'LinearBlockMinifloat_w': 1, 'LinearBlockLog': 1, 'LinearBlockLog_ebw': 1, 'LinearBlockLog_block_size': 1, 'LinearBinary_stochastic': 2, 'LinearBinary_bipolar': 0, 'LinearBinaryScaling_stochastic': 2, 'LinearBinaryScaling_bipolar': 2, 'LinearBinaryScaling_binary_training': 1, 'LinearBinaryResidualSign_stochastic': 0, 'LinearBinaryResidualSign_bipolar': 1, 'LinearBinaryResidualSign_binary_training': 1, 'bert.encoder.layer.0.attention.self.query_type': <class 'chop.nn.quantized.modules.linear.LinearBinary'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'chop.nn.quantized.modules.linear.LinearBinary'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.output.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinary'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinary'>, 'bert.encoder.layer.1.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.pooler.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinary'>, 'classifier_type': <class 'torch.nn.modules.linear.Linear'>}. Best is trial 0 with value: 0.514.\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearBinary'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 01:13, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.717300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.712200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.713200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.716800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.711300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.710400</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 00:33]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-06 15:46:09,944] Trial 2 finished with value: 0.50076 and parameters: {'linear_layer_quantization': 2, 'linear_layer_quantization_fractional': 0, 'LinearMinifloatDenorm': 0, 'LinearMinifloatDenorm_ew': 1, 'LinearMinifloatDenorm_eb': 1, 'LinearMinifloatIEEE': 1, 'LinearMinifloatIEEE_ew': 1, 'LinearMinifloatIEEE_eb': 2, 'LinearLog': 1, 'LinearLog_ew': 1, 'LinearLog_eb': 2, 'LinearBlockFP': 0, 'LinearBlockFP_block_size': 1, 'LinearBlockMinifloat_ebw': 2, 'LinearBlockMinifloat_ew': 2, 'LinearBlockMinifloat_block_size': 1, 'LinearBlockMinifloat_w': 2, 'LinearBlockLog': 0, 'LinearBlockLog_ebw': 0, 'LinearBlockLog_block_size': 2, 'LinearBinary_stochastic': 1, 'LinearBinary_bipolar': 0, 'LinearBinaryScaling_stochastic': 0, 'LinearBinaryScaling_bipolar': 0, 'LinearBinaryScaling_binary_training': 0, 'LinearBinaryResidualSign_stochastic': 1, 'LinearBinaryResidualSign_bipolar': 2, 'LinearBinaryResidualSign_binary_training': 2, 'bert.encoder.layer.0.attention.self.query_type': <class 'chop.nn.quantized.modules.linear.LinearBinary'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.output.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinary'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'chop.nn.quantized.modules.linear.LinearBinary'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.output.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinary'>, 'bert.pooler.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinary'>, 'classifier_type': <class 'torch.nn.modules.linear.Linear'>}. Best is trial 0 with value: 0.514.\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearBinary'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 01:12, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.718200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.715700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.717700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.719100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.710300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.708700</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 00:35]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-06 15:47:58,416] Trial 3 finished with value: 0.49984 and parameters: {'linear_layer_quantization': 0, 'linear_layer_quantization_fractional': 0, 'LinearMinifloatDenorm': 0, 'LinearMinifloatDenorm_ew': 0, 'LinearMinifloatDenorm_eb': 1, 'LinearMinifloatIEEE': 1, 'LinearMinifloatIEEE_ew': 1, 'LinearMinifloatIEEE_eb': 0, 'LinearLog': 1, 'LinearLog_ew': 1, 'LinearLog_eb': 1, 'LinearBlockFP': 2, 'LinearBlockFP_block_size': 0, 'LinearBlockMinifloat_ebw': 1, 'LinearBlockMinifloat_ew': 0, 'LinearBlockMinifloat_block_size': 1, 'LinearBlockMinifloat_w': 1, 'LinearBlockLog': 2, 'LinearBlockLog_ebw': 1, 'LinearBlockLog_block_size': 1, 'LinearBinary_stochastic': 2, 'LinearBinary_bipolar': 1, 'LinearBinaryScaling_stochastic': 0, 'LinearBinaryScaling_bipolar': 0, 'LinearBinaryScaling_binary_training': 1, 'LinearBinaryResidualSign_stochastic': 1, 'LinearBinaryResidualSign_bipolar': 0, 'LinearBinaryResidualSign_binary_training': 0, 'bert.encoder.layer.0.attention.self.query_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'chop.nn.quantized.modules.linear.LinearBinary'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinary'>, 'bert.encoder.layer.0.output.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinary'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'chop.nn.quantized.modules.linear.LinearBinary'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'chop.nn.quantized.modules.linear.LinearBinary'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.pooler.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinary'>, 'classifier_type': <class 'torch.nn.modules.linear.Linear'>}. Best is trial 0 with value: 0.514.\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearBinary'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 01:11, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.722100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.717700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.708100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.712500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.709600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.707500</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 00:34]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-06 15:49:44,722] Trial 4 finished with value: 0.495 and parameters: {'linear_layer_quantization': 0, 'linear_layer_quantization_fractional': 2, 'LinearMinifloatDenorm': 0, 'LinearMinifloatDenorm_ew': 2, 'LinearMinifloatDenorm_eb': 0, 'LinearMinifloatIEEE': 0, 'LinearMinifloatIEEE_ew': 2, 'LinearMinifloatIEEE_eb': 2, 'LinearLog': 1, 'LinearLog_ew': 0, 'LinearLog_eb': 0, 'LinearBlockFP': 2, 'LinearBlockFP_block_size': 2, 'LinearBlockMinifloat_ebw': 0, 'LinearBlockMinifloat_ew': 0, 'LinearBlockMinifloat_block_size': 0, 'LinearBlockMinifloat_w': 0, 'LinearBlockLog': 1, 'LinearBlockLog_ebw': 1, 'LinearBlockLog_block_size': 2, 'LinearBinary_stochastic': 1, 'LinearBinary_bipolar': 0, 'LinearBinaryScaling_stochastic': 1, 'LinearBinaryScaling_bipolar': 0, 'LinearBinaryScaling_binary_training': 0, 'LinearBinaryResidualSign_stochastic': 2, 'LinearBinaryResidualSign_bipolar': 2, 'LinearBinaryResidualSign_binary_training': 2, 'bert.encoder.layer.0.attention.self.query_type': <class 'chop.nn.quantized.modules.linear.LinearBinary'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'chop.nn.quantized.modules.linear.LinearBinary'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'chop.nn.quantized.modules.linear.LinearBinary'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.output.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinary'>, 'bert.pooler.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinary'>, 'classifier_type': <class 'torch.nn.modules.linear.Linear'>}. Best is trial 0 with value: 0.514.\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearBinary'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 01:11, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>6.733800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>6.911700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>6.097000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>6.529600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>6.295600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>6.457300</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 00:32]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-06 15:51:30,612] Trial 5 finished with value: 0.49464 and parameters: {'linear_layer_quantization': 0, 'linear_layer_quantization_fractional': 2, 'LinearMinifloatDenorm': 2, 'LinearMinifloatDenorm_ew': 0, 'LinearMinifloatDenorm_eb': 2, 'LinearMinifloatIEEE': 2, 'LinearMinifloatIEEE_ew': 1, 'LinearMinifloatIEEE_eb': 2, 'LinearLog': 1, 'LinearLog_ew': 2, 'LinearLog_eb': 1, 'LinearBlockFP': 0, 'LinearBlockFP_block_size': 0, 'LinearBlockMinifloat_ebw': 1, 'LinearBlockMinifloat_ew': 1, 'LinearBlockMinifloat_block_size': 2, 'LinearBlockMinifloat_w': 2, 'LinearBlockLog': 0, 'LinearBlockLog_ebw': 2, 'LinearBlockLog_block_size': 2, 'LinearBinary_stochastic': 2, 'LinearBinary_bipolar': 0, 'LinearBinaryScaling_stochastic': 1, 'LinearBinaryScaling_bipolar': 2, 'LinearBinaryScaling_binary_training': 0, 'LinearBinaryResidualSign_stochastic': 1, 'LinearBinaryResidualSign_bipolar': 2, 'LinearBinaryResidualSign_binary_training': 2, 'bert.encoder.layer.0.attention.self.query_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinary'>, 'bert.encoder.layer.0.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.output.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinary'>, 'bert.pooler.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinary'>, 'classifier_type': <class 'chop.nn.quantized.modules.linear.LinearBinary'>}. Best is trial 0 with value: 0.514.\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearBinary'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 01:14, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>6.505500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>6.342600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>6.643400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>6.291600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>6.547800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>6.450000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 00:31]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-06 15:53:17,023] Trial 6 finished with value: 0.51212 and parameters: {'linear_layer_quantization': 0, 'linear_layer_quantization_fractional': 1, 'LinearMinifloatDenorm': 2, 'LinearMinifloatDenorm_ew': 0, 'LinearMinifloatDenorm_eb': 0, 'LinearMinifloatIEEE': 1, 'LinearMinifloatIEEE_ew': 2, 'LinearMinifloatIEEE_eb': 0, 'LinearLog': 0, 'LinearLog_ew': 1, 'LinearLog_eb': 1, 'LinearBlockFP': 0, 'LinearBlockFP_block_size': 0, 'LinearBlockMinifloat_ebw': 1, 'LinearBlockMinifloat_ew': 1, 'LinearBlockMinifloat_block_size': 2, 'LinearBlockMinifloat_w': 0, 'LinearBlockLog': 1, 'LinearBlockLog_ebw': 0, 'LinearBlockLog_block_size': 1, 'LinearBinary_stochastic': 2, 'LinearBinary_bipolar': 2, 'LinearBinaryScaling_stochastic': 1, 'LinearBinaryScaling_bipolar': 1, 'LinearBinaryScaling_binary_training': 2, 'LinearBinaryResidualSign_stochastic': 2, 'LinearBinaryResidualSign_bipolar': 0, 'LinearBinaryResidualSign_binary_training': 0, 'bert.encoder.layer.0.attention.self.query_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinary'>, 'bert.encoder.layer.0.output.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinary'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinary'>, 'bert.encoder.layer.1.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.pooler.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinary'>, 'classifier_type': <class 'chop.nn.quantized.modules.linear.LinearBinary'>}. Best is trial 0 with value: 0.514.\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearBinary'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 01:10, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.709500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.695500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.696800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.693500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.694400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.696000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 00:31]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-06 15:55:00,087] Trial 7 finished with value: 0.49728 and parameters: {'linear_layer_quantization': 1, 'linear_layer_quantization_fractional': 1, 'LinearMinifloatDenorm': 0, 'LinearMinifloatDenorm_ew': 1, 'LinearMinifloatDenorm_eb': 2, 'LinearMinifloatIEEE': 1, 'LinearMinifloatIEEE_ew': 1, 'LinearMinifloatIEEE_eb': 0, 'LinearLog': 0, 'LinearLog_ew': 2, 'LinearLog_eb': 1, 'LinearBlockFP': 1, 'LinearBlockFP_block_size': 1, 'LinearBlockMinifloat_ebw': 0, 'LinearBlockMinifloat_ew': 1, 'LinearBlockMinifloat_block_size': 2, 'LinearBlockMinifloat_w': 1, 'LinearBlockLog': 0, 'LinearBlockLog_ebw': 2, 'LinearBlockLog_block_size': 0, 'LinearBinary_stochastic': 2, 'LinearBinary_bipolar': 2, 'LinearBinaryScaling_stochastic': 1, 'LinearBinaryScaling_bipolar': 0, 'LinearBinaryScaling_binary_training': 2, 'LinearBinaryResidualSign_stochastic': 0, 'LinearBinaryResidualSign_bipolar': 1, 'LinearBinaryResidualSign_binary_training': 0, 'bert.encoder.layer.0.attention.self.query_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinary'>, 'bert.encoder.layer.0.output.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinary'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'chop.nn.quantized.modules.linear.LinearBinary'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'chop.nn.quantized.modules.linear.LinearBinary'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinary'>, 'bert.encoder.layer.1.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.pooler.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'classifier_type': <class 'torch.nn.modules.linear.Linear'>}. Best is trial 0 with value: 0.514.\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearBinary'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 01:11, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>6.696300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>6.744500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>6.181200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>6.517000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>6.352300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>6.481300</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 00:34]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-06 15:56:47,281] Trial 8 finished with value: 0.51552 and parameters: {'linear_layer_quantization': 2, 'linear_layer_quantization_fractional': 0, 'LinearMinifloatDenorm': 1, 'LinearMinifloatDenorm_ew': 0, 'LinearMinifloatDenorm_eb': 2, 'LinearMinifloatIEEE': 1, 'LinearMinifloatIEEE_ew': 1, 'LinearMinifloatIEEE_eb': 1, 'LinearLog': 1, 'LinearLog_ew': 2, 'LinearLog_eb': 2, 'LinearBlockFP': 0, 'LinearBlockFP_block_size': 2, 'LinearBlockMinifloat_ebw': 0, 'LinearBlockMinifloat_ew': 2, 'LinearBlockMinifloat_block_size': 1, 'LinearBlockMinifloat_w': 0, 'LinearBlockLog': 0, 'LinearBlockLog_ebw': 1, 'LinearBlockLog_block_size': 2, 'LinearBinary_stochastic': 0, 'LinearBinary_bipolar': 1, 'LinearBinaryScaling_stochastic': 1, 'LinearBinaryScaling_bipolar': 2, 'LinearBinaryScaling_binary_training': 2, 'LinearBinaryResidualSign_stochastic': 2, 'LinearBinaryResidualSign_bipolar': 0, 'LinearBinaryResidualSign_binary_training': 0, 'bert.encoder.layer.0.attention.self.query_type': <class 'chop.nn.quantized.modules.linear.LinearBinary'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'chop.nn.quantized.modules.linear.LinearBinary'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.output.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinary'>, 'bert.pooler.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinary'>, 'classifier_type': <class 'chop.nn.quantized.modules.linear.LinearBinary'>}. Best is trial 8 with value: 0.51552.\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearBinary'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 01:11, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.716800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.711700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.713100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.718900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.713600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.712100</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 00:34]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-06 15:58:34,421] Trial 9 finished with value: 0.5104 and parameters: {'linear_layer_quantization': 1, 'linear_layer_quantization_fractional': 1, 'LinearMinifloatDenorm': 0, 'LinearMinifloatDenorm_ew': 0, 'LinearMinifloatDenorm_eb': 0, 'LinearMinifloatIEEE': 1, 'LinearMinifloatIEEE_ew': 2, 'LinearMinifloatIEEE_eb': 2, 'LinearLog': 2, 'LinearLog_ew': 1, 'LinearLog_eb': 2, 'LinearBlockFP': 2, 'LinearBlockFP_block_size': 1, 'LinearBlockMinifloat_ebw': 1, 'LinearBlockMinifloat_ew': 1, 'LinearBlockMinifloat_block_size': 1, 'LinearBlockMinifloat_w': 1, 'LinearBlockLog': 0, 'LinearBlockLog_ebw': 0, 'LinearBlockLog_block_size': 0, 'LinearBinary_stochastic': 2, 'LinearBinary_bipolar': 1, 'LinearBinaryScaling_stochastic': 1, 'LinearBinaryScaling_bipolar': 0, 'LinearBinaryScaling_binary_training': 1, 'LinearBinaryResidualSign_stochastic': 0, 'LinearBinaryResidualSign_bipolar': 1, 'LinearBinaryResidualSign_binary_training': 1, 'bert.encoder.layer.0.attention.self.query_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'chop.nn.quantized.modules.linear.LinearBinary'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinary'>, 'bert.encoder.layer.0.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'chop.nn.quantized.modules.linear.LinearBinary'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinary'>, 'bert.encoder.layer.1.output.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinary'>, 'bert.pooler.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinary'>, 'classifier_type': <class 'torch.nn.modules.linear.Linear'>}. Best is trial 8 with value: 0.51552.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.514, 0.514, 0.514, 0.514, 0.514, 0.514, 0.514, 0.514, 0.51552, 0.51552]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from pathlib import Path\n",
    "import dill\n",
    "\n",
    "with open(f\"{Path.home()}/tutorial_5_best_model.pkl\", \"rb\") as f:\n",
    "    base_model = dill.load(f)\n",
    "from chop.nn.quantized.modules.linear import (\n",
    "    LinearInteger,\n",
    "    LinearMinifloatDenorm,\n",
    "    LinearMinifloatIEEE,\n",
    "    LinearLog,\n",
    "    LinearBlockFP,\n",
    "    LinearBlockMinifloat,\n",
    "    LinearBlockLog,\n",
    "    LinearBinary,\n",
    "    LinearBinaryScaling,\n",
    "    LinearBinaryResidualSign,\n",
    ")\n",
    "search_space = {\n",
    "    \"linear_layer_choices\": [\n",
    "        torch.nn.Linear,\n",
    "        # LinearInteger,\n",
    "        # LinearMinifloatDenorm,\n",
    "        # LinearMinifloatIEEE,\n",
    "        # LinearLog,\n",
    "        # LinearBlockFP,\n",
    "        # # LinearBlockMinifloat,\n",
    "        # LinearBlockLog,\n",
    "        LinearBinary,\n",
    "        # LinearBinaryScaling,\n",
    "        # LinearBinaryResidualSign,\n",
    "    ],\n",
    "    'linear_layer_quantization': [8, 16, 32],\n",
    "    'linear_layer_quantization_fractional':  [2, 4, 8],\n",
    "    'LinearMinifloatDenorm': [8, 16, 32],\n",
    "    'LinearMinifloatDenorm_ew': [2, 4, 8],\n",
    "    'LinearMinifloatDenorm_eb': [2, 4, 8],\n",
    "    'LinearMinifloatIEEE': [8, 16, 32],\n",
    "    'LinearMinifloatIEEE_ew': [2, 4, 8],\n",
    "    'LinearMinifloatIEEE_eb': [2, 4, 8],\n",
    "    'LinearLog': [8, 16, 32],\n",
    "    'LinearLog_ew': [2, 4, 8],\n",
    "    'LinearLog_eb': [2, 4, 8],\n",
    "    'LinearBlockFP': [8, 16, 32],\n",
    "    'LinearBlockFP_block_size':[[4], [8], [16]] ,\n",
    "    'LinearBlockMinifloat_ebw': [2, 4, 8],\n",
    "    'LinearBlockMinifloat_ew': [2, 4, 8],\n",
    "    'LinearBlockMinifloat_block_size': [[4], [8], [16]],\n",
    "    'LinearBlockMinifloat_w': [2, 4, 8],\n",
    "    'LinearBlockLog': [8, 16, 32],\n",
    "    'LinearBlockLog_ebw': [2, 4, 8],\n",
    "    'LinearBlockLog_block_size': [[4], [8], [16]] ,\n",
    "    'LinearBinary_stochastic': [2, 4, 8],\n",
    "    'LinearBinary_bipolar': [2, 4, 8],\n",
    "    'LinearBinaryScaling_stochastic': [2, 4, 8],\n",
    "    'LinearBinaryScaling_bipolar': [2, 4, 8],\n",
    "    \"LinearBinaryScaling_binary_training\": [2, 4, 8],\n",
    "    'LinearBinaryResidualSign_stochastic': [2, 4, 8],\n",
    "    'LinearBinaryResidualSign_bipolar': [2, 4, 8],\n",
    "    \"LinearBinaryResidualSign_binary_training\": [2, 4, 8],\n",
    "}\n",
    "\n",
    "def construct_model(trial):\n",
    "    # Fetch the model\n",
    "    my_config = {}\n",
    "    trial_model = deepcopy(base_model)\n",
    "    \n",
    "    # Select quantization and fractional width configurations\n",
    "    for param in ['linear_layer_quantization', \n",
    "                  'linear_layer_quantization_fractional',\n",
    "                  'LinearMinifloatDenorm',\n",
    "                  'LinearMinifloatDenorm_ew',\n",
    "                  'LinearMinifloatDenorm_eb',\n",
    "                  'LinearMinifloatIEEE',\n",
    "                  'LinearMinifloatIEEE_ew',\n",
    "                  'LinearMinifloatIEEE_eb',\n",
    "                    'LinearLog',\n",
    "                    'LinearLog_ew',\n",
    "                    'LinearLog_eb',\n",
    "                    'LinearBlockFP',\n",
    "                    'LinearBlockFP_block_size',\n",
    "                    'LinearBlockMinifloat_ebw',\n",
    "                    'LinearBlockMinifloat_ew',\n",
    "                    'LinearBlockMinifloat_block_size',\n",
    "                    'LinearBlockMinifloat_w',\n",
    "                    'LinearBlockLog',\n",
    "                    'LinearBlockLog_ebw',\n",
    "                    'LinearBlockLog_block_size',\n",
    "                    'LinearBinary_stochastic',\n",
    "                    'LinearBinary_bipolar',\n",
    "                    'LinearBinaryScaling_stochastic',\n",
    "                    'LinearBinaryScaling_bipolar',\n",
    "                    'LinearBinaryScaling_binary_training',\n",
    "                    'LinearBinaryResidualSign_stochastic',\n",
    "                    'LinearBinaryResidualSign_bipolar',\n",
    "                    'LinearBinaryResidualSign_binary_training'\n",
    "                  ]:\n",
    "        chosen_idx = trial.suggest_int(param, 0, len(search_space[param]) - 1)\n",
    "        my_config[param] = search_space[param][chosen_idx]\n",
    "\n",
    "    # Quantize layers according to Optuna suggestions\n",
    "    for name, layer in trial_model.named_modules():\n",
    "        if isinstance(layer, torch.nn.Linear):\n",
    "            new_layer_cls = trial.suggest_categorical(\n",
    "                f\"{name}_type\",\n",
    "                search_space[\"linear_layer_choices\"],\n",
    "            )\n",
    "\n",
    "            if new_layer_cls == torch.nn.Linear:\n",
    "                continue\n",
    "\n",
    "            kwargs = {\n",
    "                \"in_features\": layer.in_features,\n",
    "                \"out_features\": layer.out_features,\n",
    "            }\n",
    "\n",
    "            try:\n",
    "                # Configure parameters for specific layer types\n",
    "                # if new_layer_cls == LinearInteger:\n",
    "                #     kwargs[\"config\"] = {\n",
    "                #         \"data_in_width\": my_config['linear_layer_quantization'],\n",
    "                #         \"data_in_frac_width\": my_config['linear_layer_quantization_fractional'],\n",
    "                #         \"weight_width\": my_config['linear_layer_quantization'],\n",
    "                #         \"weight_frac_width\": my_config['linear_layer_quantization_fractional'],\n",
    "                #         \"bias_width\": my_config['linear_layer_quantization'],\n",
    "                #         \"bias_frac_width\": my_config['linear_layer_quantization_fractional'],\n",
    "                #     }\n",
    "                # elif new_layer_cls == LinearMinifloatDenorm:\n",
    "                #     kwargs[\"config\"] = {\n",
    "                #         \"data_in_width\": my_config['LinearMinifloatDenorm'],\n",
    "                #         \"data_in_exponent_width\": my_config['LinearMinifloatDenorm_ew'],\n",
    "                #         \"data_in_exponent_bias\": my_config['LinearMinifloatDenorm_eb'],\n",
    "                #         \"weight_width\": my_config['LinearMinifloatDenorm'],\n",
    "                #         \"weight_exponent_width\": my_config['LinearMinifloatDenorm_ew'],\n",
    "                #         \"weight_exponent_bias\": my_config['LinearMinifloatDenorm_eb'],\n",
    "                #         \"bias_width\": my_config['LinearMinifloatDenorm'],\n",
    "                #         \"bias_exponent_width\": my_config['LinearMinifloatDenorm_ew'],\n",
    "                #         \"bias_exponent_bias\": my_config['LinearMinifloatDenorm_eb'],\n",
    "                #     }\n",
    "                # elif new_layer_cls == LinearMinifloatIEEE:\n",
    "                #     kwargs[\"config\"] = {\n",
    "                #         \"data_in_width\": my_config['LinearMinifloatIEEE'],\n",
    "                #         \"data_in_exponent_width\": my_config['LinearMinifloatIEEE_ew'],\n",
    "                #         \"data_in_exponent_bias\": my_config['LinearMinifloatIEEE_eb'],\n",
    "                #         \"weight_width\": my_config['LinearMinifloatIEEE'],\n",
    "                #         \"weight_exponent_width\": my_config['LinearMinifloatIEEE_ew'],\n",
    "                #         \"weight_exponent_bias\": my_config['LinearMinifloatIEEE_eb'],\n",
    "                #         \"bias_width\": my_config['LinearMinifloatIEEE'],\n",
    "                #         \"bias_exponent_width\": my_config['LinearMinifloatIEEE_ew'],\n",
    "                #         \"bias_exponent_bias\": my_config['LinearMinifloatIEEE_eb'],\n",
    "                #     }\n",
    "                # elif new_layer_cls == LinearLog:\n",
    "                #     kwargs[\"config\"] = {\n",
    "                #         \"data_in_width\": my_config['LinearLog'],\n",
    "                #         \"data_in_exponent_width\": my_config['LinearLog_ew'],\n",
    "                #         \"data_in_exponent_bias\": my_config['LinearLog_eb'],\n",
    "                #         \"weight_width\": my_config['LinearLog'],\n",
    "                #         \"weight_exponent_width\": my_config['LinearLog_ew'],\n",
    "                #         \"weight_exponent_bias\": my_config['LinearLog_eb'],\n",
    "                #         \"bias_width\": my_config['LinearLog'],\n",
    "                #         \"bias_exponent_width\": my_config['LinearLog_ew'],\n",
    "                #         \"bias_exponent_bias\": my_config['LinearLog_eb'],\n",
    "                #     }\n",
    "                # elif new_layer_cls == LinearBlockFP:\n",
    "                #     kwargs[\"config\"] = {\n",
    "                #         \"weight_width\": my_config['LinearBlockFP'],\n",
    "                #         \"weight_exponent_width\": my_config['LinearBlockFP'],\n",
    "                #         \"weight_block_size\": my_config['LinearBlockFP_block_size'],\n",
    "                #         \"weight_exponent_bias\": my_config['LinearBlockFP'],\n",
    "                #         \"data_in_width\": my_config['LinearBlockFP'],\n",
    "                #         \"data_in_exponent_width\": my_config['LinearBlockFP'],\n",
    "                #         \"data_in_block_size\": my_config['LinearBlockFP_block_size'],\n",
    "                #         \"data_in_exponent_bias\": my_config['LinearBlockFP'],\n",
    "                #         \"bias_width\": my_config['LinearBlockFP'],\n",
    "                #         \"bias_exponent_width\": my_config['LinearBlockFP'],\n",
    "                #         \"bias_block_size\": my_config['LinearBlockFP_block_size'],\n",
    "                #         \"bias_exponent_bias\": my_config['LinearBlockFP'],\n",
    "                #     }\n",
    "                # elif new_layer_cls == LinearBlockMinifloat:\n",
    "                #     kwargs[\"config\"] = {\n",
    "                #         \"weight_exponent_bias_width\" : my_config['LinearBlockMinifloat_ebw'],\n",
    "                #         \"weight_exponent_width\": my_config['LinearBlockMinifloat_ew'],\n",
    "                #         \"weight_block_size\": my_config['LinearBlockMinifloat_block_size'],\n",
    "                #         \"weight_width\": my_config['LinearBlockMinifloat_w'],\n",
    "                #         \"data_in_width\":my_config[\"LinearBlockMinifloat_w\"],\n",
    "                #         \"data_in_exponent_bias_width\":my_config[\"LinearBlockMinifloat_ew\"],\n",
    "                #         \"data_in_exponent_width\":my_config[\"LinearBlockMinifloat_ebw\"],\n",
    "                #         \"data_in_block_size\":my_config[\"LinearBlockMinifloat_block_size\"],\n",
    "                #         \"bias_width\":my_config[\"LinearBlockMinifloat_w\"],\n",
    "                #         \"bias_exponent_width\":my_config[\"LinearBlockMinifloat_ew\"],\n",
    "                #         \"bias_exponent_bias_width\":my_config[\"LinearBlockMinifloat_ebw\"],\n",
    "                #         \"bias_block_size\":my_config[\"LinearBlockMinifloat_block_size\"],\n",
    "                # #     }\n",
    "                # elif new_layer_cls == LinearBlockLog:\n",
    "                #     kwargs[\"config\"] = {\n",
    "                #         \"data_in_width\": my_config['LinearBlockLog'],\n",
    "                #         \"data_in_exponent_bias_width\": my_config['LinearBlockLog_ebw'],\n",
    "                #         \"data_in_block_size\": my_config['LinearBlockLog_block_size'],\n",
    "                #         \"weight_width\": my_config['LinearBlockLog'],\n",
    "                #         \"weight_exponent_bias_width\": my_config['LinearBlockLog_ebw'],\n",
    "                #         \"weight_block_size\": my_config['LinearBlockLog_block_size'],\n",
    "                #         \"bias_width\": my_config['LinearBlockLog'],\n",
    "                #         \"bias_exponent_bias_width\": my_config['LinearBlockLog_ebw'],\n",
    "                #         \"bias_block_size\": my_config['LinearBlockLog_block_size'],\n",
    "                #     }\n",
    "\n",
    "                if new_layer_cls == LinearBinary:\n",
    "                    kwargs[\"config\"] = {\n",
    "                        'weight_stochastic':my_config['LinearBinary_stochastic'],\n",
    "                        'weight_bipolar':my_config['LinearBinary_bipolar'],\n",
    "                    }\n",
    "\n",
    "                    \n",
    "                # elif new_layer_cls == LinearBinaryScaling:\n",
    "                #     kwargs[\"config\"] = {\n",
    "                #         'weight_stochastic':my_config['LinearBinaryScaling_stochastic'],\n",
    "                #         'weight_bipolar':my_config['LinearBinaryScaling_bipolar'],\n",
    "                #         \"data_in_stochastic\": my_config['LinearBinaryScaling_stochastic'],\n",
    "                #         \"data_in_bipolar\": my_config['LinearBinaryScaling_bipolar'],\n",
    "                #         \"bias_stochastic\": my_config['LinearBinaryScaling_stochastic'],\n",
    "                #         \"bias_bipolar\": my_config['LinearBinaryScaling_bipolar'],\n",
    "                #         \"binary_training\": my_config['LinearBinaryScaling_binary_training'],\n",
    "                #     }\n",
    "                # elif new_layer_cls == LinearBinaryResidualSign:\n",
    "                #     kwargs[\"config\"] = {\n",
    "                #         'weight_stochastic':my_config['LinearBinaryResidualSign_stochastic'],\n",
    "                #         'weight_bipolar':my_config['LinearBinaryResidualSign_bipolar'],\n",
    "                #         \"data_in_stochastic\": my_config['LinearBinaryResidualSign_stochastic'],\n",
    "                #         \"data_in_bipolar\": my_config['LinearBinaryResidualSign_bipolar'],\n",
    "                #         \"bias_stochastic\": my_config['LinearBinaryResidualSign_stochastic'],\n",
    "                #         \"bias_bipolar\": my_config['LinearBinaryResidualSign_bipolar'],\n",
    "                #         \"binary_training\": my_config['LinearBinaryResidualSign_binary_training'],\n",
    "                #     }\n",
    "                # Create the new layer (copy the weights if applicable)\n",
    "                new_layer = new_layer_cls(**kwargs)\n",
    "                new_layer.weight.data = layer.weight.data.clone()\n",
    "\n",
    "                # Replace the layer in the model\n",
    "                deepsetattr(trial_model, name, new_layer)\n",
    "            except TypeError as e:\n",
    "                print(f\"Error creating layer {name} with class {new_layer_cls}: {e}\")\n",
    "                raise\n",
    "\n",
    "    return trial_model\n",
    "from chop.tools import get_trainer\n",
    "import random\n",
    "\n",
    "\n",
    "def objective(trial):\n",
    "\n",
    "    # Define the model\n",
    "    model = construct_model(trial)\n",
    "\n",
    "    trainer = get_trainer(\n",
    "        model=model,\n",
    "        tokenized_dataset=dataset,\n",
    "        tokenizer=tokenizer,\n",
    "        evaluate_metric=\"accuracy\",\n",
    "        num_train_epochs=1,\n",
    "    )\n",
    "\n",
    "    trainer.train()\n",
    "    eval_results = trainer.evaluate()\n",
    "\n",
    "    trial.set_user_attr(\"model\", model)\n",
    "\n",
    "    return eval_results[\"eval_accuracy\"]\n",
    "\n",
    "\n",
    "from optuna.samplers import GridSampler, RandomSampler, TPESampler\n",
    "\n",
    "sampler = RandomSampler()\n",
    "\n",
    "import optuna\n",
    "\n",
    "study = optuna.create_study(\n",
    "    direction=\"maximize\",\n",
    "    study_name=\"bert-tiny-nas-study\",\n",
    "    sampler=sampler,\n",
    ")\n",
    "\n",
    "study.optimize(\n",
    "    objective,\n",
    "    n_trials=10,\n",
    "    timeout=60 * 60 * 24,\n",
    ")\n",
    "\n",
    "task8 = [max([t.value for t in study.trials[:i+1]]) for i in range(len(study.trials))]\n",
    "task8"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "task8 = [0.514, 0.514, 0.514, 0.514, 0.514, 0.514, 0.514, 0.514, 0.51552, 0.51552]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-10 20:42:30,714] A new study created in memory with name: bert-tiny-nas-study\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 01:10, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.343200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.302700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.341400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.342600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.332200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.362500</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 00:31]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-10 20:44:13,132] Trial 0 finished with value: 0.8728 and parameters: {'linear_layer_quantization': 0, 'linear_layer_quantization_fractional': 2, 'LinearMinifloatDenorm': 1, 'LinearMinifloatDenorm_ew': 2, 'LinearMinifloatDenorm_eb': 1, 'LinearMinifloatIEEE': 1, 'LinearMinifloatIEEE_ew': 2, 'LinearMinifloatIEEE_eb': 0, 'LinearLog': 2, 'LinearLog_ew': 0, 'LinearLog_eb': 0, 'LinearBlockFP': 2, 'LinearBlockFP_block_size': 1, 'LinearBlockMinifloat_ebw': 2, 'LinearBlockMinifloat_ew': 2, 'LinearBlockMinifloat_block_size': 0, 'LinearBlockMinifloat_w': 0, 'LinearBlockLog': 1, 'LinearBlockLog_ebw': 0, 'LinearBlockLog_block_size': 2, 'LinearBinary_stochastic': 0, 'LinearBinary_bipolar': 0, 'LinearBinaryScaling_stochastic': 0, 'LinearBinaryScaling_bipolar': 0, 'LinearBinaryScaling_binary_training': 1, 'LinearBinaryResidualSign_stochastic': 0, 'LinearBinaryResidualSign_bipolar': 1, 'LinearBinaryResidualSign_binary_training': 1, 'bert.encoder.layer.0.attention.self.query_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'>, 'bert.encoder.layer.0.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.output.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'>, 'bert.pooler.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'>, 'classifier_type': <class 'torch.nn.modules.linear.Linear'>}. Best is trial 0 with value: 0.8728.\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 01:17, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.340400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.301000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.334800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.344800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.333500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.363000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 00:33]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-10 20:46:05,111] Trial 1 finished with value: 0.8724 and parameters: {'linear_layer_quantization': 0, 'linear_layer_quantization_fractional': 2, 'LinearMinifloatDenorm': 1, 'LinearMinifloatDenorm_ew': 1, 'LinearMinifloatDenorm_eb': 1, 'LinearMinifloatIEEE': 2, 'LinearMinifloatIEEE_ew': 1, 'LinearMinifloatIEEE_eb': 0, 'LinearLog': 2, 'LinearLog_ew': 1, 'LinearLog_eb': 2, 'LinearBlockFP': 0, 'LinearBlockFP_block_size': 0, 'LinearBlockMinifloat_ebw': 1, 'LinearBlockMinifloat_ew': 0, 'LinearBlockMinifloat_block_size': 2, 'LinearBlockMinifloat_w': 1, 'LinearBlockLog': 0, 'LinearBlockLog_ebw': 0, 'LinearBlockLog_block_size': 0, 'LinearBinary_stochastic': 0, 'LinearBinary_bipolar': 1, 'LinearBinaryScaling_stochastic': 0, 'LinearBinaryScaling_bipolar': 1, 'LinearBinaryScaling_binary_training': 1, 'LinearBinaryResidualSign_stochastic': 1, 'LinearBinaryResidualSign_bipolar': 1, 'LinearBinaryResidualSign_binary_training': 1, 'bert.encoder.layer.0.attention.self.query_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'>, 'bert.encoder.layer.0.output.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'>, 'bert.encoder.layer.1.output.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'>, 'bert.pooler.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'classifier_type': <class 'torch.nn.modules.linear.Linear'>}. Best is trial 0 with value: 0.8728.\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 01:17, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.341800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.309400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.335800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.345100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.333700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.365500</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 00:36]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-10 20:48:00,020] Trial 2 finished with value: 0.8714 and parameters: {'linear_layer_quantization': 0, 'linear_layer_quantization_fractional': 2, 'LinearMinifloatDenorm': 2, 'LinearMinifloatDenorm_ew': 1, 'LinearMinifloatDenorm_eb': 0, 'LinearMinifloatIEEE': 0, 'LinearMinifloatIEEE_ew': 0, 'LinearMinifloatIEEE_eb': 0, 'LinearLog': 0, 'LinearLog_ew': 2, 'LinearLog_eb': 1, 'LinearBlockFP': 0, 'LinearBlockFP_block_size': 2, 'LinearBlockMinifloat_ebw': 2, 'LinearBlockMinifloat_ew': 2, 'LinearBlockMinifloat_block_size': 2, 'LinearBlockMinifloat_w': 2, 'LinearBlockLog': 0, 'LinearBlockLog_ebw': 1, 'LinearBlockLog_block_size': 2, 'LinearBinary_stochastic': 2, 'LinearBinary_bipolar': 1, 'LinearBinaryScaling_stochastic': 1, 'LinearBinaryScaling_bipolar': 0, 'LinearBinaryScaling_binary_training': 1, 'LinearBinaryResidualSign_stochastic': 1, 'LinearBinaryResidualSign_bipolar': 1, 'LinearBinaryResidualSign_binary_training': 1, 'bert.encoder.layer.0.attention.self.query_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'>, 'bert.encoder.layer.0.output.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'>, 'bert.encoder.layer.1.output.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'>, 'bert.pooler.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'classifier_type': <class 'torch.nn.modules.linear.Linear'>}. Best is trial 0 with value: 0.8728.\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 01:25, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.563200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.424800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.395000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.372700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.358500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.383000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 00:34]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-10 20:50:10,416] Trial 3 finished with value: 0.8576 and parameters: {'linear_layer_quantization': 1, 'linear_layer_quantization_fractional': 2, 'LinearMinifloatDenorm': 1, 'LinearMinifloatDenorm_ew': 1, 'LinearMinifloatDenorm_eb': 0, 'LinearMinifloatIEEE': 2, 'LinearMinifloatIEEE_ew': 2, 'LinearMinifloatIEEE_eb': 2, 'LinearLog': 0, 'LinearLog_ew': 1, 'LinearLog_eb': 1, 'LinearBlockFP': 0, 'LinearBlockFP_block_size': 0, 'LinearBlockMinifloat_ebw': 1, 'LinearBlockMinifloat_ew': 2, 'LinearBlockMinifloat_block_size': 1, 'LinearBlockMinifloat_w': 1, 'LinearBlockLog': 0, 'LinearBlockLog_ebw': 0, 'LinearBlockLog_block_size': 0, 'LinearBinary_stochastic': 0, 'LinearBinary_bipolar': 1, 'LinearBinaryScaling_stochastic': 1, 'LinearBinaryScaling_bipolar': 1, 'LinearBinaryScaling_binary_training': 0, 'LinearBinaryResidualSign_stochastic': 1, 'LinearBinaryResidualSign_bipolar': 1, 'LinearBinaryResidualSign_binary_training': 1, 'bert.encoder.layer.0.attention.self.query_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'>, 'bert.encoder.layer.0.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'>, 'bert.encoder.layer.1.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.pooler.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'>, 'classifier_type': <class 'torch.nn.modules.linear.Linear'>}. Best is trial 0 with value: 0.8728.\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 01:22, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>6.163600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>2.018500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>1.647600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>1.535100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>1.261200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>1.292900</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 00:43]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-10 20:52:19,652] Trial 4 finished with value: 0.74808 and parameters: {'linear_layer_quantization': 1, 'linear_layer_quantization_fractional': 0, 'LinearMinifloatDenorm': 1, 'LinearMinifloatDenorm_ew': 0, 'LinearMinifloatDenorm_eb': 2, 'LinearMinifloatIEEE': 2, 'LinearMinifloatIEEE_ew': 1, 'LinearMinifloatIEEE_eb': 1, 'LinearLog': 2, 'LinearLog_ew': 0, 'LinearLog_eb': 1, 'LinearBlockFP': 1, 'LinearBlockFP_block_size': 0, 'LinearBlockMinifloat_ebw': 0, 'LinearBlockMinifloat_ew': 1, 'LinearBlockMinifloat_block_size': 2, 'LinearBlockMinifloat_w': 0, 'LinearBlockLog': 0, 'LinearBlockLog_ebw': 2, 'LinearBlockLog_block_size': 0, 'LinearBinary_stochastic': 0, 'LinearBinary_bipolar': 2, 'LinearBinaryScaling_stochastic': 1, 'LinearBinaryScaling_bipolar': 0, 'LinearBinaryScaling_binary_training': 0, 'LinearBinaryResidualSign_stochastic': 0, 'LinearBinaryResidualSign_bipolar': 0, 'LinearBinaryResidualSign_binary_training': 1, 'bert.encoder.layer.0.attention.self.query_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'>, 'bert.encoder.layer.0.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.output.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'>, 'bert.pooler.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'classifier_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'>}. Best is trial 0 with value: 0.8728.\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 01:15, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.339200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.306700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.338000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.347500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.331700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.360500</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 00:32]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-10 20:54:08,883] Trial 5 finished with value: 0.87308 and parameters: {'linear_layer_quantization': 2, 'linear_layer_quantization_fractional': 2, 'LinearMinifloatDenorm': 1, 'LinearMinifloatDenorm_ew': 1, 'LinearMinifloatDenorm_eb': 1, 'LinearMinifloatIEEE': 0, 'LinearMinifloatIEEE_ew': 1, 'LinearMinifloatIEEE_eb': 1, 'LinearLog': 1, 'LinearLog_ew': 0, 'LinearLog_eb': 0, 'LinearBlockFP': 1, 'LinearBlockFP_block_size': 0, 'LinearBlockMinifloat_ebw': 1, 'LinearBlockMinifloat_ew': 2, 'LinearBlockMinifloat_block_size': 0, 'LinearBlockMinifloat_w': 1, 'LinearBlockLog': 0, 'LinearBlockLog_ebw': 2, 'LinearBlockLog_block_size': 2, 'LinearBinary_stochastic': 2, 'LinearBinary_bipolar': 1, 'LinearBinaryScaling_stochastic': 0, 'LinearBinaryScaling_bipolar': 1, 'LinearBinaryScaling_binary_training': 1, 'LinearBinaryResidualSign_stochastic': 0, 'LinearBinaryResidualSign_bipolar': 0, 'LinearBinaryResidualSign_binary_training': 0, 'bert.encoder.layer.0.attention.self.query_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.output.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'>, 'bert.encoder.layer.1.output.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'>, 'bert.pooler.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'classifier_type': <class 'torch.nn.modules.linear.Linear'>}. Best is trial 5 with value: 0.87308.\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 01:12, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.337900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.312600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.329600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.339400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.331000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.360600</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 00:32]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-10 20:55:54,392] Trial 6 finished with value: 0.87332 and parameters: {'linear_layer_quantization': 2, 'linear_layer_quantization_fractional': 1, 'LinearMinifloatDenorm': 0, 'LinearMinifloatDenorm_ew': 0, 'LinearMinifloatDenorm_eb': 0, 'LinearMinifloatIEEE': 2, 'LinearMinifloatIEEE_ew': 2, 'LinearMinifloatIEEE_eb': 0, 'LinearLog': 0, 'LinearLog_ew': 2, 'LinearLog_eb': 1, 'LinearBlockFP': 1, 'LinearBlockFP_block_size': 2, 'LinearBlockMinifloat_ebw': 2, 'LinearBlockMinifloat_ew': 0, 'LinearBlockMinifloat_block_size': 0, 'LinearBlockMinifloat_w': 1, 'LinearBlockLog': 1, 'LinearBlockLog_ebw': 1, 'LinearBlockLog_block_size': 1, 'LinearBinary_stochastic': 1, 'LinearBinary_bipolar': 2, 'LinearBinaryScaling_stochastic': 0, 'LinearBinaryScaling_bipolar': 1, 'LinearBinaryScaling_binary_training': 1, 'LinearBinaryResidualSign_stochastic': 0, 'LinearBinaryResidualSign_bipolar': 1, 'LinearBinaryResidualSign_binary_training': 0, 'bert.encoder.layer.0.attention.self.query_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.pooler.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'classifier_type': <class 'torch.nn.modules.linear.Linear'>}. Best is trial 6 with value: 0.87332.\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 01:13, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.340000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.312500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.334400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.345900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.338900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.364700</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 00:31]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-10 20:57:40,164] Trial 7 finished with value: 0.87336 and parameters: {'linear_layer_quantization': 0, 'linear_layer_quantization_fractional': 2, 'LinearMinifloatDenorm': 1, 'LinearMinifloatDenorm_ew': 0, 'LinearMinifloatDenorm_eb': 2, 'LinearMinifloatIEEE': 2, 'LinearMinifloatIEEE_ew': 1, 'LinearMinifloatIEEE_eb': 0, 'LinearLog': 0, 'LinearLog_ew': 1, 'LinearLog_eb': 1, 'LinearBlockFP': 1, 'LinearBlockFP_block_size': 1, 'LinearBlockMinifloat_ebw': 0, 'LinearBlockMinifloat_ew': 1, 'LinearBlockMinifloat_block_size': 2, 'LinearBlockMinifloat_w': 0, 'LinearBlockLog': 0, 'LinearBlockLog_ebw': 1, 'LinearBlockLog_block_size': 0, 'LinearBinary_stochastic': 2, 'LinearBinary_bipolar': 1, 'LinearBinaryScaling_stochastic': 0, 'LinearBinaryScaling_bipolar': 1, 'LinearBinaryScaling_binary_training': 1, 'LinearBinaryResidualSign_stochastic': 1, 'LinearBinaryResidualSign_bipolar': 0, 'LinearBinaryResidualSign_binary_training': 1, 'bert.encoder.layer.0.attention.self.query_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.output.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.pooler.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'classifier_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'>}. Best is trial 7 with value: 0.87336.\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 01:11, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.337700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.302200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.330600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.339100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.326900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.361900</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 00:30]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-10 20:59:23,252] Trial 8 finished with value: 0.87328 and parameters: {'linear_layer_quantization': 0, 'linear_layer_quantization_fractional': 2, 'LinearMinifloatDenorm': 1, 'LinearMinifloatDenorm_ew': 1, 'LinearMinifloatDenorm_eb': 0, 'LinearMinifloatIEEE': 1, 'LinearMinifloatIEEE_ew': 1, 'LinearMinifloatIEEE_eb': 1, 'LinearLog': 2, 'LinearLog_ew': 2, 'LinearLog_eb': 0, 'LinearBlockFP': 0, 'LinearBlockFP_block_size': 1, 'LinearBlockMinifloat_ebw': 1, 'LinearBlockMinifloat_ew': 1, 'LinearBlockMinifloat_block_size': 2, 'LinearBlockMinifloat_w': 1, 'LinearBlockLog': 0, 'LinearBlockLog_ebw': 1, 'LinearBlockLog_block_size': 0, 'LinearBinary_stochastic': 2, 'LinearBinary_bipolar': 2, 'LinearBinaryScaling_stochastic': 1, 'LinearBinaryScaling_bipolar': 0, 'LinearBinaryScaling_binary_training': 1, 'LinearBinaryResidualSign_stochastic': 0, 'LinearBinaryResidualSign_bipolar': 0, 'LinearBinaryResidualSign_binary_training': 1, 'bert.encoder.layer.0.attention.self.query_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.pooler.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'classifier_type': <class 'torch.nn.modules.linear.Linear'>}. Best is trial 7 with value: 0.87336.\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 01:28, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.570700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.670200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.688600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.637400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.621400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.576600</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 00:47]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-10 21:01:40,655] Trial 9 finished with value: 0.84972 and parameters: {'linear_layer_quantization': 2, 'linear_layer_quantization_fractional': 1, 'LinearMinifloatDenorm': 2, 'LinearMinifloatDenorm_ew': 0, 'LinearMinifloatDenorm_eb': 1, 'LinearMinifloatIEEE': 0, 'LinearMinifloatIEEE_ew': 2, 'LinearMinifloatIEEE_eb': 0, 'LinearLog': 0, 'LinearLog_ew': 0, 'LinearLog_eb': 1, 'LinearBlockFP': 1, 'LinearBlockFP_block_size': 1, 'LinearBlockMinifloat_ebw': 1, 'LinearBlockMinifloat_ew': 0, 'LinearBlockMinifloat_block_size': 2, 'LinearBlockMinifloat_w': 1, 'LinearBlockLog': 0, 'LinearBlockLog_ebw': 0, 'LinearBlockLog_block_size': 0, 'LinearBinary_stochastic': 0, 'LinearBinary_bipolar': 1, 'LinearBinaryScaling_stochastic': 1, 'LinearBinaryScaling_bipolar': 0, 'LinearBinaryScaling_binary_training': 0, 'LinearBinaryResidualSign_stochastic': 0, 'LinearBinaryResidualSign_bipolar': 1, 'LinearBinaryResidualSign_binary_training': 0, 'bert.encoder.layer.0.attention.self.query_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'>, 'bert.encoder.layer.0.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'>, 'bert.encoder.layer.1.output.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'>, 'bert.pooler.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'classifier_type': <class 'torch.nn.modules.linear.Linear'>}. Best is trial 7 with value: 0.87336.\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 01:16, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.340700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.303500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.336500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.340600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.334600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.360300</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 00:33]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-10 21:03:43,275] Trial 10 finished with value: 0.8714 and parameters: {'linear_layer_quantization': 1, 'linear_layer_quantization_fractional': 2, 'LinearMinifloatDenorm': 0, 'LinearMinifloatDenorm_ew': 0, 'LinearMinifloatDenorm_eb': 0, 'LinearMinifloatIEEE': 2, 'LinearMinifloatIEEE_ew': 1, 'LinearMinifloatIEEE_eb': 2, 'LinearLog': 2, 'LinearLog_ew': 0, 'LinearLog_eb': 1, 'LinearBlockFP': 0, 'LinearBlockFP_block_size': 1, 'LinearBlockMinifloat_ebw': 0, 'LinearBlockMinifloat_ew': 0, 'LinearBlockMinifloat_block_size': 2, 'LinearBlockMinifloat_w': 0, 'LinearBlockLog': 2, 'LinearBlockLog_ebw': 2, 'LinearBlockLog_block_size': 1, 'LinearBinary_stochastic': 1, 'LinearBinary_bipolar': 2, 'LinearBinaryScaling_stochastic': 1, 'LinearBinaryScaling_bipolar': 1, 'LinearBinaryScaling_binary_training': 1, 'LinearBinaryResidualSign_stochastic': 0, 'LinearBinaryResidualSign_bipolar': 0, 'LinearBinaryResidualSign_binary_training': 0, 'bert.encoder.layer.0.attention.self.query_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'>, 'bert.encoder.layer.0.output.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'>, 'bert.encoder.layer.1.output.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'>, 'bert.pooler.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'>, 'classifier_type': <class 'torch.nn.modules.linear.Linear'>}. Best is trial 7 with value: 0.87336.\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 01:50, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.693600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.693500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.693400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.693300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.693000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.693200</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 00:49]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-10 21:06:24,806] Trial 11 finished with value: 0.5 and parameters: {'linear_layer_quantization': 1, 'linear_layer_quantization_fractional': 0, 'LinearMinifloatDenorm': 2, 'LinearMinifloatDenorm_ew': 2, 'LinearMinifloatDenorm_eb': 2, 'LinearMinifloatIEEE': 0, 'LinearMinifloatIEEE_ew': 0, 'LinearMinifloatIEEE_eb': 1, 'LinearLog': 2, 'LinearLog_ew': 2, 'LinearLog_eb': 2, 'LinearBlockFP': 0, 'LinearBlockFP_block_size': 0, 'LinearBlockMinifloat_ebw': 0, 'LinearBlockMinifloat_ew': 1, 'LinearBlockMinifloat_block_size': 0, 'LinearBlockMinifloat_w': 0, 'LinearBlockLog': 1, 'LinearBlockLog_ebw': 2, 'LinearBlockLog_block_size': 0, 'LinearBinary_stochastic': 0, 'LinearBinary_bipolar': 2, 'LinearBinaryScaling_stochastic': 0, 'LinearBinaryScaling_bipolar': 1, 'LinearBinaryScaling_binary_training': 0, 'LinearBinaryResidualSign_stochastic': 0, 'LinearBinaryResidualSign_bipolar': 0, 'LinearBinaryResidualSign_binary_training': 0, 'bert.encoder.layer.0.attention.self.query_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.output.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.output.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'>, 'bert.pooler.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'>, 'classifier_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'>}. Best is trial 7 with value: 0.87336.\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 01:30, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.615200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.560100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.564600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.552500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.543300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.557100</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 00:51]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-10 21:08:47,919] Trial 12 finished with value: 0.75424 and parameters: {'linear_layer_quantization': 0, 'linear_layer_quantization_fractional': 1, 'LinearMinifloatDenorm': 1, 'LinearMinifloatDenorm_ew': 0, 'LinearMinifloatDenorm_eb': 2, 'LinearMinifloatIEEE': 1, 'LinearMinifloatIEEE_ew': 2, 'LinearMinifloatIEEE_eb': 1, 'LinearLog': 1, 'LinearLog_ew': 1, 'LinearLog_eb': 2, 'LinearBlockFP': 2, 'LinearBlockFP_block_size': 0, 'LinearBlockMinifloat_ebw': 1, 'LinearBlockMinifloat_ew': 2, 'LinearBlockMinifloat_block_size': 0, 'LinearBlockMinifloat_w': 1, 'LinearBlockLog': 0, 'LinearBlockLog_ebw': 2, 'LinearBlockLog_block_size': 1, 'LinearBinary_stochastic': 1, 'LinearBinary_bipolar': 1, 'LinearBinaryScaling_stochastic': 1, 'LinearBinaryScaling_bipolar': 0, 'LinearBinaryScaling_binary_training': 0, 'LinearBinaryResidualSign_stochastic': 1, 'LinearBinaryResidualSign_bipolar': 0, 'LinearBinaryResidualSign_binary_training': 1, 'bert.encoder.layer.0.attention.self.query_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.output.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.output.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'>, 'bert.pooler.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'classifier_type': <class 'torch.nn.modules.linear.Linear'>}. Best is trial 7 with value: 0.87336.\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 01:24, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.693200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.693400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.693200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.693300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.692900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.693200</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 00:38]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-10 21:10:51,547] Trial 13 finished with value: 0.5 and parameters: {'linear_layer_quantization': 0, 'linear_layer_quantization_fractional': 0, 'LinearMinifloatDenorm': 2, 'LinearMinifloatDenorm_ew': 2, 'LinearMinifloatDenorm_eb': 0, 'LinearMinifloatIEEE': 2, 'LinearMinifloatIEEE_ew': 1, 'LinearMinifloatIEEE_eb': 1, 'LinearLog': 0, 'LinearLog_ew': 0, 'LinearLog_eb': 0, 'LinearBlockFP': 2, 'LinearBlockFP_block_size': 0, 'LinearBlockMinifloat_ebw': 2, 'LinearBlockMinifloat_ew': 2, 'LinearBlockMinifloat_block_size': 2, 'LinearBlockMinifloat_w': 1, 'LinearBlockLog': 0, 'LinearBlockLog_ebw': 0, 'LinearBlockLog_block_size': 1, 'LinearBinary_stochastic': 0, 'LinearBinary_bipolar': 2, 'LinearBinaryScaling_stochastic': 0, 'LinearBinaryScaling_bipolar': 1, 'LinearBinaryScaling_binary_training': 0, 'LinearBinaryResidualSign_stochastic': 1, 'LinearBinaryResidualSign_bipolar': 1, 'LinearBinaryResidualSign_binary_training': 1, 'bert.encoder.layer.0.attention.self.query_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.output.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'>, 'bert.pooler.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'classifier_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'>}. Best is trial 7 with value: 0.87336.\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 01:12, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.342100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.309700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.333100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.342800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.332700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.360300</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 00:33]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-10 21:12:38,593] Trial 14 finished with value: 0.87216 and parameters: {'linear_layer_quantization': 0, 'linear_layer_quantization_fractional': 2, 'LinearMinifloatDenorm': 2, 'LinearMinifloatDenorm_ew': 2, 'LinearMinifloatDenorm_eb': 2, 'LinearMinifloatIEEE': 1, 'LinearMinifloatIEEE_ew': 1, 'LinearMinifloatIEEE_eb': 0, 'LinearLog': 1, 'LinearLog_ew': 2, 'LinearLog_eb': 2, 'LinearBlockFP': 1, 'LinearBlockFP_block_size': 1, 'LinearBlockMinifloat_ebw': 0, 'LinearBlockMinifloat_ew': 1, 'LinearBlockMinifloat_block_size': 1, 'LinearBlockMinifloat_w': 0, 'LinearBlockLog': 2, 'LinearBlockLog_ebw': 0, 'LinearBlockLog_block_size': 1, 'LinearBinary_stochastic': 2, 'LinearBinary_bipolar': 1, 'LinearBinaryScaling_stochastic': 0, 'LinearBinaryScaling_bipolar': 1, 'LinearBinaryScaling_binary_training': 1, 'LinearBinaryResidualSign_stochastic': 0, 'LinearBinaryResidualSign_bipolar': 0, 'LinearBinaryResidualSign_binary_training': 1, 'bert.encoder.layer.0.attention.self.query_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'>, 'bert.encoder.layer.0.output.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'>, 'bert.encoder.layer.1.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.pooler.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'classifier_type': <class 'torch.nn.modules.linear.Linear'>}. Best is trial 7 with value: 0.87336.\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 01:11, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.336400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.301800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.331700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.341100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.329200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.361300</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 00:31]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-10 21:14:22,153] Trial 15 finished with value: 0.87168 and parameters: {'linear_layer_quantization': 1, 'linear_layer_quantization_fractional': 2, 'LinearMinifloatDenorm': 1, 'LinearMinifloatDenorm_ew': 0, 'LinearMinifloatDenorm_eb': 1, 'LinearMinifloatIEEE': 0, 'LinearMinifloatIEEE_ew': 0, 'LinearMinifloatIEEE_eb': 0, 'LinearLog': 2, 'LinearLog_ew': 2, 'LinearLog_eb': 2, 'LinearBlockFP': 2, 'LinearBlockFP_block_size': 1, 'LinearBlockMinifloat_ebw': 2, 'LinearBlockMinifloat_ew': 0, 'LinearBlockMinifloat_block_size': 2, 'LinearBlockMinifloat_w': 1, 'LinearBlockLog': 2, 'LinearBlockLog_ebw': 0, 'LinearBlockLog_block_size': 1, 'LinearBinary_stochastic': 2, 'LinearBinary_bipolar': 0, 'LinearBinaryScaling_stochastic': 1, 'LinearBinaryScaling_bipolar': 1, 'LinearBinaryScaling_binary_training': 1, 'LinearBinaryResidualSign_stochastic': 0, 'LinearBinaryResidualSign_bipolar': 0, 'LinearBinaryResidualSign_binary_training': 1, 'bert.encoder.layer.0.attention.self.query_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'>, 'bert.encoder.layer.0.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.output.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'>, 'bert.pooler.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'>, 'classifier_type': <class 'torch.nn.modules.linear.Linear'>}. Best is trial 7 with value: 0.87336.\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 01:26, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>7.557000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>2.081600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>1.910800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>1.675400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>1.473000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>1.334400</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 00:46]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-10 21:16:36,335] Trial 16 finished with value: 0.78372 and parameters: {'linear_layer_quantization': 0, 'linear_layer_quantization_fractional': 2, 'LinearMinifloatDenorm': 1, 'LinearMinifloatDenorm_ew': 2, 'LinearMinifloatDenorm_eb': 0, 'LinearMinifloatIEEE': 2, 'LinearMinifloatIEEE_ew': 0, 'LinearMinifloatIEEE_eb': 0, 'LinearLog': 1, 'LinearLog_ew': 0, 'LinearLog_eb': 2, 'LinearBlockFP': 0, 'LinearBlockFP_block_size': 1, 'LinearBlockMinifloat_ebw': 0, 'LinearBlockMinifloat_ew': 1, 'LinearBlockMinifloat_block_size': 0, 'LinearBlockMinifloat_w': 2, 'LinearBlockLog': 1, 'LinearBlockLog_ebw': 2, 'LinearBlockLog_block_size': 0, 'LinearBinary_stochastic': 2, 'LinearBinary_bipolar': 0, 'LinearBinaryScaling_stochastic': 1, 'LinearBinaryScaling_bipolar': 0, 'LinearBinaryScaling_binary_training': 0, 'LinearBinaryResidualSign_stochastic': 1, 'LinearBinaryResidualSign_bipolar': 0, 'LinearBinaryResidualSign_binary_training': 0, 'bert.encoder.layer.0.attention.self.query_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'>, 'bert.encoder.layer.0.output.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'>, 'bert.encoder.layer.1.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.pooler.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'classifier_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'>}. Best is trial 7 with value: 0.87336.\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 01:26, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.449600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.465900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.514900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.462100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.432100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.430700</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 00:45]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-10 21:18:48,696] Trial 17 finished with value: 0.84676 and parameters: {'linear_layer_quantization': 2, 'linear_layer_quantization_fractional': 2, 'LinearMinifloatDenorm': 2, 'LinearMinifloatDenorm_ew': 1, 'LinearMinifloatDenorm_eb': 0, 'LinearMinifloatIEEE': 2, 'LinearMinifloatIEEE_ew': 0, 'LinearMinifloatIEEE_eb': 0, 'LinearLog': 0, 'LinearLog_ew': 0, 'LinearLog_eb': 1, 'LinearBlockFP': 2, 'LinearBlockFP_block_size': 2, 'LinearBlockMinifloat_ebw': 1, 'LinearBlockMinifloat_ew': 1, 'LinearBlockMinifloat_block_size': 1, 'LinearBlockMinifloat_w': 2, 'LinearBlockLog': 0, 'LinearBlockLog_ebw': 2, 'LinearBlockLog_block_size': 0, 'LinearBinary_stochastic': 1, 'LinearBinary_bipolar': 2, 'LinearBinaryScaling_stochastic': 1, 'LinearBinaryScaling_bipolar': 0, 'LinearBinaryScaling_binary_training': 0, 'LinearBinaryResidualSign_stochastic': 1, 'LinearBinaryResidualSign_bipolar': 0, 'LinearBinaryResidualSign_binary_training': 0, 'bert.encoder.layer.0.attention.self.query_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'>, 'bert.encoder.layer.0.output.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'>, 'bert.encoder.layer.1.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.pooler.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'classifier_type': <class 'torch.nn.modules.linear.Linear'>}. Best is trial 7 with value: 0.87336.\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 01:24, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>5.672400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>1.795700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>1.617300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>1.451000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>1.301300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>1.251500</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 00:40]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-10 21:20:54,772] Trial 18 finished with value: 0.81088 and parameters: {'linear_layer_quantization': 2, 'linear_layer_quantization_fractional': 2, 'LinearMinifloatDenorm': 1, 'LinearMinifloatDenorm_ew': 1, 'LinearMinifloatDenorm_eb': 1, 'LinearMinifloatIEEE': 1, 'LinearMinifloatIEEE_ew': 0, 'LinearMinifloatIEEE_eb': 2, 'LinearLog': 1, 'LinearLog_ew': 1, 'LinearLog_eb': 2, 'LinearBlockFP': 1, 'LinearBlockFP_block_size': 0, 'LinearBlockMinifloat_ebw': 1, 'LinearBlockMinifloat_ew': 0, 'LinearBlockMinifloat_block_size': 0, 'LinearBlockMinifloat_w': 1, 'LinearBlockLog': 0, 'LinearBlockLog_ebw': 1, 'LinearBlockLog_block_size': 1, 'LinearBinary_stochastic': 0, 'LinearBinary_bipolar': 1, 'LinearBinaryScaling_stochastic': 1, 'LinearBinaryScaling_bipolar': 0, 'LinearBinaryScaling_binary_training': 0, 'LinearBinaryResidualSign_stochastic': 0, 'LinearBinaryResidualSign_bipolar': 0, 'LinearBinaryResidualSign_binary_training': 1, 'bert.encoder.layer.0.attention.self.query_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'>, 'bert.encoder.layer.0.output.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.pooler.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'classifier_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'>}. Best is trial 7 with value: 0.87336.\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 01:22, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.617900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.451600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.404200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.389200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.364300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.393400</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 00:43]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-10 21:23:01,048] Trial 19 finished with value: 0.84092 and parameters: {'linear_layer_quantization': 2, 'linear_layer_quantization_fractional': 2, 'LinearMinifloatDenorm': 1, 'LinearMinifloatDenorm_ew': 2, 'LinearMinifloatDenorm_eb': 1, 'LinearMinifloatIEEE': 0, 'LinearMinifloatIEEE_ew': 0, 'LinearMinifloatIEEE_eb': 1, 'LinearLog': 0, 'LinearLog_ew': 0, 'LinearLog_eb': 0, 'LinearBlockFP': 1, 'LinearBlockFP_block_size': 0, 'LinearBlockMinifloat_ebw': 0, 'LinearBlockMinifloat_ew': 1, 'LinearBlockMinifloat_block_size': 0, 'LinearBlockMinifloat_w': 1, 'LinearBlockLog': 2, 'LinearBlockLog_ebw': 2, 'LinearBlockLog_block_size': 2, 'LinearBinary_stochastic': 2, 'LinearBinary_bipolar': 2, 'LinearBinaryScaling_stochastic': 1, 'LinearBinaryScaling_bipolar': 1, 'LinearBinaryScaling_binary_training': 0, 'LinearBinaryResidualSign_stochastic': 1, 'LinearBinaryResidualSign_bipolar': 0, 'LinearBinaryResidualSign_binary_training': 0, 'bert.encoder.layer.0.attention.self.query_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryScaling'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.pooler.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'classifier_type': <class 'torch.nn.modules.linear.Linear'>}. Best is trial 7 with value: 0.87336.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.8728,\n",
       " 0.8728,\n",
       " 0.8728,\n",
       " 0.8728,\n",
       " 0.8728,\n",
       " 0.87308,\n",
       " 0.87332,\n",
       " 0.87336,\n",
       " 0.87336,\n",
       " 0.87336,\n",
       " 0.87336,\n",
       " 0.87336,\n",
       " 0.87336,\n",
       " 0.87336,\n",
       " 0.87336,\n",
       " 0.87336,\n",
       " 0.87336,\n",
       " 0.87336,\n",
       " 0.87336,\n",
       " 0.87336]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from pathlib import Path\n",
    "import dill\n",
    "\n",
    "with open(f\"{Path.home()}/tutorial_5_best_model.pkl\", \"rb\") as f:\n",
    "    base_model = dill.load(f)\n",
    "from chop.nn.quantized.modules.linear import (\n",
    "    LinearInteger,\n",
    "    LinearMinifloatDenorm,\n",
    "    LinearMinifloatIEEE,\n",
    "    LinearLog,\n",
    "    LinearBlockFP,\n",
    "    LinearBlockMinifloat,\n",
    "    LinearBlockLog,\n",
    "    LinearBinary,\n",
    "    LinearBinaryScaling,\n",
    "    LinearBinaryResidualSign,\n",
    ")\n",
    "search_space = {\n",
    "    \"linear_layer_choices\": [\n",
    "        torch.nn.Linear,\n",
    "        # LinearInteger,\n",
    "        # LinearMinifloatDenorm,\n",
    "        # LinearMinifloatIEEE,\n",
    "        # LinearLog,\n",
    "        # LinearBlockFP,\n",
    "        # # LinearBlockMinifloat,\n",
    "        # LinearBlockLog,\n",
    "        # LinearBinary,\n",
    "        LinearBinaryScaling,\n",
    "        # LinearBinaryResidualSign,\n",
    "    ],\n",
    "    'linear_layer_quantization': [8, 16, 32],\n",
    "    'linear_layer_quantization_fractional':  [2, 4, 8],\n",
    "    'LinearMinifloatDenorm': [8, 16, 32],\n",
    "    'LinearMinifloatDenorm_ew': [2, 4, 8],\n",
    "    'LinearMinifloatDenorm_eb': [2, 4, 8],\n",
    "    'LinearMinifloatIEEE': [8, 16, 32],\n",
    "    'LinearMinifloatIEEE_ew': [2, 4, 8],\n",
    "    'LinearMinifloatIEEE_eb': [2, 4, 8],\n",
    "    'LinearLog': [8, 16, 32],\n",
    "    'LinearLog_ew': [2, 4, 8],\n",
    "    'LinearLog_eb': [2, 4, 8],\n",
    "    'LinearBlockFP': [8, 16, 32],\n",
    "    'LinearBlockFP_block_size':[[4], [8], [16]] ,\n",
    "    'LinearBlockMinifloat_ebw': [2, 4, 8],\n",
    "    'LinearBlockMinifloat_ew': [2, 4, 8],\n",
    "    'LinearBlockMinifloat_block_size': [[4], [8], [16]],\n",
    "    'LinearBlockMinifloat_w': [2, 4, 8],\n",
    "    'LinearBlockLog': [8, 16, 32],\n",
    "    'LinearBlockLog_ebw': [2, 4, 8],\n",
    "    'LinearBlockLog_block_size': [[4], [8], [16]] ,\n",
    "    'LinearBinary_stochastic': [2, 4, 8],\n",
    "    'LinearBinary_bipolar': [2, 4, 8],\n",
    "    'LinearBinaryScaling_stochastic': [True,False],\n",
    "    'LinearBinaryScaling_bipolar': [True,False],\n",
    "    \"LinearBinaryScaling_binary_training\": [True,False],\n",
    "    'LinearBinaryResidualSign_stochastic': [True,False],\n",
    "    'LinearBinaryResidualSign_bipolar': [True,False],\n",
    "    \"LinearBinaryResidualSign_binary_training\": [True,False],\n",
    "}\n",
    "\n",
    "def construct_model(trial):\n",
    "    # Fetch the model\n",
    "    my_config = {}\n",
    "    trial_model = deepcopy(base_model)\n",
    "    \n",
    "    # Select quantization and fractional width configurations\n",
    "    for param in ['linear_layer_quantization', \n",
    "                  'linear_layer_quantization_fractional',\n",
    "                  'LinearMinifloatDenorm',\n",
    "                  'LinearMinifloatDenorm_ew',\n",
    "                  'LinearMinifloatDenorm_eb',\n",
    "                  'LinearMinifloatIEEE',\n",
    "                  'LinearMinifloatIEEE_ew',\n",
    "                  'LinearMinifloatIEEE_eb',\n",
    "                    'LinearLog',\n",
    "                    'LinearLog_ew',\n",
    "                    'LinearLog_eb',\n",
    "                    'LinearBlockFP',\n",
    "                    'LinearBlockFP_block_size',\n",
    "                    'LinearBlockMinifloat_ebw',\n",
    "                    'LinearBlockMinifloat_ew',\n",
    "                    'LinearBlockMinifloat_block_size',\n",
    "                    'LinearBlockMinifloat_w',\n",
    "                    'LinearBlockLog',\n",
    "                    'LinearBlockLog_ebw',\n",
    "                    'LinearBlockLog_block_size',\n",
    "                    'LinearBinary_stochastic',\n",
    "                    'LinearBinary_bipolar',\n",
    "                    'LinearBinaryScaling_stochastic',\n",
    "                    'LinearBinaryScaling_bipolar',\n",
    "                    'LinearBinaryScaling_binary_training',\n",
    "                    'LinearBinaryResidualSign_stochastic',\n",
    "                    'LinearBinaryResidualSign_bipolar',\n",
    "                    'LinearBinaryResidualSign_binary_training'\n",
    "                  ]:\n",
    "        chosen_idx = trial.suggest_int(param, 0, len(search_space[param]) - 1)\n",
    "        my_config[param] = search_space[param][chosen_idx]\n",
    "\n",
    "    # Quantize layers according to Optuna suggestions\n",
    "    for name, layer in trial_model.named_modules():\n",
    "        if isinstance(layer, torch.nn.Linear):\n",
    "            new_layer_cls = trial.suggest_categorical(\n",
    "                f\"{name}_type\",\n",
    "                search_space[\"linear_layer_choices\"],\n",
    "            )\n",
    "\n",
    "            if new_layer_cls == torch.nn.Linear:\n",
    "                continue\n",
    "\n",
    "            kwargs = {\n",
    "                \"in_features\": layer.in_features,\n",
    "                \"out_features\": layer.out_features,\n",
    "            }\n",
    "\n",
    "            try:\n",
    "                # Configure parameters for specific layer types\n",
    "                # if new_layer_cls == LinearInteger:\n",
    "                #     kwargs[\"config\"] = {\n",
    "                #         \"data_in_width\": my_config['linear_layer_quantization'],\n",
    "                #         \"data_in_frac_width\": my_config['linear_layer_quantization_fractional'],\n",
    "                #         \"weight_width\": my_config['linear_layer_quantization'],\n",
    "                #         \"weight_frac_width\": my_config['linear_layer_quantization_fractional'],\n",
    "                #         \"bias_width\": my_config['linear_layer_quantization'],\n",
    "                #         \"bias_frac_width\": my_config['linear_layer_quantization_fractional'],\n",
    "                #     }\n",
    "                # elif new_layer_cls == LinearMinifloatDenorm:\n",
    "                #     kwargs[\"config\"] = {\n",
    "                #         \"data_in_width\": my_config['LinearMinifloatDenorm'],\n",
    "                #         \"data_in_exponent_width\": my_config['LinearMinifloatDenorm_ew'],\n",
    "                #         \"data_in_exponent_bias\": my_config['LinearMinifloatDenorm_eb'],\n",
    "                #         \"weight_width\": my_config['LinearMinifloatDenorm'],\n",
    "                #         \"weight_exponent_width\": my_config['LinearMinifloatDenorm_ew'],\n",
    "                #         \"weight_exponent_bias\": my_config['LinearMinifloatDenorm_eb'],\n",
    "                #         \"bias_width\": my_config['LinearMinifloatDenorm'],\n",
    "                #         \"bias_exponent_width\": my_config['LinearMinifloatDenorm_ew'],\n",
    "                #         \"bias_exponent_bias\": my_config['LinearMinifloatDenorm_eb'],\n",
    "                #     }\n",
    "                # elif new_layer_cls == LinearMinifloatIEEE:\n",
    "                #     kwargs[\"config\"] = {\n",
    "                #         \"data_in_width\": my_config['LinearMinifloatIEEE'],\n",
    "                #         \"data_in_exponent_width\": my_config['LinearMinifloatIEEE_ew'],\n",
    "                #         \"data_in_exponent_bias\": my_config['LinearMinifloatIEEE_eb'],\n",
    "                #         \"weight_width\": my_config['LinearMinifloatIEEE'],\n",
    "                #         \"weight_exponent_width\": my_config['LinearMinifloatIEEE_ew'],\n",
    "                #         \"weight_exponent_bias\": my_config['LinearMinifloatIEEE_eb'],\n",
    "                #         \"bias_width\": my_config['LinearMinifloatIEEE'],\n",
    "                #         \"bias_exponent_width\": my_config['LinearMinifloatIEEE_ew'],\n",
    "                #         \"bias_exponent_bias\": my_config['LinearMinifloatIEEE_eb'],\n",
    "                #     }\n",
    "                # elif new_layer_cls == LinearLog:\n",
    "                #     kwargs[\"config\"] = {\n",
    "                #         \"data_in_width\": my_config['LinearLog'],\n",
    "                #         \"data_in_exponent_width\": my_config['LinearLog_ew'],\n",
    "                #         \"data_in_exponent_bias\": my_config['LinearLog_eb'],\n",
    "                #         \"weight_width\": my_config['LinearLog'],\n",
    "                #         \"weight_exponent_width\": my_config['LinearLog_ew'],\n",
    "                #         \"weight_exponent_bias\": my_config['LinearLog_eb'],\n",
    "                #         \"bias_width\": my_config['LinearLog'],\n",
    "                #         \"bias_exponent_width\": my_config['LinearLog_ew'],\n",
    "                #         \"bias_exponent_bias\": my_config['LinearLog_eb'],\n",
    "                #     }\n",
    "                # elif new_layer_cls == LinearBlockFP:\n",
    "                #     kwargs[\"config\"] = {\n",
    "                #         \"weight_width\": my_config['LinearBlockFP'],\n",
    "                #         \"weight_exponent_width\": my_config['LinearBlockFP'],\n",
    "                #         \"weight_block_size\": my_config['LinearBlockFP_block_size'],\n",
    "                #         \"weight_exponent_bias\": my_config['LinearBlockFP'],\n",
    "                #         \"data_in_width\": my_config['LinearBlockFP'],\n",
    "                #         \"data_in_exponent_width\": my_config['LinearBlockFP'],\n",
    "                #         \"data_in_block_size\": my_config['LinearBlockFP_block_size'],\n",
    "                #         \"data_in_exponent_bias\": my_config['LinearBlockFP'],\n",
    "                #         \"bias_width\": my_config['LinearBlockFP'],\n",
    "                #         \"bias_exponent_width\": my_config['LinearBlockFP'],\n",
    "                #         \"bias_block_size\": my_config['LinearBlockFP_block_size'],\n",
    "                #         \"bias_exponent_bias\": my_config['LinearBlockFP'],\n",
    "                #     }\n",
    "                # # elif new_layer_cls == LinearBlockMinifloat:\n",
    "                # #     kwargs[\"config\"] = {\n",
    "                # #         \"weight_exponent_bias_width\" : my_config['LinearBlockMinifloat_ebw'],\n",
    "                # #         \"weight_exponent_width\": my_config['LinearBlockMinifloat_ew'],\n",
    "                # #         \"weight_block_size\": my_config['LinearBlockMinifloat_block_size'],\n",
    "                # #         \"weight_width\": my_config['LinearBlockMinifloat_w'],\n",
    "                # #         \"data_in_width\":my_config[\"LinearBlockMinifloat_w\"],\n",
    "                # #         \"data_in_exponent_bias_width\":my_config[\"LinearBlockMinifloat_ew\"],\n",
    "                # #         \"data_in_exponent_width\":my_config[\"LinearBlockMinifloat_ebw\"],\n",
    "                # #         \"data_in_block_size\":my_config[\"LinearBlockMinifloat_block_size\"],\n",
    "                # #         \"bias_width\":my_config[\"LinearBlockMinifloat_w\"],\n",
    "                # #         \"bias_exponent_width\":my_config[\"LinearBlockMinifloat_ew\"],\n",
    "                # #         \"bias_exponent_bias_width\":my_config[\"LinearBlockMinifloat_ebw\"],\n",
    "                # #         \"bias_block_size\":my_config[\"LinearBlockMinifloat_block_size\"],\n",
    "                # #     }\n",
    "                # elif new_layer_cls == LinearBlockLog:\n",
    "                #     kwargs[\"config\"] = {\n",
    "                #         \"data_in_width\": my_config['LinearBlockLog'],\n",
    "                #         \"data_in_exponent_bias_width\": my_config['LinearBlockLog_ebw'],\n",
    "                #         \"data_in_block_size\": my_config['LinearBlockLog_block_size'],\n",
    "                #         \"weight_width\": my_config['LinearBlockLog'],\n",
    "                #         \"weight_exponent_bias_width\": my_config['LinearBlockLog_ebw'],\n",
    "                #         \"weight_block_size\": my_config['LinearBlockLog_block_size'],\n",
    "                #         \"bias_width\": my_config['LinearBlockLog'],\n",
    "                #         \"bias_exponent_bias_width\": my_config['LinearBlockLog_ebw'],\n",
    "                #         \"bias_block_size\": my_config['LinearBlockLog_block_size'],\n",
    "                #     }\n",
    "\n",
    "                # elif new_layer_cls == LinearBinary:\n",
    "                #     kwargs[\"config\"] = {\n",
    "                #         'weight_stochastic':my_config['LinearBinary_stochastic'],\n",
    "                #         'weight_bipolar':my_config['LinearBinary_bipolar'],\n",
    "                #     }\n",
    "\n",
    "                    \n",
    "                if new_layer_cls == LinearBinaryScaling:\n",
    "                    kwargs[\"config\"] = {\n",
    "                        'weight_stochastic':my_config['LinearBinaryScaling_stochastic'],\n",
    "                        'weight_bipolar':my_config['LinearBinaryScaling_bipolar'],\n",
    "                        \"data_in_stochastic\": my_config['LinearBinaryScaling_stochastic'],\n",
    "                        \"data_in_bipolar\": my_config['LinearBinaryScaling_bipolar'],\n",
    "                        \"bias_stochastic\": my_config['LinearBinaryScaling_stochastic'],\n",
    "                        \"bias_bipolar\": my_config['LinearBinaryScaling_bipolar'],\n",
    "                        \"binary_training\": my_config['LinearBinaryScaling_binary_training'],\n",
    "                    }\n",
    "                # elif new_layer_cls == LinearBinaryResidualSign:\n",
    "                #     kwargs[\"config\"] = {\n",
    "                #         'weight_stochastic':my_config['LinearBinaryResidualSign_stochastic'],\n",
    "                #         'weight_bipolar':my_config['LinearBinaryResidualSign_bipolar'],\n",
    "                #         \"data_in_stochastic\": my_config['LinearBinaryResidualSign_stochastic'],\n",
    "                #         \"data_in_bipolar\": my_config['LinearBinaryResidualSign_bipolar'],\n",
    "                #         \"bias_stochastic\": my_config['LinearBinaryResidualSign_stochastic'],\n",
    "                #         \"bias_bipolar\": my_config['LinearBinaryResidualSign_bipolar'],\n",
    "                #         \"binary_training\": my_config['LinearBinaryResidualSign_binary_training'],\n",
    "                #     }\n",
    "                # Create the new layer (copy the weights if applicable)\n",
    "                new_layer = new_layer_cls(**kwargs)\n",
    "                new_layer.weight.data = layer.weight.data.clone()\n",
    "\n",
    "                # Replace the layer in the model\n",
    "                deepsetattr(trial_model, name, new_layer)\n",
    "            except TypeError as e:\n",
    "                print(f\"Error creating layer {name} with class {new_layer_cls}: {e}\")\n",
    "                raise\n",
    "\n",
    "    return trial_model\n",
    "from chop.tools import get_trainer\n",
    "import random\n",
    "\n",
    "\n",
    "def objective(trial):\n",
    "\n",
    "    # Define the model\n",
    "    model = construct_model(trial)\n",
    "\n",
    "    trainer = get_trainer(\n",
    "        model=model,\n",
    "        tokenized_dataset=dataset,\n",
    "        tokenizer=tokenizer,\n",
    "        evaluate_metric=\"accuracy\",\n",
    "        num_train_epochs=1,\n",
    "    )\n",
    "\n",
    "    trainer.train()\n",
    "    eval_results = trainer.evaluate()\n",
    "\n",
    "    trial.set_user_attr(\"model\", model)\n",
    "\n",
    "    return eval_results[\"eval_accuracy\"]\n",
    "\n",
    "\n",
    "from optuna.samplers import GridSampler, RandomSampler, TPESampler\n",
    "\n",
    "sampler = RandomSampler()\n",
    "\n",
    "import optuna\n",
    "\n",
    "study = optuna.create_study(\n",
    "    direction=\"maximize\",\n",
    "    study_name=\"bert-tiny-nas-study\",\n",
    "    sampler=sampler,\n",
    ")\n",
    "\n",
    "study.optimize(\n",
    "    objective,\n",
    "    n_trials=20,\n",
    "    timeout=60 * 60 * 24,\n",
    ")\n",
    "\n",
    "task9 = [max([t.value for t in study.trials[:i+1]]) for i in range(len(study.trials))]\n",
    "task9"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "task9 = [0.50072,\n",
    " 0.50072,\n",
    " 0.50072,\n",
    " 0.50072,\n",
    " 0.50308,\n",
    " 0.50308,\n",
    " 0.50308,\n",
    " 0.50308,\n",
    " 0.50308,\n",
    " 0.50608]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-10 19:46:53,709] A new study created in memory with name: bert-tiny-nas-study\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2025-02-10 19:46:59,038] [INFO] [real_accelerator.py:158:get_accelerator] Setting ds_accelerator to cuda (auto detect)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 02:13, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.340800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.306800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.318900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.312200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.294300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.332800</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 00:48]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-10 19:50:09,397] Trial 0 finished with value: 0.86908 and parameters: {'linear_layer_quantization': 2, 'linear_layer_quantization_fractional': 2, 'LinearMinifloatDenorm': 1, 'LinearMinifloatDenorm_ew': 2, 'LinearMinifloatDenorm_eb': 1, 'LinearMinifloatIEEE': 1, 'LinearMinifloatIEEE_ew': 1, 'LinearMinifloatIEEE_eb': 2, 'LinearLog': 2, 'LinearLog_ew': 1, 'LinearLog_eb': 0, 'LinearBlockFP': 2, 'LinearBlockFP_block_size': 0, 'LinearBlockMinifloat_ebw': 0, 'LinearBlockMinifloat_ew': 0, 'LinearBlockMinifloat_block_size': 0, 'LinearBlockMinifloat_w': 1, 'LinearBlockLog': 1, 'LinearBlockLog_ebw': 0, 'LinearBlockLog_block_size': 0, 'LinearBinary_stochastic': 1, 'LinearBinary_bipolar': 0, 'LinearBinaryScaling_stochastic': 1, 'LinearBinaryScaling_bipolar': 0, 'LinearBinaryScaling_binary_training': 2, 'LinearBinaryResidualSign_stochastic': 1, 'LinearBinaryResidualSign_bipolar': 1, 'LinearBinaryResidualSign_binary_training': 1, 'bert.encoder.layer.0.attention.self.query_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'>, 'bert.encoder.layer.0.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.output.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'>, 'bert.pooler.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'>, 'classifier_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'>}. Best is trial 0 with value: 0.86908.\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 01:40, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.341900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.305300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.316200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.321500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.298100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.335500</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 00:38]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-10 19:52:29,360] Trial 1 finished with value: 0.874 and parameters: {'linear_layer_quantization': 2, 'linear_layer_quantization_fractional': 1, 'LinearMinifloatDenorm': 1, 'LinearMinifloatDenorm_ew': 1, 'LinearMinifloatDenorm_eb': 1, 'LinearMinifloatIEEE': 2, 'LinearMinifloatIEEE_ew': 1, 'LinearMinifloatIEEE_eb': 1, 'LinearLog': 2, 'LinearLog_ew': 1, 'LinearLog_eb': 1, 'LinearBlockFP': 1, 'LinearBlockFP_block_size': 1, 'LinearBlockMinifloat_ebw': 2, 'LinearBlockMinifloat_ew': 2, 'LinearBlockMinifloat_block_size': 0, 'LinearBlockMinifloat_w': 0, 'LinearBlockLog': 0, 'LinearBlockLog_ebw': 1, 'LinearBlockLog_block_size': 0, 'LinearBinary_stochastic': 2, 'LinearBinary_bipolar': 1, 'LinearBinaryScaling_stochastic': 2, 'LinearBinaryScaling_bipolar': 0, 'LinearBinaryScaling_binary_training': 2, 'LinearBinaryResidualSign_stochastic': 0, 'LinearBinaryResidualSign_bipolar': 1, 'LinearBinaryResidualSign_binary_training': 1, 'bert.encoder.layer.0.attention.self.query_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'>, 'bert.encoder.layer.1.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.pooler.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'classifier_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'>}. Best is trial 1 with value: 0.874.\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 01:43, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.683000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.642600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.638900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.599100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.574400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.574100</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 00:47]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-10 19:55:01,718] Trial 2 finished with value: 0.81484 and parameters: {'linear_layer_quantization': 0, 'linear_layer_quantization_fractional': 0, 'LinearMinifloatDenorm': 0, 'LinearMinifloatDenorm_ew': 1, 'LinearMinifloatDenorm_eb': 2, 'LinearMinifloatIEEE': 1, 'LinearMinifloatIEEE_ew': 1, 'LinearMinifloatIEEE_eb': 1, 'LinearLog': 1, 'LinearLog_ew': 1, 'LinearLog_eb': 2, 'LinearBlockFP': 2, 'LinearBlockFP_block_size': 2, 'LinearBlockMinifloat_ebw': 0, 'LinearBlockMinifloat_ew': 2, 'LinearBlockMinifloat_block_size': 2, 'LinearBlockMinifloat_w': 1, 'LinearBlockLog': 2, 'LinearBlockLog_ebw': 1, 'LinearBlockLog_block_size': 1, 'LinearBinary_stochastic': 2, 'LinearBinary_bipolar': 0, 'LinearBinaryScaling_stochastic': 2, 'LinearBinaryScaling_bipolar': 1, 'LinearBinaryScaling_binary_training': 1, 'LinearBinaryResidualSign_stochastic': 0, 'LinearBinaryResidualSign_bipolar': 1, 'LinearBinaryResidualSign_binary_training': 0, 'bert.encoder.layer.0.attention.self.query_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'>, 'bert.encoder.layer.1.output.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'>, 'bert.pooler.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'classifier_type': <class 'torch.nn.modules.linear.Linear'>}. Best is trial 1 with value: 0.874.\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 02:19, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.708700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.701700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.698200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.698600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.693500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.696800</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 00:59]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-10 19:58:22,917] Trial 3 finished with value: 0.71784 and parameters: {'linear_layer_quantization': 0, 'linear_layer_quantization_fractional': 1, 'LinearMinifloatDenorm': 2, 'LinearMinifloatDenorm_ew': 0, 'LinearMinifloatDenorm_eb': 1, 'LinearMinifloatIEEE': 2, 'LinearMinifloatIEEE_ew': 0, 'LinearMinifloatIEEE_eb': 0, 'LinearLog': 0, 'LinearLog_ew': 1, 'LinearLog_eb': 2, 'LinearBlockFP': 1, 'LinearBlockFP_block_size': 1, 'LinearBlockMinifloat_ebw': 0, 'LinearBlockMinifloat_ew': 1, 'LinearBlockMinifloat_block_size': 1, 'LinearBlockMinifloat_w': 2, 'LinearBlockLog': 2, 'LinearBlockLog_ebw': 2, 'LinearBlockLog_block_size': 0, 'LinearBinary_stochastic': 2, 'LinearBinary_bipolar': 1, 'LinearBinaryScaling_stochastic': 0, 'LinearBinaryScaling_bipolar': 1, 'LinearBinaryScaling_binary_training': 1, 'LinearBinaryResidualSign_stochastic': 0, 'LinearBinaryResidualSign_bipolar': 1, 'LinearBinaryResidualSign_binary_training': 0, 'bert.encoder.layer.0.attention.self.query_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.output.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'>, 'bert.encoder.layer.1.output.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'>, 'bert.pooler.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'classifier_type': <class 'torch.nn.modules.linear.Linear'>}. Best is trial 1 with value: 0.874.\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 01:59, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.693700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.693200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.693200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.693300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.692800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.693200</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 00:54]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-10 20:01:20,197] Trial 4 finished with value: 0.50028 and parameters: {'linear_layer_quantization': 1, 'linear_layer_quantization_fractional': 2, 'LinearMinifloatDenorm': 0, 'LinearMinifloatDenorm_ew': 0, 'LinearMinifloatDenorm_eb': 1, 'LinearMinifloatIEEE': 0, 'LinearMinifloatIEEE_ew': 2, 'LinearMinifloatIEEE_eb': 2, 'LinearLog': 2, 'LinearLog_ew': 1, 'LinearLog_eb': 1, 'LinearBlockFP': 2, 'LinearBlockFP_block_size': 1, 'LinearBlockMinifloat_ebw': 1, 'LinearBlockMinifloat_ew': 0, 'LinearBlockMinifloat_block_size': 0, 'LinearBlockMinifloat_w': 1, 'LinearBlockLog': 1, 'LinearBlockLog_ebw': 0, 'LinearBlockLog_block_size': 2, 'LinearBinary_stochastic': 1, 'LinearBinary_bipolar': 0, 'LinearBinaryScaling_stochastic': 0, 'LinearBinaryScaling_bipolar': 0, 'LinearBinaryScaling_binary_training': 2, 'LinearBinaryResidualSign_stochastic': 0, 'LinearBinaryResidualSign_bipolar': 1, 'LinearBinaryResidualSign_binary_training': 0, 'bert.encoder.layer.0.attention.self.query_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.pooler.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'>, 'classifier_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'>}. Best is trial 1 with value: 0.874.\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 02:08, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.596600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.388000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.380800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.377700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.374800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.382500</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 00:52]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-10 20:04:22,173] Trial 5 finished with value: 0.84792 and parameters: {'linear_layer_quantization': 2, 'linear_layer_quantization_fractional': 2, 'LinearMinifloatDenorm': 1, 'LinearMinifloatDenorm_ew': 2, 'LinearMinifloatDenorm_eb': 0, 'LinearMinifloatIEEE': 1, 'LinearMinifloatIEEE_ew': 0, 'LinearMinifloatIEEE_eb': 0, 'LinearLog': 1, 'LinearLog_ew': 2, 'LinearLog_eb': 0, 'LinearBlockFP': 2, 'LinearBlockFP_block_size': 2, 'LinearBlockMinifloat_ebw': 2, 'LinearBlockMinifloat_ew': 0, 'LinearBlockMinifloat_block_size': 1, 'LinearBlockMinifloat_w': 1, 'LinearBlockLog': 2, 'LinearBlockLog_ebw': 0, 'LinearBlockLog_block_size': 0, 'LinearBinary_stochastic': 1, 'LinearBinary_bipolar': 0, 'LinearBinaryScaling_stochastic': 0, 'LinearBinaryScaling_bipolar': 2, 'LinearBinaryScaling_binary_training': 0, 'LinearBinaryResidualSign_stochastic': 1, 'LinearBinaryResidualSign_bipolar': 0, 'LinearBinaryResidualSign_binary_training': 0, 'bert.encoder.layer.0.attention.self.query_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.output.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'>, 'bert.pooler.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'>, 'classifier_type': <class 'torch.nn.modules.linear.Linear'>}. Best is trial 1 with value: 0.874.\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 01:52, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.432700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.350400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.343300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.333100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.319600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.350800</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 00:47]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-10 20:07:04,718] Trial 6 finished with value: 0.87036 and parameters: {'linear_layer_quantization': 0, 'linear_layer_quantization_fractional': 1, 'LinearMinifloatDenorm': 0, 'LinearMinifloatDenorm_ew': 0, 'LinearMinifloatDenorm_eb': 1, 'LinearMinifloatIEEE': 0, 'LinearMinifloatIEEE_ew': 0, 'LinearMinifloatIEEE_eb': 0, 'LinearLog': 1, 'LinearLog_ew': 1, 'LinearLog_eb': 1, 'LinearBlockFP': 2, 'LinearBlockFP_block_size': 0, 'LinearBlockMinifloat_ebw': 0, 'LinearBlockMinifloat_ew': 1, 'LinearBlockMinifloat_block_size': 0, 'LinearBlockMinifloat_w': 1, 'LinearBlockLog': 0, 'LinearBlockLog_ebw': 1, 'LinearBlockLog_block_size': 2, 'LinearBinary_stochastic': 1, 'LinearBinary_bipolar': 2, 'LinearBinaryScaling_stochastic': 0, 'LinearBinaryScaling_bipolar': 0, 'LinearBinaryScaling_binary_training': 1, 'LinearBinaryResidualSign_stochastic': 1, 'LinearBinaryResidualSign_bipolar': 1, 'LinearBinaryResidualSign_binary_training': 0, 'bert.encoder.layer.0.attention.self.query_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.output.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.pooler.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'>, 'classifier_type': <class 'torch.nn.modules.linear.Linear'>}. Best is trial 1 with value: 0.874.\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 01:53, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.337700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.297800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.306800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.313900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.297900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.334400</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 00:47]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-10 20:09:46,593] Trial 7 finished with value: 0.8716 and parameters: {'linear_layer_quantization': 2, 'linear_layer_quantization_fractional': 2, 'LinearMinifloatDenorm': 2, 'LinearMinifloatDenorm_ew': 0, 'LinearMinifloatDenorm_eb': 0, 'LinearMinifloatIEEE': 1, 'LinearMinifloatIEEE_ew': 1, 'LinearMinifloatIEEE_eb': 0, 'LinearLog': 0, 'LinearLog_ew': 1, 'LinearLog_eb': 0, 'LinearBlockFP': 0, 'LinearBlockFP_block_size': 1, 'LinearBlockMinifloat_ebw': 2, 'LinearBlockMinifloat_ew': 0, 'LinearBlockMinifloat_block_size': 1, 'LinearBlockMinifloat_w': 1, 'LinearBlockLog': 1, 'LinearBlockLog_ebw': 2, 'LinearBlockLog_block_size': 2, 'LinearBinary_stochastic': 1, 'LinearBinary_bipolar': 0, 'LinearBinaryScaling_stochastic': 2, 'LinearBinaryScaling_bipolar': 1, 'LinearBinaryScaling_binary_training': 2, 'LinearBinaryResidualSign_stochastic': 0, 'LinearBinaryResidualSign_bipolar': 0, 'LinearBinaryResidualSign_binary_training': 1, 'bert.encoder.layer.0.attention.self.query_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.output.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'>, 'bert.encoder.layer.1.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.pooler.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'classifier_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'>}. Best is trial 1 with value: 0.874.\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 01:39, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.344200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.300000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.315300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.323700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.309300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.341000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 00:41]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-10 20:12:08,457] Trial 8 finished with value: 0.87472 and parameters: {'linear_layer_quantization': 1, 'linear_layer_quantization_fractional': 0, 'LinearMinifloatDenorm': 2, 'LinearMinifloatDenorm_ew': 1, 'LinearMinifloatDenorm_eb': 1, 'LinearMinifloatIEEE': 1, 'LinearMinifloatIEEE_ew': 1, 'LinearMinifloatIEEE_eb': 2, 'LinearLog': 0, 'LinearLog_ew': 0, 'LinearLog_eb': 1, 'LinearBlockFP': 1, 'LinearBlockFP_block_size': 1, 'LinearBlockMinifloat_ebw': 0, 'LinearBlockMinifloat_ew': 0, 'LinearBlockMinifloat_block_size': 0, 'LinearBlockMinifloat_w': 1, 'LinearBlockLog': 1, 'LinearBlockLog_ebw': 0, 'LinearBlockLog_block_size': 2, 'LinearBinary_stochastic': 1, 'LinearBinary_bipolar': 2, 'LinearBinaryScaling_stochastic': 0, 'LinearBinaryScaling_bipolar': 0, 'LinearBinaryScaling_binary_training': 2, 'LinearBinaryResidualSign_stochastic': 0, 'LinearBinaryResidualSign_bipolar': 1, 'LinearBinaryResidualSign_binary_training': 1, 'bert.encoder.layer.0.attention.self.query_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'>, 'bert.encoder.layer.0.output.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.pooler.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'classifier_type': <class 'torch.nn.modules.linear.Linear'>}. Best is trial 8 with value: 0.87472.\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 01:50, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.706800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.651400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.631500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.589100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.585800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.568100</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 00:47]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-10 20:14:47,956] Trial 9 finished with value: 0.71256 and parameters: {'linear_layer_quantization': 2, 'linear_layer_quantization_fractional': 2, 'LinearMinifloatDenorm': 1, 'LinearMinifloatDenorm_ew': 2, 'LinearMinifloatDenorm_eb': 2, 'LinearMinifloatIEEE': 1, 'LinearMinifloatIEEE_ew': 0, 'LinearMinifloatIEEE_eb': 0, 'LinearLog': 0, 'LinearLog_ew': 0, 'LinearLog_eb': 0, 'LinearBlockFP': 2, 'LinearBlockFP_block_size': 1, 'LinearBlockMinifloat_ebw': 0, 'LinearBlockMinifloat_ew': 0, 'LinearBlockMinifloat_block_size': 1, 'LinearBlockMinifloat_w': 0, 'LinearBlockLog': 2, 'LinearBlockLog_ebw': 0, 'LinearBlockLog_block_size': 1, 'LinearBinary_stochastic': 0, 'LinearBinary_bipolar': 2, 'LinearBinaryScaling_stochastic': 0, 'LinearBinaryScaling_bipolar': 2, 'LinearBinaryScaling_binary_training': 0, 'LinearBinaryResidualSign_stochastic': 0, 'LinearBinaryResidualSign_bipolar': 0, 'LinearBinaryResidualSign_binary_training': 0, 'bert.encoder.layer.0.attention.self.query_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.output.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.pooler.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'classifier_type': <class 'torch.nn.modules.linear.Linear'>}. Best is trial 8 with value: 0.87472.\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 02:34, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.697000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.696100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.693400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.694000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.696500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.696600</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 01:05]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-10 20:18:31,956] Trial 10 finished with value: 0.5108 and parameters: {'linear_layer_quantization': 1, 'linear_layer_quantization_fractional': 0, 'LinearMinifloatDenorm': 1, 'LinearMinifloatDenorm_ew': 0, 'LinearMinifloatDenorm_eb': 2, 'LinearMinifloatIEEE': 2, 'LinearMinifloatIEEE_ew': 1, 'LinearMinifloatIEEE_eb': 0, 'LinearLog': 0, 'LinearLog_ew': 0, 'LinearLog_eb': 1, 'LinearBlockFP': 2, 'LinearBlockFP_block_size': 2, 'LinearBlockMinifloat_ebw': 2, 'LinearBlockMinifloat_ew': 0, 'LinearBlockMinifloat_block_size': 1, 'LinearBlockMinifloat_w': 1, 'LinearBlockLog': 0, 'LinearBlockLog_ebw': 0, 'LinearBlockLog_block_size': 0, 'LinearBinary_stochastic': 1, 'LinearBinary_bipolar': 1, 'LinearBinaryScaling_stochastic': 1, 'LinearBinaryScaling_bipolar': 0, 'LinearBinaryScaling_binary_training': 1, 'LinearBinaryResidualSign_stochastic': 0, 'LinearBinaryResidualSign_bipolar': 1, 'LinearBinaryResidualSign_binary_training': 0, 'bert.encoder.layer.0.attention.self.query_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'>, 'bert.encoder.layer.0.output.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'>, 'bert.encoder.layer.1.output.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'>, 'bert.pooler.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'>, 'classifier_type': <class 'torch.nn.modules.linear.Linear'>}. Best is trial 8 with value: 0.87472.\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 02:17, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.693000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.693000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.693000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.693300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.693400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.693200</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 00:58]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-10 20:21:54,937] Trial 11 finished with value: 0.76472 and parameters: {'linear_layer_quantization': 2, 'linear_layer_quantization_fractional': 2, 'LinearMinifloatDenorm': 2, 'LinearMinifloatDenorm_ew': 1, 'LinearMinifloatDenorm_eb': 1, 'LinearMinifloatIEEE': 1, 'LinearMinifloatIEEE_ew': 0, 'LinearMinifloatIEEE_eb': 2, 'LinearLog': 1, 'LinearLog_ew': 1, 'LinearLog_eb': 0, 'LinearBlockFP': 1, 'LinearBlockFP_block_size': 0, 'LinearBlockMinifloat_ebw': 1, 'LinearBlockMinifloat_ew': 1, 'LinearBlockMinifloat_block_size': 1, 'LinearBlockMinifloat_w': 0, 'LinearBlockLog': 2, 'LinearBlockLog_ebw': 0, 'LinearBlockLog_block_size': 1, 'LinearBinary_stochastic': 0, 'LinearBinary_bipolar': 2, 'LinearBinaryScaling_stochastic': 2, 'LinearBinaryScaling_bipolar': 2, 'LinearBinaryScaling_binary_training': 1, 'LinearBinaryResidualSign_stochastic': 0, 'LinearBinaryResidualSign_bipolar': 1, 'LinearBinaryResidualSign_binary_training': 0, 'bert.encoder.layer.0.attention.self.query_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.output.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'>, 'bert.encoder.layer.1.output.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'>, 'bert.pooler.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'classifier_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'>}. Best is trial 8 with value: 0.87472.\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 01:51, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.716100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.688600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.689800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.693700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.690400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.694500</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 00:50]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-10 20:24:39,462] Trial 12 finished with value: 0.51168 and parameters: {'linear_layer_quantization': 0, 'linear_layer_quantization_fractional': 0, 'LinearMinifloatDenorm': 2, 'LinearMinifloatDenorm_ew': 2, 'LinearMinifloatDenorm_eb': 2, 'LinearMinifloatIEEE': 1, 'LinearMinifloatIEEE_ew': 1, 'LinearMinifloatIEEE_eb': 1, 'LinearLog': 0, 'LinearLog_ew': 1, 'LinearLog_eb': 1, 'LinearBlockFP': 1, 'LinearBlockFP_block_size': 0, 'LinearBlockMinifloat_ebw': 1, 'LinearBlockMinifloat_ew': 2, 'LinearBlockMinifloat_block_size': 0, 'LinearBlockMinifloat_w': 1, 'LinearBlockLog': 2, 'LinearBlockLog_ebw': 1, 'LinearBlockLog_block_size': 0, 'LinearBinary_stochastic': 1, 'LinearBinary_bipolar': 0, 'LinearBinaryScaling_stochastic': 2, 'LinearBinaryScaling_bipolar': 2, 'LinearBinaryScaling_binary_training': 0, 'LinearBinaryResidualSign_stochastic': 0, 'LinearBinaryResidualSign_bipolar': 1, 'LinearBinaryResidualSign_binary_training': 0, 'bert.encoder.layer.0.attention.self.query_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'>, 'bert.encoder.layer.0.output.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.pooler.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'classifier_type': <class 'torch.nn.modules.linear.Linear'>}. Best is trial 8 with value: 0.87472.\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 01:23, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.338800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.321300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.344700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.345500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.325300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.372000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 00:38]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-10 20:26:45,435] Trial 13 finished with value: 0.8724 and parameters: {'linear_layer_quantization': 2, 'linear_layer_quantization_fractional': 1, 'LinearMinifloatDenorm': 2, 'LinearMinifloatDenorm_ew': 1, 'LinearMinifloatDenorm_eb': 1, 'LinearMinifloatIEEE': 2, 'LinearMinifloatIEEE_ew': 2, 'LinearMinifloatIEEE_eb': 0, 'LinearLog': 1, 'LinearLog_ew': 0, 'LinearLog_eb': 2, 'LinearBlockFP': 1, 'LinearBlockFP_block_size': 0, 'LinearBlockMinifloat_ebw': 1, 'LinearBlockMinifloat_ew': 0, 'LinearBlockMinifloat_block_size': 1, 'LinearBlockMinifloat_w': 1, 'LinearBlockLog': 0, 'LinearBlockLog_ebw': 2, 'LinearBlockLog_block_size': 2, 'LinearBinary_stochastic': 1, 'LinearBinary_bipolar': 1, 'LinearBinaryScaling_stochastic': 2, 'LinearBinaryScaling_bipolar': 2, 'LinearBinaryScaling_binary_training': 1, 'LinearBinaryResidualSign_stochastic': 1, 'LinearBinaryResidualSign_bipolar': 1, 'LinearBinaryResidualSign_binary_training': 0, 'bert.encoder.layer.0.attention.self.query_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.pooler.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'classifier_type': <class 'torch.nn.modules.linear.Linear'>}. Best is trial 8 with value: 0.87472.\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 02:23, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.340400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.297700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.301200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.297100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.289100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.318100</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 00:54]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-10 20:30:04,108] Trial 14 finished with value: 0.8712 and parameters: {'linear_layer_quantization': 1, 'linear_layer_quantization_fractional': 0, 'LinearMinifloatDenorm': 0, 'LinearMinifloatDenorm_ew': 0, 'LinearMinifloatDenorm_eb': 0, 'LinearMinifloatIEEE': 0, 'LinearMinifloatIEEE_ew': 2, 'LinearMinifloatIEEE_eb': 2, 'LinearLog': 2, 'LinearLog_ew': 1, 'LinearLog_eb': 2, 'LinearBlockFP': 2, 'LinearBlockFP_block_size': 2, 'LinearBlockMinifloat_ebw': 0, 'LinearBlockMinifloat_ew': 0, 'LinearBlockMinifloat_block_size': 0, 'LinearBlockMinifloat_w': 2, 'LinearBlockLog': 0, 'LinearBlockLog_ebw': 0, 'LinearBlockLog_block_size': 2, 'LinearBinary_stochastic': 0, 'LinearBinary_bipolar': 2, 'LinearBinaryScaling_stochastic': 2, 'LinearBinaryScaling_bipolar': 2, 'LinearBinaryScaling_binary_training': 1, 'LinearBinaryResidualSign_stochastic': 1, 'LinearBinaryResidualSign_bipolar': 0, 'LinearBinaryResidualSign_binary_training': 1, 'bert.encoder.layer.0.attention.self.query_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.output.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'>, 'bert.encoder.layer.1.output.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'>, 'bert.pooler.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'>, 'classifier_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'>}. Best is trial 8 with value: 0.87472.\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 01:30, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.334700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.307500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.313000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.317400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.307200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.332700</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 00:40]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-10 20:32:18,777] Trial 15 finished with value: 0.87172 and parameters: {'linear_layer_quantization': 0, 'linear_layer_quantization_fractional': 1, 'LinearMinifloatDenorm': 2, 'LinearMinifloatDenorm_ew': 1, 'LinearMinifloatDenorm_eb': 0, 'LinearMinifloatIEEE': 2, 'LinearMinifloatIEEE_ew': 2, 'LinearMinifloatIEEE_eb': 0, 'LinearLog': 0, 'LinearLog_ew': 0, 'LinearLog_eb': 0, 'LinearBlockFP': 1, 'LinearBlockFP_block_size': 1, 'LinearBlockMinifloat_ebw': 1, 'LinearBlockMinifloat_ew': 0, 'LinearBlockMinifloat_block_size': 0, 'LinearBlockMinifloat_w': 2, 'LinearBlockLog': 1, 'LinearBlockLog_ebw': 1, 'LinearBlockLog_block_size': 0, 'LinearBinary_stochastic': 1, 'LinearBinary_bipolar': 2, 'LinearBinaryScaling_stochastic': 1, 'LinearBinaryScaling_bipolar': 1, 'LinearBinaryScaling_binary_training': 0, 'LinearBinaryResidualSign_stochastic': 1, 'LinearBinaryResidualSign_bipolar': 0, 'LinearBinaryResidualSign_binary_training': 1, 'bert.encoder.layer.0.attention.self.query_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'>, 'bert.encoder.layer.1.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.pooler.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'classifier_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'>}. Best is trial 8 with value: 0.87472.\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 01:55, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.337900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.300800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.312000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.306500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.295600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.334400</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 00:49]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-10 20:35:05,452] Trial 16 finished with value: 0.87124 and parameters: {'linear_layer_quantization': 1, 'linear_layer_quantization_fractional': 1, 'LinearMinifloatDenorm': 1, 'LinearMinifloatDenorm_ew': 2, 'LinearMinifloatDenorm_eb': 0, 'LinearMinifloatIEEE': 0, 'LinearMinifloatIEEE_ew': 2, 'LinearMinifloatIEEE_eb': 2, 'LinearLog': 1, 'LinearLog_ew': 2, 'LinearLog_eb': 2, 'LinearBlockFP': 0, 'LinearBlockFP_block_size': 0, 'LinearBlockMinifloat_ebw': 2, 'LinearBlockMinifloat_ew': 2, 'LinearBlockMinifloat_block_size': 0, 'LinearBlockMinifloat_w': 2, 'LinearBlockLog': 0, 'LinearBlockLog_ebw': 2, 'LinearBlockLog_block_size': 1, 'LinearBinary_stochastic': 1, 'LinearBinary_bipolar': 1, 'LinearBinaryScaling_stochastic': 1, 'LinearBinaryScaling_bipolar': 2, 'LinearBinaryScaling_binary_training': 0, 'LinearBinaryResidualSign_stochastic': 1, 'LinearBinaryResidualSign_bipolar': 0, 'LinearBinaryResidualSign_binary_training': 1, 'bert.encoder.layer.0.attention.self.query_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.output.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'>, 'bert.pooler.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'classifier_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'>}. Best is trial 8 with value: 0.87472.\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 01:43, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.715500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.714200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.710400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.712500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.706700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.705200</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 00:46]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-10 20:37:36,334] Trial 17 finished with value: 0.51752 and parameters: {'linear_layer_quantization': 1, 'linear_layer_quantization_fractional': 2, 'LinearMinifloatDenorm': 2, 'LinearMinifloatDenorm_ew': 0, 'LinearMinifloatDenorm_eb': 0, 'LinearMinifloatIEEE': 2, 'LinearMinifloatIEEE_ew': 0, 'LinearMinifloatIEEE_eb': 0, 'LinearLog': 2, 'LinearLog_ew': 0, 'LinearLog_eb': 1, 'LinearBlockFP': 0, 'LinearBlockFP_block_size': 1, 'LinearBlockMinifloat_ebw': 2, 'LinearBlockMinifloat_ew': 0, 'LinearBlockMinifloat_block_size': 1, 'LinearBlockMinifloat_w': 1, 'LinearBlockLog': 2, 'LinearBlockLog_ebw': 0, 'LinearBlockLog_block_size': 0, 'LinearBinary_stochastic': 0, 'LinearBinary_bipolar': 0, 'LinearBinaryScaling_stochastic': 2, 'LinearBinaryScaling_bipolar': 0, 'LinearBinaryScaling_binary_training': 1, 'LinearBinaryResidualSign_stochastic': 0, 'LinearBinaryResidualSign_bipolar': 0, 'LinearBinaryResidualSign_binary_training': 0, 'bert.encoder.layer.0.attention.self.query_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'>, 'bert.encoder.layer.0.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.pooler.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'>, 'classifier_type': <class 'torch.nn.modules.linear.Linear'>}. Best is trial 8 with value: 0.87472.\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 01:52, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.328900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.305200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.318800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.310200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.294300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.322000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 00:46]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-10 20:40:16,865] Trial 18 finished with value: 0.87212 and parameters: {'linear_layer_quantization': 1, 'linear_layer_quantization_fractional': 2, 'LinearMinifloatDenorm': 2, 'LinearMinifloatDenorm_ew': 1, 'LinearMinifloatDenorm_eb': 0, 'LinearMinifloatIEEE': 1, 'LinearMinifloatIEEE_ew': 1, 'LinearMinifloatIEEE_eb': 2, 'LinearLog': 1, 'LinearLog_ew': 2, 'LinearLog_eb': 1, 'LinearBlockFP': 2, 'LinearBlockFP_block_size': 0, 'LinearBlockMinifloat_ebw': 0, 'LinearBlockMinifloat_ew': 2, 'LinearBlockMinifloat_block_size': 1, 'LinearBlockMinifloat_w': 0, 'LinearBlockLog': 1, 'LinearBlockLog_ebw': 2, 'LinearBlockLog_block_size': 2, 'LinearBinary_stochastic': 0, 'LinearBinary_bipolar': 2, 'LinearBinaryScaling_stochastic': 0, 'LinearBinaryScaling_bipolar': 2, 'LinearBinaryScaling_binary_training': 0, 'LinearBinaryResidualSign_stochastic': 0, 'LinearBinaryResidualSign_bipolar': 1, 'LinearBinaryResidualSign_binary_training': 1, 'bert.encoder.layer.0.attention.self.query_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'>, 'bert.encoder.layer.0.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'>, 'bert.encoder.layer.1.output.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'>, 'bert.pooler.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'classifier_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'>}. Best is trial 8 with value: 0.87472.\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 01:32, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.328800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.311700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.308000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.319800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.299000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.332700</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 00:36]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-10 20:42:26,344] Trial 19 finished with value: 0.87192 and parameters: {'linear_layer_quantization': 0, 'linear_layer_quantization_fractional': 1, 'LinearMinifloatDenorm': 1, 'LinearMinifloatDenorm_ew': 2, 'LinearMinifloatDenorm_eb': 1, 'LinearMinifloatIEEE': 1, 'LinearMinifloatIEEE_ew': 0, 'LinearMinifloatIEEE_eb': 2, 'LinearLog': 0, 'LinearLog_ew': 1, 'LinearLog_eb': 1, 'LinearBlockFP': 1, 'LinearBlockFP_block_size': 0, 'LinearBlockMinifloat_ebw': 0, 'LinearBlockMinifloat_ew': 1, 'LinearBlockMinifloat_block_size': 0, 'LinearBlockMinifloat_w': 0, 'LinearBlockLog': 0, 'LinearBlockLog_ebw': 0, 'LinearBlockLog_block_size': 1, 'LinearBinary_stochastic': 2, 'LinearBinary_bipolar': 1, 'LinearBinaryScaling_stochastic': 0, 'LinearBinaryScaling_bipolar': 2, 'LinearBinaryScaling_binary_training': 0, 'LinearBinaryResidualSign_stochastic': 0, 'LinearBinaryResidualSign_bipolar': 1, 'LinearBinaryResidualSign_binary_training': 1, 'bert.encoder.layer.0.attention.self.query_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'>, 'bert.encoder.layer.0.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'>, 'bert.encoder.layer.1.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.pooler.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'classifier_type': <class 'chop.nn.quantized.modules.linear.LinearBinaryResidualSign'>}. Best is trial 8 with value: 0.87472.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.86908,\n",
       " 0.874,\n",
       " 0.874,\n",
       " 0.874,\n",
       " 0.874,\n",
       " 0.874,\n",
       " 0.874,\n",
       " 0.874,\n",
       " 0.87472,\n",
       " 0.87472,\n",
       " 0.87472,\n",
       " 0.87472,\n",
       " 0.87472,\n",
       " 0.87472,\n",
       " 0.87472,\n",
       " 0.87472,\n",
       " 0.87472,\n",
       " 0.87472,\n",
       " 0.87472,\n",
       " 0.87472]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from pathlib import Path\n",
    "import dill\n",
    "import torch\n",
    "\n",
    "with open(f\"{Path.home()}/tutorial_5_best_model.pkl\", \"rb\") as f:\n",
    "    base_model = dill.load(f)\n",
    "from chop.nn.quantized.modules.linear import (\n",
    "    LinearInteger,\n",
    "    LinearMinifloatDenorm,\n",
    "    LinearMinifloatIEEE,\n",
    "    LinearLog,\n",
    "    LinearBlockFP,\n",
    "    LinearBlockMinifloat,\n",
    "    LinearBlockLog,\n",
    "    LinearBinary,\n",
    "    LinearBinaryScaling,\n",
    "    LinearBinaryResidualSign,\n",
    ")\n",
    "search_space = {\n",
    "    \"linear_layer_choices\": [\n",
    "        torch.nn.Linear,\n",
    "        # LinearInteger,\n",
    "        # LinearMinifloatDenorm,\n",
    "        # LinearMinifloatIEEE,\n",
    "        # LinearLog,\n",
    "        # LinearBlockFP,\n",
    "        # # LinearBlockMinifloat,\n",
    "        # LinearBlockLog,\n",
    "        # LinearBinary,\n",
    "        # LinearBinaryScaling,\n",
    "        LinearBinaryResidualSign,\n",
    "    ],\n",
    "    'linear_layer_quantization': [8, 16, 32],\n",
    "    'linear_layer_quantization_fractional':  [2, 4, 8],\n",
    "    'LinearMinifloatDenorm': [8, 16, 32],\n",
    "    'LinearMinifloatDenorm_ew': [2, 4, 8],\n",
    "    'LinearMinifloatDenorm_eb': [2, 4, 8],\n",
    "    'LinearMinifloatIEEE': [8, 16, 32],\n",
    "    'LinearMinifloatIEEE_ew': [2, 4, 8],\n",
    "    'LinearMinifloatIEEE_eb': [2, 4, 8],\n",
    "    'LinearLog': [8, 16, 32],\n",
    "    'LinearLog_ew': [2, 4, 8],\n",
    "    'LinearLog_eb': [2, 4, 8],\n",
    "    'LinearBlockFP': [8, 16, 32],\n",
    "    'LinearBlockFP_block_size':[[4], [8], [16]] ,\n",
    "    'LinearBlockMinifloat_ebw': [2, 4, 8],\n",
    "    'LinearBlockMinifloat_ew': [2, 4, 8],\n",
    "    'LinearBlockMinifloat_block_size': [[4], [8], [16]],\n",
    "    'LinearBlockMinifloat_w': [2, 4, 8],\n",
    "    'LinearBlockLog': [8, 16, 32],\n",
    "    'LinearBlockLog_ebw': [2, 4, 8],\n",
    "    'LinearBlockLog_block_size': [[4], [8], [16]] ,\n",
    "    'LinearBinary_stochastic': [2, 4, 8],\n",
    "    'LinearBinary_bipolar': [2, 4, 8],\n",
    "    'LinearBinaryScaling_stochastic': [2, 4, 8],\n",
    "    'LinearBinaryScaling_bipolar': [2, 4, 8],\n",
    "    \"LinearBinaryScaling_binary_training\": [2, 4, 8],\n",
    "    'LinearBinaryResidualSign_stochastic': [True, False],\n",
    "    'LinearBinaryResidualSign_bipolar': [True, False],\n",
    "    \"LinearBinaryResidualSign_binary_training\": [True, False],\n",
    "}\n",
    "\n",
    "def construct_model(trial):\n",
    "    # Fetch the model\n",
    "    my_config = {}\n",
    "    trial_model = deepcopy(base_model)\n",
    "    \n",
    "    # Select quantization and fractional width configurations\n",
    "    for param in ['linear_layer_quantization', \n",
    "                  'linear_layer_quantization_fractional',\n",
    "                  'LinearMinifloatDenorm',\n",
    "                  'LinearMinifloatDenorm_ew',\n",
    "                  'LinearMinifloatDenorm_eb',\n",
    "                  'LinearMinifloatIEEE',\n",
    "                  'LinearMinifloatIEEE_ew',\n",
    "                  'LinearMinifloatIEEE_eb',\n",
    "                    'LinearLog',\n",
    "                    'LinearLog_ew',\n",
    "                    'LinearLog_eb',\n",
    "                    'LinearBlockFP',\n",
    "                    'LinearBlockFP_block_size',\n",
    "                    'LinearBlockMinifloat_ebw',\n",
    "                    'LinearBlockMinifloat_ew',\n",
    "                    'LinearBlockMinifloat_block_size',\n",
    "                    'LinearBlockMinifloat_w',\n",
    "                    'LinearBlockLog',\n",
    "                    'LinearBlockLog_ebw',\n",
    "                    'LinearBlockLog_block_size',\n",
    "                    'LinearBinary_stochastic',\n",
    "                    'LinearBinary_bipolar',\n",
    "                    'LinearBinaryScaling_stochastic',\n",
    "                    'LinearBinaryScaling_bipolar',\n",
    "                    'LinearBinaryScaling_binary_training',\n",
    "                    'LinearBinaryResidualSign_stochastic',\n",
    "                    'LinearBinaryResidualSign_bipolar',\n",
    "                    \"LinearBinaryResidualSign_binary_training\"\n",
    "                  ]:\n",
    "        chosen_idx = trial.suggest_int(param, 0, len(search_space[param]) - 1)\n",
    "        my_config[param] = search_space[param][chosen_idx]\n",
    "\n",
    "    # Quantize layers according to Optuna suggestions\n",
    "    for name, layer in trial_model.named_modules():\n",
    "        if isinstance(layer, torch.nn.Linear):\n",
    "            new_layer_cls = trial.suggest_categorical(\n",
    "                f\"{name}_type\",\n",
    "                search_space[\"linear_layer_choices\"],\n",
    "            )\n",
    "\n",
    "            if new_layer_cls == torch.nn.Linear:\n",
    "                continue\n",
    "\n",
    "            kwargs = {\n",
    "                \"in_features\": layer.in_features,\n",
    "                \"out_features\": layer.out_features,\n",
    "            }\n",
    "\n",
    "            try:\n",
    "                # Configure parameters for specific layer types\n",
    "                # if new_layer_cls == LinearInteger:\n",
    "                #     kwargs[\"config\"] = {\n",
    "                #         \"data_in_width\": my_config['linear_layer_quantization'],\n",
    "                #         \"data_in_frac_width\": my_config['linear_layer_quantization_fractional'],\n",
    "                #         \"weight_width\": my_config['linear_layer_quantization'],\n",
    "                #         \"weight_frac_width\": my_config['linear_layer_quantization_fractional'],\n",
    "                #         \"bias_width\": my_config['linear_layer_quantization'],\n",
    "                #         \"bias_frac_width\": my_config['linear_layer_quantization_fractional'],\n",
    "                #     }\n",
    "                # elif new_layer_cls == LinearMinifloatDenorm:\n",
    "                #     kwargs[\"config\"] = {\n",
    "                #         \"data_in_width\": my_config['LinearMinifloatDenorm'],\n",
    "                #         \"data_in_exponent_width\": my_config['LinearMinifloatDenorm_ew'],\n",
    "                #         \"data_in_exponent_bias\": my_config['LinearMinifloatDenorm_eb'],\n",
    "                #         \"weight_width\": my_config['LinearMinifloatDenorm'],\n",
    "                #         \"weight_exponent_width\": my_config['LinearMinifloatDenorm_ew'],\n",
    "                #         \"weight_exponent_bias\": my_config['LinearMinifloatDenorm_eb'],\n",
    "                #         \"bias_width\": my_config['LinearMinifloatDenorm'],\n",
    "                #         \"bias_exponent_width\": my_config['LinearMinifloatDenorm_ew'],\n",
    "                #         \"bias_exponent_bias\": my_config['LinearMinifloatDenorm_eb'],\n",
    "                #     }\n",
    "                # elif new_layer_cls == LinearMinifloatIEEE:\n",
    "                #     kwargs[\"config\"] = {\n",
    "                #         \"data_in_width\": my_config['LinearMinifloatIEEE'],\n",
    "                #         \"data_in_exponent_width\": my_config['LinearMinifloatIEEE_ew'],\n",
    "                #         \"data_in_exponent_bias\": my_config['LinearMinifloatIEEE_eb'],\n",
    "                #         \"weight_width\": my_config['LinearMinifloatIEEE'],\n",
    "                #         \"weight_exponent_width\": my_config['LinearMinifloatIEEE_ew'],\n",
    "                #         \"weight_exponent_bias\": my_config['LinearMinifloatIEEE_eb'],\n",
    "                #         \"bias_width\": my_config['LinearMinifloatIEEE'],\n",
    "                #         \"bias_exponent_width\": my_config['LinearMinifloatIEEE_ew'],\n",
    "                #         \"bias_exponent_bias\": my_config['LinearMinifloatIEEE_eb'],\n",
    "                #     }\n",
    "                # elif new_layer_cls == LinearLog:\n",
    "                #     kwargs[\"config\"] = {\n",
    "                #         \"data_in_width\": my_config['LinearLog'],\n",
    "                #         \"data_in_exponent_width\": my_config['LinearLog_ew'],\n",
    "                #         \"data_in_exponent_bias\": my_config['LinearLog_eb'],\n",
    "                #         \"weight_width\": my_config['LinearLog'],\n",
    "                #         \"weight_exponent_width\": my_config['LinearLog_ew'],\n",
    "                #         \"weight_exponent_bias\": my_config['LinearLog_eb'],\n",
    "                #         \"bias_width\": my_config['LinearLog'],\n",
    "                #         \"bias_exponent_width\": my_config['LinearLog_ew'],\n",
    "                #         \"bias_exponent_bias\": my_config['LinearLog_eb'],\n",
    "                #     }\n",
    "                # elif new_layer_cls == LinearBlockFP:\n",
    "                #     kwargs[\"config\"] = {\n",
    "                #         \"weight_width\": my_config['LinearBlockFP'],\n",
    "                #         \"weight_exponent_width\": my_config['LinearBlockFP'],\n",
    "                #         \"weight_block_size\": my_config['LinearBlockFP_block_size'],\n",
    "                #         \"weight_exponent_bias\": my_config['LinearBlockFP'],\n",
    "                #         \"data_in_width\": my_config['LinearBlockFP'],\n",
    "                #         \"data_in_exponent_width\": my_config['LinearBlockFP'],\n",
    "                #         \"data_in_block_size\": my_config['LinearBlockFP_block_size'],\n",
    "                #         \"data_in_exponent_bias\": my_config['LinearBlockFP'],\n",
    "                #         \"bias_width\": my_config['LinearBlockFP'],\n",
    "                #         \"bias_exponent_width\": my_config['LinearBlockFP'],\n",
    "                #         \"bias_block_size\": my_config['LinearBlockFP_block_size'],\n",
    "                #         \"bias_exponent_bias\": my_config['LinearBlockFP'],\n",
    "                #     }\n",
    "                # # elif new_layer_cls == LinearBlockMinifloat:\n",
    "                #     kwargs[\"config\"] = {\n",
    "                #         \"weight_exponent_bias_width\" : my_config['LinearBlockMinifloat_ebw'],\n",
    "                #         \"weight_exponent_width\": my_config['LinearBlockMinifloat_ew'],\n",
    "                #         \"weight_block_size\": my_config['LinearBlockMinifloat_block_size'],\n",
    "                #         \"weight_width\": my_config['LinearBlockMinifloat_w'],\n",
    "                #         \"data_in_width\":my_config[\"LinearBlockMinifloat_w\"],\n",
    "                #         \"data_in_exponent_bias_width\":my_config[\"LinearBlockMinifloat_ew\"],\n",
    "                #         \"data_in_exponent_width\":my_config[\"LinearBlockMinifloat_ebw\"],\n",
    "                #         \"data_in_block_size\":my_config[\"LinearBlockMinifloat_block_size\"],\n",
    "                #         \"bias_width\":my_config[\"LinearBlockMinifloat_w\"],\n",
    "                #         \"bias_exponent_width\":my_config[\"LinearBlockMinifloat_ew\"],\n",
    "                #         \"bias_exponent_bias_width\":my_config[\"LinearBlockMinifloat_ebw\"],\n",
    "                #         \"bias_block_size\":my_config[\"LinearBlockMinifloat_block_size\"],\n",
    "                #     }\n",
    "                # elif new_layer_cls == LinearBlockLog:\n",
    "                #     kwargs[\"config\"] = {\n",
    "                #         \"data_in_width\": my_config['LinearBlockLog'],\n",
    "                #         \"data_in_exponent_bias_width\": my_config['LinearBlockLog_ebw'],\n",
    "                #         \"data_in_block_size\": my_config['LinearBlockLog_block_size'],\n",
    "                #         \"weight_width\": my_config['LinearBlockLog'],\n",
    "                #         \"weight_exponent_bias_width\": my_config['LinearBlockLog_ebw'],\n",
    "                #         \"weight_block_size\": my_config['LinearBlockLog_block_size'],\n",
    "                #         \"bias_width\": my_config['LinearBlockLog'],\n",
    "                #         \"bias_exponent_bias_width\": my_config['LinearBlockLog_ebw'],\n",
    "                #         \"bias_block_size\": my_config['LinearBlockLog_block_size'],\n",
    "                # #     }\n",
    "\n",
    "                # elif new_layer_cls == LinearBinary:\n",
    "                #     kwargs[\"config\"] = {\n",
    "                #         'weight_stochastic':my_config['LinearBinary_stochastic'],\n",
    "                #         'weight_bipolar':my_config['LinearBinary_bipolar'],\n",
    "                #     }\n",
    "\n",
    "                    \n",
    "                # elif new_layer_cls == LinearBinaryScaling:\n",
    "                #     kwargs[\"config\"] = {\n",
    "                #         'weight_stochastic':my_config['LinearBinaryScaling_stochastic'],\n",
    "                #         'weight_bipolar':my_config['LinearBinaryScaling_bipolar'],\n",
    "                #         \"data_in_stochastic\": my_config['LinearBinaryScaling_stochastic'],\n",
    "                #         \"data_in_bipolar\": my_config['LinearBinaryScaling_bipolar'],\n",
    "                #         \"bias_stochastic\": my_config['LinearBinaryScaling_stochastic'],\n",
    "                #         \"bias_bipolar\": my_config['LinearBinaryScaling_bipolar'],\n",
    "                #         \"binary_training\": my_config['LinearBinaryScaling_binary_training'],\n",
    "                #     }\n",
    "                if new_layer_cls == LinearBinaryResidualSign:\n",
    "                    kwargs[\"config\"] = {\n",
    "                        'weight_stochastic':my_config['LinearBinaryResidualSign_stochastic'],\n",
    "                        'weight_bipolar':my_config['LinearBinaryResidualSign_bipolar'],\n",
    "                        \"data_in_stochastic\": my_config['LinearBinaryResidualSign_stochastic'],\n",
    "                        \"data_in_bipolar\": my_config['LinearBinaryResidualSign_bipolar'],\n",
    "                        \"bias_stochastic\": my_config['LinearBinaryResidualSign_stochastic'],\n",
    "                        \"bias_bipolar\": my_config['LinearBinaryResidualSign_bipolar'],\n",
    "                        \"binary_training\": my_config['LinearBinaryResidualSign_binary_training'],\n",
    "                    }\n",
    "                # Create the new layer (copy the weights if applicable)\n",
    "                new_layer = new_layer_cls(**kwargs)\n",
    "                new_layer.weight.data = layer.weight.data.clone()\n",
    "\n",
    "                # Replace the layer in the model\n",
    "                deepsetattr(trial_model, name, new_layer)\n",
    "            except TypeError as e:\n",
    "                print(f\"Error creating layer {name} with class {new_layer_cls}: {e}\")\n",
    "                raise\n",
    "\n",
    "    return trial_model\n",
    "from chop.tools import get_trainer\n",
    "import random\n",
    "\n",
    "\n",
    "def objective(trial):\n",
    "\n",
    "    # Define the model\n",
    "    model = construct_model(trial)\n",
    "\n",
    "    trainer = get_trainer(\n",
    "        model=model,\n",
    "        tokenized_dataset=dataset,\n",
    "        tokenizer=tokenizer,\n",
    "        evaluate_metric=\"accuracy\",\n",
    "        num_train_epochs=1,\n",
    "    )\n",
    "\n",
    "    trainer.train()\n",
    "    eval_results = trainer.evaluate()\n",
    "\n",
    "    trial.set_user_attr(\"model\", model)\n",
    "\n",
    "    return eval_results[\"eval_accuracy\"]\n",
    "\n",
    "\n",
    "from optuna.samplers import GridSampler, RandomSampler, TPESampler\n",
    "\n",
    "sampler = RandomSampler()\n",
    "\n",
    "import optuna\n",
    "\n",
    "study = optuna.create_study(\n",
    "    direction=\"maximize\",\n",
    "    study_name=\"bert-tiny-nas-study\",\n",
    "    sampler=sampler,\n",
    ")\n",
    "\n",
    "study.optimize(\n",
    "    objective,\n",
    "    n_trials=20,\n",
    "    timeout=60 * 60 * 24,\n",
    ")\n",
    "\n",
    "task10 = [max([t.value for t in study.trials[:i+1]]) for i in range(len(study.trials))]\n",
    "task10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "task10=[0.51296,\n",
    " 0.51296,\n",
    " 0.51296,\n",
    " 0.51296,\n",
    " 0.51296,\n",
    " 0.51296,\n",
    " 0.51296,\n",
    " 0.51296,\n",
    " 0.51296,\n",
    " 0.51296]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "task1=[0.5,\n",
    " 0.8562,\n",
    " 0.8562,\n",
    " 0.8562,\n",
    " 0.8562,\n",
    " 0.8562,\n",
    " 0.8562,\n",
    " 0.8562,\n",
    " 0.87072,\n",
    " 0.87072]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "task3=[0.87256,\n",
    " 0.87268,\n",
    " 0.87268,\n",
    " 0.87268,\n",
    " 0.87284,\n",
    " 0.87284,\n",
    " 0.87296,\n",
    " 0.87296,\n",
    " 0.8738,\n",
    " 0.8738,\n",
    " 0.8738,\n",
    " 0.8738,\n",
    " 0.8738,\n",
    " 0.8738,\n",
    " 0.8738,\n",
    " 0.8738,\n",
    " 0.8738,\n",
    " 0.8738,\n",
    " 0.8738,\n",
    " 0.8738]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "task1=[\n",
    " 0.5,\n",
    " 0.8562,\n",
    " 0.8562,\n",
    " 0.8562,\n",
    " 0.8562,\n",
    " 0.8562,\n",
    " 0.8562,\n",
    " 0.8562,\n",
    " 0.87072,\n",
    " 0.87072,\n",
    " 0.87072,\n",
    " 0.87072,\n",
    " 0.87072,\n",
    " 0.87072,\n",
    " 0.87072,\n",
    " 0.87072,\n",
    " 0.87072,\n",
    " 0.87072,\n",
    " 0.87072,\n",
    " 0.87072]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "task4=[0.87332,\n",
    " 0.87348,\n",
    " 0.87348,\n",
    " 0.8738,\n",
    " 0.8738,\n",
    " 0.8738,\n",
    " 0.8738,\n",
    " 0.8738,\n",
    " 0.8738,\n",
    " 0.8738,\n",
    " 0.8738,\n",
    " 0.8738,\n",
    " 0.8738,\n",
    " 0.8738,\n",
    " 0.8738,\n",
    " 0.8738,\n",
    " 0.8738,\n",
    " 0.8738,\n",
    " 0.8738,\n",
    " 0.8738]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "task5=[0.87404,\n",
    " 0.87404,\n",
    " 0.87404,\n",
    " 0.87404,\n",
    " 0.87404,\n",
    " 0.87404,\n",
    " 0.87404,\n",
    " 0.87404,\n",
    " 0.87404,\n",
    " 0.87404,\n",
    " 0.87404,\n",
    " 0.87404,\n",
    " 0.87404,\n",
    " 0.87404,\n",
    " 0.87404,\n",
    " 0.87404,\n",
    " 0.87404,\n",
    " 0.87404,\n",
    " 0.87404,\n",
    " 0.87404]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "task6=[0.87308,\n",
    " 0.87308,\n",
    " 0.87308,\n",
    " 0.87308,\n",
    " 0.87308,\n",
    " 0.87308,\n",
    " 0.87308,\n",
    " 0.8742,\n",
    " 0.8742,\n",
    " 0.8742,\n",
    " 0.8742,\n",
    " 0.8742,\n",
    " 0.8742,\n",
    " 0.8742,\n",
    " 0.8742,\n",
    " 0.8742,\n",
    " 0.8742,\n",
    " 0.8742,\n",
    " 0.8742,\n",
    " 0.8742]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "task7=[0.87304,\n",
    "        0.874, \n",
    "        0.874, \n",
    "        0.874, \n",
    "        0.874, \n",
    "        0.874, \n",
    "        0.874, \n",
    "        0.874, \n",
    "        0.874, \n",
    "        0.874,\n",
    "        0.874,\n",
    "        0.874,\n",
    "        0.874,\n",
    "        0.874,\n",
    "        0.874,\n",
    "        0.874,\n",
    "        0.874,\n",
    "        0.874,\n",
    "        0.874,\n",
    "        0.874]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "task8=[0.50224,\n",
    " 0.86552,\n",
    " 0.8688,\n",
    " 0.8688,\n",
    " 0.8688,\n",
    " 0.8688,\n",
    " 0.8688,\n",
    " 0.8688,\n",
    " 0.8688,\n",
    " 0.8688,\n",
    " 0.87052,\n",
    " 0.87052,\n",
    " 0.87052,\n",
    " 0.87052,\n",
    " 0.87052,\n",
    " 0.87052,\n",
    " 0.87052,\n",
    " 0.87052,\n",
    " 0.87052,\n",
    " 0.87052]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "task9=[0.8728,\n",
    " 0.8728,\n",
    " 0.8728,\n",
    " 0.8728,\n",
    " 0.8728,\n",
    " 0.87308,\n",
    " 0.87332,\n",
    " 0.87336,\n",
    " 0.87336,\n",
    " 0.87336,\n",
    " 0.87336,\n",
    " 0.87336,\n",
    " 0.87336,\n",
    " 0.87336,\n",
    " 0.87336,\n",
    " 0.87336,\n",
    " 0.87336,\n",
    " 0.87336,\n",
    " 0.87336,\n",
    " 0.87336]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "task10=[0.86908,\n",
    " 0.874,\n",
    " 0.874,\n",
    " 0.874,\n",
    " 0.874,\n",
    " 0.874,\n",
    " 0.874,\n",
    " 0.874,\n",
    " 0.87472,\n",
    " 0.87472,\n",
    " 0.87472,\n",
    " 0.87472,\n",
    " 0.87472,\n",
    " 0.87472,\n",
    " 0.87472,\n",
    " 0.87472,\n",
    " 0.87472,\n",
    " 0.87472,\n",
    " 0.87472,\n",
    " 0.87472]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkAAAAHHCAYAAABXx+fLAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/SrBM8AAAACXBIWXMAAA9hAAAPYQGoP6dpAAClG0lEQVR4nOzdeVxU1fsH8M+dAWbYFwFBBVlEBAQXwJ1wwQY1t0pcKEVTc0Hza6SouYALWmqE69cN1K/lkmaaK1H4C7U0ZbEgCgXRAnFjXwZm7u+PgSvjgMywDcjz7jWvmHvPPfeZQZiHc889D8OyLAtCCCGEkDaEp+4ACCGEEEKaGyVAhBBCCGlzKAEihBBCSJtDCRAhhBBC2hxKgAghhBDS5lACRAghhJA2hxIgQgghhLQ5lAARQgghpM2hBIgQQgghbQ4lQIQQtYiKigLDMMjIyFB3KERJa9asAcMw6g6DkEZBCRAhzaDqw/63335T+dji4mKsWbMGsbGxjR9YEwkICICenp66w2hSUqkUhw4dQt++fWFiYgJ9fX107doVU6dOxS+//MK1S05Oxpo1a5os0du5cyeioqKapG9CXmeUABHSwhUXFyMkJKRVJUDKeP/991FSUoLOnTurO5R6WbhwIaZNmwZLS0usWbMGmzZtwogRI/DLL7/g4sWLXLvk5GSEhIS8FgnQp59+ipKSkmY5FyFNTUPdARBCXh9FRUXQ1dVVqi2fzwefz2/iiOpPKpVCLBZDKBQq7Hv06BF27tyJWbNmYc+ePXL7wsPD8fjx43qdk2VZlJaWQltbu17HNzUNDQ1oaNDHBnk90AgQIWpSdZnon3/+wbhx46CnpwczMzMEBQVBIpEAADIyMmBmZgYACAkJAcMwYBgGa9as4fr5888/8e6778LExARCoRAeHh44c+aMwvmSkpLg7e0NbW1tdOrUCevWrUNkZGSN83AuXLgALy8v6OrqQl9fH6NGjcIff/xRY/x3797FyJEjoa+vD39/f6Vff01zgGxsbPDWW28hLi4Offr0gVAohJ2dHQ4dOqRwfG5uLhYtWgQrKysIBAJ06dIFmzZtglQqlWu3efNmDBgwAO3atYO2tjbc3d3xzTffKPTHMAwCAwNx5MgRuLi4QCAQyI3kVJeeng6WZTFw4MAa+zE3N+de44QJEwAAQ4YM4b5/VaN5Va/30qVL8PDwgLa2Nv773/8CACIjIzF06FCYm5tDIBDA2dkZu3btkjuXjY0N/vjjD1y5coXre/DgwSq/R0+fPsX7778PAwMDGBkZYdq0aUhMTATDMHKjS7XNAfrf//4Hd3d3aGtrw8TEBJMmTcKDBw/k2vz999945513YGFhAaFQiE6dOmHSpEnIy8ur8T0mpKlRKk+IGkkkEohEIvTt2xebN2/GDz/8gC1btsDe3h5z586FmZkZdu3ahblz52L8+PF4++23AQBubm4AgD/++AMDBw5Ex44dERwcDF1dXRw/fhzjxo3DyZMnMX78eADAP//8w30AL1u2DLq6uti3bx8EAoFCTIcPH8a0adMgEomwadMmFBcXY9euXRg0aBDi4+NhY2PDta2oqIBIJMKgQYOwefNm6OjoNPg9SUtLw7vvvosPPvgA06ZNw4EDBxAQEAB3d3e4uLgAkF0W9Pb2xj///IMPP/wQ1tbWuHbtGpYtW4asrCyEh4dz/X355ZcYM2YM/P39IRaLcfToUUyYMAHff/89Ro0aJXfuH3/8EcePH0dgYCBMTU3lXmt1VZftTpw4gQkTJtT6ut944w0sXLgQERERWL58OZycnACA+z8ApKamYvLkyfjwww8xa9YsODo6AgB27doFFxcXjBkzBhoaGjh79izmzZsHqVSK+fPnA5CNNi1YsAB6enpYsWIFAKB9+/YqvUdSqRSjR4/GjRs3MHfuXHTr1g3fffcdpk2bptT3a/369Vi5ciX8/Pwwc+ZMPH78GNu2bcMbb7yB+Ph4GBkZQSwWQyQSoaysDAsWLICFhQX++ecffP/998jNzYWhoaFS5yKkUbGEkCYXGRnJAmBv3rzJbZs2bRoLgA0NDZVr26tXL9bd3Z17/vjxYxYAu3r1aoV+hw0bxrq6urKlpaXcNqlUyg4YMIB1cHDgti1YsIBlGIaNj4/ntj19+pQ1MTFhAbDp6eksy7JsQUEBa2RkxM6aNUvuPNnZ2ayhoaHc9qr4g4ODFeKaNm0aq6urq9R7UnVulmXZzp07swDY//u//+O25eTksAKBgP3444+5bWvXrmV1dXXZv/76S67P4OBgls/ns5mZmdy24uJiuTZisZjt3r07O3ToULntAFgej8f+8ccfr4y7ytSpU1kArLGxMTt+/Hh28+bNbEpKikK7EydOsADYn376SWFf1eu9ePGiwr6X42ZZlhWJRKydnZ3cNhcXF9bb21uhrbLv0cmTJ1kAbHh4ONdGIpGwQ4cOZQGwkZGR3PbVq1ez1T82MjIyWD6fz65fv17uHHfu3GE1NDS47fHx8SwA9sSJEwpxEqIudAmMEDWbM2eO3HMvLy/cu3evzuOePXuGH3/8EX5+figoKMCTJ0/w5MkTPH36FCKRCH///Tf++ecfAMDFixfRv39/9OzZkzvexMRE4ZJVdHQ0cnNzMXnyZK6/J0+egM/no2/fvvjpp58U4pg7d249XnXtnJ2d4eXlxT03MzODo6Oj3Hty4sQJeHl5wdjYWC5OHx8fSCQS/N///R/Xtvp8mufPnyMvLw9eXl64ffu2wrm9vb3h7OysVJyRkZHYvn07bG1t8e233yIoKAhOTk4YNmwY974rw9bWFiKRSGF79bjz8vLw5MkTeHt74969e0pdNlL2Pbp48SI0NTUxa9Ys7lgej8eNMr3KqVOnIJVK4efnJ3cOCwsLODg4cP9eqkZ4Ll26hOLi4jr7JaQ50CUwQtRIKBRyc3yqGBsb4/nz53Uem5aWBpZlsXLlSqxcubLGNjk5OejYsSPu37+P/v37K+zv0qWL3PO///4bADB06NAa+zMwMJB7rqGhgU6dOtUZqyqsra0Vtr38nvz9999ISkpSeO+q5OTkcF9///33WLduHRISElBWVsZtr2kui62trdJxViUJ8+fPx9OnT3H16lXs3r0bFy5cwKRJk/Dzzz8r1U9t57x69SpWr16N69evKyQNeXl5dV42UvY9un//PiwtLRUu4738b6O2c7AsCwcHhxr3a2pqApC9xsWLF2Pr1q04cuQIvLy8MGbMGLz33nt0+YuoDSVAhKhRQ+6CqprIGhQUVOMIAqDch1hNfR4+fBgWFhYK+1++A0ggEIDHa9yB5NreE5Zlua+lUimGDx+OJUuW1Ni2a9euAICff/4ZY8aMwRtvvIGdO3fC0tISmpqaiIyMxFdffaVwXH3vvmrXrh3GjBmDMWPGYPDgwbhy5Qru37+v1C3+NZ3z7t27GDZsGLp164atW7fCysoKWlpaOH/+PL744guFScw1UfY9agipVAqGYXDhwoUav2/V14LasmULAgIC8N133+Hy5ctYuHAhwsLC8MsvvzR6Ek2IMigBIqSFq23lXTs7OwCyv7J9fHxe2Ufnzp2RlpamsP3lbfb29gAAc3PzOvtUJ3t7exQWFtYZ48mTJyEUCnHp0iW5Cd+RkZFNFpuHhweuXLmCrKwsdO7cuV4rJ589exZlZWU4c+aM3IhYTZcga+tf2feoc+fO+Omnn1BcXCw3ClTTv5eazsGyLGxtbZVKqFxdXeHq6opPP/0U165dw8CBA7F7926sW7euzmMJaWw0B4iQFq7qQyk3N1duu7m5OQYPHoz//ve/yMrKUjiu+lo0IpEI169fR0JCArft2bNnOHLkiNwxIpEIBgYG2LBhA8rLy1/Zpzr5+fnh+vXruHTpksK+3NxcVFRUAJCNJjEMwy0rAMiWFjh9+nSDzp+dnY3k5GSF7WKxGDExMeDxeNzoW9W6SC9//16lajSl+qhXXl5ejYmbrq5ujX0r+x6JRCKUl5dj79693H6pVIodO3bUGefbb78NPp+PkJAQuVirYn/69CkAID8/nztfFVdXV/B4PLnLkoQ0JxoBIqSF09bWhrOzM44dO4auXbvCxMQE3bt3R/fu3bFjxw4MGjQIrq6umDVrFuzs7PDo0SNcv34dDx8+RGJiIgBgyZIl+N///ofhw4djwYIF3G3w1tbWePbsGTeKYGBggF27duH9999H7969MWnSJJiZmSEzMxPnzp3DwIEDsX37dqXiLi8vr/EvexMTE8ybN69B78knn3yCM2fO4K233uJukS8qKsKdO3fwzTffICMjA6amphg1ahS2bt0KX19fTJkyBTk5OdixYwe6dOmCpKSkep//4cOH6NOnD4YOHYphw4bBwsICOTk5+Prrr5GYmIhFixbB1NQUANCzZ0/w+Xxs2rQJeXl5EAgE3Po+tXnzzTehpaWF0aNH48MPP0RhYSH27t0Lc3NzhWTX3d0du3btwrp169ClSxeYm5tj6NChSr9H48aNQ58+ffDxxx8jLS0N3bp1w5kzZ/Ds2TMAtY8wAbIRoHXr1mHZsmXIyMjAuHHjoK+vj/T0dHz77beYPXs2goKC8OOPPyIwMBATJkxA165dUVFRgcOHD4PP5+Odd96p9/eBkAZR4x1ohLQZtd0GX9Ot4i/fasyyLHvt2jXW3d2d1dLSUrgl/u7du+zUqVNZCwsLVlNTk+3YsSP71ltvsd98841cH/Hx8ayXlxcrEAjYTp06sWFhYWxERAQLgM3OzpZr+9NPP7EikYg1NDRkhUIha29vzwYEBLC//fZbnfFX7QNQ48Pe3l7uPXn5NvhRo0Yp9Oft7a1wq3dBQQG7bNkytkuXLqyWlhZramrKDhgwgN28eTMrFou5dvv372cdHBxYgUDAduvWjY2MjKzxPQbAzp8/v8bX87L8/Hz2yy+/ZEUiEdupUydWU1OT1dfXZ/v378/u3buXlUqlcu337t3L2tnZsXw+X+6W+NpeL8uy7JkzZ1g3NzdWKBSyNjY27KZNm9gDBw4ovGfZ2dnsqFGjWH19fRaA3Puk7Hv0+PFjdsqUKay+vj5raGjIBgQEsFevXmUBsEePHuXa1fS+sazsVvpBgwaxurq6rK6uLtutWzd2/vz5bGpqKsuyLHvv3j12xowZrL29PSsUClkTExN2yJAh7A8//KDU+01IU2BY9qVxS0JIm7Fo0SL897//RWFhYYsuS0Ga3+nTpzF+/HjExcXVuOI1Ia0dzQEipI14uYjl06dPcfjwYQwaNIiSnzbu5X8bEokE27Ztg4GBAXr37q2mqAhpWjQHiJA2on///hg8eDCcnJzw6NEj7N+/H/n5+bWuIUTajgULFqCkpAT9+/dHWVkZTp06hWvXrmHDhg0ttjArIQ1Fl8AIaSOWL1+Ob775Bg8fPgTDMOjduzdWr17dom93J83jq6++wpYtW5CWlobS0lJ06dIFc+fORWBgoLpDI6TJUAJECCGEkDaH5gARQgghpM2hBIgQQgghbQ5Ngq6BVCrFv//+C319/XotY08IIYSQ5seyLAoKCtChQ4c66xRSAlSDf//9F1ZWVuoOgxBCCCH18ODBgzqL7FICVAN9fX0AsjfQwMBAzdEQQgghRBn5+fmwsrLiPsdfhRKgGlSvi0QJECGEENK6KDN9hSZBE0IIIaTNoQSIEEIIIW0OJUCEEEIIaXMoASKEEEJIm0MJECGEEELaHEqACCGEENLmUAJECCGEkDaHEiBCCCGEtDmUABFCCCGkzaEEiBBCCCFtDiVAhBBCCGlzKAEihBBCSJtDxVAJIY2KZVmwbAWkUjFYVgypVAyptELdYRFCWhgNDT1oahqq7/xqO3MbxLIs2JISpdtLJRJUiMWoKBdz/y8vK5N7XlEmRnl5KcrKH0IsuQ8J8wwA24SvAQBY2X+s7KuqLWBlz2T7XmyX7QNYSGWtq7WTPeVakYZgAUgrv0cswEor///y8xq3s2B4UjA8CRieBKj2NVPr16/YV3chZkJIG1eROxCitw+p7fyUADWjXV9sRGlKluyjnmHBQvZJJEsMWLmvq9q8jKdRAR2jomqPQmgbFoOvLW3211MTpvJBSBWWBVgpDyz9yyCEVPPoyRO1np8SoGZU+PgZijuaKNmahUBQDF3dZ9DVew493efQ1XsObe2CGltLJHwUFRmjpNgALEtTu4jqpCwPrJQHKcuX/V/Kr9zGh/Tl7VI+WLbq61ccI+WD0mJCSE30+cVqPT8lQM2oi6UB0v7Nk10eYAAGjOyjgcdCqJsPHb08aOvnQqifB6FeLjQ0y2vsp7xMG2WFRigtMq78vxEqSvQA8JvldTA1fN24H281naH6ly/tVzj5SxuqXY+p/lHMVO7ivg8vtWHk2lTfx8jtrwqB21e5seq53Pe6MhaGYar1ybz6uMq2PB4LPp+FBk8CPk8KDb4UGvwKaPCk3HYNvhQ8Rip3Caqmy1E1fr/4WgBfs/L/ghdfa1R+raEF8AQAX6Nym9ZLD02Ap/hvMONxEa7fe4pyacu/yMmwLDQkgKaEhabkxde8lh86Ia2OVbcOaj0/JUDN6O2FK1Fe/hwFBSkoLPwTBYXJKCz8E0VFaWBZxWSHYTSgq2MPPT0n6Ok7QV/PCXp6TtDSUnYUiRD1+Te3BFsu/4VT8Q8r5441Dy0A+mC4hx73f/nt1fdVfa1HN8YS0mz+QBEwVH3npwSoGaXd3Yz793fVuE9DQx96es7Q0+sGfT1n6Os7QVe3C3g8QTNHCbAVUkjyylDxvBSS5/L/Z8skzR4PaV0kUhZPCsvwrEiMtwC8BV3oCzSgpdF0yQVPCggqpNCqYMFvhGSrnAeINXgQazAo02Ag5dElPEIam6mV+u4AAygBalbawk4AAKHQCvr6TtDTc4a+Xjfo6TlDKOzAXR5pamyFFBW5ZZA8L+WSG9nXsv9LCsRNeSMZaQNMAJhUvyRbxgLNmTwzAE9bA4xQAzxt+Yf8Nj542pqV2/kv2vBpJIiQ1x0lQM2offu30L79KGho6DfpedhySWWCUzV68yK5qXheBmmBuO5ONHjQMBaAbyx88X8jAXg6mk0aO2l9WJbFjYznOPHbAzwqKAUAdDAUYqKnNXpZGTVPYs9n5JMcLT4YGrUhhLwCJUDNSENDr0n7L7r1CHkX0yEtqHnydHWMJk8+uTEWgG8kBN9YAA1jIXh6ms02IkVarxvpz7DhfAoSHuQCAEz1BFg8vCv8PDpBg0ZRCCEtGCVAr5GiG9lc8sNo8blkRu7/lUkOT5cSHFJ/aTmF2HTxT0QnPwIA6GjxMfsNO8zysoOugH6tEEJaPvpN9RqRFsuSn3YBLhA6GlOCQxpdTkEpvvzhbxy9+QASKQs+j8FETyss8nGAub5Q3eERQojSKAF6jUiLZfWW+IYCSn5IoyoWV2Dv/6Xjv/93F8Vi2WRmH6f2CB7hiC7mTTunjRBCmgIlQK8JlmUhLZGNAPF16NtKGkeFRIoTtx5ia/RfeFxQBgDoYWWE5SO6oa9dOzVHRwgh9UeflK8JtkwCVJYD41ECRBqIZVn8+GcONl74E3/nFAIArE10sMTXEaNcLWmEkRDS6tEn5WtCWlQ5+VmTB0azeUpikNdT0sNcrD+Xgl/TnwEAjHQ0sXCoA/z7WUOgQf+2CCGvB0qAXhPSEtn8Hxr9IfX14FkxPruUirOJ/wIAtDR4mDHQFnMH28NQm9Z/IoS8XujT8jVRNQH6qUSKry6nqjka0to8LijDydsPUS5hwTDA+F4d8fGbjuhopK3u0AghpElQAvSaqLoF/m5hKbb9mKbmaEhr5eVgiuAR3eDSQb01egghpKlRAvSaqBoBygeLN53bowP95U5UwGMYDHY0wxtdzdQdCiGENAu1J0A7duzA559/juzsbPTo0QPbtm1Dnz59am0fHh6OXbt2ITMzE6ampnj33XcRFhYGoVC2CNuaNWsQEhIid4yjoyP+/PPPJn0d6lY1ApQPFoFDu8Ctk5F6AyKEEEJaMLUmQMeOHcPixYuxe/du9O3bF+Hh4RCJREhNTYW5ublC+6+++grBwcE4cOAABgwYgL/++gsBAQFgGAZbt27l2rm4uOCHH37gnmtoqD3Pa3JVI0B5YGGso6XmaAghhJCWTa3VCrdu3YpZs2Zh+vTpcHZ2xu7du6Gjo4MDBw7U2P7atWsYOHAgpkyZAhsbG7z55puYPHkybty4IddOQ0MDFhYW3MPU1LQ5Xo5aiQtlFd7zwcKIKrYTQgghr6S2BEgsFuPWrVvw8fF5EQyPBx8fH1y/fr3GYwYMGIBbt25xCc+9e/dw/vx5jBw5Uq7d33//jQ4dOsDOzg7+/v7IzMx8ZSxlZWXIz8+Xe7Q2VQlQIcNCj4pREkIIIa+ktk/KJ0+eQCKRoH379nLb27dvX+t8nSlTpuDJkycYNGgQWJZFRUUF5syZg+XLl3Nt+vbti6ioKDg6OiIrKwshISHw8vLC77//Dn39mmsWhYWFKcwbam0qisqhAUAi0KBVegkhhJA6qPUSmKpiY2OxYcMG7Ny5E7dv38apU6dw7tw5rF27lmszYsQITJgwAW5ubhCJRDh//jxyc3Nx/PjxWvtdtmwZ8vLyuMeDBw+a4+U0KrZyIUQIaaVeQgghpC5qGwEyNTUFn8/Ho0eP5LY/evQIFhYWNR6zcuVKvP/++5g5cyYAwNXVFUVFRZg9ezZWrFgBHk8xnzMyMkLXrl2Rllb72jgCgQACgaABr0b9mBJZhW4+zf8hhBBC6qS2ESAtLS24u7sjJiaG2yaVShETE4P+/fvXeExxcbFCksPny0Y8WJat8ZjCwkLcvXsXlpaWjRR5y8NKWWiUyyqhauhRAkQIIYTURa2zZRcvXoxp06bBw8MDffr0QXh4OIqKijB9+nQAwNSpU9GxY0eEhYUBAEaPHo2tW7eiV69e6Nu3L9LS0rBy5UqMHj2aS4SCgoIwevRodO7cGf/++y9Wr14NPp+PyZMnq+11NrWqOmAAINSnW+AJIYSQuqg1AZo4cSIeP36MVatWITs7Gz179sTFixe5idGZmZlyIz6ffvopGIbBp59+in/++QdmZmYYPXo01q9fz7V5+PAhJk+ejKdPn8LMzAyDBg3CL7/8AjOz13eF26pFEAvBwlCPEiBCCCGkLgxb27WjNiw/Px+GhobIy8uDgYGBusOpU9n9fDzelYh/IcXtEZ0wx9te3SERQgghzU6Vz+9WdRcYqVnVCJBsFWiaA0QIIYTUhRKg10D1QqhGVAaDEEIIqRMlQK+B6gkQ1QEjhBBC6kYJ0GuALoERQgghqqEE6DUgKZIlQHQJjBBCCFEOJUCvgTKqBE8IIYSohBKg10B5ZQIk1mCgyadvKSGEEFIX+rR8DUgqJ0FLqRAqIYQQohRKgF4HlaUwGB21LuxNCCGEtBqUAL0GeKVVleBpAjQhhBCiDEqAWjm2Qgq+RFbNRIsqwRNCCCFKoQSolataBFECFjpUCZ4QQghRCiVArVzVIogFYGGkSwkQIYQQogxKgFo5KoNBCCGEqI4SoFauehkMWgSREEIIUQ4lQK0cjQARQgghqqMEqJWTlryoA0YJECGEEKIcSoBauapCqHQJjBBCCFEeJUCtnLjwxV1gxnQXGCGEEKIUSoBaOXFlIdRCBtDVolpghBBCiDIoAWrlKipHgCoEfDAMo+ZoCCGEkNaBEqBWruo2eIYqwRNCCCFKowSotSuTFUJlaAI0IYQQojRKgFoxlmWhUZkAaehqqDkaQgghpPWgBKgVY8VS8KSyr4X6AvUGQwghhLQilAC1YlWLIIrBQlePboEnhBBClEUJUCsmLXpRBsOEEiBCCCFEaZQAtWJVd4Dlg4URlcEghBBClEYJUCsmLZGNAOVRHTBCCCFEJZQAtWLVR4CM6TZ4QgghRGmUALVi1ecA0SUwQgghRHmUALViFTQCRAghhNQLJUCtWGl+GQBZAmSoTQkQIYQQoixKgFqx8spCqGINBhp8+lYSQgghylL7p+aOHTtgY2MDoVCIvn374saNG69sHx4eDkdHR2hra8PKygr/+c9/UFpa2qA+WytJ5SUwqZDKYBBCCCGqUGsCdOzYMSxevBirV6/G7du30aNHD4hEIuTk5NTY/quvvkJwcDBWr16NlJQU7N+/H8eOHcPy5cvr3WdrxlbeBg9tSoAIIYQQVag1Adq6dStmzZqF6dOnw9nZGbt374aOjg4OHDhQY/tr165h4MCBmDJlCmxsbPDmm29i8uTJciM8qvbZmvFKZYVQ+VQIlRBCCFGJ2hIgsViMW7duwcfH50UwPB58fHxw/fr1Go8ZMGAAbt26xSU89+7dw/nz5zFy5Mh69wkAZWVlyM/Pl3u0dKyUhYZYVglVi8pgEEIIISpROQHy9vbGoUOHUFJS0qATP3nyBBKJBO3bt5fb3r59e2RnZ9d4zJQpUxAaGopBgwZBU1MT9vb2GDx4MHcJrD59AkBYWBgMDQ25h5WVVYNeW3NgSyvAVH4t0KcEiBBCCFGFyglQr169EBQUBAsLC8yaNQu//PJLU8RVo9jYWGzYsAE7d+7E7du3cerUKZw7dw5r165tUL/Lli1DXl4e93jw4EEjRdx0qspgFIOFoZ5AzdEQQgghrYvKCVB4eDj+/fdfREZGIicnB2+88QacnZ2xefNmPHr0SOl+TE1NwefzFY559OgRLCwsajxm5cqVeP/99zFz5ky4urpi/Pjx2LBhA8LCwiCVSuvVJwAIBAIYGBjIPVo6afGLVaBpEURCCCFENfWaA6ShoYG3334b3333HR4+fIgpU6Zg5cqVsLKywrhx4/Djjz/W2YeWlhbc3d0RExPDbZNKpYiJiUH//v1rPKa4uBg8nnzIfD4fAMCybL36bK0kVAmeEEIIqbcG3T5048YNREZG4ujRozA3N0dAQAD++ecfvPXWW5g3bx42b978yuMXL16MadOmwcPDA3369EF4eDiKioowffp0AMDUqVPRsWNHhIWFAQBGjx6NrVu3olevXujbty/S0tKwcuVKjB49mkuE6urzdcFWGwEypQSIEEIIUYnKCVBOTg4OHz6MyMhI/P333xg9ejS+/vpriEQiMIxsWm5AQAB8fX3rTIAmTpyIx48fY9WqVcjOzkbPnj1x8eJFbhJzZmam3IjPp59+CoZh8Omnn+Kff/6BmZkZRo8ejfXr1yvd5+uiagQoDyy60CUwQgghRCUMy7KsKgdoaWnB3t4eM2bMQEBAAMzMzBTa5OfnY+zYsfjpp58aLdDmlJ+fD0NDQ+Tl5bXY+UB5lzNQ8OMDfAsx3g4egI5G2uoOiRBCCFErVT6/VR4BiomJgZeX1yvbGBgYtNrkp7UoK6RK8IQQQkh9qTwJulOnTvj7778Vtv/999/IyMhojJiIEsoKZJXgi3iAtiZfzdEQQgghrYvKCVBAQACuXbumsP3XX39FQEBAY8RElFBROQIk0eJxc68IIYQQohyVE6D4+HgMHDhQYXu/fv2QkJDQGDERJUgqF0JkqRI8IYQQojKVEyCGYVBQUKCwPS8vDxKJpFGCInVjKhMgRocSIEIIIURVKidAb7zxBsLCwuSSHYlEgrCwMAwaNKhRgyO145fJ3n8NXZoATQghhKhK5eGDTZs24Y033oCjoyN3N9jPP/+M/Px8pVaAJg3HSqTQqJCtXkCV4AkhhBDVqTwC5OzsjKSkJPj5+SEnJwcFBQWYOnUq/vzzT3Tv3r0pYiQvqSqECgA6BpQAEUIIIaqq1wSSDh06YMOGDY0dC1FSVSHUArAw0qVK8IQQQoiq6j2Dtri4GJmZmRCLxXLb3dzcGhwUeTVptTIYRrQIIiGEEKIylROgx48fY/r06bhw4UKN++lOsKYnrVYI1ZgKoRJCCCEqU3kO0KJFi5Cbm4tff/0V2trauHjxIg4ePAgHBwecOXOmKWIkL6kaAcoHC2O6C4wQQghRmcojQD/++CO+++47eHh4gMfjoXPnzhg+fDgMDAwQFhaGUaNGNUWcpJqqEaA8SGFEI0CEEEKIylQeASoqKoK5uTkAwNjYGI8fPwYAuLq64vbt240bHalReVH1QqiUABFCCCGqUjkBcnR0RGpqKgCgR48e+O9//4t//vkHu3fvhqWlZaMHSBSV5ssKoeaDhaE2XQIjhBBCVKXyJbCPPvoIWVlZAIDVq1fD19cXR44cgZaWFqKioho7PlKD8kIx+ADEmjzweVQIlRBCCFGVygnQe++9x33t7u6O+/fv488//4S1tTVMTU0bNThSs4qiCvABSAV8dYdCCCGEtEoqXQIrLy+Hvb09UlJSuG06Ojro3bs3JT/NSFoimwMEqgRPCCGE1ItKCZCmpiZKS0ubKhaiJF6pbK0lni4lQIQQQkh9qDwJev78+di0aRMqKirqbkyahEaZFACgpUt3gBFCCCH1ofIQws2bNxETE4PLly/D1dUVurq6cvtPnTrVaMERRWy5BHxpZSV4fUqACCGEkPpQOQEyMjLCO++80xSxECVULYJYARZ6lAARQggh9aJyAhQZGdkUcRAlSapXgtejSvCEEEJIfag8B4ioV/VK8MZUCZ4QQgipF5VHgGxtbcEwtS++d+/evQYFRF6NKsETQgghDadyArRo0SK55+Xl5YiPj8fFixfxySefNFZcpBbVK8F3phEgQgghpF7qVQqjJjt27MBvv/3W4IDIq1W/BGZCt8ETQggh9dJoc4BGjBiBkydPNlZ3pBalBWIAdAmMEEIIaYhGS4C++eYbmJiYNFZ3pBZl+bIEqJgHCDWpFhghhBBSHypfAuvVq5fcJGiWZZGdnY3Hjx9j586djRocUVReVA4tABValPwQQggh9aVyAjRu3Di55zweD2ZmZhg8eDC6devWWHGRWkgq5wCxVAiVEEIIqTeVP0VXr17dFHEQZZXIboNntGkEiBBCCKkvlecAnT9/HpcuXVLYfunSJVy4cKFRgiK145XJKsFr6NIt8IQQQkh9qZwABQcHQyKRKGxnWRbBwcH1CmLHjh2wsbGBUChE3759cePGjVrbDh48GAzDKDxGjRrFtQkICFDY7+vrW6/YWhKWZaEpllWC19SjO8AIIYSQ+lL5Etjff/8NZ2dnhe3dunVDWlqaygEcO3YMixcvxu7du9G3b1+Eh4dDJBIhNTUV5ubmCu1PnToFsVjMPX/69Cl69OiBCRMmyLXz9fWVq1smELT+ullsmQQ8WSF4aBu0/tdDCCGEqIvKI0CGhoY1lrtIS0uDrq6uygFs3boVs2bNwvTp0+Hs7Izdu3dDR0cHBw4cqLG9iYkJLCwsuEd0dDR0dHQUEiCBQCDXztjYWOXYWpqqMhhlYKFPleAJIYSQelM5ARo7diwWLVqEu3fvctvS0tLw8ccfY8yYMSr1JRaLcevWLfj4+LwIiMeDj48Prl+/rlQf+/fvx6RJkxSSr9jYWJibm8PR0RFz587F06dPVYqtJapeBoMWQSSEEELqT+UE6LPPPoOuri66desGW1tb2NrawsnJCe3atcPmzZtV6uvJkyeQSCRo37693Pb27dsjOzu7zuNv3LiB33//HTNnzpTb7uvri0OHDiEmJgabNm3ClStXMGLEiBrnLgFAWVkZ8vPz5R4tkbTyDrA8sDCmSdCEEEJIvak8B8jQ0BDXrl1DdHQ0EhMToa2tDTc3N7zxxhtNEd8r7d+/H66urujTp4/c9kmTJnFfu7q6ws3NDfb29oiNjcWwYcMU+gkLC0NISEiTx9tQ1UeAOtEIECGEEFJv9VpNj2EYvPnmm3jzzTcbdHJTU1Pw+Xw8evRIbvujR49gYWHxymOLiopw9OhRhIaG1nkeOzs7mJqaIi0trcYEaNmyZVi8eDH3PD8/H1ZWVkq+iuZTNQeILoERQgghDaPyJbCFCxciIiJCYfv27duxaNEilfrS0tKCu7s7YmJiuG1SqRQxMTHo37//K489ceIEysrK8N5779V5nocPH+Lp06ewtLSscb9AIICBgYHcoyUqL6w+B4gugRFCCCH1pXICdPLkSQwcOFBh+4ABA/DNN9+oHMDixYuxd+9eHDx4ECkpKZg7dy6Kioowffp0AMDUqVOxbNkyheP279+PcePGoV27dnLbCwsL8cknn+CXX35BRkYGYmJiMHbsWHTp0gUikUjl+FqSkvwyALIEyEBICRAhhBBSXypfAnv69CkMDQ0VthsYGODJkycqBzBx4kQ8fvwYq1atQnZ2Nnr27ImLFy9yE6MzMzPB48nnaampqYiLi8Ply5cV+uPz+UhKSsLBgweRm5uLDh064M0338TatWtb/VpA4gIxNACINXng8Zg62xNCCCGkZionQF26dMHFixcRGBgot/3ChQuws7OrVxCBgYEK/VWJjY1V2Obo6AiWZWtsr62tXWOpjtdBeVE5NABIBVQHjBBCCGkIlROgxYsXIzAwEI8fP8bQoUMBADExMdiyZQvCw8MbOz5SDVtSVQmeEiBCCCGkIVROgGbMmIGysjKsX78ea9euBQDY2Nhg165dmDp1aqMHSF5gSmTrGPFoAjQhhBDSIPW6DX7u3LmYO3cuHj9+DG1tbejp6QEAnj17BhMTk0YNkLzAF1MleEIIIaQxqHwXWHVmZmbQ09PD5cuX4efnh44dOzZWXOQlrJSFZrls3pOA6oARQgghDVLvBOj+/ftYvXo1bGxsMGHCBPB4PBw6dKgxYyPVSEsqUHXfl45h676bjRBCCFE3lS6BicVinDp1Cvv27cPVq1fh4+ODhw8fIj4+Hq6urk0VI8GLOmBFYGGoRyNAhBBCSEMoPQK0YMECdOjQAV9++SXGjx+Phw8f4uzZs2AYBnw+3ZXU1KrqgOVRGQxCCCGkwZQeAdq1axeWLl2K4OBg6OvrN2VMpAZVdcAKwMKI7gIjhBBCGkTpEaDDhw/jxo0bsLS0xMSJE/H9999DIpE0ZWykGhoBIoQQQhqP0gnQ5MmTER0djTt37qBbt26YP38+LCwsIJVKkZyc3JQxEgDSouqFUCkBIoQQQhpC5bvAbG1tERISgoyMDPzvf//DO++8g/feew+dOnXCwoULmyJGAqAkXwxAlgDRJTBCCCGkYeq1ECIAMAwDkUgEkUiEZ8+e4dChQ4iMjGzM2Eg1pQVl4AEo5gFCTZp0TgghhDREgxZCrGJiYoJFixYhMTGxMbojNSgvlF0Cq9BqlG8ZIYQQ0qbRp2krIamcBC0V1HvQjhBCCCGVKAFqJdjKhRChQ5e/CCGEkIaiBKiV4JXKlhzg0wRoQgghpMEoAWolNMRSAICWHiVAhBBCSEMpNaEkKSlJ6Q7d3NzqHQypGVshhaZEVgleaECFUAkhhJCGUioB6tmzJxiGAcuyYBjmlW1pdejGV1UIVQoWOpQAEUIIIQ2m1CWw9PR03Lt3D+np6Th58iRsbW2xc+dOxMfHIz4+Hjt37oS9vT1OnjzZ1PG2SVVlMArAwliXVoEmhBBCGkqpEaDOnTtzX0+YMAEREREYOXIkt83NzQ1WVlZYuXIlxo0b1+hBtnVVhVCpDAYhhBDSOFSeBH3nzh3Y2toqbLe1taWaYE2kagSIymAQQgghjUPlBMjJyQlhYWEQi8XcNrFYjLCwMDg5OTVqcESmagSIKsETQgghjUPlZYV3796N0aNHo1OnTtwdX0lJSWAYBmfPnm30AAlQVviiEColQIQQQkjDqZwA9enTB/fu3cORI0fw559/AgAmTpyIKVOmQFdXt9EDJEBJXhkA2SRofSGVwiCEEEIaql6fprq6upg9e3Zjx0JqUVYohhYAsSYPPN6rlyEghBBCSN3qtRL04cOHMWjQIHTo0AH3798HAHzxxRf47rvvGjU4IlNRVQleQHXACCGEkMagcgK0a9cuLF68GCNGjMDz58+5hQ+NjY0RHh7e2PERvLgLDEJKgAghhJDGoHICtG3bNuzduxcrVqyAhsaLK2geHh64c+dOowZHKlUWQmW06RZ4QgghpDGonAClp6ejV69eCtsFAgGKiooaJSgij18mS4A0qBAqIYQQ0ihUToBsbW2RkJCgsP3ixYu0DlATYFkWWpWV4AX6dAs8IYQQ0hhUvgts8eLFmD9/PkpLS8GyLG7cuIGvv/4aYWFh2LdvX1PE2Kax5VLwZYXgITSgBIgQQghpDConQDNnzoS2tjY+/fRTFBcXY8qUKejQoQO+/PJLTJo0qSlibNOqVoGuAAsDfaoETwghhDSGeq0D5O/vD39/fxQXF6OwsBDm5uaNHRepVHUHWB5VgieEEEIajcoJ0Lp16+Dv7w9bW1vo6OhAR0enwUHs2LEDn3/+ObKzs9GjRw9s27YNffr0qbHt4MGDceXKFYXtI0eOxLlz5wDI5s2sXr0ae/fuRW5uLgYOHIhdu3bBwcGhwbE2t+qV4I2oDAYhLYJEIkF5ebm6wyCkzdHU1ASf3zhLwqicAJ04cQKrV69G37598d5778HPzw+mpqb1DuDYsWNYvHgxdu/ejb59+yI8PBwikQipqak1jiydOnVKrhDr06dP0aNHD0yYMIHb9tlnnyEiIgIHDx6Era0tVq5cCZFIhOTkZAiFwnrHqg7VK8F3oQSIELViWRbZ2dnIzc1VdyiEtFlGRkawsLAAwzSsMgLDsiyr6kF//PEHjhw5gqNHj+Lhw4cYPnw4/P39MW7cOJVHhPr27QtPT09s374dACCVSmFlZYUFCxYgODi4zuPDw8OxatUqZGVlQVdXFyzLokOHDvj4448RFBQEAMjLy0P79u0RFRWl1Dyl/Px8GBoaIi8vDwYGBiq9nsZWcP1f5H13F/+HcvgsHwBzg9aVwBHyOsnKykJubi7Mzc2ho6PT4F/AhBDlsSyL4uJi5OTkwMjICJaWlgptVPn8rtccIBcXF2zYsAEbNmzA1atX8dVXX2HRokWYM2cO8vPzle5HLBbj1q1bWLZsGbeNx+PBx8cH169fV6qP/fv3Y9KkSVwh1vT0dGRnZ8PHx4drY2hoiL59++L69eutbqJ2Sb6sECpdAiNEvSQSCZf8tGvXTt3hENImaWtrAwBycnJgbm7eoMthDS4trqurC21tbWhpaaGgoEClY588eQKJRIL27dvLbW/fvj1Xaf5Vbty4gd9//x379+/ntmVnZ3N9vNxn1b6XlZWVoaysjHuuShLX1EryxeADKOEBWhr1Kt1GCGkEVXN+GmPeIyGk/qp+BsvLyxuUANXrEzU9PR3r16+Hi4sLPDw8EB8fj5CQkFoTjKayf/9+uLq61jphWllhYWEwNDTkHlZWVo0UYcOJi2Tzncq1qA4YIS0BXfYiRL0a62dQ5QSoX79+6NKlC7755htMnz4d9+/fR0xMDD744AMYGhqq1JepqSn4fD4ePXokt/3Ro0ewsLB45bFFRUU4evQoPvjgA7ntVcep0ueyZcuQl5fHPR48eKDS62hKkiLZX51SKoRKCCGENBqVE6Bhw4bhzp07iI+PR1BQEDp27Fjvk2tpacHd3R0xMTHcNqlUipiYGPTv3/+Vx544cQJlZWV477335Lbb2trCwsJCrs/8/Hz8+uuvtfYpEAhgYGAg92gp2Mrb4CFs8NVKQgh5bWVkZIBhmBpLNRFSE5UToPXr18PZ2RlisRipqamoqKhoUACLFy/G3r17cfDgQaSkpGDu3LkoKirC9OnTAQBTp06VmyRdZf/+/Rg3bpzCZESGYbBo0SKsW7cOZ86cwZ07dzB16lR06NAB48aNa1Cs6sBUVoLn6VICRAipn4CAAJV//zEMg9OnTzdJPPU1ePBgLFq0qMZ9VlZWyMrKQvfu3Zs3KNJqqfypWlJSgsDAQBw8eBAA8Ndff8HOzg4LFixAx44dlbp1vbqJEyfi8ePHWLVqFbKzs9GzZ09cvHiRm8ScmZkJHk8+T0tNTUVcXBwuX75cY59LlixBUVERZs+ejdzcXAwaNAgXL15sdWsAAYCGWJYAadIq0ISQNkosFkNL69W/A/l8fp1TJ5qDRCIBwzAKn1uk5VH5OxQcHIzExETExsbKJRQ+Pj44duxYvYIIDAzE/fv3UVZWhl9//RV9+/bl9sXGxiIqKkquvaOjI1iWxfDhw2vsj2EYhIaGIjs7G6Wlpfjhhx/QtWvXesWmTizLQqtctkyTkCrBE0IayeDBg7Fw4UIsWbIEJiYmsLCwwJo1a7j9NjY2AIDx48eDYRjuOQB899136N27N4RCIezs7BASEiJ3JeDPP//EoEGDIBQK4ezsjB9++EFhNOnBgwfw8/ODkZERTExMMHbsWGRkZHD7q0as1q9fjw4dOsDR0bHO1/TyJbDY2FgwDIOYmBh4eHhAR0cHAwYMQGpqqtxxdb2erVu3wtXVFbq6urCyssK8efNQWFjI7Y+KioKRkRHOnDkDZ2dnCAQCZGZm1hkvUT+VR4BOnz6NY8eOoV+/fnIzsV1cXHD37t1GDa6tY8skXIaqY0iFUAlpaViWRUm5pNnPq63Jb/CdMAcPHsTixYvx66+/4vr16wgICMDAgQMxfPhw3Lx5E+bm5oiMjISvry93q/HPP/+MqVOnIiIiAl5eXrh79y5mz54NAFi9ejUkEgnGjRsHa2tr/PrrrygoKMDHH38sd97y8nKIRCL0798fP//8MzQ0NLBu3Tr4+voiKSmJG+mJiYmBgYEBoqOjG/Q6V6xYgS1btsDMzAxz5szBjBkzcPXqVaVeDyBbmy4iIgK2tra4d+8e5s2bhyVLlmDnzp3cOYqLi7Fp0ybs27cP7dq1o/qYrYTKCdDjx49r/OYWFRXR7aGNTFp5B1gJWBhSJXhCWpyScgmcV11q9vMmh4qgo9WweYFubm7ch7yDgwO2b9+OmJgYDB8+HGZmZgBelByoEhISguDgYEybNg0AYGdnh7Vr12LJkiVYvXo1oqOjcffuXcTGxnLHrV+/Xm60/tixY5BKpdi3bx/3mREZGQkjIyPExsbizTffBCBbY27fvn11Xvqqy/r16+Ht7Q1AdgVj1KhRKC0thVAorPP1AJCbc2RjY4N169Zhzpw5cglQeXk5du7ciR49ejQoVtK8VP4J8vDwwLlz57BgwQIAL+7H37dvX513bhHVyBdC1VRzNISQ14mbm5vcc0tLS+Tk5LzymMTERFy9ehXr16/ntkkkEpSWlqK4uBipqamwsrKSS5peXqctMTERaWlp0NfXl9teWloqdxXB1dW1wckPIP86q0on5OTkwNraus7Xo6Ojgx9++AFhYWH4888/kZ+fj4qKCrn9gOyO5pffT9LyqZwAbdiwASNGjEBycjIqKirw5ZdfIjk5GdeuXauxSjupP2nJiwTIhMpgENLiaGvykRwqUst5G0pTU/6PKoZhIJVKX3lMYWEhQkJC8PbbbyvsU/Ymk8LCQri7u+PIkSMK+6pGngBw5Y0aqvrrrPqDvep11vV6MjIy8NZbb2Hu3LlYv349TExMEBcXhw8++ABisZhLgLS1tekKSCukcgI0aNAgJCQkYOPGjXB1dcXly5fRu3dvXL9+Ha6urk0RY5tVVQk+DyzsKQEipMVhGKbBl6JaKk1NTUgk8vObevfujdTUVHTp0qXGYxwdHfHgwQM8evSIu5P35s2bCn0cO3YM5ubmal9zra7Xc+vWLUilUmzZsoW7q+v48ePNGSJpQvX6ybW3t8fevXsbOxbykrICWRmMArAw0qVLYISQ5mNjY4OYmBgMHDgQAoEAxsbGWLVqFd566y1YW1vj3XffBY/HQ2JiIn7//XesW7cOw4cPh729PaZNm4bPPvsMBQUF+PTTTwG8GH3x9/fH559/jrFjxyI0NBSdOnXC/fv3cerUKSxZsgSdOnV6ZVyPHz9WWOywpqrgyqjr9XTp0gXl5eXYtm0bRo8ejatXr2L37t31OhdpeZS6Db56cdD8/PxXPkjjKc6TFWgtAAt9wev5VyYhpGXasmULoqOjYWVlhV69egEARCIRvv/+e1y+fBmenp7o168fvvjiC3Tu3BmAbC2e06dPo7CwEJ6enpg5cyZWrFgB4MUlMh0dHfzf//0frK2t8fbbb8PJyQkffPABSktLlRoR+uqrr9CrVy+5R33/IK/r9fTo0QNbt27Fpk2b0L17dxw5cgRhYWH1OhdpeRiWZdm6GvH5fGRlZcHc3Bw8Hq/Ga50sy4JhGIUh09YoPz8fhoaGyMvLU+sQ7d2vUyBIfIJvNCuwaO0QtcVBCJFN0k1PT4etrW2rXFRVXa5evYpBgwYhLS0N9vb26g6HvAZe9bOoyue3UsMKP/74I0xMTAAAP/30Uz1DJqqqKBRDAKBCQCuKEkJah2+//RZ6enpwcHBAWloaPvroIwwcOJCSH9LiKJUAVa2h8PLXpGlJKm+DlwqoEjwhpHUoKCjA0qVLkZmZCVNTU/j4+GDLli3qDosQBfWaWJKbm4sbN24gJydH4bbJqVOnNkpgBEDlbfAMrQFECGklpk6dSp8DpFVQOQE6e/Ys/P39UVhYCAMDA7n5QAzD0D/8RsQvk82n0tChCdCEEEJIY1J5csnHH3+MGTNmoLCwELm5uXj+/Dn3ePbsWVPE2GZpiGWja1pUCJUQQghpVConQP/88w8WLlzIrYBJmgYrYSGQVFWCpzpghBBCSGNSOQESiUT47bffmiIWUo20tIL7WpcqwRNCCCGNSqnJJWfOnOG+HjVqFD755BMkJyfD1dVVoZ7MmDFjGjfCNqqqDEYBWBjpUQJECCGENCalEqBx48YpbAsNDVXY9roshNgSVFWCLwALY7oLjBBCCGlUSl0Ck0qlSj0o+Wk81QuhGuvSJGhCSNNgGAanT59WdxgqsbGxQXh4eIvtb/DgwVi0aJHS7TMyMsAwjFyNs6tXr3JXWcaNG4fY2FgwDIPc3NxGi7OtoyWGW6iKIlkClA8WRjQCRAhpgICAgBpH8gEgKysLI0aMaN6AahEVFQWGYbiHnp4e3N3dcerUKbXGFRAQAIZhMGfOHIV98+fPB8MwCAgI4LadOnUKa9euVbp/KysrZGVloXv37ty2xYsXo2fPnkhPT0dUVFRDwq9VW0+qVE6AFi5ciIiICIXt27dvVynjJa9WUlkINR8sjLRpBIgQ0jQsLCwgEKh3niHLsqiokF32NzAwQFZWFrKyshAfHw+RSAQ/Pz+kpqaqNUYrKyscPXoUJSUl3LbS0lJ89dVXsLa2lmtrYmICfX19pfvm8/mwsLCAhsaLWSl3797F0KFD0alTJxgZGTU4fqJI5QTo5MmTGDhwoML2AQMG4JtvvmmUoAhQki9LgIp5gJYGDdQRQppG9UtgVZdiTp06hSFDhkBHRwc9evTA9evX5Y6Ji4uDl5cXtLW1YWVlhYULF6KoqIjbf/jwYXh4eEBfXx8WFhaYMmUKcnJyuP1VIw8XLlyAu7s7BAIB4uLiuHgsLCxgYWEBBwcHrFu3DjweD0lJSbW+hszMTIwdOxZ6enowMDCAn58fHj16JNfm7Nmz8PT0hFAohKmpKcaPH19rf/v27YORkRFiYmK4bb1794aVlZXcaNSpU6dgbW2NXr16yR3/8iUwGxsbbNiwATNmzIC+vj6sra2xZ88ebn/1S2BVXz99+hQzZswAwzC1jgCdPHkSLi4uEAgEsLGxUSg58qrvQ0ZGBoYMkRXZNjY2VhjFagtU/mR9+vQpDA0NFbYbGBjgyZMnjRIUAcoKZJfAyrUo+SGkxWJZQFzU/A+WbdKXtWLFCgQFBSEhIQFdu3bF5MmTuRGau3fvwtfXF++88w6SkpJw7NgxxMXFITAwkDu+vLwca9euRWJiIk6fPo2MjIwaP1yDg4OxceNGpKSkwM3NTWG/RCLBwYMHAcgSkJpIpVKMHTsWz549w5UrVxAdHY179+5h4sSJXJtz585h/PjxGDlyJOLj4xETE4M+ffrU2N9nn32G4OBgXL58GcOGDZPbN2PGDERGRnLPDxw4gOnTp9fyLsrbsmULPDw8EB8fj3nz5mHu3Lk1jmpVXQ4zMDBAeHg4srKy5F5LlVu3bsHPzw+TJk3CnTt3sGbNGqxcuVIuWXrV98HKygonT54EAKSmpiIrKwtffvmlUq/ldaFyjYUuXbrg4sWLcv/YAeDChQuws7NrtMDauooiMQAqhEpIi1ZeDGzo0PznXf4voKXbZN0HBQVh1KhRAICQkBC4uLggLS0N3bp1Q1hYGPz9/bkRDgcHB0RERMDb2xu7du2CUCjEjBkzuL7s7OwQEREBT09PFBYWQk9Pj9sXGhqK4cOHy507Ly+Pa1NSUgJNTU3s2bOn1mryMTExuHPnDtLT02FlZQUAOHToEFxcXHDz5k14enpi/fr1mDRpEkJCQrjjevToodDX0qVLcfjwYVy5cgUuLi4K+9977z0sW7YM9+/fByCbqHz06FHExsbW9ZZi5MiRmDdvHneeL774Aj/99BMcHR3l2lVdDmMYBoaGhrCwsKixv61bt2LYsGFYuXIlAKBr165ITk7G559/ziU5dX0fTExMAADm5uZt8jKbygnQ4sWLERgYiMePH2Po0KEAZP8At2zZ0qiz6Ns6aWUhVFZICRAhpHlVH42xtLQEAOTk5KBbt25ITExEUlISjhw5wrVhWRZSqRTp6elwcnLCrVu3sGbNGiQmJuL58+dc0ezMzEw4Oztzx3l4eCicW19fH7dv3wYAFBcX44cffsCcOXPQrl07jB49WqF9SkoKrKysuOQHAJydnWFkZISUlBR4enoiISEBs2bNeuVr3rJlC4qKivDbb7/V+se8mZkZRo0ahaioKLAsi1GjRsHU1PSV/Vap/p5WXearfllQVSkpKRg7dqzctoEDByI8PBwSiQR8Pl/p70NbpXICNGPGDJSVlWH9+vXcLHcbGxvs2rWLCqE2Il6pbEkBHt0BRkjLpakjG41Rx3mbsvtqC9xWFbyu+vAsLCzEhx9+iIULFyocZ21tjaKiIohEIohEIhw5cgRmZmbIzMyESCSCWCyWa6+rqziKxePx0KVLF+65m5sbLl++jE2bNtWYAClDW1u7zjZeXl44d+4cjh8/juDg4FrbzZgxg7sCsmPHDqVjeHnRYIZhuPe0KajyfWir6lVmfO7cuZg7dy4eP34MbW1tbrjy2bNn3JAaaZiqSvCaupQAEdJiMUyTXopqiXr37o3k5GS5JKW6O3fu4OnTp9i4cSM3KtPQ8kl8Pl/u7qvqnJyc8ODBAzx48IA7X3JyMnJzc7lRDjc3N8TExLxyvk6fPn0QGBgIX19faGhoICgoqMZ2vr6+EIvFYBgGIpGoQa+rIZycnHD16lW5bVevXkXXrl3B5/Px559/1vl90NKS3WHcVtfwq1cCVMXMzAwAcPnyZezbtw9nz56t9R8pUY1WOVWCJ4Q0nry8PLmF9gCgXbt2KvezdOlS9OvXD4GBgZg5cyZ0dXWRnJyM6OhobN++HdbW1tDS0sK2bdswZ84c/P777yqticOyLLKzswHI5gBFR0fj0qVLWLVqVY3tfXx84OrqCn9/f4SHh6OiogLz5s2Dt7c3d4lt9erVGDZsGOzt7TFp0iRUVFTg/PnzWLp0qVxfAwYMwPnz5zFixAhoaGjUuLQLn89HSkoK97W6fPzxx/D09MTatWsxceJEXL9+Hdu3b8fOnTsBQKnvQ+fOncEwDL7//nuMHDlSbkCjLaj3LUb379/H6tWrYWNjgwkTJoDH4+HQoUONGVubxVZIoVk5MqprJFRvMISQ10JsbCx69eol96g+KVhZbm5uuHLlCv766y94eXmhV69eWLVqFTp0kE0GNzMzQ1RUFE6cOAFnZ2ds3LgRmzdvVrr//Px8WFpawtLSEk5OTtiyZQtCQ0OxYsWKGtszDIPvvvsOxsbGeOONN+Dj4wM7OzscO3aMazN48GCcOHECZ86cQc+ePTF06FDcuHGjxv4GDRqEc+fO4dNPP8W2bdtqbGNgYAADAwOlX1NT6N27N44fP46jR4+ie/fuWLVqFUJDQ7kJ0Mp8Hzp27IiQkBAEBwejffv2Cjc3ve4YllX+fkqxWIxTp05h3759uHr1Knx8fHDhwgXEx8fD1dW1KeNsVvn5+TA0NEReXp5a/pFL8suQteEGJGCRMNEOY3t1avYYCCHySktLkZ6eDltbWwiF9IcJIeryqp9FVT6/lR4BWrBgATp06IAvv/wS48ePx8OHD3H27FkwDKPWYcDXUVUh1HywMNalSvCEEEJIY1N6DtCuXbuwdOlSBAcHq7TEN1FdVSFUWSV4mgNECCGENDalR4AOHz6MGzduwNLSEhMnTsT333/fZmeONzVJ0YtK8FQIlRBCCGl8SidAkydPRnR0NO7cuYNu3bph/vz5sLCwgFQqRXJyclPG2OaUFsrWaJBdAqMRIEIIIaSxqXwXmK2tLUJCQpCRkYH//e9/eOedd/Dee++hU6dONS6MRVRXXFkJvpABdLVofhUhhBDS2Op9G3zVIlDHjx/Hv//+i6CgIFy5ckXlfnbs2AEbGxsIhUL07du31lsTq+Tm5mL+/PmwtLSEQCBA165dcf78eW7/mjVrwDCM3KNbt24qx6VOpfmyEaAyDYZbhZUQQgghjadBCyFWMTExwaJFi2pcNOpVjh07hsWLF2P37t3o27cvwsPDIRKJkJqaCnNzc4X2YrEYw4cPh7m5Ob755ht07NgR9+/fVyji5uLigh9++IF7rqHRKC+z2ZQXiiEEUEGV4AkhhJAmodbMYOvWrZg1axa3PPnu3btx7tw5HDhwoMZaLAcOHMCzZ89w7do1rq6KjY2NQjsNDY1aK+i2BpLK2+ClwtaVuBFCCCGthdqGGMRiMW7dugUfH58XwfB48PHxwfXr12s85syZM+jfvz/mz5+P9u3bo3v37tiwYYPC3Wh///03OnToADs7O/j7+yMzM/OVsZSVlSE/P1/uoVaVleAZbZr/QwghhDQFtSVAT548gUQiQfv27eW2t2/fnqsD87J79+7hm2++gUQiwfnz57Fy5Ups2bIF69at49r07dsXUVFRuHjxInbt2oX09HR4eXmhoKCg1ljCwsJgaGjIPaoKx6kLU1kJnk93gBFCmhjDMDh9+rS6w6iXNWvWoGfPniodY2Njg/DwcO55dnY2hg8fDl1dXW46RWt+T4jyWtUkE6lUCnNzc+zZswfu7u6YOHEiVqxYgd27d3NtRowYgQkTJsDNzQ0ikQjnz59Hbm4ujh8/Xmu/y5YtQ15eHvd48OBBc7ycWmmKqRI8IaTxBAQEYNy4cTXuy8rKwogRI5o3oFpERUWBYRg4OTkp7Dtx4gQYhpGb9hAUFISYmBiVznHz5k3Mnj2be/7FF18gKysLCQkJ+Ouvv+ode11qSqqioqLk5rBWvf6XH9XLPQQEBNTYxtfXl2tjY2NTY5uNGzc22etrjeo1yaS0tBRJSUnIycmBVCqV2zdmzBil+jA1NQWfz8ejR4/ktj969KjW+TuWlpbQ1NSUK73h5OSE7OxsiMViaGkpjpgYGRmha9euSEtLqzUWgUAAgaBllJxgWRZaFbLybEIDGgEihDStljBfkmVZbiqDrq4ucnJycP36dfTv359rs3//flhbW8sdp6enp3L1cjMzM7nnd+/ehbu7OxwcHOoZfeMyMDBAamqq3LaX7wb29fVFZGSk3LaXP8NCQ0Mxa9YsuW1UxUGeyiNAFy9ehLW1Nfr164cxY8Zg3Lhx3GP8+PFK96OlpQV3d3e57F0qlSImJkbuH311AwcORFpamlzS9ddff8HS0rLG5AcACgsLcffuXVhaWiodmzqxYik0KsvT6hpSwUVCSNOqPjKRkZEBhmFw6tQpDBkyBDo6OujRo4fCvMy4uDh4eXlBW1sbVlZWWLhwIYqKirj9hw8fhoeHB/T19WFhYYEpU6YgJyeH2x8bGwuGYXDhwgW4u7tDIBAgLi4OgOwmlilTpuDAgQNc+4cPHyI2NhZTpkyRi+PlS2BVI12bN2+GpaUl2rVrh/nz56O8vJxrU/0SmI2NDU6ePIlDhw6BYRiukvrL7ty5g6FDh0JbWxvt2rXD7NmzUVhYyO2/efMmhg8fDlNTUxgaGsLb2xu3b9+WOycAjB8/XmEU62UMw8DCwkLu8fJUEYFAoNDG2NhYrk3Ve1/9oaurW+t52yKVE6AFCxZgwoQJyMrKglQqlXuoWhpj8eLF2Lt3Lw4ePIiUlBTMnTsXRUVF3F1hU6dOxbJly7j2c+fOxbNnz/DRRx/hr7/+wrlz57BhwwbMnz+fa1O1HlFGRgauXbuG8ePHg8/nY/Lkyaq+VLWoqgMmBgsDfRoBIqQlY1kWxeXFzf5gWbZJX9eKFSsQFBSEhIQEdO3aFZMnT0ZFhezmjLt378LX1xfvvPMOkpKScOzYMcTFxSEwMJA7vry8HGvXrkViYiJOnz6NjIyMGpOL4OBgbNy4ESkpKXBzc+O2z5gxA8ePH0dxcTEA2aUhX19fhUSgJj/99BPu3r2Ln376CQcPHkRUVBSioqJqbHvz5k34+vrCz88PWVlZ+PLLLxXaFBUVQSQSwdjYGDdv3sSJEyfwww8/yL3egoICTJs2DXFxcfjll1/g4OCAkSNHcnNPb968CQCIjIxEVlYW95yol8qXwB49eoTFixcr9Q+xLhMnTsTjx4+xatUqZGdno2fPnrh48SLXd2ZmJni8FzmalZUVLl26hP/85z9wc3NDx44d8dFHH2Hp0qVcm4cPH2Ly5Ml4+vQpzMzMMGjQIPzyyy8Kw54tlVwleL2WcVmOEFKzkooS9P2qb7Of99cpv0JHU6fJ+g8KCsKoUaMAACEhIXBxcUFaWhq6deuGsLAw+Pv7c+u+OTg4ICIiAt7e3ti1axeEQiFmzJjB9WVnZ4eIiAh4enqisLBQ7pJVaGgohg8frnD+Xr16wc7ODt988w3ef/99REVFYevWrbh3716dsRsbG2P79u3g8/no1q0bRo0ahZiYGIXLQYDscphAIIC2tnatlwK/+uorlJaW4tChQ9wIyvbt2zF69Ghs2rQJ7du3x9ChQ+WO2bNnD4yMjHDlyhW89dZb3OePkZFRnZcc8/LyFC7reXl54cKFC9zz77//XqHN8uXLsXz5cu750qVL8emnn8q1uXDhAry8vF55/rZE5QTo3XffRWxsLOzt7RslgMDAQLlMurrY2FiFbf3798cvv/xSa39Hjx5tlLjUpWoEKA8sOlIhVEKIGlQfjamaPpCTk4Nu3bohMTERSUlJOHLkCNeGZVlIpVKkp6fDyckJt27dwpo1a5CYmIjnz59z0xYyMzPh7OzMHefh4VFrDDNmzEBkZCSsra1RVFSEkSNHYvv27XXG7uLiIjdP1NLSEnfu3FH+xb8kJSUFPXr0kLt8NHDgQEilUqSmpqJ9+/Z49OgRPv30U8TGxiInJwcSiQTFxcV1LsFSE319fbnLZwCgra0t93zIkCHYtWuX3DYTExO555988onCqFvHjh1Vjud1pnICtH37dkyYMAE///wzXF1duQUJq1A9sIYpr1YI1UWHLoER0pJpa2jj1ym/quW8Tan67/WqCbhVSUxhYSE+/PDDGn/XVyUrIpEIIpEIR44cgZmZGTIzMyESiSAWi+Xav2pOir+/P5YsWYI1a9bg/fffV3pF/5c/kxiGUbhZp7FNmzYNT58+xZdffonOnTtDIBCgf//+Cq9XGTweD126dHllG11d3TrbmJqa1tmmrVM5Afr6669x+fJlCIVCbiJbFYZhKAFqoOJ8WSHUfLAw0qYRIEJaMoZhmvRSVEvUu3dvJCcn1/rheufOHTx9+hQbN27k1lT77bffVD6PiYkJxowZg+PHj8stddLcnJycEBUVhaKiIi5hu3r1Kng8HhwdHbnnO3fuxMiRIwEADx48wJMnT+T60dTUVHmeLGlaKidAK1asQEhICIKDg+Xm55DGUZwnhiaAYj4DDT69v4SQxpGXl4eEhAS5be3atVO5n6VLl6Jfv34IDAzEzJkzoauri+TkZERHR2P79u2wtraGlpYWtm3bhjlz5uD333/H2rVr6xVzVFQUdu7cWa84G4u/vz9Wr16NadOmYc2aNXj8+DEWLFiA999/n5uv6uDgwN35lp+fj08++UThspWNjQ1iYmIwcOBACAQChbu2qrAsW+NiwObm5txnbllZmUIbDQ0NmJqacs8LCgoU2ujo6MDAwED1N+E1pfInrFgsxsSJEyn5aSLiAtkIULkmvb+EkMYTGxuLXr16yT1CQkJU7sfNzQ1XrlzBX3/9BS8vL/Tq1QurVq1Chw4dAMgmFkdFReHEiRNwdnbGxo0bsXnz5nrFXHXbuTrp6Ojg0qVLePbsGTw9PfHuu+9i2LBhcvOR9u/fj+fPn6N37954//33sXDhQoWC3lu2bEF0dDSsrKzQq1evWs+Xn58PS0tLhUf1ZQQuXryosH/QoEFy/axatUqhzZIlSxrpXXk9MKyK91P+5z//gZmZmdxs89dNfn4+DA0NkZeX1+zZ8u97EmF0Lx/fGgILltFsfUJaitLSUqSnp8PW1lZuZV5CSPN61c+iKp/fKl8Ck0gk+Oyzz3Dp0iW4ubkpTDjbunWrql2SaqruAmOpEjwhhBDSZFT+lL1z5w43fPf777/L7Xt5uW6iOqZUtg4QT5sSIEIIIaSpqPwp+9NPPzVFHKQSv0x2u6YGFUIlhBBCmgzNtG1hNMtlCZCAymAQQgghTUblEaAhQ4a88lLXjz/+2KCA2jJWykJQWQle25DKYBBCCCFNReUEqHrlXUBW9C4hIQG///47pk2b1lhxtUlsaQU3JEeV4AkhhJCmo3IC9MUXX9S4fc2aNSgsLGxwQG1ZVSHUYrAwoktghBBCSJNptDlA7733Hg4cONBY3bVJkspb4PPBwpjqgBFCCCFNptESoOvXr9PiYA1UvRK8EVWCJ4QQQpqMypfA3n77bbnnLMsiKysLv/32G1auXNlogbVFJfmyysEFNAJECGkmDMPg22+/xbhx49QdCiHNSuUEyNDQUO55VUXc0NBQvPnmm40WWFtUnCerA1bAsNDR4qs5GkLI6yIgIAC5ubk4ffq0wr6srKxaC3M2t6ioKCxatAi5ubnqDoW0ASonQJGRkU0RBwFQml8GIYAyDR6tqk0IaRYWFhbqDgEsy0Iikag7DNLGNGgOUGFhIfLz8+UepP7EhbJLYBVatD4lIaR5MAzDjQxlZGSAYRicOnUKQ4YMgY6ODnr06IHr16/LHRMXFwcvLy9oa2vDysoKCxcuRFFREbf/8OHD8PDwgL6+PiwsLDBlyhS5auaxsbFgGAYXLlyAu7s7BAIB4uLi6ow1MzMTY8eOhZ6eHgwMDODn54dHjx7JtVm3bh3Mzc2hr6+PmTNnIjg4WGH5FkKAeiRA6enpGDVqFHR1dWFoaAhjY2MYGxvDyMioxQyjtlYVlbfBSwR0+YuQ1oBlWUiLi5v9wbJsk76uFStWICgoCAkJCejatSsmT56MigrZ76e7d+/C19cX77zzDpKSknDs2DHExcUhMDCQO768vBxr165FYmIiTp8+jYyMDAQEBCicJzg4GBs3bkRKSgrc3NxeGZNUKsXYsWPx7NkzXLlyBdHR0bh37x4mTpzItTly5AjWr1+PTZs24datW7C2tsauXbsa500hrx2VL4G99957YFkWBw4cQPv27elSTSNiKxMgUCFUQloFtqQEqb3dm/28jrdvgdHRabL+g4KCMGrUKABASEgIXFxckJaWhm7duiEsLAz+/v5YtGgRAMDBwQERERHw9vbGrl27IBQKMWPGDK4vOzs7REREwNPTE4WFhdDT0+P2hYaGYvjw4UrFFBMTgzt37iA9PR1WVlYAgEOHDsHFxQU3b96Ep6cntm3bhg8++ADTp08HAKxatQqXL1+mNepIjVT+pE1MTMStW7fg6OjYFPG0abzKSvB8HUqACCHqU300xtLSEgCQk5ODbt26ITExEUlJSThy5AjXhmVZSKVSpKenw8nJCbdu3cKaNWuQmJiI58+fQyqV1TjMzMyEs7Mzd5yHh4fSMaWkpMDKyopLfgDA2dkZRkZGSElJgaenJ1JTUzFv3jy54/r06UMlmkiNVP6k9fT0xIMHDygBagJ8seyXhKYe3QJPSGvAaGvD8fYttZy3KWlqvliHrGqUvyqJKSwsxIcffoiFCxcqHGdtbY2ioiKIRCKIRCIcOXIEZmZmyMzMhEgkglgslmuvq6vbhK+CkFdTOQHat28f5syZg3/++Qfdu3eX+0EBUOd1XFI7QWUleCGVwSCkVWAYpkkvRbVEvXv3RnJyMrp06VLj/jt37uDp06fYuHEjN1rz22+/Nfi8Tk5OePDgAR48eMD1m5ycjNzcXG5UydHRETdv3sTUqVO5427evNngc5PXk8oJ0OPHj3H37l3uGisg+yXAsiwYhqFbGeuJlUghkOU/VAmeENLo8vLykJCQILetXbt2KvezdOlS9OvXD4GBgZg5cyZ0dXWRnJyM6OhobN++HdbW1tDS0sK2bdswZ84c/P7771i7dq3S/UskEoU4BQIBfHx84OrqCn9/f4SHh6OiogLz5s2Dt7c3dyltwYIFmDVrFjw8PDBgwAAcO3YMSUlJsLOzU/l1ktefygnQjBkz0KtXL3z99dc0CboRSUtk83+kYKFPleAJIY0sNjYWvXr1ktv2wQcfqNyPm5sbrly5ghUrVsDLywssy8Le3p67G8vMzAxRUVFYvnw5IiIi0Lt3b2zevBljxoxRqv/CwkKFOO3t7ZGWlobvvvsOCxYswBtvvAEejwdfX19s27aNa+fv74979+4hKCgIpaWl8PPzQ0BAAG7cuKHy6ySvP4ZV8X5KXV1dJCYm1jr8+TrIz8+HoaEh8vLyYGBg0CznLM8pxqOtt5APFsVzusPDxqRZzksIUU5paSnS09Nha2tLdQ9bkeHDh8PCwgKHDx9WdyikkbzqZ1GVz2+VR4CGDh362idA6iCtVgnelOqAEUKIyoqLi7F7926IRCLw+Xx8/fXX+OGHHxAdHa3u0EgLpHICNHr0aPznP//BnTt34OrqqjAJWtlhTiKvahXofLDoqksJECGEqIphGJw/fx7r169HaWkpHB0dcfLkSfj4+Kg7NNICqZwAzZkzB4BsAauX0STo+ivKlRVCzQcLQ23NOloTQgh5mba2Nn744Qd1h0FaCZUToKq1IEjjKskrgyaAEj7A59HEckIIIaQpUdXNFqK0QHYJTKxJ3xJCCCGkqSk1AhQREYHZs2dDKBQiIiLilW1rWh2U1K28qKoSPBVCJYQQQpqaUgnQF198AX9/fwiFQnzxxRe1tmMYhhKgepJWFkJlhZQAEUIIIU1NqQQoPT29xq9JI6pcCJGhSvCEEEJIk1N5wklpaWmt+7KyslQOYMeOHbCxsYFQKETfvn3rXLEzNzcX8+fPh6WlJQQCAbp27Yrz5883qM+WgFcmu3tOQ5fuACOEEEKamsoJUO/evRXqtADAyZMnVS6EeuzYMSxevBirV6/G7du30aNHD4hEIuTk5NTYXiwWY/jw4cjIyMA333yD1NRU7N27Fx07dqx3ny2FVmUleC2qBE8IaUYMw+D06dPqDkMlNjY2CA8Pb7H9kdZB5QRo8ODB6NevHzZt2gQAKCoqQkBAAN5//30sX75cpb62bt2KWbNmYfr06XB2dsbu3buho6ODAwcO1Nj+wIEDePbsGU6fPo2BAwfCxsYG3t7e6NGjR737bCkEFbKKJEIqhEoIaWQBAQEYN25cjfuysrIwYsSI5g2oFlFRUWAYhnvo6enB3d0dp06dUmtcAQEBcnFVPdLS0hT2a2lpoUuXLggNDUVFRYVa4yavpnICtHPnTpw8eRLh4eHw8vJCjx49kJCQgBs3buA///mP0v2IxWLcunVLboVOHo8HHx8fXL9+vcZjzpw5g/79+2P+/Plo3749unfvjg0bNnCLL9anTwAoKytDfn6+3KM5seUSaFZWZNM1pBEgQkjzsbCwgECg3j+8WJblkgUDAwNkZWUhKysL8fHxEIlE8PPzQ2pqqlpj9PX15eKqetja2irs//vvv/Hxxx9jzZo1+Pzzz9UYMalLvRadGTFiBN5++21cvXoVmZmZ2LRpE7p3765SH0+ePIFEIkH79u3ltrdv3x7Z2dk1HnPv3j188803kEgkOH/+PFauXIktW7Zg3bp19e4TAMLCwmBoaMg9rKysVHotDVV1B1gFWBhSJXhCSDOqfgksIyMDDMPg1KlTGDJkCHR0dNCjRw+FPyDj4uLg5eUFbW1tWFlZYeHChSgqKuL2Hz58GB4eHtDX14eFhQWmTJkiNw0hNjYWDMPgwoULcHd3h0AgQFxcHBePhYUFLCws4ODggHXr1oHH4yEpKanW15CZmYmxY8dCT08PBgYG8PPzw6NHj+TanD17Fp6enhAKhTA1NcX48eNr7W/fvn0wMjJCTEwMt00gEHBxVT34fL7C/s6dO2Pu3Lnw8fHBmTNnXvHOE3VTOQG6e/cu+vfvj++//x6XLl3CkiVLMGbMGCxZsgTl5eVNESNHKpXC3Nwce/bsgbu7OyZOnIgVK1Zg9+7dDep32bJlyMvL4x4PHjxopIiVI6lMgPLBwojqgBHSarAsi/IySbM/WJZt0te1YsUKBAUFISEhAV27dsXkyZO5EZq7d+/C19cX77zzDpKSknDs2DHExcUhMDCQO768vBxr165FYmIiTp8+jYyMDAQEBCicJzg4GBs3bkRKSkqNc0glEgkOHjwIQDb/tCZSqRRjx47Fs2fPcOXKFURHR+PevXuYOHEi1+bcuXMYP348Ro4cifj4eMTExKBPnz419vfZZ58hODgYly9fxrBhw5R+z16mra0NsVhc7+NJ01P5nuuePXti1KhRuHTpEoyMjDB8+HCMHDkSU6dORXR0NOLj45Xqx9TUFHw+XyFLf/ToESwsLGo8xtLSEpqamnJZt5OTE7KzsyEWi+vVJyDL3NU5BCwtelEI1Y4qwRPSalSIpdjz0ZVmP+/sL72hKWi6NcOCgoIwatQoAEBISAhcXFyQlpaGbt26ISwsDP7+/li0aBEAwMHBAREREfD29sauXbsgFAoxY8YMri87OztERETA09MThYWF0NPT4/aFhoZi+PDhcufOy8vj2pSUlEBTUxN79uyBvb19jbHGxMTgzp07SE9P50bvDx06BBcXF9y8eROenp5Yv349Jk2ahJCQEO646nNHqyxduhSHDx/GlStX4OLiIrfv+++/l4t9xIgROHHihEIfLMsiJiYGly5dwoIFC2qMmbQM9ZoDdPToURgZGXHbBgwYgPj4+Foz9JpoaWnB3d1dbohRKpUiJiYG/fv3r/GYgQMHIi0tTa4e2V9//QVLS0toaWnVq8+WoDj/RSFUY0qACCFqVn00xtLSEgC4S1iJiYmIioqCnp4e9xCJRJBKpdw6cbdu3cLo0aNhbW0NfX19eHt7A5BdqqrOw8ND4dz6+vpISEhAQkIC4uPjsWHDBsyZMwdnz56tMdaUlBRYWVnJTV1wdnaGkZERUlJSAAAJCQl1juZs2bIFe/fuRVxcnELyAwBDhgzh4kpISFCoilCVIAmFQowYMQITJ07EmjVrXnlOol4qjwC9//77NW7X19fH/v37Vepr8eLFmDZtGjw8PNCnTx+Eh4ejqKgI06dPBwBMnToVHTt2RFhYGABg7ty52L59Oz766CMsWLAAf//9NzZs2CC3+nRdfbZERbllYAAUMIA2lcIgpNXQ0OJh9pfeajlvU9LUfLEeGcPIijNX/eFZWFiIDz/8sMZV/62trVFUVASRSASRSIQjR47AzMwMmZmZEIlECpeEdHV1Ffrg8Xjo0qUL99zNzQ2XL1/Gpk2bMHr06Hq9Hm1t7TrbeHl54dy5czh+/DiCg4MV9uvq6srF9bIhQ4Zg165d0NLSQocOHaChQYvatnT1/g4lJycjMzNT7h80wzAq/QOdOHEiHj9+jFWrViE7Oxs9e/bExYsXuUnMmZmZ4PFe/KBbWVnh0qVL+M9//gM3Nzd07NgRH330EZYuXap0ny1RSX4ZdACUaVAVeEJaE4ZhmvRSVEvUu3dvJCcn15oM3LlzB0+fPsXGjRu5UZnffvutQefk8/koKSmpcZ+TkxMePHiABw8ecOdLTk5Gbm4unJ2dAciSqJiYmFf+IdynTx8EBgbC19cXGhoaCAoKUinGuhIk0vKonADdu3cP48ePx507d8AwDDcZr+qvhKpb0pUVGBgoN3muutjYWIVt/fv3xy+//FLvPlsicWE5dACUUyV4QkgTycvLU1jEtl27dir3s3TpUvTr1w+BgYGYOXMmdHV1kZycjOjoaGzfvh3W1tbQ0tLCtm3bMGfOHPz+++9Yu3at0v2zLMvdtVtSUoLo6GhcunQJq1atqrG9j48PXF1d4e/vj/DwcFRUVGDevHnw9vbmLrGtXr0aw4YNg729PSZNmoSKigqcP39e7o9nQDad4/z58xgxYgQ0NDS4eU7k9aTyJ+5HH30EW1tb5OTkQEdHB3/88Qf+7//+Dx4eHjUmLKRuFZWToCVt7C9JQkjziY2NRa9eveQe1ScFK8vNzQ1XrlzBX3/9BS8vL/Tq1QurVq1Chw4dAABmZmaIiorCiRMn4OzsjI0bN2Lz5s1K95+fnw9LS0tYWlrCyckJW7ZsQWhoKFasWFFje4Zh8N1338HY2BhvvPEGfHx8YGdnh2PHjnFtBg8ejBMnTuDMmTPo2bMnhg4dWmuJpEGDBuHcuXP49NNPsW3bNhXeGdLaMKyK91Oamprixx9/hJubGwwNDXHjxg04Ojrixx9/xMcff6z0XWAtWX5+PgwNDZGXlwcDA4MmP1/8Fzdh9qgUFzpoYtbCfk1+PkKI6kpLS5Geng5bW1sIhbReFyHq8qqfRVU+v1UeAZJIJNDX1wcgS4b+/fdfAEDnzp3VvlJnq1Uqu2zI6FAhVEIIIaQ5qDwHqHv37khMTIStrS369u2Lzz77DFpaWtizZw/s7OyaIsbXnoZYlgBp6lECRAghhDQHlROgTz/9lFvyPDQ0FG+99Ra8vLzQrl07uWuuRHla5ZWFUPWpECohhBDSHFROgEQiEfd1ly5d8Oeff+LZs2cwNjbm7gQjymNZFsLKSvDaBpQAEUIIIc2hUVZqMjExaYxu2iS2TIKqe7/0jCgBIoQQQpqD0glQ9dour3LgwIF6B9MWVVWCLwMLQwO6s4QQQghpDkonQFFRUejcuTN69erV5JWI2xJpcTkAIA8sjOkuMEIIIaRZKJ0AzZ07F19//TXS09Mxffp0vPfee3TpqxGUFbyoBG9LhVAJIYSQZqH0OkA7duxAVlYWlixZgrNnz8LKygp+fn64dOkSjQg1QGFeKQBZAmSgTSNAhBBCSHNQaSFEgUCAyZMnIzo6GsnJyXBxccG8efNgY2ODwsLCporxtVaSVyb7P58Bn0d30RFCmhfDMDh9+rS6w1CJjY0NwsPDW2x/pHWod/VNHo/HFUNVtQAqeaE0X3YJTEyV4AkhTSQgIADjxo2rcV9WVhZGjBjRvAHVIioqCgzDcA89PT24u7vj1KlTao3rVe8fab1USoDKysrw9ddfY/jw4ejatSvu3LmD7du3IzMzE3p6ek0V42tNXCibBF0uoErwhJDmZ2FhAYFAvUtwsCyLigrZHbEGBgbIyspCVlYW4uPjIRKJ4OfnR6WWSKNT+lN33rx5sLS0xMaNG/HWW2/hwYMHOHHiBEaOHAkejz6866vqLjBW0ChLMhFCiEqqXwLLyMgAwzA4deoUhgwZAh0dHfTo0QPXr1+XOyYuLg5eXl7Q1taGlZUVFi5cyFUIAIDDhw/Dw8MD+vr6sLCwwJQpU5CTk8Ptj42NBcMwuHDhAtzd3SEQCBAXF8fFY2FhAQsLCzg4OGDdunXg8XhISkqq9TVkZmZi7Nix0NPTg4GBAfz8/PDo0SO5NmfPnoWnpyeEQiFMTU0xfvz4Wvvbt28fjIyMEBMTo9R7eOXKFfTp0wcCgQCWlpYIDg7mEjoAKCgogL+/P3R1dWFpaYkvvvgCgwcPxqJFi5TqnzQNpTOX3bt3w8DAAHZ2drhy5Qpmz56Nt99+W+FBVMOWVP6QaFMCREhrw7IsyktLm/3R1DeerFixAkFBQUhISEDXrl0xefJk7gP97t278PX1xTvvvIOkpCQcO3YMcXFxCAwM5I4vLy/H2rVrkZiYiNOnTyMjIwMBAQEK5wkODsbGjRuRkpICNzc3hf0SiQQHDx4EAPTu3bvGWKVSKcaOHYtnz57hypUriI6Oxr179zBx4kSuzblz5zB+/HiMHDkS8fHxiImJQZ8+fWrs77PPPkNwcDAuX76MYcOG1fle/fPPPxg5ciQ8PT2RmJiIXbt2Yf/+/Vi3bh3XZvHixbh69SrOnDmD6Oho/Pzzz7h9+3adfZOmpfSn7tSpU6nURRPglcnmT2no0h1ghLQ2FWVliJj2brOfd+HBb6ApbLqFU4OCgjBq1CgAQEhICFxcXJCWloZu3bohLCwM/v7+3OiFg4MDIiIi4O3tjV27dkEoFMotnGtnZ4eIiAh4enqisLBQbrpEaGgohg8fLnfuvLw8rk1JSQk0NTWxZ88e2Nvb1xhrTEwM7ty5g/T0dFhZWQEADh06BBcXF9y8eROenp5Yv349Jk2ahJCQEO64Hj16KPS1dOlSHD58GFeuXIGLi4tS79XOnTthZWWF7du3g2EYdOvWDf/++y+WLl2KVatWoaioCAcPHsRXX33FJVSRkZHo0KGDUv2TpqPSQoik8WmKpbL/UyV4QkgLUX00xtLSEgCQk5ODbt26ITExEUlJSThy5AjXhmVZSKVSpKenw8nJCbdu3cKaNWuQmJiI58+fQyqV/Z7LzMyEs7Mzd5yHh4fCufX19bnRkeLiYvzwww+YM2cO2rVrh9GjRyu0T0lJgZWVFZf8AICzszOMjIyQkpICT09PJCQkYNasWa98zVu2bEFRURF+++032NnZKfM2cefv37+/3ADBwIEDUVhYiIcPH+L58+coLy+XG3EyNDSEo6Oj0ucgTYOuu6iZoFz2i4EqwRPS+mgIBFh48Bu1nLcpaWq++IOs6oO9KokpLCzEhx9+iIULFyocZ21tjaKiIohEIohEIhw5cgRmZmbIzMyESCSCWCyWa6+rq6vQB4/HQ5cuXbjnbm5uuHz5MjZt2lRjAqQMbW3tOtt4eXnh3LlzOH78OIKDg+t1HtK6UAKkRqyUhUD2OwW6hpQAEdLaMAzTpJeiWqLevXsjOTlZLkmp7s6dO3j69Ck2btzIjcr89ttvDTonn89HSUlJjfucnJzw4MEDPHjwgDtfcnIycnNzudEmNzc3xMTEYPr06bWeo0+fPggMDISvry80NDQQFBSkVGxOTk44efIkWJblksWrV69CX18fnTp1grGxMTQ1NXHz5k1YW1sDkF3m++uvv/DGG28o/R6QxkcJkBpJSyq4Wej6xm3rlyghpHnl5eUhISFBblu7du1U7mfp0qXo168fAgMDMXPmTOjq6iI5ORnR0dHYvn07rK2toaWlhW3btmHOnDn4/fffsXbtWqX7Z1kW2dnZAGRzgKKjo3Hp0iWsWrWqxvY+Pj5wdXWFv78/wsPDUVFRgXnz5sHb25u7xLZ69WoMGzYM9vb2mDRpEioqKnD+/HksXbpUrq8BAwbg/PnzGDFiBDQ0NOTu0qrt/Zs3bx7Cw8OxYMECBAYGIjU1FatXr8bixYvB4/Ggr6+PadOm4ZNPPoGJiQnMzc2xevVqbi09oj6UAKlR1S3wRWBhpEcjQISQphMbG4tevXrJbfvggw9U7sfNzQ1XrlzBihUr4OXlBZZlYW9vz911ZWZmhqioKCxfvhwRERHo3bs3Nm/ejDFjxijVf35+PjfvSCAQoHPnzggNDVVIVqowDIPvvvsOCxYswBtvvAEejwdfX19s27aNazN48GCcOHECa9euxcaNG2FgYFDr6MugQYNw7tw5jBw5Enw+HwsWLABQ+/u3b98+nD9/Hp988gl69OgBExMTfPDBB/j000+5dlu3bsWcOXPw1ltvwcDAAEuWLMGDBw8gbGOjhy0Nw1IhLwX5+fkwNDREXl4eDAwMmuw8Zffz8HhXEv6FFFbL+sDSsO7r1IQQ9SgtLUV6ejpsbW3pg4s0SFFRETp27IgtW7bUKwlt6171s6jK5zeNAKlRYWUdsHywMKZK8IQQ8lqKj4/Hn3/+iT59+iAvLw+hoaEAgLFjx6o5sraNEiA1KnpeBh6AQoaFUJOv7nAIIYQ0kc2bNyM1NRVaWlpwd3fHzz//DFNTU3WH1aZRAqRGJQVl0AVQSoVQCSHktdWrVy/cunVL3WGQl1ARLzUqK6isBE+jP4QQQkizogRIjSoKZQmQlCrBE0IIIc2KPnnVSFJZCJWlQqiEEEJIs6IESI2YygSIRwkQIYQQ0qwoAVIjflllIVSqBE8IIYQ0K0qA1EirshCqlj6tAUQIIYQ0J0qA1EhYIVuEW9uAymAQQtSDYRicPn1a3WGoxMbGBuHh4eoOg7RylACpCVshhaCyCImeES2rTwhpOgEBARg3blyN+7KysjBixIjmDagWUVFRYBiGe+jp6cHd3R2nTp2Sa3fz5k3Mnj1bTVGS10WLSIB27NgBGxsbCIVC9O3bFzdu3Ki17cs/IAzDKNQCCQgIUGjj6+vb1C9DJdLKCdBSsDAwohEgQoh6WFhYQCBQ7+8glmVRUSH7nWhgYICsrCxkZWUhPj4eIpEIfn5+SE1N5dqbmZlBR0enSWMSi8VN2j9RP7UnQMeOHcPixYuxevVq3L59Gz169IBIJEJOTk6tx1T/AcnKysL9+/cV2vj6+sq1+frrr5vyZaisqhJ8AVgY61ICREhrxLIspGJJsz8as4Z19UtgGRkZYBgGp06dwpAhQ6Cjo4MePXrg+vXrcsfExcXBy8sL2trasLKywsKFC1FUVMTtP3z4MDw8PKCvrw8LCwtMmTJF7nd6bGwsGIbBhQsX4O7uDoFAgLi4OC4eCwsLWFhYwMHBAevWrQOPx0NSUhJ3/MuXwBiGwb59+zB+/Hjo6OjAwcEBZ86c4fZLJBJ88MEHsLW1hba2NhwdHfHll1/KvaaqUbL169ejQ4cOcHR0RGhoKLp3767wnvXs2RMrV65U/c0mLYra77/eunUrZs2ahenTpwMAdu/ejXPnzuHAgQMIDg6u8ZiqH5BXEQgEdbZRp9LKVaDzwaITFUIlpFViy6X4d9W1Zj9vh9ABYLSabgX5FStWYPPmzXBwcMCKFSswefJkpKWlQUNDA3fv3oWvry/WrVuHAwcO4PHjxwgMDERgYCAiIyMBAOXl5Vi7di0cHR2Rk5ODxYsXIyAgAOfPn5c7T3BwMDZv3gw7OzsYGxsjIyNDbr9EIsGhQ4cAAL17935lzCEhIfjss8/w+eefY9u2bfD398f9+/dhYmICqVSKTp064cSJE2jXrh2uXbuG2bNnw9LSEn5+flwfMTExMDAwQHR0NADA0NAQISEhuHnzJjw9PQHICpsmJSUpXJYjrY9aEyCxWIxbt25h2bJl3DYejwcfHx+FvziqKywsROfOnSGVStG7d29s2LABLi4ucm1iY2Nhbm4OY2NjDB06FOvWrUO7du1q7K+srAxlZWXc8/z8/Aa+sroVPi8FAOSBhb5Q7XkoIYRwgoKCMGrUKACyxMLFxQVpaWno1q0bwsLC4O/vj0WLFgEAHBwcEBERAW9vb+zatQtCoRAzZszg+rKzs0NERAQ8PT1RWFgIPT09bl9oaCiGDx8ud+68vDyuTUlJCTQ1NbFnzx7Y29u/MuaAgABMnjwZALBhwwZERETgxo0b8PX1haamJkJCQri2tra2uH79Oo4fPy6XAOnq6mLfvn3Q0nrxR6lIJEJkZCSXAEVGRsLb2xt2dnZKv5+kZVLrJ++TJ08gkUjQvn17ue3t27fHn3/+WeMxjo6OOHDgANzc3JCXl4fNmzdjwIAB+OOPP9CpUycAsstfb7/9NmxtbXH37l0sX74cI0aMwPXr18HnK/7VFBYWJvfD0RyK8kohBFDCZ8DjUTFUQlojRpOHDqED1HLepuTm5sZ9bWlpCQDIyclBt27dkJiYiKSkJBw5coRrw7IspFIp0tPT4eTkhFu3bmHNmjVITEzE8+fPIZXKlvzIzMyEs7Mzd5yHh4fCufX19XH79m0AQHFxMX744QfMmTMH7dq1w+jRo5WKWVdXFwYGBnKX3Xbs2IEDBw4gMzMTJSUlEIvF6Nmzp1wfrq6ucskPAMyaNQszZszA1q1bwePx8NVXX+GLL76oNQ7SerS6oYf+/fujf//+3PMBAwbAyckJ//3vf7F27VoAwKRJk7j9rq6ucHNzg729PWJjYzFs2DCFPpctW4bFixdzz/Pz82FlZdWErwIozRdDCKCMKsET0moxDNOkl6LURVPzxeKsDCP7HVWVxBQWFuLDDz/EwoULFY6ztrZGUVERRCIRRCIRjhw5AjMzM2RmZkIkEilMLNbV1VXog8fjoUuXLtxzNzc3XL58GZs2bXplAlQ95qq4q2I+evQogoKCsGXLFvTv3x/6+vr4/PPP8euvv9YZz+jRoyEQCPDtt99CS0sL5eXlePfdd2uNg7Qeak2ATE1Nwefz8ejRI7ntjx49Unr+jqamJnr16oW0tLRa29jZ2cHU1BRpaWk1JkACgaDZ74IQVxZCrXgNf3kSQl5fvXv3RnJyslySUt2dO3fw9OlTbNy4kftD8rfffmvQOfl8PkpKSup9/NWrVzFgwADMmzeP23b37l2ljtXQ0MC0adMQGRkJLS0tTJo0Cdra2vWOhbQcak2AtLS04O7ujpiYGG6NCqlUipiYGAQGBirVh0QiwZ07dzBy5Mha2zx8+BBPnz7lhnJbAklR5W3wAkqACCFNLy8vDwkJCXLbapsX+SpLly5Fv379EBgYiJkzZ0JXVxfJycmIjo7G9u3bYW1tDS0tLWzbtg1z5szB77//zo3OK4NlWWRnZwOQzQGKjo7GpUuXsGrVKpVjreLg4IBDhw7h0qVLsLW1xeHDh3Hz5k3Y2toqdfzMmTPh5OQEQJZMkdeD2i+BLV68GNOmTYOHhwf69OmD8PBwFBUVcXeFTZ06FR07dkRYWBgA2aS5fv36oUuXLsjNzcXnn3+O+/fvY+bMmQBkw7MhISF45513YGFhgbt372LJkiXo0qULRCKR2l7ny9jKdYBAhVAJIc0gNjYWvXr1ktv2wQcfqNyPm5sbrly5ghUrVsDLywssy8Le3h4TJ04EIFujJyoqCsuXL0dERAR69+6NzZs3Y8yYMUr1n5+fz/2xKhAI0LlzZ4SGhmLp0qUqx1rlww8/RHx8PCZOnAiGYTB58mTMmzcPFy5cUOp4BwcHDBgwAM+ePUPfvn3rHQdpYdgWYNu2bay1tTWrpaXF9unTh/3ll1+4fd7e3uy0adO454sWLeLatm/fnh05ciR7+/Ztbn9xcTH75ptvsmZmZqympibbuXNndtasWWx2drbS8eTl5bEA2Ly8vEZ5fTW5ueEa+2Dp/7H/i7xdd2NCiNqVlJSwycnJbElJibpDIc1MKpWy9vb27JYtW9QdCmFf/bOoyuc3w7KNuKLWayI/Px+GhobIy8uDgYFBk5wjYc1VmJZKcc3dBH4TXOo+gBCiVqWlpUhPT4etra3C6vPk9fX48WMcPXoUy5Ytw4MHD2BsbKzukNq8V/0sqvL5Tddf1ESrQnZ3gtCAFkEkhJCWytzcHKamptizZw8lP68ZSoDUgGVZaFdOAdKhSvCEENJi0UWS15faa4G1RWy5FFUrVlAleEIIIaT5UQKkBtJi2fBPOVgYUSV4QgghpNlRAqQGksIXhVCpEjwhhBDS/CgBUoPCXFnh1XywMNLRrKM1IYQQQhobJUBqUJgnqwRfyAACDVoJmhBCCGlulACpQXGebASolE+FUAkh6sUwDE6fPq3uMFRiY2OD8PBwdYfRbAICArhyUQAwePBgLFq0SG3xvC4oAVKDsgLZHKAyLXr7CSFN7+UP0OqysrIwYsSI5g2oFlFRUWAYhnvo6enB3d0dp06dkmt38+ZNzJ49W01RyisuLsayZctgb28PoVAIMzMzeHt747vvvmuyc546dUql+mqkZrQOkBqUF8kSIAklQIQQNbOwsFB3CGBZFhKJBABgYGCA1NRUAEBBQQEiIyPh5+eHP/74A46OjgBk9caamlgshpZW3QvVzpkzB7/++iu2bdsGZ2dnPH36FNeuXcPTp0+bLDYTE5Mm67stoU9gNai6DZ4V0vwfQoh6Vb8ElpGRAYZhcOrUKQwZMgQ6Ojro0aMHrl+/LndMXFwcvLy8oK2tDSsrKyxcuBBFRUXc/sOHD8PDwwP6+vqwsLDAlClTkJOTw+2PjY0FwzC4cOEC3N3dIRAIEBcXx8VjYWEBCwsLODg4YN26deDxeEhKSuKOf/kSGMMw2LdvH8aPHw8dHR04ODjgzJkz3H6JRIIPPvgAtra20NbWhqOjI7788ku511Q1SrZ+/Xp06NABjo6OCA0NRffu3RXes549e2LlypUAgDNnzmD58uUYOXIkbGxs4O7ujgULFmDGjBlc+7KyMixduhRWVlYQCATo0qUL9u/fr3RsL3v5EpiNjQ02bNiAGTNmQF9fH9bW1tizZ4/cMdeuXUPPnj0hFArh4eGB06dPg2EYJCQkvPJcrzNKgNShshI8o013gBHSmrEsC7FY3OyPpl6deMWKFQgKCkJCQgK6du2KyZMno6JC9nvr7t278PX1xTvvvIOkpCQcO3YMcXFxCAwM5I4vLy/H2rVrkZiYiNOnTyMjIwMBAQEK5wkODsbGjRuRkpICNzc3hf0SiQQHDx4EAPTu3fuVMYeEhMDPzw9JSUkYOXIk/P398ezZMwCAVCpFp06dcOLECSQnJ2PVqlVYvnw5jh8/LtdHTEwMUlNTER0dje+//x4zZsxASkoKbt68ybWJj49HUlISpk+fDkA2gnb+/HkUFBTUGtvUqVPx9ddfIyIiAikpKfjvf/8LPT09lWKry5YtW+Dh4YH4+HjMmzcPc+fO5UbS8vPzMXr0aLi6uuL27dtYu3Ytli5dqlL/ryO6BKYG/DLZUK+GLr39hLRm5eXl2LBhQ7Ofd/ny5UpdnqmvoKAgjBo1CoAssXBxcUFaWhq6deuGsLAw+Pv7cyMQDg4OiIiIgLe3N3bt2gWhUCg3+mFnZ4eIiAh4enqisLCQ++AHgNDQUAwfPlzu3Hl5eVybkpISaGpqYs+ePbC3t39lzAEBAZg8eTIAYMOGDYiIiMCNGzfg6+sLTU1NhISEcG1tbW1x/fp1HD9+HH5+ftx2XV1d7Nu3T+69FYlEiIyMhKenJwAgMjIS3t7esLOzAwDs2bMH/v7+aNeuHXr06IFBgwbh3XffxcCBAwEAf/31F44fP47o6Gj4+Phw70kVZWOry8iRIzFv3jwAwNKlS/HFF1/gp59+gqOjI7766iswDIO9e/dCKBTC2dkZ//zzD2bNmqV0/68jGgFSA81yWSFULT0qhEoIaXmqj8ZYWloCAHcJKzExEVFRUdDT0+MeIpEIUqkU6enpAIBbt25h9OjRsLa2hr6+Pry9vQEAmZmZcufx8PBQOLe+vj4SEhKQkJCA+Ph4bNiwAXPmzMHZs2eVjllXVxcGBgZyl9127NgBd3d3mJmZQU9PD3v27FGIx9XVVSGxnDVrFr7++muUlpZCLBbjq6++kkvw3njjDdy7dw8xMTF499138ccff8DLy4ubpJyQkAA+n8+9BzVRJra6VH/9VZcRq15/amoq3Nzc5Cqn9+nTR6X+X0c0BKEGggrZ8LW2Ia0CTUhrpqmpieXLl6vlvM3VP8PIluuQSmV/uBUWFuLDDz/EwoULFY6ztrZGUVERRCIRRCIRjhw5AjMzM2RmZkIkEkEsFsu119XVVeiDx+OhS5cu3HM3NzdcvnwZmzZtwujRo5WKuSruqpiPHj2KoKAgbNmyBf3794e+vj4+//xz/Prrr3XGM3r0aAgEAnz77bfQ0tJCeXk53n33XYVze3l5wcvLC0uXLsW6desQGhqKpUuXQltbu9aYVYmtLq96/aRmlAA1M5ZloS27AgYdKoRKSKvGMEyTXopqiXr37o3k5GS5JKW6O3fu4OnTp9i4cSOsrKwAAL/99luDzsnn81FSUlLv469evYoBAwZwl4gA2VwmZWhoaGDatGmIjIyElpYWJk2aVGdS4+zsjIqKCpSWlsLV1RVSqRRXrlzhLoE1VmzKcnR0xP/+9z+UlZVBIJD94V19XlNbRQlQM2NLJai698uAEiBCSDPJy8tTuOOnXbt2KvezdOlS9OvXD4GBgZg5cyZ0dXWRnJyM6OhobN++HdbW1tDS0sK2bdswZ84c/P777yqtWcOyLLKzswHI5gBFR0fj0qVLWLVqlcqxVnFwcMChQ4dw6dIl2Nra4vDhw7h58yZsbW2VOn7mzJlwcnICIEtYqhs8eDAmT54MDw8PtGvXDsnJyVi+fDmGDBkCAwMDGBgYYNq0aZgxYwYiIiLQo0cP3L9/Hzk5OfDz82twbMqYMmUKVqxYgdmzZyM4OBiZmZnYvHkzgBcjfG0RJUDNTFpcDgAoAQtjA7oERghpHrGxsejVq5fctg8++EDlftzc3HDlyhWsWLECXl5eYFkW9vb2mDhxIgDZGj1RUVFYvnw5IiIi0Lt3b2zevBljxoxRqv/8/Hxu3pFAIEDnzp25y0n19eGHHyI+Ph4TJ04EwzCYPHky5s2bhwsXLih1vIODAwYMGIBnz56hb9++cvtEIhEOHjyI5cuXo7i4GB06dMBbb70ll7Dt2rULy5cvx7x58/D06VNYW1tzl04bGpsyDAwMcPbsWcydOxc9e/aEq6srVq1ahSlTpsjNC2prGLap76dshfLz82FoaIi8vDwYGBg0at8F6bnI++8dPIIUXVYPgCHdCk9Iq1BaWor09HTY2tq26Q+NtohlWTg4OGDevHlYvHixusNpFEeOHMH06dORl5dX5yW9luZVP4uqfH7TCFAzK3j+ohK8gZDefkIIackeP36Mo0ePIjs7m1v7pzU6dOgQ7Ozs0LFjRyQmJmLp0qXw8/NrdclPY6JP4GZWlFcKbQDFvLZ97ZUQQloDc3NzmJqaYs+ePTA2NlZ3OPWWnZ2NVatWITs7G5aWlpgwYQLWr1+v7rDUihKgZlaSXwZtAGWalPwQQkhL97rMElmyZAmWLFmi7jBaFFoIsZmVF8rWwSjXpDpghBBCiLpQAtTMyotkd4FJBJQAEUIIIepCCVAzYysLoUKbEiBCCCFEXSgBama8Etky0Hwduv2dEEIIURdKgJoZX1xZCV6PEiBCCCFEXSgBamZa5bI7CoT6bat+ECGEENKSUALUzIRVleANaCVZQoj6MQyD06dPqzsMldjY2CA8PFzdYTSbur5HGRkZYBhGodZbQ6n6PsfGxoJhGOTm5jZqHE2FEqBmxEpY6FQuKaFvRHXACCHNIyAgAOPGjatxX1ZWFkaMGNG8AdUiKioKDMNwDz09Pbi7u+PUqVNy7W7evInZs2erKUp5a9as4eLl8/mwsrLC7Nmz8ezZs0Y7R0v5HiUmJmLMmDEwNzeHUCiEjY0NJk6ciJycHADAgAEDkJWVBUNDQzVHqhxKgJqRtKSc+9rQpO0uP04IaTksLCwgEKj3DzKWZVFRIbtD1sDAAFlZWcjKykJ8fDxEIhH8/PyQmprKtTczM4OOjk6TxiQWi5Vu6+LigqysLGRmZiIyMhIXL17E3LlzGy2WlvA9evz4MYYNGwYTExNcunQJKSkpiIyMRIcOHVBUVAQA0NLSgoWFRaupckAJUDOqKJQlQAVgYaRHc4AIIepX/fJK1aWUU6dOYciQIdDR0UGPHj1w/fp1uWPi4uLg5eUFbW1tWFlZYeHChdyHIAAcPnwYHh4e0NfXh4WFBaZMmcKNEgAvLpVcuHAB7u7uEAgEiIuL4+KxsLCAhYUFHBwcsG7dOvB4PCQlJXHHv3xphmEY7Nu3D+PHj4eOjg4cHBxw5swZbr9EIsEHH3wAW1tbaGtrw9HREV9++aXca6oaJVu/fj06dOgAR0dHhIaGonv37grvWc+ePbFy5UruuYaGBiwsLNCxY0f4+PhgwoQJiI6Oljtm3759cHJyglAoRLdu3bBz505un1gsRmBgICwtLSEUCtG5c2eEhYXV+D0CgBs3bqBXr14QCoXw8PBAfHy83LmioqJgZGQkt+306dNyicndu3cxduxYtG/fHnp6evD09MQPP/yg8FqrXL16FXl5edi3bx969eoFW1tbDBkyBF988QVsbW0B1HwJbO/evbCysoKOjg7Gjx+PrVu3ysW2Zs0a9OzZE4cPH4aNjQ0MDQ0xadIkFBQU1BpLY6EEqBkV5JYCkBVCNdKmBIiQ1o5lWUgkxc3+aOryDCtWrEBQUBASEhLQtWtXTJ48mRuhuXv3Lnx9ffHOO+8gKSkJx44dQ1xcHAIDA7njy8vLsXbtWiQmJuL06dPIyMhAQECAwnmCg4OxceNGpKSkwM3NTWG/RCLBwYMHAQC9e/d+ZcwhISHw8/NDUlISRo4cCX9/f+4ylFQqRadOnXDixAkkJydj1apVWL58OY4fPy7XR0xMDFJTUxEdHY3vv/8eM2bMQEpKCm7evMm1iY+PR1JSUq2FUTMyMnDp0iVoab34HX/kyBGsWrUK69evR0pKCjZs2ICVK1dyry0iIgJnzpzB8ePHkZqaiiNHjsDGxqbG/gsLC/HWW2/B2dkZt27dwpo1axAUFPTK96a2fkaOHImYmBjEx8fD19cXo0ePRmZmZo3tLSwsUFFRgW+//Vbpf39Xr17FnDlz8NFHHyEhIQHDhw+vsf7Y3bt3cfr0aXz//ff4/vvvceXKFWzcuFHl16QqqgXWjApyS6EJoIhhoaVBuSchrZ1UWoLYK67Nft7B3nfA5zfdJaCgoCCMGjUKgCyxcHFxQVpaGrp164awsDD4+/tj0aJFAAAHBwdERETA29sbu3btglAoxIwZM7i+7OzsEBERAU9PTxQWFkJPT4/bFxoaiuHDh8udOy8vj2tTUlICTU1N7NmzB/b29q+MOSAgAJMnTwYAbNiwAREREbhx4wZ8fX2hqamJkJAQrq2trS2uX7+O48ePw8/Pj9uuq6uLffv2ySUvIpEIkZGR8PT8//buPCqKK+0D8K9BGpBVRZZ2QVREJIgIimDykUPQdhnEcRRcRmFU4m6MS3SSKAYSBUXG4GFERwUTI6KZgBviLomgqAgGFxYJbgcbFUUWg2D3/f7gUGPbCzQCDfT7nFPHrqpbt+/t16LfrrpVNQwAEBsbC09PT/Tt25crk5OTA0NDQ4jFYlRX1/3QjYyM5NYHBwdjy5YtmDRpEvf+t2/fxo4dOxAQEIAHDx7A1tYWH374IXg8HqytrRX2c//+/ZBIJNi9ezf09PTg4OCAR48eqXzKzcnJCU5OTtx8aGgoEhMTceTIEalktt6IESPw5ZdfYvr06Zg/fz6GDx8OLy8vzJo1CxYWFnLfY9u2bRg7diyXoA0YMADp6ek4duyYVDmJRIK4uDgYGRkBAGbOnImzZ8+2+MNa28S3cHR0NPr06QM9PT24ubnhypUrCsu+O0iOx+NBT0/6iirGGNatWwcrKyvo6+vD29sbBQUFLd2NBr16+bruX+32cX6UEKKZ3j4aY2VlBQDcKawbN24gLi4OhoaG3CQUCiGRSFBUVAQAyMzMhI+PD3r37g0jIyN4enoCgMzRBVdXV5n3NjIyQnZ2NrKzs5GVlYUNGzZg/vz5OHr0aKPbbGBgAGNjY6nTbtHR0XBxcUH37t1haGiInTt3yrTH0dFRKvkBgKCgIMTHx6O6uho1NTXYv3+/VIIHAHZ2dsjOzsbVq1exevVqCIVCLFmyBABQVVWFwsJCzJkzR+oz+/bbb1FYWAigLnnLzs6GnZ0dli5dilOnTinsZ/3Rsre/99zd3ZV+NvJUVlZi5cqVsLe3h6mpKQwNDXHnzh2FR4AA4LvvvoNIJEJMTAwcHBwQExODgQMHIicnR275vLw8DB8+XGrZu/NA3SnN+uQHqPs/93bsWorajwAlJCRg+fLliImJgZubG7Zu3QqhUIi8vDyYm5vL3cbY2FhqQNy7A642bdqEqKgo7N27FzY2Nli7di2EQiFu374tkyy1ptcVdYPqanTaRN5JCHlPWlr6+NhT/h//ln7flqSj878btdb/fZVIJADqvjjnzZuHpUuXymzXu3dvVFVVQSgUQigU4qeffkL37t3x4MEDCIVCmYHFBgYGMnVoaWmhf//+3PzgwYNx6tQphIeHw8fHp1Ftrm93fZsPHDiAlStXYsuWLXB3d4eRkRE2b96MjIyMBtvj4+MDXV1dJCYmgs/no7a2FpMnT5Yqw+fzuTaHhYVh/Pjx+OabbxAaGorKykoAdWNh3NzcpLbT1q57JNLQoUNRVFSEEydO4MyZM/Dz84O3tzd+/vlnhf1VRktLS+Y0VW1trdT8ypUrcfr0aURERKB///7Q19fH5MmTGxz83a1bN0yZMgVTpkzBhg0b4OzsjIiICO50XlMoi11LUnsCFBkZiaCgIO58akxMDI4fP449e/ZgzZo1crepHyQnD2MMW7duxddffw1fX18AwA8//AALCwskJSVh6tSpLdORRvjz9Ru8AsMbPj0HjJCOoO7S55a9GqmtGTp0KG7fvi2VpLwtJycHpaWlCAsLQ69evQAA165de6/31NbWxp9//tnk7dPS0uDh4YGFCxdyy+qPvjSkU6dOCAgIQGxsLPh8PqZOnQp9feUJ6Ndffw0vLy8sWLAAAoEAAoEAf/zxB2bMmKFwG2NjY/j7+8Pf3x+TJ0/GmDFj8Pz5c3Tt2lWqnL29PX788UdUV1dzP+gvX74sVaZ79+6oqKhAVVUVl9S9e4+gtLQ0BAYG4q9//SuAusT23r17jflIOHw+H/369ZMaAP82Ozs7qfFTAGTm1UmthyJqamqQmZkJb29vbpmWlha8vb1lrjp4W2VlJaytrdGrVy/4+vri1q1b3LqioiKIRCKpOk1MTODm5qawztevX6O8vFxqagl3enfGaFTgojVdAk8IaV0vX77kTi3VTw8fPlS5ntWrVyM9PR2LFy9GdnY2CgoKcPjwYW7cSO/evcHn87Ft2zb88ccfOHLkCEJDQxtdP2MMIpEIIpEIRUVF2LlzJ06ePMn9oG0KW1tbXLt2DSdPnkR+fj7Wrl2r0hfx3Llzce7cOaSkpMic/pLH3d0dgwcPxoYNGwDUjaPauHEjoqKikJ+fj5ycHMTGxnLjhCIjIxEfH4/c3Fzk5+fj0KFDsLS0lLmSCwCmT58OHo+HoKAg3L59G8nJyYiIiJAq4+bmhs6dO+PLL79EYWEh9u/fj7i4OJnP5JdffkF2djZu3LiB6dOnKz3qcuzYMfz973/HsWPHkJ+fj7y8PERERCA5OVlhbJYsWYLk5GRERkaioKAAO3bswIkTJ9rMZfJqTYCePXsGsVgsM4DKwsICIpFI7jZ2dnbYs2cPDh8+jH379kEikcDDwwOPHj0CAG47VercuHEjTExMuKn+V0tzqxUz6OlooYsBXQFGCGldFy5cgLOzs9T09sDgxho8eDBSU1ORn5+Pjz76CM7Ozli3bh0EAgGAuqMPcXFxOHToEAYNGoSwsDCZL2hlysvLYWVlBSsrK9jb22PLli0ICQnBV199pXJb682bNw+TJk2Cv78/3NzcUFpaKnU0qCG2trbw8PDAwIEDZU5jKfL5559j165dePjwIebOnYtdu3YhNjYWjo6O8PT0RFxcHHf5uJGRETZt2gRXV1cMGzYM9+7dQ3JyMrS0ZL+iDQ0NcfToUeTk5MDZ2RlfffUVwsPDpcp07doV+/btQ3JyMhwdHREfH4/169dLlYmMjESXLl3g4eEBHx8fCIVCpVfaDRo0CJ07d8aKFSswZMgQjBgxAgcPHsSuXbswc+ZMuduMHDkSMTExiIyMhJOTE1JSUvD555+rdSjK23ispa+nVKK4uBg9evRAenq61CCuL774AqmpqTLnZ+Wpra2Fvb09pk2bhtDQUKSnp2PkyJEoLi7mBu8BgJ+fH3g8HhISEmTqeP36NV6/fs3Nl5eXo1evXnj58iWMjY3fs5eyxBIGba22kQETQhqnuroaRUVFsLGxaTN/wEnrYIzB1tYWCxcuxPLly9XdnHYtKCgIubm5+O2335pch7J9sby8HCYmJo36/lbrGCAzMzNoa2ujpKREanlJSYnCMT7v0tHRgbOzM+7evQsA3HYlJSVSCVBJSQmGDBkitw5dXd1WvcsmJT+EENI+PH36FAcOHIBIJFJ47x+iWEREBEaNGgUDAwOcOHECe/fulboJpDqp9RQYn8+Hi4sLzp49yy2TSCQ4e/Zsoy/rE4vFyMnJ4ZIdGxsbWFpaStVZXl6OjIyMJl0qSAghRHOZm5sjJCQEO3fuRJcuXdTdnHbnypUrGDVqFBwdHRETE4OoqCjMnTtX3c0C0AauAlu+fDkCAgLg6uqK4cOHY+vWraiqquIy7VmzZqFHjx7cbcFDQkIwYsQI9O/fH2VlZdi8eTPu37/PfaA8Hg/Lli3Dt99+C1tbW+4yeIFAoPBhgIQQQog8ahwl0iG8e7fttkTtCZC/vz+ePn2KdevWQSQSYciQIUhJSeEGMT948EBqINiLFy8QFBQEkUiELl26wMXFBenp6Rg0aBBX5osvvkBVVRU+/fRTlJWV4cMPP0RKSgqdtyeEEEIIADUPgm6rVBlERQjRDDQImpC2obkGQdMtiQkhRAX0m5EQ9WqufZASIEIIaYT62/W/evVKzS0hRLPV74PvPkJDVWofA0QIIe2BtrY2TE1NuYc0du7cuc3c0ZYQTcAYw6tXr/DkyROYmppyz1JrKkqACCGkkervM9YaT6omhMhnamra6HsFKkMJECGENBKPx4OVlRXMzc1lnq5NCGl5Ojo6733kpx4lQIQQoiJtbe1m+yNMCFEPGgRNCCGEEI1DCRAhhBBCNA4lQIQQQgjRODQGSI76myyVl5eruSWEEEIIaaz67+3G3CyREiA5KioqAAC9evVSc0sIIYQQoqqKigqYmJgoLUPPApNDIpGguLgYRkZGzX6js/LycvTq1QsPHz7s8M8Zo752XJrUX+prx6VJ/dWUvjLGUFFRAYFAIPUgdXnoCJAcWlpa6NmzZ4u+h7GxcYf+T/g26mvHpUn9pb52XJrUX03oa0NHfurRIGhCCCGEaBxKgAghhBCicSgBamW6uroIDg6Grq6uupvS4qivHZcm9Zf62nFpUn81qa+NRYOgCSGEEKJx6AgQIYQQQjQOJUCEEEII0TiUABFCCCFE41ACRAghhBCNQwlQC4iOjkafPn2gp6cHNzc3XLlyRWn5Q4cOYeDAgdDT04OjoyOSk5NbqaVNt3HjRgwbNgxGRkYwNzfHxIkTkZeXp3SbuLg48Hg8qUlPT6+VWtx069evl2n3wIEDlW7THmNar0+fPjL95fF4WLRokdzy7Smuv/76K3x8fCAQCMDj8ZCUlCS1njGGdevWwcrKCvr6+vD29kZBQUGD9aq6z7cGZX2tra3F6tWr4ejoCAMDAwgEAsyaNQvFxcVK62zKvtBaGoptYGCgTNvHjBnTYL3tLbYA5O6/PB4PmzdvVlhnW45tS6EEqJklJCRg+fLlCA4OxvXr1+Hk5AShUIgnT57ILZ+eno5p06Zhzpw5yMrKwsSJEzFx4kTcvHmzlVuumtTUVCxatAiXL1/G6dOnUVtbi9GjR6OqqkrpdsbGxnj8+DE33b9/v5Va/H4cHByk2n3x4kWFZdtrTOtdvXpVqq+nT58GAEyZMkXhNu0lrlVVVXByckJ0dLTc9Zs2bUJUVBRiYmKQkZEBAwMDCIVCVFdXK6xT1X2+tSjr66tXr3D9+nWsXbsW169fxy+//IK8vDxMmDChwXpV2RdaU0OxBYAxY8ZItT0+Pl5pne0xtgCk+vj48WPs2bMHPB4Pf/vb35TW21Zj22IYaVbDhw9nixYt4ubFYjETCARs48aNcsv7+fmx8ePHSy1zc3Nj8+bNa9F2NrcnT54wACw1NVVhmdjYWGZiYtJ6jWomwcHBzMnJqdHlO0pM63322WesX79+TCKRyF3fXuMKgCUmJnLzEomEWVpass2bN3PLysrKmK6uLouPj1dYj6r7vDq821d5rly5wgCw+/fvKyyj6r6gLvL6GxAQwHx9fVWqp6PE1tfXl3l5eSkt015i25zoCFAzqqmpQWZmJry9vbllWlpa8Pb2xqVLl+Ruc+nSJanyACAUChWWb6tevnwJAOjatavScpWVlbC2tkavXr3g6+uLW7dutUbz3ltBQQEEAgH69u2LGTNm4MGDBwrLdpSYAnX/p/ft24fZs2crfTBwe43r24qKiiASiaRiZ2JiAjc3N4Wxa8o+31a9fPkSPB4PpqamSsupsi+0NRcuXIC5uTns7OywYMEClJaWKizbUWJbUlKC48ePY86cOQ2Wbc+xbQpKgJrRs2fPIBaLYWFhIbXcwsICIpFI7jYikUil8m2RRCLBsmXLMHLkSHzwwQcKy9nZ2WHPnj04fPgw9u3bB4lEAg8PDzx69KgVW6s6Nzc3xMXFISUlBdu3b0dRURE++ugjVFRUyC3fEWJaLykpCWVlZQgMDFRYpr3G9V318VEldk3Z59ui6upqrF69GtOmTVP6oExV94W2ZMyYMfjhhx9w9uxZhIeHIzU1FWPHjoVYLJZbvqPEdu/evTAyMsKkSZOUlmvPsW0qeho8eW+LFi3CzZs3Gzxf7O7uDnd3d27ew8MD9vb22LFjB0JDQ1u6mU02duxY7vXgwYPh5uYGa2trHDx4sFG/qtqz3bt3Y+zYsRAIBArLtNe4kjq1tbXw8/MDYwzbt29XWrY97wtTp07lXjs6OmLw4MHo168fLly4gE8++USNLWtZe/bswYwZMxq8MKE9x7ap6AhQMzIzM4O2tjZKSkqklpeUlMDS0lLuNpaWliqVb2sWL16MY8eO4fz58+jZs6dK2+ro6MDZ2Rl3795toda1DFNTUwwYMEBhu9t7TOvdv38fZ86cwdy5c1Xarr3GtT4+qsSuKft8W1Kf/Ny/fx+nT59WevRHnob2hbasb9++MDMzU9j29h5bAPjtt9+Ql5en8j4MtO/YNhYlQM2Iz+fDxcUFZ8+e5ZZJJBKcPXtW6hfy29zd3aXKA8Dp06cVlm8rGGNYvHgxEhMTce7cOdjY2Khch1gsRk5ODqysrFqghS2nsrIShYWFCtvdXmP6rtjYWJibm2P8+PEqbdde42pjYwNLS0up2JWXlyMjI0Nh7Jqyz7cV9clPQUEBzpw5g27duqlcR0P7Qlv26NEjlJaWKmx7e45tvd27d8PFxQVOTk4qb9ueY9to6h6F3dEcOHCA6erqsri4OHb79m326aefMlNTUyYSiRhjjM2cOZOtWbOGK5+WlsY6derEIiIi2J07d1hwcDDT0dFhOTk56upCoyxYsICZmJiwCxcusMePH3PTq1evuDLv9vWbb75hJ0+eZIWFhSwzM5NNnTqV6enpsVu3bqmjC422YsUKduHCBVZUVMTS0tKYt7c3MzMzY0+ePGGMdZyYvk0sFrPevXuz1atXy6xrz3GtqKhgWVlZLCsriwFgkZGRLCsri7vyKSwsjJmamrLDhw+z33//nfn6+jIbGxv2559/cnV4eXmxbdu2cfMN7fPqoqyvNTU1bMKECaxnz54sOztbah9+/fo1V8e7fW1oX1AnZf2tqKhgK1euZJcuXWJFRUXszJkzbOjQoczW1pZVV1dzdXSE2NZ7+fIl69y5M9u+fbvcOtpTbFsKJUAtYNu2bax3796Mz+ez4cOHs8uXL3PrPD09WUBAgFT5gwcPsgEDBjA+n88cHBzY8ePHW7nFqgMgd4qNjeXKvNvXZcuWcZ+LhYUFGzduHLt+/XrrN15F/v7+zMrKivH5fNajRw/m7+/P7t69y63vKDF928mTJxkAlpeXJ7OuPcf1/Pnzcv/f1vdHIpGwtWvXMgsLC6arq8s++eQTmc/A2tqaBQcHSy1Tts+ri7K+FhUVKdyHz58/z9Xxbl8b2hfUSVl/X716xUaPHs26d+/OdHR0mLW1NQsKCpJJZDpCbOvt2LGD6evrs7KyMrl1tKfYthQeY4y16CEmQgghhJA2hsYAEUIIIUTjUAJECCGEEI1DCRAhhBBCNA4lQIQQQgjROJQAEUIIIUTjUAJECCGEEI1DCRAhhBBCNA4lQISQNufevXvg8XjIzs5Wd1M4ubm5GDFiBPT09DBkyJBmqXP9+vUq18Xj8ZCUlNQs70+IJqMEiBAiIzAwEDweD2FhYVLLk5KSwOPx1NQq9QoODoaBgQHy8vJknvUG1CUmyqb169fLbLNy5Uq5dRFCWl4ndTeAENI26enpITw8HPPmzUOXLl3U3ZxmUVNTAz6f36RtCwsLMX78eFhbW8td//jxY+51QkIC1q1bh7y8PG6ZoaEh95oxBrFYDENDQ6nlhJDWQ0eACCFyeXt7w9LSEhs3blRYRt4pnK1bt6JPnz7cfGBgICZOnIgNGzbAwsICpqamCAkJwZs3b7Bq1Sp07doVPXv2RGxsrEz9ubm58PDwgJ6eHj744AOkpqZKrb958ybGjh0LQ0NDWFhYYObMmXj27Bm3/uOPP8bixYuxbNkymJmZQSgUyu2HRCJBSEgIevbsCV1dXQwZMgQpKSnceh6Ph8zMTISEhCg8mmNpaclNJiYm4PF43Hxubi6MjIxw4sQJuLi4QFdXFxcvXpT5/K5evYpRo0bBzMwMJiYm8PT0xPXr1xV+/jU1NVi8eDGsrKygp6cHa2trpfEihPwPJUCEELm0tbWxYcMGbNu2DY8ePXqvus6dO4fi4mL8+uuviIyMRHBwMP7yl7+gS5cuyMjIwPz58zFv3jyZ91m1ahVWrFiBrKwsuLu7w8fHB6WlpQCAsrIyeHl5wdnZGdeuXUNKSgpKSkrg5+cnVcfevXvB5/ORlpaGmJgYue37/vvvsWXLFkREROD333+HUCjEhAkTUFBQAKDu6I6DgwNWrFiBx48fY+XKlU36HNasWYOwsDDcuXMHgwcPlllfUVGBgIAAXLx4EZcvX4atrS3GjRuHiooKufVFRUXhyJEjOHjwIPLy8vDTTz9JJZ+EECXU/DBWQkgbFBAQwHx9fRljjI0YMYLNnj2bMcZYYmIie/vPRnBwMHNycpLa9l//+heztraWqsva2pqJxWJumZ2dHfvoo4+4+Tdv3jADAwMWHx/PGGPc08rDwsK4MrW1taxnz54sPDycMcZYaGgoGz16tNR7P3z4UOop9p6enszZ2bnB/goEAvbdd99JLRs2bBhbuHAhN+/k5CTzpHBFYmNjmYmJCTdf//TupKQkqXLyPr+3icViZmRkxI4ePcotA8ASExMZY4wtWbKEeXl5MYlE0qh2EUL+h44AEUKUCg8Px969e3Hnzp0m1+Hg4AAtrf/9ubGwsICjoyM3r62tjW7duuHJkydS27m7u3OvO3XqBFdXV64dN27cwPnz57lxNIaGhhg4cCCAuvE69VxcXJS2rby8HMXFxRg5cqTU8pEjR75Xn+VxdXVVur6kpARBQUGwtbWFiYkJjI2NUVlZiQcPHsgtHxgYiOzsbNjZ2WHp0qU4depUs7aXkI6MBkETQpT6v//7PwiFQvzzn/9EYGCg1DotLS0wxqSW1dbWytSho6MjNc/j8eQuk0gkjW5XZWUlfHx8EB4eLrPOysqKe21gYNDoOltaQ20JCAhAaWkpvv/+e1hbW0NXVxfu7u6oqamRW37o0KEoKirCiRMncObMGfj5+cHb2xs///xzSzSfkA6FjgARQhoUFhaGo0eP4tKlS1LLu3fvDpFIJJUENee9ey5fvsy9fvPmDTIzM2Fvbw+g7sv/1q1b6NOnD/r37y81qZL0GBsbQyAQIC0tTWp5WloaBg0a1DwdaaS0tDQsXboU48aNg4ODA3R1daUGdctjbGwMf39//Oc//0FCQgL++9//4vnz563UYkLaL0qACCENcnR0xIwZMxAVFSW1/OOPP8bTp0+xadMmFBYWIjo6GidOnGi2942OjkZiYiJyc3OxaNEivHjxArNnzwYALFq0CM+fP8e0adNw9epVFBYW4uTJk/jHP/4BsVis0vusWrUK4eHhSEhIQF5eHtasWYPs7Gx89tlnzdaXxrC1tcWPP/6IO3fuICMjAzNmzIC+vr7C8pGRkYiPj0dubi7y8/Nx6NAhWFpawtTUtPUaTUg7RQkQIaRRQkJCZE5R2dvb49///jeio6Ph5OSEK1euNPkKKXnCwsIQFhYGJycnXLx4EUeOHIGZmRkAcEdtxGIxRo8eDUdHRyxbtgympqZS440aY+nSpVi+fDlWrFgBR0dHpKSk4MiRI7C1tW22vjTG7t278eLFCwwdOhQzZ87E0qVLYW5urrC8kZERNm3aBFdXVwwbNgz37t1DcnKyyv0nRBPx2Lsn8AkhhBBCOjj6mUAIIYQQjUMJECGEEEI0DiVAhBBCCNE4lAARQgghRONQAkQIIYQQjUMJECGEEEI0DiVAhBBCCNE4lAARQgghRONQAkQIIYQQjUMJECGEEEI0DiVAhBBCCNE4lAARQgghROP8P7WHO16EJXizAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# 横轴：试验次数\n",
    "x_task1_len= range(len(task1))\n",
    "\n",
    "# 绘制曲线\n",
    "plt.plot(x_task1_len, task1, label=\"IntegerLinear \")\n",
    "plt.plot(x_task1_len, task3, label=\"LinearBlockMinifloat\")\n",
    "plt.plot(x_task1_len, task4, label=\"LinearMinifloatIEEE\")\n",
    "plt.plot(x_task1_len, task5, label=\"LinearLog\")\n",
    "plt.plot(x_task1_len, task6, label=\"LinearBlockFP\")\n",
    "plt.plot(x_task1_len, task7, label=\"LinearBlockLog\")\n",
    "plt.plot(x_task1_len, task8, label=\"LinearBinary\")\n",
    "plt.plot(x_task1_len, task9, label=\"LinearBinaryScaling\")\n",
    "plt.plot(x_task1_len, task10, label=\"LinearBinaryResidualSign\")\n",
    "\n",
    "\n",
    "# 图形设置\n",
    "plt.xlabel(\"Number of Trials\")\n",
    "plt.ylabel(\"Maximum Achieved Accuracy\")\n",
    "plt.title(\"IntegerLinear Strategies\")\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32mINFO    \u001b[0m \u001b[34mTokenizing dataset imdb with AutoTokenizer for bert-base-uncased.\u001b[0m\n",
      "[I 2025-02-10 21:23:13,595] A new study created in memory with name: bert-tiny-nas-study\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearBinary'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 01:25, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.444600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.318600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.347100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.349700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.330800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.373800</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 00:36]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-10 21:25:17,495] Trial 0 finished with value: 0.87052 and parameters: {'linear_layer_quantization': 2, 'linear_layer_quantization_fractional': 2, 'LinearMinifloatDenorm': 0, 'LinearMinifloatDenorm_ew': 1, 'LinearMinifloatDenorm_eb': 2, 'LinearMinifloatIEEE': 2, 'LinearMinifloatIEEE_ew': 2, 'LinearMinifloatIEEE_eb': 2, 'LinearLog': 1, 'LinearLog_ew': 2, 'LinearLog_eb': 0, 'LinearBlockFP': 1, 'LinearBlockFP_block_size': 0, 'LinearBlockMinifloat_ebw': 1, 'LinearBlockMinifloat_ew': 0, 'LinearBlockMinifloat_block_size': 0, 'LinearBlockMinifloat_w': 0, 'LinearBlockLog': 1, 'LinearBlockLog_ebw': 2, 'LinearBlockLog_block_size': 2, 'LinearBinary_stochastic': 1, 'LinearBinary_bipolar': 1, 'bert.encoder.layer.0.attention.self.query_type': <class 'chop.nn.quantized.modules.linear.LinearBinary'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.output.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinary'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'chop.nn.quantized.modules.linear.LinearBinary'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.pooler.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinary'>, 'classifier_type': <class 'chop.nn.quantized.modules.linear.LinearBinary'>}. Best is trial 0 with value: 0.87052.\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearBinary'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 01:21, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.478300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.321100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.339400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.334500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.323200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.365000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 00:38]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-10 21:27:19,388] Trial 1 finished with value: 0.8704 and parameters: {'linear_layer_quantization': 1, 'linear_layer_quantization_fractional': 2, 'LinearMinifloatDenorm': 1, 'LinearMinifloatDenorm_ew': 2, 'LinearMinifloatDenorm_eb': 1, 'LinearMinifloatIEEE': 0, 'LinearMinifloatIEEE_ew': 0, 'LinearMinifloatIEEE_eb': 2, 'LinearLog': 1, 'LinearLog_ew': 2, 'LinearLog_eb': 2, 'LinearBlockFP': 1, 'LinearBlockFP_block_size': 0, 'LinearBlockMinifloat_ebw': 0, 'LinearBlockMinifloat_ew': 1, 'LinearBlockMinifloat_block_size': 0, 'LinearBlockMinifloat_w': 0, 'LinearBlockLog': 1, 'LinearBlockLog_ebw': 0, 'LinearBlockLog_block_size': 1, 'LinearBinary_stochastic': 1, 'LinearBinary_bipolar': 1, 'bert.encoder.layer.0.attention.self.query_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinary'>, 'bert.encoder.layer.0.output.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinary'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'chop.nn.quantized.modules.linear.LinearBinary'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'chop.nn.quantized.modules.linear.LinearBinary'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinary'>, 'bert.encoder.layer.1.output.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.pooler.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinary'>, 'classifier_type': <class 'chop.nn.quantized.modules.linear.LinearBinary'>}. Best is trial 0 with value: 0.87052.\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'torch.nn.modules.linear.Linear'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/distributions.py:524: UserWarning: Choices for a categorical distribution should be a tuple of None, bool, int, float and str for persistent storage but contains <class 'chop.nn.quantized.modules.linear.LinearBinary'> which is of type type.\n",
      "  warnings.warn(message)\n",
      "/root/local/mase/src/chop/tools/huggingface.py:157: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='1195' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [1195/3125 00:30 < 00:49, 39.24 it/s, Epoch 0.38/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.421700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.347700</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[W 2025-02-10 21:27:50,896] Trial 2 failed with parameters: {'linear_layer_quantization': 1, 'linear_layer_quantization_fractional': 2, 'LinearMinifloatDenorm': 1, 'LinearMinifloatDenorm_ew': 0, 'LinearMinifloatDenorm_eb': 1, 'LinearMinifloatIEEE': 1, 'LinearMinifloatIEEE_ew': 1, 'LinearMinifloatIEEE_eb': 0, 'LinearLog': 2, 'LinearLog_ew': 0, 'LinearLog_eb': 0, 'LinearBlockFP': 1, 'LinearBlockFP_block_size': 0, 'LinearBlockMinifloat_ebw': 0, 'LinearBlockMinifloat_ew': 2, 'LinearBlockMinifloat_block_size': 2, 'LinearBlockMinifloat_w': 2, 'LinearBlockLog': 1, 'LinearBlockLog_ebw': 2, 'LinearBlockLog_block_size': 0, 'LinearBinary_stochastic': 1, 'LinearBinary_bipolar': 0, 'bert.encoder.layer.0.attention.self.query_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.0.attention.self.value_type': <class 'chop.nn.quantized.modules.linear.LinearBinary'>, 'bert.encoder.layer.0.intermediate.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinary'>, 'bert.encoder.layer.0.output.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinary'>, 'bert.encoder.layer.1.attention.self.key_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.attention.self.value_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.intermediate.dense_type': <class 'torch.nn.modules.linear.Linear'>, 'bert.encoder.layer.1.output.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinary'>, 'bert.pooler.dense_type': <class 'chop.nn.quantized.modules.linear.LinearBinary'>, 'classifier_type': <class 'torch.nn.modules.linear.Linear'>} because of the following error: KeyboardInterrupt().\n",
      "Traceback (most recent call last):\n",
      "  File \"/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/study/_optimize.py\", line 200, in _run_trial\n",
      "    value_or_values = func(trial)\n",
      "                      ^^^^^^^^^^^\n",
      "  File \"/tmp/ipykernel_946/528676388.py\", line 267, in objective\n",
      "    trainer.train()\n",
      "  File \"/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/transformers/trainer.py\", line 2171, in train\n",
      "    return inner_training_loop(\n",
      "           ^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/transformers/trainer.py\", line 2480, in _inner_training_loop\n",
      "    batch_samples, num_items_in_batch = self.get_batch_samples(epoch_iterator, num_batches)\n",
      "                                        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/transformers/trainer.py\", line 5153, in get_batch_samples\n",
      "    batch_samples += [next(epoch_iterator)]\n",
      "                      ^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/accelerate/data_loader.py\", line 461, in __iter__\n",
      "    next_batch = next(dataloader_iter)\n",
      "                 ^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/torch/utils/data/dataloader.py\", line 630, in __next__\n",
      "    data = self._next_data()\n",
      "           ^^^^^^^^^^^^^^^^^\n",
      "  File \"/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/torch/utils/data/dataloader.py\", line 674, in _next_data\n",
      "    data = self._dataset_fetcher.fetch(index)  # may raise StopIteration\n",
      "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/torch/utils/data/_utils/fetch.py\", line 49, in fetch\n",
      "    data = self.dataset.__getitems__(possibly_batched_index)\n",
      "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/datasets/arrow_dataset.py\", line 2807, in __getitems__\n",
      "    batch = self.__getitem__(keys)\n",
      "            ^^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/datasets/arrow_dataset.py\", line 2803, in __getitem__\n",
      "    return self._getitem(key)\n",
      "           ^^^^^^^^^^^^^^^^^^\n",
      "  File \"/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/datasets/arrow_dataset.py\", line 2788, in _getitem\n",
      "    formatted_output = format_table(\n",
      "                       ^^^^^^^^^^^^^\n",
      "  File \"/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/datasets/formatting/formatting.py\", line 629, in format_table\n",
      "    return formatter(pa_table, query_type=query_type)\n",
      "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/datasets/formatting/formatting.py\", line 400, in __call__\n",
      "    return self.format_batch(pa_table)\n",
      "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/datasets/formatting/formatting.py\", line 448, in format_batch\n",
      "    batch = self.python_arrow_extractor().extract_batch(pa_table)\n",
      "            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/datasets/formatting/formatting.py\", line 150, in extract_batch\n",
      "    return pa_table.to_pydict()\n",
      "           ^^^^^^^^^^^^^^^^^^^^\n",
      "KeyboardInterrupt\n",
      "[W 2025-02-10 21:27:50,921] Trial 2 failed with value None.\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[13], line 287\u001b[0m\n\u001b[1;32m    279\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01moptuna\u001b[39;00m\n\u001b[1;32m    281\u001b[0m study \u001b[38;5;241m=\u001b[39m optuna\u001b[38;5;241m.\u001b[39mcreate_study(\n\u001b[1;32m    282\u001b[0m     direction\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmaximize\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    283\u001b[0m     study_name\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mbert-tiny-nas-study\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    284\u001b[0m     sampler\u001b[38;5;241m=\u001b[39msampler,\n\u001b[1;32m    285\u001b[0m )\n\u001b[0;32m--> 287\u001b[0m \u001b[43mstudy\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43moptimize\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    288\u001b[0m \u001b[43m    \u001b[49m\u001b[43mobjective\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    289\u001b[0m \u001b[43m    \u001b[49m\u001b[43mn_trials\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m20\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m    290\u001b[0m \u001b[43m    \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m60\u001b[39;49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43m \u001b[49m\u001b[38;5;241;43m60\u001b[39;49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43m \u001b[49m\u001b[38;5;241;43m24\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m    291\u001b[0m \u001b[43m)\u001b[49m\n\u001b[1;32m    293\u001b[0m task8 \u001b[38;5;241m=\u001b[39m [\u001b[38;5;28mmax\u001b[39m([t\u001b[38;5;241m.\u001b[39mvalue \u001b[38;5;28;01mfor\u001b[39;00m t \u001b[38;5;129;01min\u001b[39;00m study\u001b[38;5;241m.\u001b[39mtrials[:i\u001b[38;5;241m+\u001b[39m\u001b[38;5;241m1\u001b[39m]]) \u001b[38;5;28;01mfor\u001b[39;00m i \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mrange\u001b[39m(\u001b[38;5;28mlen\u001b[39m(study\u001b[38;5;241m.\u001b[39mtrials))]\n\u001b[1;32m    294\u001b[0m task8\n",
      "File \u001b[0;32m/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/study/study.py:451\u001b[0m, in \u001b[0;36mStudy.optimize\u001b[0;34m(self, func, n_trials, timeout, n_jobs, catch, callbacks, gc_after_trial, show_progress_bar)\u001b[0m\n\u001b[1;32m    348\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21moptimize\u001b[39m(\n\u001b[1;32m    349\u001b[0m     \u001b[38;5;28mself\u001b[39m,\n\u001b[1;32m    350\u001b[0m     func: ObjectiveFuncType,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    357\u001b[0m     show_progress_bar: \u001b[38;5;28mbool\u001b[39m \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mFalse\u001b[39;00m,\n\u001b[1;32m    358\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m    359\u001b[0m \u001b[38;5;250m    \u001b[39m\u001b[38;5;124;03m\"\"\"Optimize an objective function.\u001b[39;00m\n\u001b[1;32m    360\u001b[0m \n\u001b[1;32m    361\u001b[0m \u001b[38;5;124;03m    Optimization is done by choosing a suitable set of hyperparameter values from a given\u001b[39;00m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    449\u001b[0m \u001b[38;5;124;03m            If nested invocation of this method occurs.\u001b[39;00m\n\u001b[1;32m    450\u001b[0m \u001b[38;5;124;03m    \"\"\"\u001b[39;00m\n\u001b[0;32m--> 451\u001b[0m     \u001b[43m_optimize\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    452\u001b[0m \u001b[43m        \u001b[49m\u001b[43mstudy\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m    453\u001b[0m \u001b[43m        \u001b[49m\u001b[43mfunc\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mfunc\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    454\u001b[0m \u001b[43m        \u001b[49m\u001b[43mn_trials\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mn_trials\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    455\u001b[0m \u001b[43m        \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtimeout\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    456\u001b[0m \u001b[43m        \u001b[49m\u001b[43mn_jobs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mn_jobs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    457\u001b[0m \u001b[43m        \u001b[49m\u001b[43mcatch\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mtuple\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mcatch\u001b[49m\u001b[43m)\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mif\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[38;5;28;43misinstance\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mcatch\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mIterable\u001b[49m\u001b[43m)\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01melse\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43m(\u001b[49m\u001b[43mcatch\u001b[49m\u001b[43m,\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    458\u001b[0m \u001b[43m        \u001b[49m\u001b[43mcallbacks\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcallbacks\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    459\u001b[0m \u001b[43m        \u001b[49m\u001b[43mgc_after_trial\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mgc_after_trial\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    460\u001b[0m \u001b[43m        \u001b[49m\u001b[43mshow_progress_bar\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mshow_progress_bar\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    461\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/study/_optimize.py:66\u001b[0m, in \u001b[0;36m_optimize\u001b[0;34m(study, func, n_trials, timeout, n_jobs, catch, callbacks, gc_after_trial, show_progress_bar)\u001b[0m\n\u001b[1;32m     64\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m     65\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m n_jobs \u001b[38;5;241m==\u001b[39m \u001b[38;5;241m1\u001b[39m:\n\u001b[0;32m---> 66\u001b[0m         \u001b[43m_optimize_sequential\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m     67\u001b[0m \u001b[43m            \u001b[49m\u001b[43mstudy\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     68\u001b[0m \u001b[43m            \u001b[49m\u001b[43mfunc\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     69\u001b[0m \u001b[43m            \u001b[49m\u001b[43mn_trials\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     70\u001b[0m \u001b[43m            \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     71\u001b[0m \u001b[43m            \u001b[49m\u001b[43mcatch\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     72\u001b[0m \u001b[43m            \u001b[49m\u001b[43mcallbacks\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     73\u001b[0m \u001b[43m            \u001b[49m\u001b[43mgc_after_trial\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     74\u001b[0m \u001b[43m            \u001b[49m\u001b[43mreseed_sampler_rng\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m     75\u001b[0m \u001b[43m            \u001b[49m\u001b[43mtime_start\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m     76\u001b[0m \u001b[43m            \u001b[49m\u001b[43mprogress_bar\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mprogress_bar\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     77\u001b[0m \u001b[43m        \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     78\u001b[0m     \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m     79\u001b[0m         \u001b[38;5;28;01mif\u001b[39;00m n_jobs \u001b[38;5;241m==\u001b[39m \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m:\n",
      "File \u001b[0;32m/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/study/_optimize.py:163\u001b[0m, in \u001b[0;36m_optimize_sequential\u001b[0;34m(study, func, n_trials, timeout, catch, callbacks, gc_after_trial, reseed_sampler_rng, time_start, progress_bar)\u001b[0m\n\u001b[1;32m    160\u001b[0m         \u001b[38;5;28;01mbreak\u001b[39;00m\n\u001b[1;32m    162\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 163\u001b[0m     frozen_trial \u001b[38;5;241m=\u001b[39m \u001b[43m_run_trial\u001b[49m\u001b[43m(\u001b[49m\u001b[43mstudy\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mfunc\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcatch\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    164\u001b[0m \u001b[38;5;28;01mfinally\u001b[39;00m:\n\u001b[1;32m    165\u001b[0m     \u001b[38;5;66;03m# The following line mitigates memory problems that can be occurred in some\u001b[39;00m\n\u001b[1;32m    166\u001b[0m     \u001b[38;5;66;03m# environments (e.g., services that use computing containers such as GitHub Actions).\u001b[39;00m\n\u001b[1;32m    167\u001b[0m     \u001b[38;5;66;03m# Please refer to the following PR for further details:\u001b[39;00m\n\u001b[1;32m    168\u001b[0m     \u001b[38;5;66;03m# https://github.com/optuna/optuna/pull/325.\u001b[39;00m\n\u001b[1;32m    169\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m gc_after_trial:\n",
      "File \u001b[0;32m/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/study/_optimize.py:251\u001b[0m, in \u001b[0;36m_run_trial\u001b[0;34m(study, func, catch)\u001b[0m\n\u001b[1;32m    244\u001b[0m         \u001b[38;5;28;01massert\u001b[39;00m \u001b[38;5;28;01mFalse\u001b[39;00m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mShould not reach.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    246\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m (\n\u001b[1;32m    247\u001b[0m     frozen_trial\u001b[38;5;241m.\u001b[39mstate \u001b[38;5;241m==\u001b[39m TrialState\u001b[38;5;241m.\u001b[39mFAIL\n\u001b[1;32m    248\u001b[0m     \u001b[38;5;129;01mand\u001b[39;00m func_err \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m    249\u001b[0m     \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(func_err, catch)\n\u001b[1;32m    250\u001b[0m ):\n\u001b[0;32m--> 251\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m func_err\n\u001b[1;32m    252\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m frozen_trial\n",
      "File \u001b[0;32m/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/optuna/study/_optimize.py:200\u001b[0m, in \u001b[0;36m_run_trial\u001b[0;34m(study, func, catch)\u001b[0m\n\u001b[1;32m    198\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m get_heartbeat_thread(trial\u001b[38;5;241m.\u001b[39m_trial_id, study\u001b[38;5;241m.\u001b[39m_storage):\n\u001b[1;32m    199\u001b[0m     \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 200\u001b[0m         value_or_values \u001b[38;5;241m=\u001b[39m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtrial\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    201\u001b[0m     \u001b[38;5;28;01mexcept\u001b[39;00m exceptions\u001b[38;5;241m.\u001b[39mTrialPruned \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[1;32m    202\u001b[0m         \u001b[38;5;66;03m# TODO(mamu): Handle multi-objective cases.\u001b[39;00m\n\u001b[1;32m    203\u001b[0m         state \u001b[38;5;241m=\u001b[39m TrialState\u001b[38;5;241m.\u001b[39mPRUNED\n",
      "Cell \u001b[0;32mIn[13], line 267\u001b[0m, in \u001b[0;36mobjective\u001b[0;34m(trial)\u001b[0m\n\u001b[1;32m    257\u001b[0m model \u001b[38;5;241m=\u001b[39m construct_model(trial)\n\u001b[1;32m    259\u001b[0m trainer \u001b[38;5;241m=\u001b[39m get_trainer(\n\u001b[1;32m    260\u001b[0m     model\u001b[38;5;241m=\u001b[39mmodel,\n\u001b[1;32m    261\u001b[0m     tokenized_dataset\u001b[38;5;241m=\u001b[39mdataset,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    264\u001b[0m     num_train_epochs\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m1\u001b[39m,\n\u001b[1;32m    265\u001b[0m )\n\u001b[0;32m--> 267\u001b[0m \u001b[43mtrainer\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtrain\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    268\u001b[0m eval_results \u001b[38;5;241m=\u001b[39m trainer\u001b[38;5;241m.\u001b[39mevaluate()\n\u001b[1;32m    270\u001b[0m trial\u001b[38;5;241m.\u001b[39mset_user_attr(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmodel\u001b[39m\u001b[38;5;124m\"\u001b[39m, model)\n",
      "File \u001b[0;32m/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/transformers/trainer.py:2171\u001b[0m, in \u001b[0;36mTrainer.train\u001b[0;34m(self, resume_from_checkpoint, trial, ignore_keys_for_eval, **kwargs)\u001b[0m\n\u001b[1;32m   2169\u001b[0m         hf_hub_utils\u001b[38;5;241m.\u001b[39menable_progress_bars()\n\u001b[1;32m   2170\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 2171\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43minner_training_loop\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m   2172\u001b[0m \u001b[43m        \u001b[49m\u001b[43margs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   2173\u001b[0m \u001b[43m        \u001b[49m\u001b[43mresume_from_checkpoint\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mresume_from_checkpoint\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   2174\u001b[0m \u001b[43m        \u001b[49m\u001b[43mtrial\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtrial\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   2175\u001b[0m \u001b[43m        \u001b[49m\u001b[43mignore_keys_for_eval\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mignore_keys_for_eval\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   2176\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/transformers/trainer.py:2480\u001b[0m, in \u001b[0;36mTrainer._inner_training_loop\u001b[0;34m(self, batch_size, args, resume_from_checkpoint, trial, ignore_keys_for_eval)\u001b[0m\n\u001b[1;32m   2478\u001b[0m update_step \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;241m1\u001b[39m\n\u001b[1;32m   2479\u001b[0m num_batches \u001b[38;5;241m=\u001b[39m args\u001b[38;5;241m.\u001b[39mgradient_accumulation_steps \u001b[38;5;28;01mif\u001b[39;00m update_step \u001b[38;5;241m!=\u001b[39m (total_updates \u001b[38;5;241m-\u001b[39m \u001b[38;5;241m1\u001b[39m) \u001b[38;5;28;01melse\u001b[39;00m remainder\n\u001b[0;32m-> 2480\u001b[0m batch_samples, num_items_in_batch \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_batch_samples\u001b[49m\u001b[43m(\u001b[49m\u001b[43mepoch_iterator\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mnum_batches\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   2481\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m i, inputs \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28menumerate\u001b[39m(batch_samples):\n\u001b[1;32m   2482\u001b[0m     step \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;241m1\u001b[39m\n",
      "File \u001b[0;32m/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/transformers/trainer.py:5153\u001b[0m, in \u001b[0;36mTrainer.get_batch_samples\u001b[0;34m(self, epoch_iterator, num_batches)\u001b[0m\n\u001b[1;32m   5151\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m _ \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mrange\u001b[39m(num_batches):\n\u001b[1;32m   5152\u001b[0m     \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m-> 5153\u001b[0m         batch_samples \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m [\u001b[38;5;28mnext\u001b[39m(epoch_iterator)]\n\u001b[1;32m   5154\u001b[0m     \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mStopIteration\u001b[39;00m:\n\u001b[1;32m   5155\u001b[0m         \u001b[38;5;28;01mbreak\u001b[39;00m\n",
      "File \u001b[0;32m/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/accelerate/data_loader.py:461\u001b[0m, in \u001b[0;36mDataLoaderShard.__iter__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    459\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdevice \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m    460\u001b[0m     current_batch \u001b[38;5;241m=\u001b[39m send_to_device(current_batch, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdevice)\n\u001b[0;32m--> 461\u001b[0m next_batch \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mnext\u001b[39m(dataloader_iter)\n\u001b[1;32m    462\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m batch_index \u001b[38;5;241m>\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mskip_batches:\n\u001b[1;32m    463\u001b[0m     \u001b[38;5;28;01myield\u001b[39;00m current_batch\n",
      "File \u001b[0;32m/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/torch/utils/data/dataloader.py:630\u001b[0m, in \u001b[0;36m_BaseDataLoaderIter.__next__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    627\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_sampler_iter \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m    628\u001b[0m     \u001b[38;5;66;03m# TODO(https://github.com/pytorch/pytorch/issues/76750)\u001b[39;00m\n\u001b[1;32m    629\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_reset()  \u001b[38;5;66;03m# type: ignore[call-arg]\u001b[39;00m\n\u001b[0;32m--> 630\u001b[0m data \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_next_data\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    631\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_num_yielded \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;241m1\u001b[39m\n\u001b[1;32m    632\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_dataset_kind \u001b[38;5;241m==\u001b[39m _DatasetKind\u001b[38;5;241m.\u001b[39mIterable \u001b[38;5;129;01mand\u001b[39;00m \\\n\u001b[1;32m    633\u001b[0m         \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_IterableDataset_len_called \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m \\\n\u001b[1;32m    634\u001b[0m         \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_num_yielded \u001b[38;5;241m>\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_IterableDataset_len_called:\n",
      "File \u001b[0;32m/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/torch/utils/data/dataloader.py:674\u001b[0m, in \u001b[0;36m_SingleProcessDataLoaderIter._next_data\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    672\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21m_next_data\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n\u001b[1;32m    673\u001b[0m     index \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_next_index()  \u001b[38;5;66;03m# may raise StopIteration\u001b[39;00m\n\u001b[0;32m--> 674\u001b[0m     data \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_dataset_fetcher\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfetch\u001b[49m\u001b[43m(\u001b[49m\u001b[43mindex\u001b[49m\u001b[43m)\u001b[49m  \u001b[38;5;66;03m# may raise StopIteration\u001b[39;00m\n\u001b[1;32m    675\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_pin_memory:\n\u001b[1;32m    676\u001b[0m         data \u001b[38;5;241m=\u001b[39m _utils\u001b[38;5;241m.\u001b[39mpin_memory\u001b[38;5;241m.\u001b[39mpin_memory(data, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_pin_memory_device)\n",
      "File \u001b[0;32m/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/torch/utils/data/_utils/fetch.py:49\u001b[0m, in \u001b[0;36m_MapDatasetFetcher.fetch\u001b[0;34m(self, possibly_batched_index)\u001b[0m\n\u001b[1;32m     47\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mauto_collation:\n\u001b[1;32m     48\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mhasattr\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdataset, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m__getitems__\u001b[39m\u001b[38;5;124m\"\u001b[39m) \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdataset\u001b[38;5;241m.\u001b[39m__getitems__:\n\u001b[0;32m---> 49\u001b[0m         data \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdataset\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m__getitems__\u001b[49m\u001b[43m(\u001b[49m\u001b[43mpossibly_batched_index\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     50\u001b[0m     \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m     51\u001b[0m         data \u001b[38;5;241m=\u001b[39m [\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdataset[idx] \u001b[38;5;28;01mfor\u001b[39;00m idx \u001b[38;5;129;01min\u001b[39;00m possibly_batched_index]\n",
      "File \u001b[0;32m/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/datasets/arrow_dataset.py:2807\u001b[0m, in \u001b[0;36mDataset.__getitems__\u001b[0;34m(self, keys)\u001b[0m\n\u001b[1;32m   2805\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21m__getitems__\u001b[39m(\u001b[38;5;28mself\u001b[39m, keys: List) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m List:\n\u001b[1;32m   2806\u001b[0m \u001b[38;5;250m    \u001b[39m\u001b[38;5;124;03m\"\"\"Can be used to get a batch using a list of integers indices.\"\"\"\u001b[39;00m\n\u001b[0;32m-> 2807\u001b[0m     batch \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[38;5;21;43m__getitem__\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mkeys\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   2808\u001b[0m     n_examples \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mlen\u001b[39m(batch[\u001b[38;5;28mnext\u001b[39m(\u001b[38;5;28miter\u001b[39m(batch))])\n\u001b[1;32m   2809\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m [{col: array[i] \u001b[38;5;28;01mfor\u001b[39;00m col, array \u001b[38;5;129;01min\u001b[39;00m batch\u001b[38;5;241m.\u001b[39mitems()} \u001b[38;5;28;01mfor\u001b[39;00m i \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mrange\u001b[39m(n_examples)]\n",
      "File \u001b[0;32m/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/datasets/arrow_dataset.py:2803\u001b[0m, in \u001b[0;36mDataset.__getitem__\u001b[0;34m(self, key)\u001b[0m\n\u001b[1;32m   2801\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21m__getitem__\u001b[39m(\u001b[38;5;28mself\u001b[39m, key):  \u001b[38;5;66;03m# noqa: F811\u001b[39;00m\n\u001b[1;32m   2802\u001b[0m \u001b[38;5;250m    \u001b[39m\u001b[38;5;124;03m\"\"\"Can be used to index columns (by string names) or rows (by integer index or iterable of indices or bools).\"\"\"\u001b[39;00m\n\u001b[0;32m-> 2803\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_getitem\u001b[49m\u001b[43m(\u001b[49m\u001b[43mkey\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/datasets/arrow_dataset.py:2788\u001b[0m, in \u001b[0;36mDataset._getitem\u001b[0;34m(self, key, **kwargs)\u001b[0m\n\u001b[1;32m   2786\u001b[0m formatter \u001b[38;5;241m=\u001b[39m get_formatter(format_type, features\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_info\u001b[38;5;241m.\u001b[39mfeatures, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mformat_kwargs)\n\u001b[1;32m   2787\u001b[0m pa_subtable \u001b[38;5;241m=\u001b[39m query_table(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_data, key, indices\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_indices \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_indices \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m)\n\u001b[0;32m-> 2788\u001b[0m formatted_output \u001b[38;5;241m=\u001b[39m \u001b[43mformat_table\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m   2789\u001b[0m \u001b[43m    \u001b[49m\u001b[43mpa_subtable\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mkey\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mformatter\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mformatter\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mformat_columns\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mformat_columns\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43moutput_all_columns\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43moutput_all_columns\u001b[49m\n\u001b[1;32m   2790\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   2791\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m formatted_output\n",
      "File \u001b[0;32m/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/datasets/formatting/formatting.py:629\u001b[0m, in \u001b[0;36mformat_table\u001b[0;34m(table, key, formatter, format_columns, output_all_columns)\u001b[0m\n\u001b[1;32m    627\u001b[0m python_formatter \u001b[38;5;241m=\u001b[39m PythonFormatter(features\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m)\n\u001b[1;32m    628\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m format_columns \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m--> 629\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mformatter\u001b[49m\u001b[43m(\u001b[49m\u001b[43mpa_table\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mquery_type\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mquery_type\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    630\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m query_type \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcolumn\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n\u001b[1;32m    631\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m key \u001b[38;5;129;01min\u001b[39;00m format_columns:\n",
      "File \u001b[0;32m/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/datasets/formatting/formatting.py:400\u001b[0m, in \u001b[0;36mFormatter.__call__\u001b[0;34m(self, pa_table, query_type)\u001b[0m\n\u001b[1;32m    398\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mformat_column(pa_table)\n\u001b[1;32m    399\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m query_type \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mbatch\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n\u001b[0;32m--> 400\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mformat_batch\u001b[49m\u001b[43m(\u001b[49m\u001b[43mpa_table\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/datasets/formatting/formatting.py:448\u001b[0m, in \u001b[0;36mPythonFormatter.format_batch\u001b[0;34m(self, pa_table)\u001b[0m\n\u001b[1;32m    446\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mlazy:\n\u001b[1;32m    447\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m LazyBatch(pa_table, \u001b[38;5;28mself\u001b[39m)\n\u001b[0;32m--> 448\u001b[0m batch \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mpython_arrow_extractor\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mextract_batch\u001b[49m\u001b[43m(\u001b[49m\u001b[43mpa_table\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    449\u001b[0m batch \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpython_features_decoder\u001b[38;5;241m.\u001b[39mdecode_batch(batch)\n\u001b[1;32m    450\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m batch\n",
      "File \u001b[0;32m/srcPkgs/miniconda3/envs/mase/lib/python3.11/site-packages/datasets/formatting/formatting.py:150\u001b[0m, in \u001b[0;36mPythonArrowExtractor.extract_batch\u001b[0;34m(self, pa_table)\u001b[0m\n\u001b[1;32m    149\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21mextract_batch\u001b[39m(\u001b[38;5;28mself\u001b[39m, pa_table: pa\u001b[38;5;241m.\u001b[39mTable) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m \u001b[38;5;28mdict\u001b[39m:\n\u001b[0;32m--> 150\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mpa_table\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mto_pydict\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "from chop.tools import get_tokenized_dataset\n",
    "checkpoint = \"prajjwal1/bert-tiny\"\n",
    "tokenizer_checkpoint = \"bert-base-uncased\"\n",
    "dataset_name = \"imdb\"\n",
    "dataset, tokenizer = get_tokenized_dataset(\n",
    "    dataset=dataset_name,\n",
    "    checkpoint=tokenizer_checkpoint,\n",
    "    return_tokenizer=True,\n",
    ")\n",
    "from transformers import AutoModel\n",
    "\n",
    "model = AutoModel.from_pretrained(checkpoint)\n",
    "from chop.tools.utils import deepsetattr\n",
    "from copy import deepcopy\n",
    "from transformers import AutoConfig, AutoModelForSequenceClassification\n",
    "\n",
    "from pathlib import Path\n",
    "import dill\n",
    "import torch\n",
    "with open(f\"{Path.home()}/tutorial_5_best_model.pkl\", \"rb\") as f:\n",
    "    base_model = dill.load(f)\n",
    "from chop.nn.quantized.modules.linear import (\n",
    "    LinearInteger,\n",
    "    LinearMinifloatDenorm,\n",
    "    LinearMinifloatIEEE,\n",
    "    LinearLog,\n",
    "    LinearBlockFP,\n",
    "    LinearBlockMinifloat,\n",
    "    LinearBlockLog,\n",
    "    LinearBinary,\n",
    "    LinearBinaryScaling,\n",
    "    LinearBinaryResidualSign,\n",
    ")\n",
    "search_space = {\n",
    "    \"linear_layer_choices\": [\n",
    "        torch.nn.Linear,\n",
    "        # LinearInteger,\n",
    "        # LinearMinifloatDenorm,\n",
    "        # LinearMinifloatIEEE,\n",
    "        # LinearLog,\n",
    "        # LinearBlockFP,\n",
    "        # # LinearBlockMinifloat,\n",
    "        # LinearBlockLog,\n",
    "        LinearBinary,\n",
    "        # LinearBinaryScaling,\n",
    "        # LinearBinaryResidualSign,\n",
    "    ],\n",
    "    'linear_layer_quantization': [8, 16, 32],\n",
    "    'linear_layer_quantization_fractional':  [2, 4, 8],\n",
    "    'LinearMinifloatDenorm': [8, 16, 32],\n",
    "    'LinearMinifloatDenorm_ew': [2, 4, 8],\n",
    "    'LinearMinifloatDenorm_eb': [2, 4, 8],\n",
    "    'LinearMinifloatIEEE': [8, 16, 32],\n",
    "    'LinearMinifloatIEEE_ew': [2, 4, 8],\n",
    "    'LinearMinifloatIEEE_eb': [2, 4, 8],\n",
    "    'LinearLog': [8, 16, 32],\n",
    "    'LinearLog_ew': [2, 4, 8],\n",
    "    'LinearLog_eb': [2, 4, 8],\n",
    "    'LinearBlockFP': [8, 16, 32],\n",
    "    'LinearBlockFP_block_size':[[4], [8], [16]] ,\n",
    "    'LinearBlockMinifloat_ebw': [2, 4, 8],\n",
    "    'LinearBlockMinifloat_ew': [2, 4, 8],\n",
    "    'LinearBlockMinifloat_block_size': [[4], [8], [16]],\n",
    "    'LinearBlockMinifloat_w': [2, 4, 8],\n",
    "    'LinearBlockLog': [8, 16, 32],\n",
    "    'LinearBlockLog_ebw': [2, 4, 8],\n",
    "    'LinearBlockLog_block_size': [[4], [8], [16]] ,\n",
    "    'LinearBinary_stochastic': [1, 0],\n",
    "    'LinearBinary_bipolar': [1, 0],\n",
    "}\n",
    "\n",
    "def construct_model(trial):\n",
    "    # Fetch the model\n",
    "    my_config = {}\n",
    "    trial_model = deepcopy(base_model)\n",
    "    \n",
    "    # Select quantization and fractional width configurations\n",
    "    for param in ['linear_layer_quantization', \n",
    "                  'linear_layer_quantization_fractional',\n",
    "                  'LinearMinifloatDenorm',\n",
    "                  'LinearMinifloatDenorm_ew',\n",
    "                  'LinearMinifloatDenorm_eb',\n",
    "                  'LinearMinifloatIEEE',\n",
    "                  'LinearMinifloatIEEE_ew',\n",
    "                  'LinearMinifloatIEEE_eb',\n",
    "                    'LinearLog',\n",
    "                    'LinearLog_ew',\n",
    "                    'LinearLog_eb',\n",
    "                    'LinearBlockFP',\n",
    "                    'LinearBlockFP_block_size',\n",
    "                    'LinearBlockMinifloat_ebw',\n",
    "                    'LinearBlockMinifloat_ew',\n",
    "                    'LinearBlockMinifloat_block_size',\n",
    "                    'LinearBlockMinifloat_w',\n",
    "                    'LinearBlockLog',\n",
    "                    'LinearBlockLog_ebw',\n",
    "                    'LinearBlockLog_block_size',\n",
    "                    'LinearBinary_stochastic',\n",
    "                    'LinearBinary_bipolar',\n",
    "                  ]:\n",
    "        chosen_idx = trial.suggest_int(param, 0, len(search_space[param]) - 1)\n",
    "        my_config[param] = search_space[param][chosen_idx]\n",
    "\n",
    "    # Quantize layers according to Optuna suggestions\n",
    "    for name, layer in trial_model.named_modules():\n",
    "        if isinstance(layer, torch.nn.Linear):\n",
    "            new_layer_cls = trial.suggest_categorical(\n",
    "                f\"{name}_type\",\n",
    "                search_space[\"linear_layer_choices\"],\n",
    "            )\n",
    "\n",
    "            if new_layer_cls == torch.nn.Linear:\n",
    "                continue\n",
    "\n",
    "            kwargs = {\n",
    "                \"in_features\": layer.in_features,\n",
    "                \"out_features\": layer.out_features,\n",
    "            }\n",
    "\n",
    "            try:\n",
    "                # Configure parameters for specific layer types\n",
    "                # if new_layer_cls == LinearInteger:\n",
    "                #     kwargs[\"config\"] = {\n",
    "                #         \"data_in_width\": my_config['linear_layer_quantization'],\n",
    "                #         \"data_in_frac_width\": my_config['linear_layer_quantization_fractional'],\n",
    "                #         \"weight_width\": my_config['linear_layer_quantization'],\n",
    "                #         \"weight_frac_width\": my_config['linear_layer_quantization_fractional'],\n",
    "                #         \"bias_width\": my_config['linear_layer_quantization'],\n",
    "                #         \"bias_frac_width\": my_config['linear_layer_quantization_fractional'],\n",
    "                #     }\n",
    "                # elif new_layer_cls == LinearMinifloatDenorm:\n",
    "                #     kwargs[\"config\"] = {\n",
    "                #         \"data_in_width\": my_config['LinearMinifloatDenorm'],\n",
    "                #         \"data_in_exponent_width\": my_config['LinearMinifloatDenorm_ew'],\n",
    "                #         \"data_in_exponent_bias\": my_config['LinearMinifloatDenorm_eb'],\n",
    "                #         \"weight_width\": my_config['LinearMinifloatDenorm'],\n",
    "                #         \"weight_exponent_width\": my_config['LinearMinifloatDenorm_ew'],\n",
    "                #         \"weight_exponent_bias\": my_config['LinearMinifloatDenorm_eb'],\n",
    "                #         \"bias_width\": my_config['LinearMinifloatDenorm'],\n",
    "                #         \"bias_exponent_width\": my_config['LinearMinifloatDenorm_ew'],\n",
    "                #         \"bias_exponent_bias\": my_config['LinearMinifloatDenorm_eb'],\n",
    "                #     }\n",
    "                # elif new_layer_cls == LinearMinifloatIEEE:\n",
    "                #     kwargs[\"config\"] = {\n",
    "                #         \"data_in_width\": my_config['LinearMinifloatIEEE'],\n",
    "                #         \"data_in_exponent_width\": my_config['LinearMinifloatIEEE_ew'],\n",
    "                #         \"data_in_exponent_bias\": my_config['LinearMinifloatIEEE_eb'],\n",
    "                #         \"weight_width\": my_config['LinearMinifloatIEEE'],\n",
    "                #         \"weight_exponent_width\": my_config['LinearMinifloatIEEE_ew'],\n",
    "                #         \"weight_exponent_bias\": my_config['LinearMinifloatIEEE_eb'],\n",
    "                #         \"bias_width\": my_config['LinearMinifloatIEEE'],\n",
    "                #         \"bias_exponent_width\": my_config['LinearMinifloatIEEE_ew'],\n",
    "                #         \"bias_exponent_bias\": my_config['LinearMinifloatIEEE_eb'],\n",
    "                #     }\n",
    "                # elif new_layer_cls == LinearLog:\n",
    "                #     kwargs[\"config\"] = {\n",
    "                #         \"data_in_width\": my_config['LinearLog'],\n",
    "                #         \"data_in_exponent_width\": my_config['LinearLog_ew'],\n",
    "                #         \"data_in_exponent_bias\": my_config['LinearLog_eb'],\n",
    "                #         \"weight_width\": my_config['LinearLog'],\n",
    "                #         \"weight_exponent_width\": my_config['LinearLog_ew'],\n",
    "                #         \"weight_exponent_bias\": my_config['LinearLog_eb'],\n",
    "                #         \"bias_width\": my_config['LinearLog'],\n",
    "                #         \"bias_exponent_width\": my_config['LinearLog_ew'],\n",
    "                #         \"bias_exponent_bias\": my_config['LinearLog_eb'],\n",
    "                #     }\n",
    "                # elif new_layer_cls == LinearBlockFP:\n",
    "                #     kwargs[\"config\"] = {\n",
    "                #         \"weight_width\": my_config['LinearBlockFP'],\n",
    "                #         \"weight_exponent_width\": my_config['LinearBlockFP'],\n",
    "                #         \"weight_block_size\": my_config['LinearBlockFP_block_size'],\n",
    "                #         \"weight_exponent_bias\": my_config['LinearBlockFP'],\n",
    "                #         \"data_in_width\": my_config['LinearBlockFP'],\n",
    "                #         \"data_in_exponent_width\": my_config['LinearBlockFP'],\n",
    "                #         \"data_in_block_size\": my_config['LinearBlockFP_block_size'],\n",
    "                #         \"data_in_exponent_bias\": my_config['LinearBlockFP'],\n",
    "                #         \"bias_width\": my_config['LinearBlockFP'],\n",
    "                #         \"bias_exponent_width\": my_config['LinearBlockFP'],\n",
    "                #         \"bias_block_size\": my_config['LinearBlockFP_block_size'],\n",
    "                #         \"bias_exponent_bias\": my_config['LinearBlockFP'],\n",
    "                #     }\n",
    "                # elif new_layer_cls == LinearBlockMinifloat:\n",
    "                #     kwargs[\"config\"] = {\n",
    "                #         \"weight_exponent_bias_width\" : my_config['LinearBlockMinifloat_ebw'],\n",
    "                #         \"weight_exponent_width\": my_config['LinearBlockMinifloat_ew'],\n",
    "                #         \"weight_block_size\": my_config['LinearBlockMinifloat_block_size'],\n",
    "                #         \"weight_width\": my_config['LinearBlockMinifloat_w'],\n",
    "                #         \"data_in_width\":my_config[\"LinearBlockMinifloat_w\"],\n",
    "                #         \"data_in_exponent_bias_width\":my_config[\"LinearBlockMinifloat_ew\"],\n",
    "                #         \"data_in_exponent_width\":my_config[\"LinearBlockMinifloat_ebw\"],\n",
    "                #         \"data_in_block_size\":my_config[\"LinearBlockMinifloat_block_size\"],\n",
    "                #         \"bias_width\":my_config[\"LinearBlockMinifloat_w\"],\n",
    "                #         \"bias_exponent_width\":my_config[\"LinearBlockMinifloat_ew\"],\n",
    "                #         \"bias_exponent_bias_width\":my_config[\"LinearBlockMinifloat_ebw\"],\n",
    "                #         \"bias_block_size\":my_config[\"LinearBlockMinifloat_block_size\"],\n",
    "                # #     }\n",
    "                # elif new_layer_cls == LinearBlockLog:\n",
    "                #     kwargs[\"config\"] = {\n",
    "                #         \"data_in_width\": my_config['LinearBlockLog'],\n",
    "                #         \"data_in_exponent_bias_width\": my_config['LinearBlockLog_ebw'],\n",
    "                #         \"data_in_block_size\": my_config['LinearBlockLog_block_size'],\n",
    "                #         \"weight_width\": my_config['LinearBlockLog'],\n",
    "                #         \"weight_exponent_bias_width\": my_config['LinearBlockLog_ebw'],\n",
    "                #         \"weight_block_size\": my_config['LinearBlockLog_block_size'],\n",
    "                #         \"bias_width\": my_config['LinearBlockLog'],\n",
    "                #         \"bias_exponent_bias_width\": my_config['LinearBlockLog_ebw'],\n",
    "                #         \"bias_block_size\": my_config['LinearBlockLog_block_size'],\n",
    "                #     }\n",
    "\n",
    "                if new_layer_cls == LinearBinary:\n",
    "                    kwargs[\"config\"] = {\n",
    "                        'weight_stochastic':my_config['LinearBinary_stochastic'],\n",
    "                        'weight_bipolar':my_config['LinearBinary_bipolar'],\n",
    "                    }\n",
    "                    # print(my_config['LinearBinary_stochastic'])\n",
    "                    # print(my_config['LinearBinary_bipolar'])\n",
    "\n",
    "                    \n",
    "                # elif new_layer_cls == LinearBinaryScaling:\n",
    "                #     kwargs[\"config\"] = {\n",
    "                #         'weight_stochastic':my_config['LinearBinaryScaling_stochastic'],\n",
    "                #         'weight_bipolar':my_config['LinearBinaryScaling_bipolar'],\n",
    "                #         \"data_in_stochastic\": my_config['LinearBinaryScaling_stochastic'],\n",
    "                #         \"data_in_bipolar\": my_config['LinearBinaryScaling_bipolar'],\n",
    "                #         \"bias_stochastic\": my_config['LinearBinaryScaling_stochastic'],\n",
    "                #         \"bias_bipolar\": my_config['LinearBinaryScaling_bipolar'],\n",
    "                #         \"binary_training\": my_config['LinearBinaryScaling_binary_training'],\n",
    "                #     }\n",
    "                # elif new_layer_cls == LinearBinaryResidualSign:\n",
    "                #     kwargs[\"config\"] = {\n",
    "                #         'weight_stochastic':my_config['LinearBinaryResidualSign_stochastic'],\n",
    "                #         'weight_bipolar':my_config['LinearBinaryResidualSign_bipolar'],\n",
    "                #         \"data_in_stochastic\": my_config['LinearBinaryResidualSign_stochastic'],\n",
    "                #         \"data_in_bipolar\": my_config['LinearBinaryResidualSign_bipolar'],\n",
    "                #         \"bias_stochastic\": my_config['LinearBinaryResidualSign_stochastic'],\n",
    "                #         \"bias_bipolar\": my_config['LinearBinaryResidualSign_bipolar'],\n",
    "                #         \"binary_training\": my_config['LinearBinaryResidualSign_binary_training'],\n",
    "                #     }\n",
    "                # Create the new layer (copy the weights if applicable)\n",
    "                new_layer = new_layer_cls(**kwargs)\n",
    "                new_layer.weight.data = layer.weight.data.clone()\n",
    "\n",
    "                # Replace the layer in the model\n",
    "                deepsetattr(trial_model, name, new_layer)\n",
    "            except TypeError as e:\n",
    "                print(f\"Error creating layer {name} with class {new_layer_cls}: {e}\")\n",
    "                raise\n",
    "\n",
    "    return trial_model\n",
    "from chop.tools import get_trainer\n",
    "import random\n",
    "\n",
    "\n",
    "def objective(trial):\n",
    "\n",
    "    # Define the model\n",
    "    model = construct_model(trial)\n",
    "\n",
    "    trainer = get_trainer(\n",
    "        model=model,\n",
    "        tokenized_dataset=dataset,\n",
    "        tokenizer=tokenizer,\n",
    "        evaluate_metric=\"accuracy\",\n",
    "        num_train_epochs=1,\n",
    "    )\n",
    "\n",
    "    trainer.train()\n",
    "    eval_results = trainer.evaluate()\n",
    "\n",
    "    trial.set_user_attr(\"model\", model)\n",
    "\n",
    "    return eval_results[\"eval_accuracy\"]\n",
    "\n",
    "\n",
    "from optuna.samplers import GridSampler, RandomSampler, TPESampler\n",
    "\n",
    "sampler = TPESampler()\n",
    "\n",
    "import optuna\n",
    "\n",
    "study = optuna.create_study(\n",
    "    direction=\"maximize\",\n",
    "    study_name=\"bert-tiny-nas-study\",\n",
    "    sampler=sampler,\n",
    ")\n",
    "\n",
    "study.optimize(\n",
    "    objective,\n",
    "    n_trials=20,\n",
    "    timeout=60 * 60 * 24,\n",
    ")\n",
    "\n",
    "task8 = [max([t.value for t in study.trials[:i+1]]) for i in range(len(study.trials))]\n",
    "task8"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "task8=[0.50224,\n",
    " 0.86552,\n",
    " 0.8688,\n",
    " 0.8688,\n",
    " 0.8688,\n",
    " 0.8688,\n",
    " 0.8688,\n",
    " 0.8688,\n",
    " 0.8688,\n",
    " 0.8688]"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "mase",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}