Train_Binarized_SNN.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "machine_shape": "hm",
      "gpuType": "V100"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuromorphs/osn23-huge-tapeout/blob/main/Train_Binarized_SNN.ipynb)"
      ],
      "metadata": {
        "id": "w5wJQOqFjbwO"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install snntorch"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zadn76o6NGl4",
        "outputId": "0848f48a-6e57-4a64-9e23-0481e2275f66"
      },
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting snntorch\n",
            "  Downloading snntorch-0.6.4-py2.py3-none-any.whl (107 kB)\n",
            "\u001b[?25l     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m0.0/107.8 kB\u001b[0m \u001b[31m?\u001b[0m eta \u001b[36m-:--:--\u001b[0m\r\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m107.8/107.8 kB\u001b[0m \u001b[31m4.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: torch>=1.1.0 in /usr/local/lib/python3.10/dist-packages (from snntorch) (2.0.1+cu118)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from snntorch) (1.5.3)\n",
            "Requirement already satisfied: matplotlib in /usr/local/lib/python3.10/dist-packages (from snntorch) (3.7.1)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from snntorch) (1.22.4)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from torch>=1.1.0->snntorch) (3.12.2)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.10/dist-packages (from torch>=1.1.0->snntorch) (4.6.3)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch>=1.1.0->snntorch) (1.11.1)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch>=1.1.0->snntorch) (3.1)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch>=1.1.0->snntorch) (3.1.2)\n",
            "Requirement already satisfied: triton==2.0.0 in /usr/local/lib/python3.10/dist-packages (from torch>=1.1.0->snntorch) (2.0.0)\n",
            "Requirement already satisfied: cmake in /usr/local/lib/python3.10/dist-packages (from triton==2.0.0->torch>=1.1.0->snntorch) (3.25.2)\n",
            "Requirement already satisfied: lit in /usr/local/lib/python3.10/dist-packages (from triton==2.0.0->torch>=1.1.0->snntorch) (16.0.6)\n",
            "Requirement already satisfied: contourpy>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib->snntorch) (1.1.0)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.10/dist-packages (from matplotlib->snntorch) (0.11.0)\n",
            "Requirement already satisfied: fonttools>=4.22.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib->snntorch) (4.40.0)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib->snntorch) (1.4.4)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib->snntorch) (23.1)\n",
            "Requirement already satisfied: pillow>=6.2.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib->snntorch) (8.4.0)\n",
            "Requirement already satisfied: pyparsing>=2.3.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib->snntorch) (3.1.0)\n",
            "Requirement already satisfied: python-dateutil>=2.7 in /usr/local/lib/python3.10/dist-packages (from matplotlib->snntorch) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->snntorch) (2022.7.1)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.7->matplotlib->snntorch) (1.16.0)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch>=1.1.0->snntorch) (2.1.3)\n",
            "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch>=1.1.0->snntorch) (1.3.0)\n",
            "Installing collected packages: snntorch\n",
            "Successfully installed snntorch-0.6.4\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import torch\n",
        "config = {\n",
        "        'model' : 'NetFC',\n",
        "\n",
        "        'exp_name' : 'mnist_tha',\n",
        "        'num_trials' : 5,\n",
        "        'num_epochs' : 50, #500,\n",
        "        'binarize' : True,\n",
        "        'binarize_input' : True,\n",
        "        'post_quantize' : True,\n",
        "        'enable_bias' : True,\n",
        "        'enable_batch_norm' : True,\n",
        "        'enable_dropout' : True,\n",
        "        'enable_threshold' : True,\n",
        "        'enable_slope' : True,\n",
        "        'on_spike_reset_to_zero' : False,\n",
        "        'data_dir' : \"~/data/mnist\",\n",
        "        'batch_size' : 128,\n",
        "        'seed' : 0,\n",
        "        'num_workers' : 0,\n",
        "\n",
        "        # final run sweeps\n",
        "        'save_csv' : True,\n",
        "        'save_model' : True,\n",
        "        'early_stopping': True,\n",
        "        'patience': 100,\n",
        "\n",
        "        # final params\n",
        "        'grad_clip' : False,\n",
        "        'weight_clip' : False,\n",
        "        'dropout1' : 0.02856,\n",
        "        'beta' : 0.992187, # was 0.99, 0.9921875 (1/2^7) other possible values = 0.75, 0.875, 0.9375\n",
        "        'lr' : 9.97e-3,\n",
        "        'slope': 10.22,\n",
        "\n",
        "        # threshold annealing. note: thr_final = threshold + thr_final\n",
        "        'threshold1' : 11.666,\n",
        "        'alpha_thr1' : 0.024,\n",
        "        'thr_final1' : 4.317,\n",
        "\n",
        "        'threshold2' : 14.105,\n",
        "        'alpha_thr2' : 0.119,\n",
        "        'thr_final2' : 16.29,\n",
        "\n",
        "        'threshold3' : 0.6656,\n",
        "        'alpha_thr3' : 0.0011,\n",
        "        'thr_final3' : 3.496,\n",
        "\n",
        "        # fixed params\n",
        "        'num_steps' : 100,\n",
        "        'correct_rate': 0.8,\n",
        "        'incorrect_rate' : 0.2,\n",
        "        'betas' : (0.9, 0.999),\n",
        "        't_0' : 4688,\n",
        "        'eta_min' : 0,\n",
        "        'df_lr' : True, # return learning rate. Useful for scheduling\n",
        "\n",
        "        # debug params\n",
        "        'print_weights' : False\n",
        "    }\n",
        "\n",
        "def optim_func(net, config):\n",
        "    optimizer = torch.optim.Adam(net.parameters(), lr=config[\"lr\"], betas=config['betas'])\n",
        "    scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=config['t_0'], eta_min=config['eta_min'], last_epoch=-1)\n",
        "    return optimizer, scheduler\n"
      ],
      "metadata": {
        "id": "O0L13aqxMYtN"
      },
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import torch\n",
        "import torch.nn as nn\n",
        "from torch.autograd import Function\n",
        "\n",
        "class BinarizeF(Function):\n",
        "    @staticmethod\n",
        "    def forward(ctx, input):\n",
        "        output = input.new(input.size())\n",
        "        output[input >= 0] = 1\n",
        "        output[input < 0] = -1\n",
        "        return output\n",
        "\n",
        "    @staticmethod\n",
        "    def backward(ctx, grad_output):\n",
        "        grad_input = grad_output.clone()\n",
        "        return grad_input\n",
        "\n",
        "# aliases\n",
        "binarize = BinarizeF.apply\n",
        "\n",
        "def binarize_activations(input):\n",
        "    output = input.new(input.size())\n",
        "    output[input >= 0.5] = 1\n",
        "    output[input < 0.5] = 0\n",
        "    return output"
      ],
      "metadata": {
        "id": "vb6YCUbWMfG6"
      },
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "id": "wED_xSVyMTfi"
      },
      "outputs": [],
      "source": [
        "import math\n",
        "\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "from torch.autograd import Variable\n",
        "\n",
        "class BinaryTanh(nn.Module):\n",
        "    def __init__(self):\n",
        "        super(BinaryTanh, self).__init__()\n",
        "        self.hardtanh = nn.Hardtanh()\n",
        "\n",
        "    def forward(self, input):\n",
        "        output = self.hardtanh(input)\n",
        "        output = binarize(output)\n",
        "        return output\n",
        "\n",
        "class BinaryLinear(nn.Linear):\n",
        "    def forward(self, input):\n",
        "        binary_weight = binarize(self.weight)\n",
        "        if self.bias is None:\n",
        "            return F.linear(input, binary_weight)\n",
        "        else:\n",
        "            return F.linear(input, binary_weight, self.bias)\n",
        "\n",
        "    def reset_parameters(self):\n",
        "        # Glorot initialization\n",
        "        in_features, out_features = self.weight.size()\n",
        "        stdv = math.sqrt(1.5 / (in_features + out_features))\n",
        "        self.weight.data.uniform_(-stdv, stdv)\n",
        "        if self.bias is not None:\n",
        "            self.bias.data.zero_()\n",
        "\n",
        "        self.weight.lr_scale = 1. / stdv\n",
        "\n",
        "class SparseBinaryLinear(nn.Linear):\n",
        "    def __init__(self, in_features, out_features, sparsity=0, bias=True, device=None, dtype=None):\n",
        "        super(SparseBinaryLinear, self).__init__(in_features, out_features, bias, device, dtype)\n",
        "        self.mask = torch.bernoulli(torch.ones_like(self.weight) * (1-sparsity))\n",
        "        self.register_buffer('weight_mask_const', self.mask)\n",
        "        print(self.mask.mean())\n",
        "\n",
        "    def forward(self, input):\n",
        "        input = binarize_activations(input)\n",
        "        binary_weight = binarize(self.weight).mul(Variable(self.weight_mask_const))\n",
        "        #print(self.weight_mask_const.mean(), binary_weight.mean())\n",
        "        if self.bias is None:\n",
        "            return F.linear(input, binary_weight)\n",
        "        else:\n",
        "            return F.linear(input, binary_weight, self.bias)\n",
        "\n",
        "    def reset_parameters(self):\n",
        "        # Glorot initialization\n",
        "        in_features, out_features = self.weight.size()\n",
        "        stdv = math.sqrt(1.5 / (in_features + out_features))\n",
        "        self.weight.data.uniform_(-stdv, stdv)\n",
        "        if self.bias is not None:\n",
        "            self.bias.data.zero_()\n",
        "\n",
        "        self.weight.lr_scale = 1. / stdv\n",
        "\n",
        "class BinaryConv2d(nn.Conv2d):\n",
        "\n",
        "    def forward(self, input):\n",
        "        bw = binarize(self.weight)\n",
        "        return F.conv2d(input, bw, self.bias, self.stride,\n",
        "                               self.padding, self.dilation, self.groups)\n",
        "\n",
        "    def reset_parameters(self):\n",
        "        # Glorot initialization\n",
        "        in_features = self.in_channels\n",
        "        out_features = self.out_channels\n",
        "        for k in self.kernel_size:\n",
        "            in_features *= k\n",
        "            out_features *= k\n",
        "        stdv = math.sqrt(1.5 / (in_features + out_features))\n",
        "        self.weight.data.uniform_(-stdv, stdv)\n",
        "        if self.bias is not None:\n",
        "            self.bias.data.zero_()\n",
        "\n",
        "        self.weight.lr_scale = 1. / stdv\n",
        "\n",
        "\n",
        "# class QuantizedBatchNorm1d(nn.BatchNorm1d):\n",
        "#     def forward(self, input):\n",
        "#         if self.momentum is None:\n",
        "#             exponential_average_factor = 0.0\n",
        "#         else:\n",
        "#             exponential_average_factor = self.momentum\n",
        "\n",
        "#         if self.training and self.track_running_stats:\n",
        "#             # TODO: if statement only here to tell the jit to skip emitting this when it is None\n",
        "#             if self.num_batches_tracked is not None:  # type: ignore[has-type]\n",
        "#                 self.num_batches_tracked.add_(1)  # type: ignore[has-type]\n",
        "#                 if self.momentum is None:  # use cumulative moving average\n",
        "#                     exponential_average_factor = 1.0 / float(self.num_batches_tracked)\n",
        "#                 else:  # use exponential moving average\n",
        "#                     exponential_average_factor = self.momentum\n",
        "\n",
        "#         if self.training:\n",
        "#             bn_training = True\n",
        "#         else:\n",
        "#             bn_training = (self.running_mean is None) and (self.running_var is None)\n",
        "\n",
        "#         return F.batch_norm(\n",
        "#             input,\n",
        "#             # If buffers are not to be tracked, ensure that they won't be updated\n",
        "#             self.running_mean\n",
        "#             if not self.training or self.track_running_stats\n",
        "#             else None,\n",
        "#             self.running_var if not self.training or self.track_running_stats else None,\n",
        "#             quantize(self.weight),\n",
        "#             self.bias,\n",
        "#             bn_training,\n",
        "#             exponential_average_factor,\n",
        "#             self.eps,\n",
        "#         )\n"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import torch\n",
        "import torch.nn as nn\n",
        "from torch.utils.data import DataLoader\n",
        "from torchvision import datasets, transforms\n",
        "\n",
        "MNIST_INPUT_RESOLUTION = 16\n",
        "\n",
        "def load_data(config):\n",
        "        data_dir = config['data_dir']\n",
        "\n",
        "        transform = transforms.Compose([\n",
        "                transforms.Resize((MNIST_INPUT_RESOLUTION, MNIST_INPUT_RESOLUTION)),\n",
        "                transforms.Grayscale(),\n",
        "                transforms.ToTensor(),\n",
        "                transforms.Normalize((0,), (1,))])\n",
        "\n",
        "        trainset = datasets.MNIST(data_dir, train=True, download=True, transform=transform)\n",
        "        testset = datasets.MNIST(data_dir, train=False, download=True, transform=transform)\n",
        "\n",
        "        return trainset, testset"
      ],
      "metadata": {
        "id": "TPJW3wqPMbEX"
      },
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import torch\n",
        "\n",
        "\n",
        "class EarlyStopping_acc:\n",
        "    \"\"\"Early stops the training if test acc doesn't improve after a given patience.\"\"\"\n",
        "    def __init__(self, patience=7, verbose=False, delta=0, path='checkpoint.pt', trace_func=print):\n",
        "        \"\"\"\n",
        "        Args:\n",
        "            patience (int): How long to wait after last time validation loss improved.\n",
        "                            Default: 7\n",
        "            verbose (bool): If True, prints a message for each validation loss improvement.\n",
        "                            Default: False\n",
        "            delta (float): Minimum change in the monitored quantity to qualify as an improvement.\n",
        "                            Default: 0\n",
        "            path (str): Path for the checkpoint to be saved to.\n",
        "                            Default: 'checkpoint.pt'\n",
        "            trace_func (function): trace print function.\n",
        "                            Default: print\n",
        "        \"\"\"\n",
        "        self.patience = patience\n",
        "        self.verbose = verbose\n",
        "        self.counter = 0\n",
        "        self.best_score = None\n",
        "        self.early_stop = False\n",
        "        self.test_loss_min = 0\n",
        "        self.delta = delta\n",
        "        self.path = path\n",
        "        self.trace_func = trace_func\n",
        "    def __call__(self, test_loss, model):\n",
        "\n",
        "        score = test_loss\n",
        "\n",
        "        if self.best_score is None:\n",
        "            self.best_score = score\n",
        "            self.save_checkpoint(test_loss, model)\n",
        "        elif score <= self.best_score + self.delta:\n",
        "            self.counter += 1\n",
        "            self.trace_func(f'EarlyStopping counter: {self.counter} out of {self.patience}')\n",
        "            if self.counter >= self.patience:\n",
        "                self.early_stop = True\n",
        "                self.counter = 0\n",
        "        else:\n",
        "            self.best_score = score\n",
        "            self.save_checkpoint(test_loss, model)\n",
        "            self.counter = 0\n",
        "\n",
        "    def save_checkpoint(self, test_loss, model):\n",
        "        '''Saves model when test acc increases.'''\n",
        "        if self.verbose:\n",
        "            self.trace_func(f'Test acc increased ({self.test_loss_min:.6f} --> {test_loss:.6f}).  Saving model ...')\n",
        "        torch.save(model.state_dict(), self.path)\n",
        "        self.test_loss_min = test_loss"
      ],
      "metadata": {
        "id": "K4S2cHioMdrY"
      },
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import torch\n",
        "import snntorch as snn\n",
        "from snntorch import functional as SF\n",
        "\n",
        "def test_accuracy(config, net, testloader, device=\"cpu\"):\n",
        "    correct = 0\n",
        "    total = 0\n",
        "    with torch.no_grad():\n",
        "        net.eval()\n",
        "        for data in testloader:\n",
        "            images, labels = data\n",
        "            images, labels = images.to(device), labels.to(device)\n",
        "\n",
        "            outputs, _ = net(images)\n",
        "            accuracy = SF.accuracy_rate(outputs, labels)\n",
        "\n",
        "            total += labels.size(0)\n",
        "            correct += accuracy * labels.size(0)\n",
        "\n",
        "    return 100 * correct / total"
      ],
      "metadata": {
        "id": "VXtq5vi-MlL6"
      },
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# exp relaxation implementation of THA based on Eq (4)\n",
        "\n",
        "def thr_annealing(config, network):\n",
        "    alpha_thr1 = config['alpha_thr1']\n",
        "    alpha_thr2 = config['alpha_thr2']\n",
        "    alpha_thr3 = config['alpha_thr3']\n",
        "\n",
        "    ### to address conditional parameters, s.t. thr_final > threshold\n",
        "    thr_final1 = config['thr_final1'] + config['threshold1']\n",
        "    thr_final2 = config['thr_final2'] + config['threshold2']\n",
        "    thr_final3 = config['thr_final3'] + config['threshold3']\n",
        "\n",
        "    network.lif1.threshold += (thr_final1 - network.lif1.threshold) * alpha_thr1\n",
        "    network.lif2.threshold += (thr_final2 - network.lif2.threshold) * alpha_thr2\n",
        "    network.lif3.threshold += (thr_final3 - network.lif3.threshold) * alpha_thr3\n",
        "\n",
        "    return"
      ],
      "metadata": {
        "id": "DOU3fLVkMmJO"
      },
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# snntorch\n",
        "import snntorch as snn\n",
        "from snntorch import spikegen\n",
        "from snntorch import surrogate\n",
        "from snntorch import functional as SF\n",
        "\n",
        "# torch\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "from torch.utils.data import DataLoader\n",
        "from torchvision import datasets, transforms\n",
        "import torch.nn.functional as F\n",
        "from torch.optim.lr_scheduler import StepLR\n",
        "\n",
        "# misc\n",
        "import os\n",
        "import numpy as np\n",
        "import math\n",
        "import itertools\n",
        "import matplotlib.pyplot as plt\n",
        "import pandas as pd\n",
        "import shutil\n",
        "import time\n",
        "from tqdm import tqdm\n",
        "\n",
        "def train(config, net, epoch, trainloader, testloader, criterion, optimizer, scheduler, device):\n",
        "    net.train()\n",
        "    loss_accum = []\n",
        "    lr_accum = []\n",
        "\n",
        "    # TRAIN\n",
        "    progress_bar = tqdm(trainloader)\n",
        "    loss_current = None\n",
        "\n",
        "    #for data, labels in trainloader:\n",
        "    for data, labels in progress_bar:\n",
        "        data, labels = data.to(device), labels.to(device)\n",
        "\n",
        "        spk_rec2, _ = net(data)\n",
        "        loss = criterion(spk_rec2, labels)\n",
        "        optimizer.zero_grad()\n",
        "        loss.backward()\n",
        "        if loss_current is None:\n",
        "            loss_current = loss.item()\n",
        "        else:\n",
        "            loss_current = 0.9 * loss_current + 0.1 * loss.item()\n",
        "        progress_bar.set_description(f\"loss: {loss_current:.4f}\")\n",
        "\n",
        "        if config['grad_clip']:\n",
        "            nn.utils.clip_grad_norm_(net.parameters(), 1.0)\n",
        "        if config['weight_clip']:\n",
        "            with torch.no_grad():\n",
        "                for param in net.parameters():\n",
        "                    param.clamp_(-1, 1)\n",
        "\n",
        "        optimizer.step()\n",
        "        scheduler.step()\n",
        "        thr_annealing(config, net)\n",
        "\n",
        "\n",
        "        loss_accum.append(loss.item()/config['num_steps'])\n",
        "        lr_accum.append(optimizer.param_groups[0][\"lr\"])\n",
        "\n",
        "\n",
        "    return loss_accum, lr_accum"
      ],
      "metadata": {
        "id": "VD1SnCZgMo2m"
      },
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# snntorch\n",
        "import snntorch as snn\n",
        "from snntorch import spikegen\n",
        "from snntorch import surrogate\n",
        "\n",
        "# torch\n",
        "import copy\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "from torch.utils.data import DataLoader\n",
        "\n",
        "# misc\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import time\n",
        "import logging\n",
        "\n",
        "def run(config):\n",
        "    print(config)\n",
        "    file_name = config['exp_name']\n",
        "\n",
        "    for trial in range(config['num_trials']):\n",
        "        # file names\n",
        "        SAVE_CSV = config['save_csv']\n",
        "        SAVE_MODEL = config['save_model']\n",
        "        csv_name = file_name + '_t' + str(trial) + '.csv'\n",
        "        log_name = file_name + '_t' + str(trial) + '.log'\n",
        "        model_name = file_name + '_t' + str(trial) + '.pt'\n",
        "        num_epochs = config['num_epochs']\n",
        "        torch.manual_seed(config['seed'])\n",
        "\n",
        "        # dataframes\n",
        "        df_train_loss = pd.DataFrame()\n",
        "        df_test_acc = pd.DataFrame(columns=['epoch', 'test_acc', 'train_time'])\n",
        "        df_lr = pd.DataFrame()\n",
        "\n",
        "        # initialize network\n",
        "        net = None\n",
        "        net_desc = config['model']\n",
        "        if net_desc in globals():\n",
        "            klass = globals()[net_desc]\n",
        "            net = klass(config)\n",
        "        else:\n",
        "            net = eval(net_desc)\n",
        "        if trial == 0:\n",
        "            print(net)\n",
        "        device = \"cpu\"\n",
        "        #device = \"mps\"\n",
        "        if torch.cuda.is_available():\n",
        "            device = \"cuda:0\"\n",
        "            if torch.cuda.device_count() > 1:\n",
        "                net = nn.DataParallel(net)\n",
        "        net.to(device)\n",
        "\n",
        "        # net params\n",
        "        criterion = SF.mse_count_loss(correct_rate=config['correct_rate'], incorrect_rate=config['incorrect_rate'])\n",
        "        optimizer, scheduler = optim_func(net, config)\n",
        "\n",
        "        # early stopping condition\n",
        "        if config['early_stopping']:\n",
        "            early_stopping = EarlyStopping_acc(patience=config['patience'], verbose=True, path=model_name)\n",
        "            early_stopping.early_stop = False\n",
        "            early_stopping.best_score = None\n",
        "\n",
        "        # load data\n",
        "        trainset, testset = load_data(config)\n",
        "        config['dataset_length'] = len(trainset)\n",
        "        trainloader = DataLoader(trainset, batch_size=int(config[\"batch_size\"]), shuffle=True)\n",
        "        testloader = DataLoader(testset, batch_size=int(config[\"batch_size\"]), shuffle=False)\n",
        "\n",
        "        print(f\"=======Trial: {trial}=======\")\n",
        "\n",
        "        for epoch in range(num_epochs):\n",
        "\n",
        "            # train\n",
        "            start_time = time.time()\n",
        "            loss_list, lr_list = train(config, net, epoch, trainloader, testloader, criterion, optimizer, scheduler, device)\n",
        "            epoch_time = time.time() - start_time\n",
        "\n",
        "            # test\n",
        "            test_acc = test_accuracy(config, net, testloader, device)\n",
        "            print(f'Epoch: {epoch} \\tTest Accuracy: {test_acc}')\n",
        "\n",
        "            if config['df_lr']:\n",
        "                df_lr = pd.concat([df_lr, pd.DataFrame(lr_list)])\n",
        "            df_train_loss = pd.concat([df_train_loss, pd.DataFrame(loss_list)])\n",
        "            test_data = pd.DataFrame([[epoch, test_acc, epoch_time]], columns = ['epoch', 'test_acc', 'train_time'])\n",
        "            df_test_acc = pd.concat([df_test_acc, test_data])\n",
        "\n",
        "            if SAVE_CSV:\n",
        "                df_train_loss.to_csv('loss_' + csv_name, index=False)\n",
        "                df_test_acc.to_csv('acc_' + csv_name, index=False)\n",
        "                if config['df_lr']:\n",
        "                    df_lr.to_csv('lr_' + csv_name, index=False)\n",
        "\n",
        "            if config['early_stopping']:\n",
        "                early_stopping(test_acc, net)\n",
        "\n",
        "                if early_stopping.early_stop:\n",
        "                    print(\"Early stopping\")\n",
        "                    early_stopping.early_stop = False\n",
        "                    early_stopping.best_score = None\n",
        "                    break\n",
        "\n",
        "            if SAVE_MODEL and not config['early_stopping']:\n",
        "                torch.save(net.state_dict(), model_name)\n",
        "\n",
        "            if config['print_weights']:\n",
        "                for name, param in net.named_parameters():\n",
        "                    print(name, param)\n",
        "\n",
        "            if config['post_quantize']:\n",
        "                net_quantized = post_quantize(copy.deepcopy(net))\n",
        "                test_acc = test_accuracy(config, net_quantized, testloader, device)\n",
        "                print(f'Epoch: {epoch} \\tTest Quantized Accuracy: {test_acc}')\n",
        "\n",
        "        # net.load_state_dict(torch.load(model_name))\n"
      ],
      "metadata": {
        "id": "S7d0Bj7qMjyp"
      },
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# snntorch\n",
        "import snntorch as snn\n",
        "from snntorch import surrogate\n",
        "\n",
        "# torch\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "\n",
        "class NetConv(nn.Module):\n",
        "    def __init__(self, config):\n",
        "        super().__init__()\n",
        "\n",
        "        self.thr1 = config['threshold1']\n",
        "        self.thr2 = config['threshold2']\n",
        "        self.thr3 = config['threshold3']\n",
        "        slope = config['slope']\n",
        "        beta = config['beta']\n",
        "        self.num_steps = config['num_steps']\n",
        "        self.batch_norm = config['enable_batch_norm']\n",
        "        p1 = config['dropout1']\n",
        "        self.binarize = config['binarize']\n",
        "        self.binarize_input = config['binarize_input']\n",
        "        self.bias = config['enable_bias']\n",
        "        self.reset_mechanism = 'zero' if config['on_spike_reset_to_zero'] else 'subtract'\n",
        "\n",
        "        spike_grad = surrogate.fast_sigmoid(slope)\n",
        "        # Initialize layers with spike operator\n",
        "        self.bconv1 = BinaryConv2d(1, 16, 5, bias=self.bias)\n",
        "        self.conv1 = nn.Conv2d(1, 16, 5, bias=self.bias)\n",
        "        self.conv1_bn = nn.BatchNorm2d(16)\n",
        "\n",
        "        self.lif1 = snn.Leaky(beta, threshold=self.thr1, reset_mechanism=self.reset_mechanism, spike_grad=spike_grad)\n",
        "        self.bconv2 = BinaryConv2d(16, 64, 5, bias=self.bias)\n",
        "        self.conv2 = nn.Conv2d(16, 64, 5, bias=self.bias)\n",
        "        self.conv2_bn = nn.BatchNorm2d(64)\n",
        "        self.lif2 = snn.Leaky(beta, threshold=self.thr2, reset_mechanism=self.reset_mechanism, spike_grad=spike_grad)\n",
        "        self.bfc1 = BinaryLinear(64 * 4 * 4, 10, bias=self.bias)\n",
        "        self.fc1 = nn.Linear(64 * 4 * 4, 10, bias=self.bias)\n",
        "        self.lif3 = snn.Leaky(beta, threshold=self.thr3, reset_mechanism=self.reset_mechanism, spike_grad=spike_grad)\n",
        "        self.dropout = nn.Dropout(p1)\n",
        "\n",
        "    def forward(self, x):\n",
        "\n",
        "        # Initialize hidden states and outputs at t=0\n",
        "        mem1 = self.lif1.init_leaky()\n",
        "        mem2 = self.lif2.init_leaky()\n",
        "        mem3 = self.lif3.init_leaky()\n",
        "\n",
        "        # Record the final layer\n",
        "        spk3_rec = []\n",
        "        mem3_rec = []\n",
        "\n",
        "        # Binarized\n",
        "        if self.binarize:\n",
        "\n",
        "            for step in range(self.num_steps):\n",
        "\n",
        "                if self.binarize_input:\n",
        "                    x = binarize_activations(x)\n",
        "                cur1 = F.avg_pool2d(self.bconv1(x), 2)\n",
        "                if self.batch_norm:\n",
        "                    cur1 = self.conv1_bn(cur1)\n",
        "                spk1, mem1 = self.lif1(cur1, mem1)\n",
        "                cur2 = F.avg_pool2d(self.bconv2(spk1), 2)\n",
        "                if self.batch_norm:\n",
        "                    cur2 = self.conv2_bn(cur2)\n",
        "                spk2, mem2 = self.lif2(cur2, mem2)\n",
        "                cur3 = self.dropout(self.bfc1(spk2.flatten(1)))\n",
        "                spk3, mem3 = self.lif3(cur3, mem3)\n",
        "\n",
        "                spk3_rec.append(spk3)\n",
        "                mem3_rec.append(mem3)\n",
        "\n",
        "            return torch.stack(spk3_rec, dim=0), torch.stack(mem3_rec, dim=0)\n",
        "\n",
        "        # Full Precision\n",
        "        else:\n",
        "\n",
        "            for step in range(self.num_steps):\n",
        "\n",
        "                cur1 = F.avg_pool2d(self.conv1(x), 2)\n",
        "                if self.batch_norm:\n",
        "                    cur1 = self.conv1_bn(cur1)\n",
        "                spk1, mem1 = self.lif1(cur1, mem1)\n",
        "                cur2 = F.avg_pool2d(self.conv2(spk1), 2)\n",
        "                if self.batch_norm:\n",
        "                    cur2 = self.conv2_bn(cur2)\n",
        "                spk2, mem2 = self.lif2(cur2, mem2)\n",
        "                cur3 = self.dropout(self.fc1(spk2.flatten(1)))\n",
        "                spk3, mem3 = self.lif3(cur3, mem3)\n",
        "\n",
        "                spk3_rec.append(spk3)\n",
        "                mem3_rec.append(mem3)\n",
        "\n",
        "            return torch.stack(spk3_rec, dim=0), torch.stack(mem3_rec, dim=0)\n",
        "\n",
        "class NetFC(nn.Module):\n",
        "    def __init__(self, config, neurons=[256, 128], sparsity=[0.0, 0.0]):\n",
        "        super().__init__()\n",
        "\n",
        "        self.thr1 = config['threshold1']\n",
        "        self.thr2 = config['threshold2']\n",
        "        self.thr3 = config['threshold3']\n",
        "        slope = config['slope']\n",
        "        beta = config['beta']\n",
        "        self.num_steps = config['num_steps']\n",
        "        self.batch_norm = config['enable_batch_norm']\n",
        "        p1 = config['dropout1']\n",
        "        self.binarize = config['binarize']\n",
        "        self.binarize_input = config['binarize_input']\n",
        "        self.bias = config['enable_bias']\n",
        "        self.reset_mechanism = 'zero' if config['on_spike_reset_to_zero'] else 'subtract'\n",
        "\n",
        "        spike_grad = surrogate.fast_sigmoid(slope)\n",
        "        # Initialize layers with spike operator\n",
        "        if self.binarize:\n",
        "            self.bfc1 = SparseBinaryLinear(MNIST_INPUT_RESOLUTION * MNIST_INPUT_RESOLUTION, neurons[0], sparsity[0], bias=(self.bias and not self.batch_norm))\n",
        "        else:\n",
        "            self.fc1 = nn.Linear(MNIST_INPUT_RESOLUTION * MNIST_INPUT_RESOLUTION, neurons[0], bias=(self.bias and not self.batch_norm))\n",
        "        if self.batch_norm:\n",
        "            self.bn1 = nn.BatchNorm1d(neurons[0])\n",
        "        self.lif1 = snn.Leaky(beta, threshold=self.thr1, reset_mechanism=self.reset_mechanism, spike_grad=spike_grad)\n",
        "\n",
        "        if self.binarize:\n",
        "            self.bfc2 = SparseBinaryLinear(neurons[0], neurons[1], sparsity[1], bias=(self.bias and not self.batch_norm))\n",
        "        else:\n",
        "            self.fc2 = nn.Linear(neurons[0], neurons[1], bias=(self.bias and not self.batch_norm))\n",
        "        if self.batch_norm:\n",
        "            self.bn2 = nn.BatchNorm1d(neurons[1])\n",
        "            #self.bn2 = QuantizedBatchNorm1d(neurons[1])\n",
        "        self.lif2 = snn.Leaky(beta, threshold=self.thr2, reset_mechanism=self.reset_mechanism, spike_grad=spike_grad)\n",
        "\n",
        "        if self.binarize:\n",
        "            self.bfc3 = BinaryLinear(neurons[1], 10, bias=self.bias)\n",
        "        else:\n",
        "            self.fc3 = nn.Linear(neurons[1], 10, bias=self.bias)\n",
        "        self.lif3 = snn.Leaky(beta, threshold=self.thr3, reset_mechanism=self.reset_mechanism, spike_grad=spike_grad)\n",
        "        self.dropout = nn.Dropout(p1)\n",
        "\n",
        "    def forward(self, x):\n",
        "\n",
        "        # Initialize hidden states and outputs at t=0\n",
        "        mem1 = self.lif1.init_leaky()\n",
        "        mem2 = self.lif2.init_leaky()\n",
        "        mem3 = self.lif3.init_leaky()\n",
        "\n",
        "        # Record the final layer\n",
        "        spk3_rec = []\n",
        "        mem3_rec = []\n",
        "\n",
        "        # Binarized\n",
        "        if self.binarize:\n",
        "            for step in range(self.num_steps):\n",
        "\n",
        "                x = x.flatten(1)\n",
        "                if self.binarize_input:\n",
        "                    x = binarize_activations(x)\n",
        "                cur1 = self.bfc1(x)\n",
        "                if self.batch_norm:\n",
        "                   cur1 = self.bn1(cur1)\n",
        "                spk1, mem1 = self.lif1(cur1, mem1)\n",
        "                cur2 = self.bfc2(spk1)\n",
        "                if self.batch_norm:\n",
        "                    cur2 = self.bn2(cur2)\n",
        "                spk2, mem2 = self.lif2(cur2, mem2)\n",
        "                cur3 = self.dropout(self.bfc3(spk2))\n",
        "                spk3, mem3 = self.lif3(cur3, mem3)\n",
        "\n",
        "                spk3_rec.append(spk3)\n",
        "                mem3_rec.append(mem3)\n",
        "\n",
        "            return torch.stack(spk3_rec, dim=0), torch.stack(mem3_rec, dim=0)\n",
        "\n",
        "        # Full Precision\n",
        "        else:\n",
        "\n",
        "            for step in range(self.num_steps):\n",
        "\n",
        "                cur1 = self.fc1(x.flatten(1))\n",
        "                if self.batch_norm:\n",
        "                    cur1 = self.bn1(cur1)\n",
        "                spk1, mem1 = self.lif1(cur1, mem1)\n",
        "                cur2 = self.fc2(spk1)\n",
        "                if self.batch_norm:\n",
        "                    cur2 = self.bn2(cur2)\n",
        "                spk2, mem2 = self.lif2(cur2, mem2)\n",
        "                cur3 = self.dropout(self.fc3(spk2))\n",
        "                spk3, mem3 = self.lif3(cur3, mem3)\n",
        "\n",
        "                spk3_rec.append(spk3)\n",
        "                mem3_rec.append(mem3)\n",
        "\n",
        "            return torch.stack(spk3_rec, dim=0), torch.stack(mem3_rec, dim=0)\n",
        "\n",
        "class NetFC_FirstConv(nn.Module):\n",
        "    def __init__(self, config, neurons=[16, 256]):\n",
        "        super().__init__()\n",
        "\n",
        "        self.thr1 = config['threshold1']\n",
        "        self.thr2 = config['threshold2']\n",
        "        self.thr3 = config['threshold3']\n",
        "        slope = config['slope']\n",
        "\n",
        "        beta = config['beta']\n",
        "        self.num_steps = config['num_steps']\n",
        "        self.batch_norm = config['enable_batch_norm']\n",
        "        p1 = config['dropout1']\n",
        "        self.binarize = config['binarize']\n",
        "        self.binarize_input = config['binarize_input']\n",
        "        self.bias = config['enable_bias']\n",
        "        self.reset_mechanism = 'zero' if config['on_spike_reset_to_zero'] else 'subtract'\n",
        "\n",
        "        spike_grad = surrogate.fast_sigmoid(slope)\n",
        "        # Initialize layers with spike operator\n",
        "        self.bconv1 = BinaryConv2d(1, neurons[0], 5, bias=self.bias)\n",
        "        self.conv1 = nn.Conv2d(1, neurons[0], 5, bias=self.bias)\n",
        "        self.conv1_bn = nn.BatchNorm2d(neurons[0])\n",
        "        self.lif1 = snn.Leaky(beta, threshold=self.thr1, reset_mechanism=self.reset_mechanism, spike_grad=spike_grad)\n",
        "\n",
        "        n = ((28-(5//2))/2)**2\n",
        "        print(n)\n",
        "        self.bfc2 = BinaryLinear(n, neurons[1], bias=self.bias)\n",
        "        self.fc2 = nn.Linear(n, neurons[1], bias=self.bias)\n",
        "        self.bn2 = nn.BatchNorm1d(neurons[1])\n",
        "        self.lif2 = snn.Leaky(beta, threshold=self.thr2, reset_mechanism=self.reset_mechanism, spike_grad=spike_grad)\n",
        "\n",
        "        self.bfc3 = BinaryLinear(neurons[1], 10, bias=self.bias)\n",
        "        self.fc3 = nn.Linear(neurons[1], 10, bias=self.bias)\n",
        "        self.lif3 = snn.Leaky(beta, threshold=self.thr3, reset_mechanism=self.reset_mechanism, spike_grad=spike_grad)\n",
        "        self.dropout = nn.Dropout(p1)\n",
        "\n",
        "    def forward(self, x):\n",
        "\n",
        "        # Initialize hidden states and outputs at t=0\n",
        "        mem1 = self.lif1.init_leaky()\n",
        "        mem2 = self.lif2.init_leaky()\n",
        "        mem3 = self.lif3.init_leaky()\n",
        "\n",
        "        # Record the final layer\n",
        "        spk3_rec = []\n",
        "        mem3_rec = []\n",
        "\n",
        "        # Binarized\n",
        "        if self.binarize:\n",
        "\n",
        "            for step in range(self.num_steps):\n",
        "\n",
        "                if self.binarize_input:\n",
        "                    x = binarize_activations(x)\n",
        "                cur1 = F.avg_pool2d(self.bconv1(x), 2)\n",
        "                if self.batch_norm:\n",
        "                    cur1 = self.conv1_bn(cur1)\n",
        "                spk1, mem1 = self.lif1(cur1, mem1)\n",
        "                spk2, mem2 = self.lif2(self.bfc2(spk1.flatten(1)), mem2)\n",
        "                if self.batch_norm:\n",
        "                    spk2 = self.bn2(spk2)\n",
        "                cur3 = self.dropout(self.bfc3(spk2))\n",
        "                spk3, mem3 = self.lif3(cur3, mem3)\n",
        "\n",
        "                spk3_rec.append(spk3)\n",
        "                mem3_rec.append(mem3)\n",
        "\n",
        "            return torch.stack(spk3_rec, dim=0), torch.stack(mem3_rec, dim=0)\n",
        "\n",
        "        # Full Precision\n",
        "        else:\n",
        "\n",
        "            for step in range(self.num_steps):\n",
        "\n",
        "                cur1 = F.avg_pool2d(self.conv1(x), 2)\n",
        "                if self.batch_norm:\n",
        "                    cur1 = self.conv1_bn(cur1)\n",
        "                spk1, mem1 = self.lif1(cur1, mem1)\n",
        "                spk2, mem2 = self.lif2(self.fc2(spk1.flatten(1)), mem2)\n",
        "                if self.batch_norm:\n",
        "                    spk2 = self.bn2(spk2)\n",
        "                cur3 = self.dropout(self.fc3(spk2))\n",
        "                spk3, mem3 = self.lif3(cur3, mem3)\n",
        "\n",
        "                spk3_rec.append(spk3)\n",
        "                mem3_rec.append(mem3)\n",
        "\n",
        "            return torch.stack(spk3_rec, dim=0), torch.stack(mem3_rec, dim=0)"
      ],
      "metadata": {
        "id": "U_IRbNKuMgho"
      },
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def quantize(input):\n",
        "    #0.5, 0.75, 1, 1.5, 2, 3, 4, 6, 8\n",
        "    output = input.new(input.size())\n",
        "    output[input < (6.0+8.0)/2] = 8\n",
        "    output[input < (4.0+6.0)/2] = 6\n",
        "    output[input < (4.0+3.0)/2] = 4\n",
        "    output[input < (3.0+4.0)/2] = 3\n",
        "    output[input < (2.0+3.0)/2] = 2\n",
        "    output[input < (1.5+2.0)/2] = 1.5\n",
        "    output[input < (1.0+1.5)/2] = 1.0\n",
        "    output[input < (0.75+1.0)/2] = 0.75\n",
        "    output[input < (0.5+0.75)/2] = 0.5\n",
        "    #output = input\n",
        "    #print(output)\n",
        "    return output\n",
        "    #return input\n",
        "    #return torch.ones_like(input)\n",
        "    #return torch.round(torch.maximum(input, torch.ones_like(input)))\n",
        "\n",
        "\n",
        "class QuantizedBatchNorm1d(nn.BatchNorm1d):\n",
        "    def forward(self, input):\n",
        "        if self.momentum is None:\n",
        "            exponential_average_factor = 0.0\n",
        "        else:\n",
        "            exponential_average_factor = self.momentum\n",
        "\n",
        "        if self.training and self.track_running_stats:\n",
        "            # TODO: if statement only here to tell the jit to skip emitting this when it is None\n",
        "            if self.num_batches_tracked is not None:  # type: ignore[has-type]\n",
        "                self.num_batches_tracked.add_(1)  # type: ignore[has-type]\n",
        "                if self.momentum is None:  # use cumulative moving average\n",
        "                    exponential_average_factor = 1.0 / float(self.num_batches_tracked)\n",
        "                else:  # use exponential moving average\n",
        "                    exponential_average_factor = self.momentum\n",
        "\n",
        "        if self.training:\n",
        "            bn_training = True\n",
        "        else:\n",
        "            bn_training = (self.running_mean is None) and (self.running_var is None)\n",
        "\n",
        "        F.batch_norm(\n",
        "            input,\n",
        "            # If buffers are not to be tracked, ensure that they won't be updated\n",
        "            self.running_mean\n",
        "            if not self.training or self.track_running_stats\n",
        "            else None,\n",
        "            self.running_var if not self.training or self.track_running_stats else None,\n",
        "            self.weight,\n",
        "            self.bias,\n",
        "            bn_training,\n",
        "            exponential_average_factor,\n",
        "            self.eps,\n",
        "        )\n",
        "\n",
        "        return input.clone() * quantize(self.weight.clone()) + self.bias\n"
      ],
      "metadata": {
        "id": "_y4JoueIUpnr"
      },
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def post_quantize(model):\n",
        "    print(\"post quantize model\")\n",
        "    with torch.no_grad():\n",
        "        #model.bfc1.weight = nn.Parameter(binarize(model.bfc1.weight) * model.bfc1.mask)\n",
        "        for child in model.children():\n",
        "            if type(child) == SparseBinaryLinear:\n",
        "                print(\"sfc\", child)\n",
        "                child.weight = nn.Parameter(binarize(child.weight).to(child.weight.device) * child.mask.to(child.weight.device))\n",
        "            if type(child) == BinaryLinear:\n",
        "                print(\"bfc\", child)\n",
        "                child.weight = nn.Parameter(binarize(child.weight).to(child.weight.device))\n",
        "            if type(child) == nn.BatchNorm1d:\n",
        "                print(\"qbn\", child)\n",
        "                child.weight = nn.Parameter(quantize(child.weight).to(child.weight.device))\n",
        "    # for n, p in model.named_parameters():\n",
        "    #     print(n, p)\n",
        "    return model\n",
        "#net = NetFC(config)\n",
        "#post_quantize(net)"
      ],
      "metadata": {
        "id": "8tqPq60lfaPa"
      },
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "cfg = config.copy()\n",
        "cfg['model'] = 'NetFC'\n",
        "#cfg['num_steps'] = 100\n",
        "cfg['enable_batch_norm'] = True\n",
        "cfg['enable_bias'] = True\n",
        "#cfg['print_weights'] = True\n",
        "run(cfg)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "ilgG9-7Oda39",
        "outputId": "6023986e-e1a0-4ec8-cab2-16d6c0bb38b6"
      },
      "execution_count": 14,
      "outputs": [
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'model': 'NetFC', 'exp_name': 'mnist_tha', 'num_trials': 5, 'num_epochs': 50, 'binarize': True, 'binarize_input': True, 'post_quantize': True, 'enable_bias': True, 'enable_batch_norm': True, 'enable_dropout': True, 'enable_threshold': True, 'enable_slope': True, 'on_spike_reset_to_zero': False, 'data_dir': '~/data/mnist', 'batch_size': 128, 'seed': 0, 'num_workers': 0, 'save_csv': True, 'save_model': True, 'early_stopping': True, 'patience': 100, 'grad_clip': False, 'weight_clip': False, 'dropout1': 0.02856, 'beta': 0.992187, 'lr': 0.00997, 'slope': 10.22, 'threshold1': 11.666, 'alpha_thr1': 0.024, 'thr_final1': 4.317, 'threshold2': 14.105, 'alpha_thr2': 0.119, 'thr_final2': 16.29, 'threshold3': 0.6656, 'alpha_thr3': 0.0011, 'thr_final3': 3.496, 'num_steps': 100, 'correct_rate': 0.8, 'incorrect_rate': 0.2, 'betas': (0.9, 0.999), 't_0': 4688, 'eta_min': 0, 'df_lr': True, 'print_weights': False}\n",
            "tensor(1.)\n",
            "tensor(1.)\n",
            "NetFC(\n",
            "  (bfc1): SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "  (bn1): BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "  (lif1): Leaky()\n",
            "  (bfc2): SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "  (bn2): BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "  (lif2): Leaky()\n",
            "  (bfc3): BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "  (lif3): Leaky()\n",
            "  (dropout): Dropout(p=0.02856, inplace=False)\n",
            ")\n",
            "=======Trial: 0=======\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.2687: 100%|██████████| 469/469 [02:26<00:00,  3.20it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 0 \tTest Accuracy: 87.33\n",
            "Test acc increased (0.000000 --> 87.330000).  Saving model ...\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 0 \tTest Quantized Accuracy: 87.39\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.2105: 100%|██████████| 469/469 [02:24<00:00,  3.24it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 1 \tTest Accuracy: 88.67\n",
            "Test acc increased (87.330000 --> 88.670000).  Saving model ...\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 1 \tTest Quantized Accuracy: 88.57\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.2066: 100%|██████████| 469/469 [02:26<00:00,  3.21it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 2 \tTest Accuracy: 88.4\n",
            "EarlyStopping counter: 1 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 2 \tTest Quantized Accuracy: 86.18\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.2146: 100%|██████████| 469/469 [02:25<00:00,  3.22it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 3 \tTest Accuracy: 88.34\n",
            "EarlyStopping counter: 2 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 3 \tTest Quantized Accuracy: 87.87\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.1451: 100%|██████████| 469/469 [02:24<00:00,  3.25it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 4 \tTest Accuracy: 89.03\n",
            "Test acc increased (88.670000 --> 89.030000).  Saving model ...\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 4 \tTest Quantized Accuracy: 88.14\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.1446: 100%|██████████| 469/469 [02:21<00:00,  3.32it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 5 \tTest Accuracy: 89.56\n",
            "Test acc increased (89.030000 --> 89.560000).  Saving model ...\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 5 \tTest Quantized Accuracy: 89.18\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.1457: 100%|██████████| 469/469 [02:20<00:00,  3.35it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 6 \tTest Accuracy: 90.3\n",
            "Test acc increased (89.560000 --> 90.300000).  Saving model ...\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 6 \tTest Quantized Accuracy: 89.97\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.1056: 100%|██████████| 469/469 [02:20<00:00,  3.33it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 7 \tTest Accuracy: 90.55\n",
            "Test acc increased (90.300000 --> 90.550000).  Saving model ...\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 7 \tTest Quantized Accuracy: 90.35\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.0908: 100%|██████████| 469/469 [02:20<00:00,  3.34it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 8 \tTest Accuracy: 90.54\n",
            "EarlyStopping counter: 1 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 8 \tTest Quantized Accuracy: 89.73\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 0.9753: 100%|██████████| 469/469 [02:19<00:00,  3.35it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 9 \tTest Accuracy: 90.8\n",
            "Test acc increased (90.550000 --> 90.800000).  Saving model ...\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 9 \tTest Quantized Accuracy: 90.35\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.0746: 100%|██████████| 469/469 [02:21<00:00,  3.31it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 10 \tTest Accuracy: 90.64\n",
            "EarlyStopping counter: 1 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 10 \tTest Quantized Accuracy: 90.2\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.0933: 100%|██████████| 469/469 [02:20<00:00,  3.33it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 11 \tTest Accuracy: 90.15\n",
            "EarlyStopping counter: 2 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 11 \tTest Quantized Accuracy: 89.42\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.1009: 100%|██████████| 469/469 [02:19<00:00,  3.35it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 12 \tTest Accuracy: 90.42\n",
            "EarlyStopping counter: 3 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 12 \tTest Quantized Accuracy: 89.92\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.1445: 100%|██████████| 469/469 [02:20<00:00,  3.33it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 13 \tTest Accuracy: 89.96\n",
            "EarlyStopping counter: 4 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 13 \tTest Quantized Accuracy: 89.48\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.1570: 100%|██████████| 469/469 [02:20<00:00,  3.33it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 14 \tTest Accuracy: 89.66\n",
            "EarlyStopping counter: 5 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 14 \tTest Quantized Accuracy: 89.12\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.1836: 100%|██████████| 469/469 [02:19<00:00,  3.36it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 15 \tTest Accuracy: 89.8\n",
            "EarlyStopping counter: 6 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 15 \tTest Quantized Accuracy: 88.58\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.1137: 100%|██████████| 469/469 [02:20<00:00,  3.34it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 16 \tTest Accuracy: 89.24\n",
            "EarlyStopping counter: 7 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 16 \tTest Quantized Accuracy: 87.53\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.1437: 100%|██████████| 469/469 [02:19<00:00,  3.35it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 17 \tTest Accuracy: 90.53\n",
            "EarlyStopping counter: 8 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 17 \tTest Quantized Accuracy: 89.9\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.1271: 100%|██████████| 469/469 [02:19<00:00,  3.36it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 18 \tTest Accuracy: 91.27\n",
            "Test acc increased (90.800000 --> 91.270000).  Saving model ...\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 18 \tTest Quantized Accuracy: 88.87\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.1404: 100%|██████████| 469/469 [02:19<00:00,  3.37it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 19 \tTest Accuracy: 91.09\n",
            "EarlyStopping counter: 1 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 19 \tTest Quantized Accuracy: 87.09\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.1291: 100%|██████████| 469/469 [02:21<00:00,  3.32it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 20 \tTest Accuracy: 90.79\n",
            "EarlyStopping counter: 2 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 20 \tTest Quantized Accuracy: 81.24\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.0890: 100%|██████████| 469/469 [02:20<00:00,  3.34it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 21 \tTest Accuracy: 91.0\n",
            "EarlyStopping counter: 3 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 21 \tTest Quantized Accuracy: 68.38\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.1359: 100%|██████████| 469/469 [02:23<00:00,  3.27it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 22 \tTest Accuracy: 90.85\n",
            "EarlyStopping counter: 4 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 22 \tTest Quantized Accuracy: 63.93\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.1031: 100%|██████████| 469/469 [02:27<00:00,  3.17it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 23 \tTest Accuracy: 90.7\n",
            "EarlyStopping counter: 5 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 23 \tTest Quantized Accuracy: 59.7\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.1427: 100%|██████████| 469/469 [02:28<00:00,  3.15it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 24 \tTest Accuracy: 91.27\n",
            "EarlyStopping counter: 6 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 24 \tTest Quantized Accuracy: 55.47\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.0961: 100%|██████████| 469/469 [02:30<00:00,  3.11it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 25 \tTest Accuracy: 91.49\n",
            "Test acc increased (91.270000 --> 91.490000).  Saving model ...\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 25 \tTest Quantized Accuracy: 54.96\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.0844: 100%|██████████| 469/469 [02:29<00:00,  3.14it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 26 \tTest Accuracy: 91.9\n",
            "Test acc increased (91.490000 --> 91.900000).  Saving model ...\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 26 \tTest Quantized Accuracy: 44.46\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.0089: 100%|██████████| 469/469 [02:27<00:00,  3.19it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 27 \tTest Accuracy: 91.94\n",
            "Test acc increased (91.900000 --> 91.940000).  Saving model ...\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 27 \tTest Quantized Accuracy: 44.93\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.0664: 100%|██████████| 469/469 [02:22<00:00,  3.30it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 28 \tTest Accuracy: 91.55\n",
            "EarlyStopping counter: 1 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 28 \tTest Quantized Accuracy: 43.16\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 0.9506: 100%|██████████| 469/469 [02:18<00:00,  3.38it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 29 \tTest Accuracy: 91.9\n",
            "EarlyStopping counter: 2 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 29 \tTest Quantized Accuracy: 45.84\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.0771: 100%|██████████| 469/469 [02:23<00:00,  3.27it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 30 \tTest Accuracy: 91.48\n",
            "EarlyStopping counter: 3 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 30 \tTest Quantized Accuracy: 60.98\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.0530: 100%|██████████| 469/469 [02:24<00:00,  3.25it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 31 \tTest Accuracy: 91.64\n",
            "EarlyStopping counter: 4 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 31 \tTest Quantized Accuracy: 44.08\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.0921: 100%|██████████| 469/469 [02:24<00:00,  3.24it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 32 \tTest Accuracy: 91.62\n",
            "EarlyStopping counter: 5 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 32 \tTest Quantized Accuracy: 45.86\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.1548: 100%|██████████| 469/469 [02:23<00:00,  3.27it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 33 \tTest Accuracy: 91.1\n",
            "EarlyStopping counter: 6 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 33 \tTest Quantized Accuracy: 40.58\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.1507: 100%|██████████| 469/469 [02:21<00:00,  3.32it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 34 \tTest Accuracy: 91.96\n",
            "Test acc increased (91.940000 --> 91.960000).  Saving model ...\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 34 \tTest Quantized Accuracy: 43.58\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.0757: 100%|██████████| 469/469 [02:20<00:00,  3.33it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 35 \tTest Accuracy: 91.91\n",
            "EarlyStopping counter: 1 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 35 \tTest Quantized Accuracy: 43.03\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.1368: 100%|██████████| 469/469 [02:22<00:00,  3.29it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 36 \tTest Accuracy: 91.77\n",
            "EarlyStopping counter: 2 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 36 \tTest Quantized Accuracy: 46.63\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.0848: 100%|██████████| 469/469 [02:21<00:00,  3.31it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 37 \tTest Accuracy: 91.8\n",
            "EarlyStopping counter: 3 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 37 \tTest Quantized Accuracy: 27.07\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.1265: 100%|██████████| 469/469 [02:22<00:00,  3.30it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 38 \tTest Accuracy: 91.95\n",
            "EarlyStopping counter: 4 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 38 \tTest Quantized Accuracy: 21.67\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.1046: 100%|██████████| 469/469 [02:21<00:00,  3.32it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 39 \tTest Accuracy: 91.97\n",
            "Test acc increased (91.960000 --> 91.970000).  Saving model ...\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 39 \tTest Quantized Accuracy: 38.42\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.0895: 100%|██████████| 469/469 [02:21<00:00,  3.30it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 40 \tTest Accuracy: 92.24\n",
            "Test acc increased (91.970000 --> 92.240000).  Saving model ...\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 40 \tTest Quantized Accuracy: 26.86\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.1452: 100%|██████████| 469/469 [02:28<00:00,  3.15it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 41 \tTest Accuracy: 91.69\n",
            "EarlyStopping counter: 1 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 41 \tTest Quantized Accuracy: 24.19\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.1178: 100%|██████████| 469/469 [02:30<00:00,  3.12it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 42 \tTest Accuracy: 91.78\n",
            "EarlyStopping counter: 2 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 42 \tTest Quantized Accuracy: 18.91\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.1195: 100%|██████████| 469/469 [02:30<00:00,  3.11it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 43 \tTest Accuracy: 92.18\n",
            "EarlyStopping counter: 3 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 43 \tTest Quantized Accuracy: 28.25\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.0643: 100%|██████████| 469/469 [02:30<00:00,  3.12it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 44 \tTest Accuracy: 92.0\n",
            "EarlyStopping counter: 4 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 44 \tTest Quantized Accuracy: 24.21\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.1164: 100%|██████████| 469/469 [02:30<00:00,  3.12it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 45 \tTest Accuracy: 91.9\n",
            "EarlyStopping counter: 5 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 45 \tTest Quantized Accuracy: 26.81\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.1084: 100%|██████████| 469/469 [02:31<00:00,  3.09it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 46 \tTest Accuracy: 91.81\n",
            "EarlyStopping counter: 6 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 46 \tTest Quantized Accuracy: 16.98\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.0956: 100%|██████████| 469/469 [02:30<00:00,  3.11it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 47 \tTest Accuracy: 92.2\n",
            "EarlyStopping counter: 7 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 47 \tTest Quantized Accuracy: 21.7\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.0251: 100%|██████████| 469/469 [02:30<00:00,  3.12it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 48 \tTest Accuracy: 91.68\n",
            "EarlyStopping counter: 8 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 48 \tTest Quantized Accuracy: 21.66\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 0.9180: 100%|██████████| 469/469 [02:32<00:00,  3.08it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 49 \tTest Accuracy: 92.73\n",
            "Test acc increased (92.240000 --> 92.730000).  Saving model ...\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 49 \tTest Quantized Accuracy: 21.72\n",
            "tensor(1.)\n",
            "tensor(1.)\n",
            "=======Trial: 1=======\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.2687: 100%|██████████| 469/469 [02:31<00:00,  3.10it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 0 \tTest Accuracy: 87.33\n",
            "Test acc increased (0.000000 --> 87.330000).  Saving model ...\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 0 \tTest Quantized Accuracy: 87.39\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.2105: 100%|██████████| 469/469 [02:30<00:00,  3.11it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 1 \tTest Accuracy: 88.67\n",
            "Test acc increased (87.330000 --> 88.670000).  Saving model ...\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 1 \tTest Quantized Accuracy: 88.57\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.2066: 100%|██████████| 469/469 [02:32<00:00,  3.08it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 2 \tTest Accuracy: 88.4\n",
            "EarlyStopping counter: 1 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 2 \tTest Quantized Accuracy: 86.18\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.2146: 100%|██████████| 469/469 [02:31<00:00,  3.10it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 3 \tTest Accuracy: 88.34\n",
            "EarlyStopping counter: 2 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 3 \tTest Quantized Accuracy: 87.87\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.1451: 100%|██████████| 469/469 [02:31<00:00,  3.11it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 4 \tTest Accuracy: 89.03\n",
            "Test acc increased (88.670000 --> 89.030000).  Saving model ...\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 4 \tTest Quantized Accuracy: 88.14\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.1446: 100%|██████████| 469/469 [02:31<00:00,  3.10it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 5 \tTest Accuracy: 89.56\n",
            "Test acc increased (89.030000 --> 89.560000).  Saving model ...\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 5 \tTest Quantized Accuracy: 89.18\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.1457: 100%|██████████| 469/469 [02:30<00:00,  3.12it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 6 \tTest Accuracy: 90.3\n",
            "Test acc increased (89.560000 --> 90.300000).  Saving model ...\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 6 \tTest Quantized Accuracy: 89.97\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.1056: 100%|██████████| 469/469 [02:20<00:00,  3.35it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 7 \tTest Accuracy: 90.55\n",
            "Test acc increased (90.300000 --> 90.550000).  Saving model ...\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 7 \tTest Quantized Accuracy: 90.35\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.0908: 100%|██████████| 469/469 [02:20<00:00,  3.34it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 8 \tTest Accuracy: 90.54\n",
            "EarlyStopping counter: 1 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 8 \tTest Quantized Accuracy: 89.73\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 0.9753: 100%|██████████| 469/469 [02:19<00:00,  3.35it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 9 \tTest Accuracy: 90.8\n",
            "Test acc increased (90.550000 --> 90.800000).  Saving model ...\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 9 \tTest Quantized Accuracy: 90.35\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.0746: 100%|██████████| 469/469 [02:20<00:00,  3.33it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 10 \tTest Accuracy: 90.64\n",
            "EarlyStopping counter: 1 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 10 \tTest Quantized Accuracy: 90.2\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.0933: 100%|██████████| 469/469 [02:20<00:00,  3.34it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 11 \tTest Accuracy: 90.15\n",
            "EarlyStopping counter: 2 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 11 \tTest Quantized Accuracy: 89.42\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.1009: 100%|██████████| 469/469 [02:21<00:00,  3.30it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 12 \tTest Accuracy: 90.42\n",
            "EarlyStopping counter: 3 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 12 \tTest Quantized Accuracy: 89.92\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.1445: 100%|██████████| 469/469 [02:23<00:00,  3.28it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 13 \tTest Accuracy: 89.96\n",
            "EarlyStopping counter: 4 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 13 \tTest Quantized Accuracy: 89.48\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.1570: 100%|██████████| 469/469 [02:21<00:00,  3.31it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 14 \tTest Accuracy: 89.66\n",
            "EarlyStopping counter: 5 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 14 \tTest Quantized Accuracy: 89.12\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.1836: 100%|██████████| 469/469 [02:21<00:00,  3.30it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 15 \tTest Accuracy: 89.8\n",
            "EarlyStopping counter: 6 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 15 \tTest Quantized Accuracy: 88.68\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.1137: 100%|██████████| 469/469 [02:21<00:00,  3.31it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 16 \tTest Accuracy: 89.24\n",
            "EarlyStopping counter: 7 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 16 \tTest Quantized Accuracy: 87.53\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.1437: 100%|██████████| 469/469 [02:21<00:00,  3.32it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 17 \tTest Accuracy: 90.53\n",
            "EarlyStopping counter: 8 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 17 \tTest Quantized Accuracy: 89.9\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.1271: 100%|██████████| 469/469 [02:20<00:00,  3.33it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 18 \tTest Accuracy: 91.27\n",
            "Test acc increased (90.800000 --> 91.270000).  Saving model ...\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 18 \tTest Quantized Accuracy: 88.75\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.1404: 100%|██████████| 469/469 [02:19<00:00,  3.36it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 19 \tTest Accuracy: 91.09\n",
            "EarlyStopping counter: 1 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 19 \tTest Quantized Accuracy: 87.09\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.1291: 100%|██████████| 469/469 [02:19<00:00,  3.36it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 20 \tTest Accuracy: 90.79\n",
            "EarlyStopping counter: 2 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 20 \tTest Quantized Accuracy: 82.54\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.0890: 100%|██████████| 469/469 [02:20<00:00,  3.34it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 21 \tTest Accuracy: 91.0\n",
            "EarlyStopping counter: 3 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 21 \tTest Quantized Accuracy: 68.38\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.1359: 100%|██████████| 469/469 [02:19<00:00,  3.36it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 22 \tTest Accuracy: 90.85\n",
            "EarlyStopping counter: 4 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 22 \tTest Quantized Accuracy: 66.73\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.1031: 100%|██████████| 469/469 [02:19<00:00,  3.36it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 23 \tTest Accuracy: 90.7\n",
            "EarlyStopping counter: 5 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 23 \tTest Quantized Accuracy: 71.75\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.1427: 100%|██████████| 469/469 [02:19<00:00,  3.35it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 24 \tTest Accuracy: 91.27\n",
            "EarlyStopping counter: 6 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 24 \tTest Quantized Accuracy: 52.79\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.0961: 100%|██████████| 469/469 [02:20<00:00,  3.33it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 25 \tTest Accuracy: 91.49\n",
            "Test acc increased (91.270000 --> 91.490000).  Saving model ...\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 25 \tTest Quantized Accuracy: 52.18\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.0844: 100%|██████████| 469/469 [02:20<00:00,  3.34it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 26 \tTest Accuracy: 91.9\n",
            "Test acc increased (91.490000 --> 91.900000).  Saving model ...\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 26 \tTest Quantized Accuracy: 41.95\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.0089: 100%|██████████| 469/469 [02:20<00:00,  3.33it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 27 \tTest Accuracy: 91.94\n",
            "Test acc increased (91.900000 --> 91.940000).  Saving model ...\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 27 \tTest Quantized Accuracy: 44.93\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.0664: 100%|██████████| 469/469 [02:21<00:00,  3.31it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 28 \tTest Accuracy: 91.55\n",
            "EarlyStopping counter: 1 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 28 \tTest Quantized Accuracy: 41.02\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 0.9506: 100%|██████████| 469/469 [02:20<00:00,  3.34it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 29 \tTest Accuracy: 91.9\n",
            "EarlyStopping counter: 2 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 29 \tTest Quantized Accuracy: 45.84\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.0771: 100%|██████████| 469/469 [02:23<00:00,  3.27it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 30 \tTest Accuracy: 91.48\n",
            "EarlyStopping counter: 3 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 30 \tTest Quantized Accuracy: 48.78\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.0530: 100%|██████████| 469/469 [02:29<00:00,  3.15it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 31 \tTest Accuracy: 91.64\n",
            "EarlyStopping counter: 4 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 31 \tTest Quantized Accuracy: 41.29\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.0921: 100%|██████████| 469/469 [02:28<00:00,  3.15it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 32 \tTest Accuracy: 91.62\n",
            "EarlyStopping counter: 5 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 32 \tTest Quantized Accuracy: 42.69\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.1548: 100%|██████████| 469/469 [02:23<00:00,  3.26it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 33 \tTest Accuracy: 91.1\n",
            "EarlyStopping counter: 6 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 33 \tTest Quantized Accuracy: 38.54\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.1507: 100%|██████████| 469/469 [02:20<00:00,  3.33it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 34 \tTest Accuracy: 91.96\n",
            "Test acc increased (91.940000 --> 91.960000).  Saving model ...\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 34 \tTest Quantized Accuracy: 41.15\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.0757: 100%|██████████| 469/469 [02:20<00:00,  3.33it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 35 \tTest Accuracy: 91.91\n",
            "EarlyStopping counter: 1 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 35 \tTest Quantized Accuracy: 43.03\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.1368: 100%|██████████| 469/469 [02:18<00:00,  3.39it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 36 \tTest Accuracy: 91.77\n",
            "EarlyStopping counter: 2 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 36 \tTest Quantized Accuracy: 29.96\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.0848: 100%|██████████| 469/469 [02:20<00:00,  3.35it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 37 \tTest Accuracy: 91.8\n",
            "EarlyStopping counter: 3 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 37 \tTest Quantized Accuracy: 27.07\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.1265: 100%|██████████| 469/469 [02:22<00:00,  3.30it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 38 \tTest Accuracy: 91.95\n",
            "EarlyStopping counter: 4 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 38 \tTest Quantized Accuracy: 23.06\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.1046: 100%|██████████| 469/469 [02:27<00:00,  3.18it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 39 \tTest Accuracy: 91.97\n",
            "Test acc increased (91.960000 --> 91.970000).  Saving model ...\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 39 \tTest Quantized Accuracy: 38.42\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.0895: 100%|██████████| 469/469 [02:27<00:00,  3.18it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 40 \tTest Accuracy: 92.24\n",
            "Test acc increased (91.970000 --> 92.240000).  Saving model ...\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 40 \tTest Quantized Accuracy: 26.86\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.1452: 100%|██████████| 469/469 [02:27<00:00,  3.17it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 41 \tTest Accuracy: 91.69\n",
            "EarlyStopping counter: 1 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 41 \tTest Quantized Accuracy: 25.86\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.1178: 100%|██████████| 469/469 [02:28<00:00,  3.16it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 42 \tTest Accuracy: 91.78\n",
            "EarlyStopping counter: 2 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 42 \tTest Quantized Accuracy: 27.64\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.1195: 100%|██████████| 469/469 [02:27<00:00,  3.18it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 43 \tTest Accuracy: 92.18\n",
            "EarlyStopping counter: 3 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 43 \tTest Quantized Accuracy: 28.25\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.0643: 100%|██████████| 469/469 [02:27<00:00,  3.17it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 44 \tTest Accuracy: 92.0\n",
            "EarlyStopping counter: 4 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 44 \tTest Quantized Accuracy: 26.61\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.1164: 100%|██████████| 469/469 [02:29<00:00,  3.14it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 45 \tTest Accuracy: 91.9\n",
            "EarlyStopping counter: 5 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 45 \tTest Quantized Accuracy: 26.81\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.1084: 100%|██████████| 469/469 [02:27<00:00,  3.18it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 46 \tTest Accuracy: 91.81\n",
            "EarlyStopping counter: 6 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 46 \tTest Quantized Accuracy: 19.72\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.0956: 100%|██████████| 469/469 [02:27<00:00,  3.18it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 47 \tTest Accuracy: 92.2\n",
            "EarlyStopping counter: 7 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 47 \tTest Quantized Accuracy: 21.7\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.0251: 100%|██████████| 469/469 [02:27<00:00,  3.18it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 48 \tTest Accuracy: 91.68\n",
            "EarlyStopping counter: 8 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 48 \tTest Quantized Accuracy: 21.66\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 0.9180: 100%|██████████| 469/469 [02:28<00:00,  3.16it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 49 \tTest Accuracy: 92.73\n",
            "Test acc increased (92.240000 --> 92.730000).  Saving model ...\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 49 \tTest Quantized Accuracy: 21.72\n",
            "tensor(1.)\n",
            "tensor(1.)\n",
            "=======Trial: 2=======\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.2687: 100%|██████████| 469/469 [02:29<00:00,  3.13it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 0 \tTest Accuracy: 87.33\n",
            "Test acc increased (0.000000 --> 87.330000).  Saving model ...\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 0 \tTest Quantized Accuracy: 87.39\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.2105: 100%|██████████| 469/469 [02:30<00:00,  3.12it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 1 \tTest Accuracy: 88.67\n",
            "Test acc increased (87.330000 --> 88.670000).  Saving model ...\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 1 \tTest Quantized Accuracy: 88.57\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.2066: 100%|██████████| 469/469 [02:30<00:00,  3.12it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 2 \tTest Accuracy: 88.4\n",
            "EarlyStopping counter: 1 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 2 \tTest Quantized Accuracy: 86.18\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.2146: 100%|██████████| 469/469 [02:30<00:00,  3.11it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 3 \tTest Accuracy: 88.34\n",
            "EarlyStopping counter: 2 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 3 \tTest Quantized Accuracy: 87.87\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.1451: 100%|██████████| 469/469 [02:30<00:00,  3.11it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 4 \tTest Accuracy: 89.03\n",
            "Test acc increased (88.670000 --> 89.030000).  Saving model ...\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 4 \tTest Quantized Accuracy: 88.14\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.1446: 100%|██████████| 469/469 [02:31<00:00,  3.10it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 5 \tTest Accuracy: 89.56\n",
            "Test acc increased (89.030000 --> 89.560000).  Saving model ...\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 5 \tTest Quantized Accuracy: 89.18\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.1457: 100%|██████████| 469/469 [02:32<00:00,  3.08it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 6 \tTest Accuracy: 90.3\n",
            "Test acc increased (89.560000 --> 90.300000).  Saving model ...\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 6 \tTest Quantized Accuracy: 89.97\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.1056: 100%|██████████| 469/469 [02:31<00:00,  3.09it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 7 \tTest Accuracy: 90.55\n",
            "Test acc increased (90.300000 --> 90.550000).  Saving model ...\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 7 \tTest Quantized Accuracy: 90.35\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.0908: 100%|██████████| 469/469 [02:30<00:00,  3.12it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 8 \tTest Accuracy: 90.54\n",
            "EarlyStopping counter: 1 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 8 \tTest Quantized Accuracy: 89.73\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 0.9753: 100%|██████████| 469/469 [02:30<00:00,  3.11it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 9 \tTest Accuracy: 90.8\n",
            "Test acc increased (90.550000 --> 90.800000).  Saving model ...\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 9 \tTest Quantized Accuracy: 90.35\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.0746: 100%|██████████| 469/469 [02:30<00:00,  3.12it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 10 \tTest Accuracy: 90.64\n",
            "EarlyStopping counter: 1 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 10 \tTest Quantized Accuracy: 90.2\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.0933: 100%|██████████| 469/469 [02:26<00:00,  3.20it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 11 \tTest Accuracy: 90.15\n",
            "EarlyStopping counter: 2 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 11 \tTest Quantized Accuracy: 89.42\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.1009: 100%|██████████| 469/469 [02:20<00:00,  3.35it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 12 \tTest Accuracy: 90.42\n",
            "EarlyStopping counter: 3 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 12 \tTest Quantized Accuracy: 89.92\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.1445: 100%|██████████| 469/469 [02:20<00:00,  3.34it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 13 \tTest Accuracy: 89.96\n",
            "EarlyStopping counter: 4 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 13 \tTest Quantized Accuracy: 89.48\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.1570: 100%|██████████| 469/469 [02:19<00:00,  3.37it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 14 \tTest Accuracy: 89.66\n",
            "EarlyStopping counter: 5 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 14 \tTest Quantized Accuracy: 89.12\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.1836: 100%|██████████| 469/469 [02:17<00:00,  3.41it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 15 \tTest Accuracy: 89.8\n",
            "EarlyStopping counter: 6 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 15 \tTest Quantized Accuracy: 88.58\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.1137: 100%|██████████| 469/469 [02:17<00:00,  3.40it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 16 \tTest Accuracy: 89.24\n",
            "EarlyStopping counter: 7 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 16 \tTest Quantized Accuracy: 87.53\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.1437: 100%|██████████| 469/469 [02:19<00:00,  3.36it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 17 \tTest Accuracy: 90.53\n",
            "EarlyStopping counter: 8 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 17 \tTest Quantized Accuracy: 89.88\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.1271: 100%|██████████| 469/469 [02:22<00:00,  3.28it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 18 \tTest Accuracy: 91.27\n",
            "Test acc increased (90.800000 --> 91.270000).  Saving model ...\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 18 \tTest Quantized Accuracy: 88.87\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.1404: 100%|██████████| 469/469 [02:21<00:00,  3.32it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 19 \tTest Accuracy: 91.09\n",
            "EarlyStopping counter: 1 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 19 \tTest Quantized Accuracy: 87.09\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.1291: 100%|██████████| 469/469 [02:21<00:00,  3.31it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 20 \tTest Accuracy: 90.79\n",
            "EarlyStopping counter: 2 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 20 \tTest Quantized Accuracy: 82.54\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.0890: 100%|██████████| 469/469 [02:23<00:00,  3.27it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 21 \tTest Accuracy: 91.0\n",
            "EarlyStopping counter: 3 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 21 \tTest Quantized Accuracy: 68.38\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.1359: 100%|██████████| 469/469 [02:23<00:00,  3.27it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 22 \tTest Accuracy: 90.85\n",
            "EarlyStopping counter: 4 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 22 \tTest Quantized Accuracy: 66.73\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.1031: 100%|██████████| 469/469 [02:24<00:00,  3.25it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 23 \tTest Accuracy: 90.7\n",
            "EarlyStopping counter: 5 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 23 \tTest Quantized Accuracy: 59.7\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.1427: 100%|██████████| 469/469 [02:23<00:00,  3.28it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 24 \tTest Accuracy: 91.27\n",
            "EarlyStopping counter: 6 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 24 \tTest Quantized Accuracy: 52.79\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.0961: 100%|██████████| 469/469 [02:19<00:00,  3.36it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 25 \tTest Accuracy: 91.49\n",
            "Test acc increased (91.270000 --> 91.490000).  Saving model ...\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 25 \tTest Quantized Accuracy: 52.18\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.0844: 100%|██████████| 469/469 [02:18<00:00,  3.38it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 26 \tTest Accuracy: 91.9\n",
            "Test acc increased (91.490000 --> 91.900000).  Saving model ...\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 26 \tTest Quantized Accuracy: 44.46\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.0089: 100%|██████████| 469/469 [02:19<00:00,  3.35it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 27 \tTest Accuracy: 91.94\n",
            "Test acc increased (91.900000 --> 91.940000).  Saving model ...\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 27 \tTest Quantized Accuracy: 44.93\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.0664: 100%|██████████| 469/469 [02:19<00:00,  3.37it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 28 \tTest Accuracy: 91.55\n",
            "EarlyStopping counter: 1 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 28 \tTest Quantized Accuracy: 55.25\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 0.9506: 100%|██████████| 469/469 [02:20<00:00,  3.33it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 29 \tTest Accuracy: 91.9\n",
            "EarlyStopping counter: 2 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 29 \tTest Quantized Accuracy: 45.84\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.0771: 100%|██████████| 469/469 [02:19<00:00,  3.36it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 30 \tTest Accuracy: 91.48\n",
            "EarlyStopping counter: 3 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 30 \tTest Quantized Accuracy: 60.98\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.0530: 100%|██████████| 469/469 [02:19<00:00,  3.36it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 31 \tTest Accuracy: 91.64\n",
            "EarlyStopping counter: 4 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 31 \tTest Quantized Accuracy: 42.29\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.0921: 100%|██████████| 469/469 [02:19<00:00,  3.36it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 32 \tTest Accuracy: 91.62\n",
            "EarlyStopping counter: 5 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 32 \tTest Quantized Accuracy: 45.86\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.1548: 100%|██████████| 469/469 [02:19<00:00,  3.36it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 33 \tTest Accuracy: 91.1\n",
            "EarlyStopping counter: 6 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 33 \tTest Quantized Accuracy: 38.54\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.1507: 100%|██████████| 469/469 [02:22<00:00,  3.28it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 34 \tTest Accuracy: 91.96\n",
            "Test acc increased (91.940000 --> 91.960000).  Saving model ...\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 34 \tTest Quantized Accuracy: 43.58\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.0757: 100%|██████████| 469/469 [02:20<00:00,  3.33it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 35 \tTest Accuracy: 91.91\n",
            "EarlyStopping counter: 1 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 35 \tTest Quantized Accuracy: 56.65\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.1368: 100%|██████████| 469/469 [02:19<00:00,  3.37it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 36 \tTest Accuracy: 91.77\n",
            "EarlyStopping counter: 2 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 36 \tTest Quantized Accuracy: 30.52\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.0848: 100%|██████████| 469/469 [02:19<00:00,  3.35it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 37 \tTest Accuracy: 91.8\n",
            "EarlyStopping counter: 3 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 37 \tTest Quantized Accuracy: 24.69\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.1265: 100%|██████████| 469/469 [02:20<00:00,  3.34it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 38 \tTest Accuracy: 91.95\n",
            "EarlyStopping counter: 4 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 38 \tTest Quantized Accuracy: 23.06\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.1046: 100%|██████████| 469/469 [02:19<00:00,  3.36it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 39 \tTest Accuracy: 91.97\n",
            "Test acc increased (91.960000 --> 91.970000).  Saving model ...\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 39 \tTest Quantized Accuracy: 38.42\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.0895: 100%|██████████| 469/469 [02:20<00:00,  3.35it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 40 \tTest Accuracy: 92.24\n",
            "Test acc increased (91.970000 --> 92.240000).  Saving model ...\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 40 \tTest Quantized Accuracy: 26.13\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.1452: 100%|██████████| 469/469 [02:19<00:00,  3.37it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 41 \tTest Accuracy: 91.69\n",
            "EarlyStopping counter: 1 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 41 \tTest Quantized Accuracy: 25.86\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.1178: 100%|██████████| 469/469 [02:19<00:00,  3.36it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 42 \tTest Accuracy: 91.78\n",
            "EarlyStopping counter: 2 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 42 \tTest Quantized Accuracy: 18.91\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.1195: 100%|██████████| 469/469 [02:19<00:00,  3.36it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 43 \tTest Accuracy: 92.18\n",
            "EarlyStopping counter: 3 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 43 \tTest Quantized Accuracy: 26.46\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.0643: 100%|██████████| 469/469 [02:19<00:00,  3.36it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 44 \tTest Accuracy: 92.0\n",
            "EarlyStopping counter: 4 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 44 \tTest Quantized Accuracy: 23.94\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.1164: 100%|██████████| 469/469 [02:19<00:00,  3.35it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 45 \tTest Accuracy: 91.9\n",
            "EarlyStopping counter: 5 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 45 \tTest Quantized Accuracy: 26.81\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.1084: 100%|██████████| 469/469 [02:21<00:00,  3.32it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 46 \tTest Accuracy: 91.81\n",
            "EarlyStopping counter: 6 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 46 \tTest Quantized Accuracy: 19.72\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.0956: 100%|██████████| 469/469 [02:19<00:00,  3.36it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 47 \tTest Accuracy: 92.2\n",
            "EarlyStopping counter: 7 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 47 \tTest Quantized Accuracy: 19.57\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.0251: 100%|██████████| 469/469 [02:19<00:00,  3.35it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 48 \tTest Accuracy: 91.68\n",
            "EarlyStopping counter: 8 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 48 \tTest Quantized Accuracy: 18.88\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 0.9180: 100%|██████████| 469/469 [02:18<00:00,  3.40it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 49 \tTest Accuracy: 92.73\n",
            "Test acc increased (92.240000 --> 92.730000).  Saving model ...\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 49 \tTest Quantized Accuracy: 18.35\n",
            "tensor(1.)\n",
            "tensor(1.)\n",
            "=======Trial: 3=======\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.2687: 100%|██████████| 469/469 [02:18<00:00,  3.39it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 0 \tTest Accuracy: 87.33\n",
            "Test acc increased (0.000000 --> 87.330000).  Saving model ...\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 0 \tTest Quantized Accuracy: 87.39\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.2105: 100%|██████████| 469/469 [02:18<00:00,  3.38it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 1 \tTest Accuracy: 88.67\n",
            "Test acc increased (87.330000 --> 88.670000).  Saving model ...\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 1 \tTest Quantized Accuracy: 88.57\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.2066: 100%|██████████| 469/469 [02:18<00:00,  3.38it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 2 \tTest Accuracy: 88.4\n",
            "EarlyStopping counter: 1 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 2 \tTest Quantized Accuracy: 86.18\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.2146: 100%|██████████| 469/469 [02:18<00:00,  3.38it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 3 \tTest Accuracy: 88.34\n",
            "EarlyStopping counter: 2 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 3 \tTest Quantized Accuracy: 87.87\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.1451: 100%|██████████| 469/469 [02:20<00:00,  3.33it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 4 \tTest Accuracy: 89.03\n",
            "Test acc increased (88.670000 --> 89.030000).  Saving model ...\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 4 \tTest Quantized Accuracy: 88.14\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.1446: 100%|██████████| 469/469 [02:22<00:00,  3.29it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 5 \tTest Accuracy: 89.56\n",
            "Test acc increased (89.030000 --> 89.560000).  Saving model ...\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 5 \tTest Quantized Accuracy: 89.18\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "loss: 1.1457: 100%|██████████| 469/469 [02:28<00:00,  3.16it/s]\n"
          ]
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-14-9706361b8ccd>\u001b[0m in \u001b[0;36m<cell line: 7>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0mcfg\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'enable_bias'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[0;31m#cfg['print_weights'] = True\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 7\u001b[0;31m \u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcfg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m<ipython-input-10-aaa8e1168407>\u001b[0m in \u001b[0;36mrun\u001b[0;34m(config)\u001b[0m\n\u001b[1;32m     79\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     80\u001b[0m             \u001b[0;31m# test\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 81\u001b[0;31m             \u001b[0mtest_acc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtest_accuracy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnet\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtestloader\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     82\u001b[0m             \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf'Epoch: {epoch} \\tTest Accuracy: {test_acc}'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     83\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-7-98d5f6844a42>\u001b[0m in \u001b[0;36mtest_accuracy\u001b[0;34m(config, net, testloader, device)\u001b[0m\n\u001b[1;32m     12\u001b[0m             \u001b[0mimages\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabels\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mimages\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabels\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     13\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 14\u001b[0;31m             \u001b[0moutputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnet\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mimages\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     15\u001b[0m             \u001b[0maccuracy\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mSF\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maccuracy_rate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabels\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     16\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1499\u001b[0m                 \u001b[0;32mor\u001b[0m \u001b[0m_global_backward_pre_hooks\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0m_global_backward_hooks\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1500\u001b[0m                 or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[0;32m-> 1501\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1502\u001b[0m         \u001b[0;31m# Do not call functions when jit is used\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1503\u001b[0m         \u001b[0mfull_backward_hooks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnon_full_backward_hooks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-11-16fcdb2afccc>\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, x)\u001b[0m\n\u001b[1;32m    161\u001b[0m                    \u001b[0mcur1\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbn1\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcur1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    162\u001b[0m                 \u001b[0mspk1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmem1\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlif1\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcur1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmem1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 163\u001b[0;31m                 \u001b[0mcur2\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbfc2\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mspk1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    164\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbatch_norm\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    165\u001b[0m                     \u001b[0mcur2\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbn2\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcur2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1499\u001b[0m                 \u001b[0;32mor\u001b[0m \u001b[0m_global_backward_pre_hooks\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0m_global_backward_hooks\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1500\u001b[0m                 or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[0;32m-> 1501\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1502\u001b[0m         \u001b[0;31m# Do not call functions when jit is used\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1503\u001b[0m         \u001b[0mfull_backward_hooks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnon_full_backward_hooks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-4-196fa6f57f18>\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m     46\u001b[0m         \u001b[0;31m#print(self.weight_mask_const.mean(), binary_weight.mean())\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     47\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbias\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 48\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mF\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlinear\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbinary_weight\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     49\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     50\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mF\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlinear\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbinary_weight\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbias\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "cfg = config.copy()\n",
        "cfg['model'] = 'NetFC(config,sparsity=[0.25, 0.5])'\n",
        "#cfg['num_steps'] = 100\n",
        "cfg['enable_batch_norm'] = True\n",
        "cfg['enable_bias'] = True\n",
        "#cfg['print_weights'] = True\n",
        "run(cfg)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 615
        },
        "id": "WVyEVOdmN1gu",
        "outputId": "d358beff-5913-43d2-b75b-b835ae916475"
      },
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{'model': 'NetFC(config,sparsity=[0.25, 0.5])', 'exp_name': 'mnist_tha', 'num_trials': 5, 'num_epochs': 50, 'binarize': True, 'binarize_input': True, 'post_quantize': True, 'enable_bias': True, 'enable_batch_norm': True, 'enable_dropout': True, 'enable_threshold': True, 'enable_slope': True, 'on_spike_reset_to_zero': False, 'data_dir': '~/data/mnist', 'batch_size': 128, 'seed': 0, 'num_workers': 0, 'save_csv': True, 'save_model': True, 'early_stopping': True, 'patience': 100, 'grad_clip': False, 'weight_clip': False, 'dropout1': 0.02856, 'beta': 0.992187, 'lr': 0.00997, 'slope': 10.22, 'threshold1': 11.666, 'alpha_thr1': 0.024, 'thr_final1': 4.317, 'threshold2': 14.105, 'alpha_thr2': 0.119, 'thr_final2': 16.29, 'threshold3': 0.6656, 'alpha_thr3': 0.0011, 'thr_final3': 3.496, 'num_steps': 100, 'correct_rate': 0.8, 'incorrect_rate': 0.2, 'betas': (0.9, 0.999), 't_0': 4688, 'eta_min': 0, 'df_lr': True, 'print_weights': False}\n",
            "tensor(0.7486)\n",
            "tensor(0.4983)\n",
            "NetFC(\n",
            "  (bfc1): SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "  (bn1): BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "  (lif1): Leaky()\n",
            "  (bfc2): SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "  (bn2): BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "  (lif2): Leaky()\n",
            "  (bfc3): BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "  (lif3): Leaky()\n",
            "  (dropout): Dropout(p=0.02856, inplace=False)\n",
            ")\n",
            "=======Trial: 0=======\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "loss: 1.5706:  17%|█▋        | 82/469 [00:28<02:12,  2.92it/s]\n"
          ]
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-16-49e5788f39a3>\u001b[0m in \u001b[0;36m<cell line: 7>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0mcfg\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'enable_bias'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[0;31m#cfg['print_weights'] = True\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 7\u001b[0;31m \u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcfg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m<ipython-input-10-aaa8e1168407>\u001b[0m in \u001b[0;36mrun\u001b[0;34m(config)\u001b[0m\n\u001b[1;32m     75\u001b[0m             \u001b[0;31m# train\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     76\u001b[0m             \u001b[0mstart_time\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 77\u001b[0;31m             \u001b[0mloss_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlr_list\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnet\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepoch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrainloader\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtestloader\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcriterion\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mscheduler\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     78\u001b[0m             \u001b[0mepoch_time\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mstart_time\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     79\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-9-e3ed6f056ebd>\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(config, net, epoch, trainloader, testloader, criterion, optimizer, scheduler, device)\u001b[0m\n\u001b[1;32m     37\u001b[0m         \u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabels\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabels\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     38\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 39\u001b[0;31m         \u001b[0mspk_rec2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnet\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     40\u001b[0m         \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcriterion\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mspk_rec2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabels\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     41\u001b[0m         \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mzero_grad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1499\u001b[0m                 \u001b[0;32mor\u001b[0m \u001b[0m_global_backward_pre_hooks\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0m_global_backward_hooks\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1500\u001b[0m                 or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[0;32m-> 1501\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1502\u001b[0m         \u001b[0;31m# Do not call functions when jit is used\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1503\u001b[0m         \u001b[0mfull_backward_hooks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnon_full_backward_hooks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-11-16fcdb2afccc>\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, x)\u001b[0m\n\u001b[1;32m    160\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbatch_norm\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    161\u001b[0m                    \u001b[0mcur1\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbn1\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcur1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 162\u001b[0;31m                 \u001b[0mspk1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmem1\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlif1\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcur1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmem1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    163\u001b[0m                 \u001b[0mcur2\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbfc2\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mspk1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    164\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbatch_norm\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1499\u001b[0m                 \u001b[0;32mor\u001b[0m \u001b[0m_global_backward_pre_hooks\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0m_global_backward_hooks\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1500\u001b[0m                 or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[0;32m-> 1501\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1502\u001b[0m         \u001b[0;31m# Do not call functions when jit is used\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1503\u001b[0m         \u001b[0mfull_backward_hooks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnon_full_backward_hooks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/snntorch/_neurons/leaky.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input_, mem)\u001b[0m\n\u001b[1;32m    175\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minit_hidden\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    176\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreset\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmem_reset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmem\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 177\u001b[0;31m             \u001b[0mmem\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_build_state_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmem\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    178\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    179\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstate_quant\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/snntorch/_neurons/leaky.py\u001b[0m in \u001b[0;36m_build_state_function\u001b[0;34m(self, input_, mem)\u001b[0m\n\u001b[1;32m    214\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreset_mechanism_val\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m  \u001b[0;31m# reset by subtraction\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    215\u001b[0m             state_fn = self._base_state_function(\n\u001b[0;32m--> 216\u001b[0;31m                 \u001b[0minput_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmem\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreset\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mthreshold\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    217\u001b[0m             )\n\u001b[1;32m    218\u001b[0m         \u001b[0;32melif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreset_mechanism_val\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m  \u001b[0;31m# reset to zero\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__getattr__\u001b[0;34m(self, name)\u001b[0m\n\u001b[1;32m   1599\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_backward_pre_hooks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mOrderedDict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1600\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1601\u001b[0;31m     \u001b[0;32mdef\u001b[0m \u001b[0m__getattr__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mUnion\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mTensor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'Module'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1602\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;34m'_parameters'\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__dict__\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1603\u001b[0m             \u001b[0m_parameters\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__dict__\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'_parameters'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "cfg = config.copy()\n",
        "cfg['model'] = 'NetFC(config,sparsity=[0.125, 0.5])'\n",
        "#cfg['num_steps'] = 100\n",
        "cfg['enable_batch_norm'] = True\n",
        "cfg['enable_bias'] = True\n",
        "#cfg['print_weights'] = True\n",
        "run(cfg)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nbVgiO2uB4HC",
        "outputId": "b146ddb0-9bb8-4c38-e16b-addcb2a35077"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{'model': 'NetFC(config,sparsity=[0.125, 0.5])', 'exp_name': 'mnist_tha', 'num_trials': 5, 'num_epochs': 50, 'binarize': True, 'binarize_input': True, 'post_quantize': True, 'enable_bias': True, 'enable_batch_norm': True, 'enable_dropout': True, 'enable_threshold': True, 'enable_slope': True, 'on_spike_reset_to_zero': False, 'data_dir': '~/data/mnist', 'batch_size': 128, 'seed': 0, 'num_workers': 0, 'save_csv': True, 'save_model': True, 'early_stopping': True, 'patience': 100, 'grad_clip': False, 'weight_clip': False, 'dropout1': 0.02856, 'beta': 0.992187, 'lr': 0.00997, 'slope': 10.22, 'threshold1': 11.666, 'alpha_thr1': 0.024, 'thr_final1': 4.317, 'threshold2': 14.105, 'alpha_thr2': 0.119, 'thr_final2': 16.29, 'threshold3': 0.6656, 'alpha_thr3': 0.0011, 'thr_final3': 3.496, 'num_steps': 100, 'correct_rate': 0.8, 'incorrect_rate': 0.2, 'betas': (0.9, 0.999), 't_0': 4688, 'eta_min': 0, 'df_lr': True, 'print_weights': False}\n",
            "tensor(0.8753)\n",
            "tensor(0.4983)\n",
            "NetFC(\n",
            "  (bfc1): SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "  (bn1): BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "  (lif1): Leaky()\n",
            "  (bfc2): SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "  (bn2): BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "  (lif2): Leaky()\n",
            "  (bfc3): BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "  (lif3): Leaky()\n",
            "  (dropout): Dropout(p=0.02856, inplace=False)\n",
            ")\n",
            "Downloading http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz\n",
            "Downloading http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz to /root/data/mnist/MNIST/raw/train-images-idx3-ubyte.gz\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 9912422/9912422 [00:00<00:00, 95368291.35it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Extracting /root/data/mnist/MNIST/raw/train-images-idx3-ubyte.gz to /root/data/mnist/MNIST/raw\n",
            "\n",
            "Downloading http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz\n",
            "Downloading http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz to /root/data/mnist/MNIST/raw/train-labels-idx1-ubyte.gz\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 28881/28881 [00:00<00:00, 27828094.15it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Extracting /root/data/mnist/MNIST/raw/train-labels-idx1-ubyte.gz to /root/data/mnist/MNIST/raw\n",
            "\n",
            "Downloading http://yann.lecun.com/exdb/mnist/t10k-images-idx3-ubyte.gz\n",
            "Downloading http://yann.lecun.com/exdb/mnist/t10k-images-idx3-ubyte.gz to /root/data/mnist/MNIST/raw/t10k-images-idx3-ubyte.gz\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "100%|██████████| 1648877/1648877 [00:00<00:00, 25904543.47it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Extracting /root/data/mnist/MNIST/raw/t10k-images-idx3-ubyte.gz to /root/data/mnist/MNIST/raw\n",
            "\n",
            "Downloading http://yann.lecun.com/exdb/mnist/t10k-labels-idx1-ubyte.gz\n",
            "Downloading http://yann.lecun.com/exdb/mnist/t10k-labels-idx1-ubyte.gz to /root/data/mnist/MNIST/raw/t10k-labels-idx1-ubyte.gz\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 4542/4542 [00:00<00:00, 4156781.32it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Extracting /root/data/mnist/MNIST/raw/t10k-labels-idx1-ubyte.gz to /root/data/mnist/MNIST/raw\n",
            "\n",
            "=======Trial: 0=======\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "loss: 1.4420: 100%|██████████| 469/469 [02:29<00:00,  3.15it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch: 0 \tTest Accuracy: 85.52\n",
            "Test acc increased (0.000000 --> 85.520000).  Saving model ...\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 0 \tTest Quantized Accuracy: 85.45\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "loss: 1.4387: 100%|██████████| 469/469 [02:26<00:00,  3.20it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch: 1 \tTest Accuracy: 86.11\n",
            "Test acc increased (85.520000 --> 86.110000).  Saving model ...\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 1 \tTest Quantized Accuracy: 85.83\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "loss: 1.4139: 100%|██████████| 469/469 [02:26<00:00,  3.20it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch: 2 \tTest Accuracy: 86.49\n",
            "Test acc increased (86.110000 --> 86.490000).  Saving model ...\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "qbn BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 2 \tTest Quantized Accuracy: 86.24\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "loss: 1.3275:  37%|███▋      | 175/469 [00:53<01:28,  3.31it/s]"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "cfg = config.copy()\n",
        "cfg['model'] = 'NetFC(config, neurons=[128, 128], sparsity=[0.5, 0.5])'\n",
        "#cfg['num_steps'] = 100\n",
        "cfg['enable_batch_norm'] = True\n",
        "cfg['enable_bias'] = True\n",
        "#cfg['print_weights'] = True\n",
        "run(cfg)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "bjYdHrOsN1Wd",
        "outputId": "a38e4e43-70c7-4325-a6e6-51117f29a965"
      },
      "execution_count": 17,
      "outputs": [
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'model': 'NetFC(config, neurons=[128, 128], sparsity=[0.5, 0.5])', 'exp_name': 'mnist_tha', 'num_trials': 5, 'num_epochs': 50, 'binarize': True, 'binarize_input': True, 'post_quantize': True, 'enable_bias': True, 'enable_batch_norm': True, 'enable_dropout': True, 'enable_threshold': True, 'enable_slope': True, 'on_spike_reset_to_zero': False, 'data_dir': '~/data/mnist', 'batch_size': 128, 'seed': 0, 'num_workers': 0, 'save_csv': True, 'save_model': True, 'early_stopping': True, 'patience': 100, 'grad_clip': False, 'weight_clip': False, 'dropout1': 0.02856, 'beta': 0.992187, 'lr': 0.00997, 'slope': 10.22, 'threshold1': 11.666, 'alpha_thr1': 0.024, 'thr_final1': 4.317, 'threshold2': 14.105, 'alpha_thr2': 0.119, 'thr_final2': 16.29, 'threshold3': 0.6656, 'alpha_thr3': 0.0011, 'thr_final3': 3.496, 'num_steps': 100, 'correct_rate': 0.8, 'incorrect_rate': 0.2, 'betas': (0.9, 0.999), 't_0': 4688, 'eta_min': 0, 'df_lr': True, 'print_weights': False}\n",
            "tensor(0.5037)\n",
            "tensor(0.4942)\n",
            "NetFC(\n",
            "  (bfc1): SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "  (bn1): BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "  (lif1): Leaky()\n",
            "  (bfc2): SparseBinaryLinear(in_features=128, out_features=128, bias=False)\n",
            "  (bn2): BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "  (lif2): Leaky()\n",
            "  (bfc3): BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "  (lif3): Leaky()\n",
            "  (dropout): Dropout(p=0.02856, inplace=False)\n",
            ")\n",
            "=======Trial: 0=======\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.7288: 100%|██████████| 469/469 [02:33<00:00,  3.06it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 0 \tTest Accuracy: 80.93\n",
            "Test acc increased (0.000000 --> 80.930000).  Saving model ...\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=128, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 0 \tTest Quantized Accuracy: 81.11\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.6848: 100%|██████████| 469/469 [02:29<00:00,  3.13it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch: 1 \tTest Accuracy: 80.82\n",
            "EarlyStopping counter: 1 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=128, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "Epoch: 1 \tTest Quantized Accuracy: 80.31\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "loss: 1.6855: 100%|██████████| 469/469 [02:31<00:00,  3.10it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch: 2 \tTest Accuracy: 80.76\n",
            "EarlyStopping counter: 2 out of 100\n",
            "post quantize model\n",
            "sfc SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "sfc SparseBinaryLinear(in_features=128, out_features=128, bias=False)\n",
            "qbn BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "bfc BinaryLinear(in_features=128, out_features=10, bias=True)\n"
          ]
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-17-1b95346886a1>\u001b[0m in \u001b[0;36m<cell line: 7>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0mcfg\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'enable_bias'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[0;31m#cfg['print_weights'] = True\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 7\u001b[0;31m \u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcfg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m<ipython-input-10-aaa8e1168407>\u001b[0m in \u001b[0;36mrun\u001b[0;34m(config)\u001b[0m\n\u001b[1;32m    112\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mconfig\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'post_quantize'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    113\u001b[0m                 \u001b[0mnet_quantized\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpost_quantize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcopy\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdeepcopy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnet\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 114\u001b[0;31m                 \u001b[0mtest_acc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtest_accuracy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnet_quantized\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtestloader\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    115\u001b[0m                 \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf'Epoch: {epoch} \\tTest Quantized Accuracy: {test_acc}'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    116\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-7-98d5f6844a42>\u001b[0m in \u001b[0;36mtest_accuracy\u001b[0;34m(config, net, testloader, device)\u001b[0m\n\u001b[1;32m      8\u001b[0m     \u001b[0;32mwith\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mno_grad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m         \u001b[0mnet\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0meval\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 10\u001b[0;31m         \u001b[0;32mfor\u001b[0m \u001b[0mdata\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtestloader\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     11\u001b[0m             \u001b[0mimages\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabels\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     12\u001b[0m             \u001b[0mimages\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabels\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mimages\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabels\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/torch/utils/data/dataloader.py\u001b[0m in \u001b[0;36m__next__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    631\u001b[0m                 \u001b[0;31m# TODO(https://github.com/pytorch/pytorch/issues/76750)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    632\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_reset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# type: ignore[call-arg]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 633\u001b[0;31m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_next_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    634\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_num_yielded\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    635\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_dataset_kind\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0m_DatasetKind\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mIterable\u001b[0m \u001b[0;32mand\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m\\\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/torch/utils/data/dataloader.py\u001b[0m in \u001b[0;36m_next_data\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    675\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_next_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    676\u001b[0m         \u001b[0mindex\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_next_index\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# may raise StopIteration\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 677\u001b[0;31m         \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_dataset_fetcher\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfetch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mindex\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# may raise StopIteration\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    678\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_pin_memory\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    679\u001b[0m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_utils\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpin_memory\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpin_memory\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_pin_memory_device\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/torch/utils/data/_utils/fetch.py\u001b[0m in \u001b[0;36mfetch\u001b[0;34m(self, possibly_batched_index)\u001b[0m\n\u001b[1;32m     49\u001b[0m                 \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__getitems__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpossibly_batched_index\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     50\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 51\u001b[0;31m                 \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0midx\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0midx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mpossibly_batched_index\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     52\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     53\u001b[0m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mpossibly_batched_index\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/torch/utils/data/_utils/fetch.py\u001b[0m in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m     49\u001b[0m                 \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__getitems__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpossibly_batched_index\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     50\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 51\u001b[0;31m                 \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0midx\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0midx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mpossibly_batched_index\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     52\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     53\u001b[0m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mpossibly_batched_index\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/torchvision/datasets/mnist.py\u001b[0m in \u001b[0;36m__getitem__\u001b[0;34m(self, index)\u001b[0m\n\u001b[1;32m    143\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    144\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtransform\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 145\u001b[0;31m             \u001b[0mimg\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtransform\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mimg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    146\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    147\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtarget_transform\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/torchvision/transforms/transforms.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, img)\u001b[0m\n\u001b[1;32m     93\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__call__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mimg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     94\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mt\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtransforms\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 95\u001b[0;31m             \u001b[0mimg\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mimg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     96\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mimg\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     97\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1499\u001b[0m                 \u001b[0;32mor\u001b[0m \u001b[0m_global_backward_pre_hooks\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0m_global_backward_hooks\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1500\u001b[0m                 or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[0;32m-> 1501\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1502\u001b[0m         \u001b[0;31m# Do not call functions when jit is used\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1503\u001b[0m         \u001b[0mfull_backward_hooks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnon_full_backward_hooks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/torchvision/transforms/transforms.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, tensor)\u001b[0m\n\u001b[1;32m    275\u001b[0m             \u001b[0mTensor\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mNormalized\u001b[0m \u001b[0mTensor\u001b[0m \u001b[0mimage\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    276\u001b[0m         \"\"\"\n\u001b[0;32m--> 277\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mF\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnormalize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtensor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmean\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstd\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minplace\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    278\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    279\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__repr__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/torchvision/transforms/functional.py\u001b[0m in \u001b[0;36mnormalize\u001b[0;34m(tensor, mean, std, inplace)\u001b[0m\n\u001b[1;32m    361\u001b[0m         \u001b[0;32mraise\u001b[0m \u001b[0mTypeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf\"img should be Tensor Image. Got {type(tensor)}\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    362\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 363\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mF_t\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnormalize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtensor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmean\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmean\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstd\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mstd\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minplace\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minplace\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    364\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    365\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/torchvision/transforms/_functional_tensor.py\u001b[0m in \u001b[0;36mnormalize\u001b[0;34m(tensor, mean, std, inplace)\u001b[0m\n\u001b[1;32m    920\u001b[0m     \u001b[0mmean\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_tensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmean\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdevice\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtensor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    921\u001b[0m     \u001b[0mstd\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_tensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstd\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdevice\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtensor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 922\u001b[0;31m     \u001b[0;32mif\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mstd\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0many\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    923\u001b[0m         \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf\"std evaluated to zero after conversion to {dtype}, leading to division by zero.\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    924\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mmean\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mndim\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# bn2.weight Parameter containing:\n",
        "# tensor([1.9990, 2.1707, 2.0983, 1.9927, 1.9176, 1.5810, 1.7940, 2.1587, 1.9185,\n",
        "#         1.7227, 1.9048, 2.1231, 1.8211, 1.9565, 2.4194, 1.8210, 1.8736, 2.0975,\n",
        "#         2.1799, 2.1952, 1.8520, 1.7353, 1.8138, 2.1197, 1.8280, 1.9596, 2.3288,\n",
        "#         2.4837, 1.7084, 1.9816, 1.7141, 2.0004, 2.3567, 1.6929, 1.8008, 1.8295,\n",
        "#         1.8755, 2.2669, 2.1998, 2.1144, 2.2599, 1.7555, 1.9367, 1.7001, 1.7601,\n",
        "#         2.3178, 2.0300, 2.2487, 1.9230, 1.9055, 2.1833, 2.0571, 1.9264, 1.8677,\n",
        "#         2.1385, 1.3140, 1.9290, 1.7938, 2.1273, 2.0558, 1.7818, 1.9466, 1.8962,\n",
        "#         2.0188, 1.9801, 1.6973, 1.8316, 2.1479, 1.7928, 1.6150, 2.1462, 2.0311,\n",
        "#         2.0386, 2.0987, 2.0303, 1.9735, 1.8485, 1.7809, 2.1801, 1.8313, 1.8320,\n",
        "#         1.8230, 2.2195, 2.3736, 2.1923, 2.2304, 2.1159, 1.7465, 2.1706, 1.8011,\n",
        "#         1.6034, 1.7145, 2.1939, 1.9043, 2.1121, 1.8974, 1.9840, 1.7288, 1.7071,\n",
        "#         2.1992, 2.0279, 2.1737, 1.8198, 2.2155, 1.6175, 2.3012, 1.8829, 2.0970,\n",
        "#         2.0961, 2.3487, 1.7499, 1.8511, 2.1443, 1.9805, 1.5925, 1.9896, 1.9556,\n",
        "#         2.1271, 2.0737, 2.2048, 2.0302, 1.5531, 2.0186, 1.8186, 1.9623, 2.0262,\n",
        "#         1.6641, 1.9470], device='cuda:0', requires_grad=True)\n",
        "# bn2.bias Parameter containing:\n",
        "# tensor([0.6498, 0.3817, 0.2181, 0.2873, 0.4080, 0.1720, 0.0317, 0.5161, 0.5260,\n",
        "#         0.2901, 0.3405, 0.4849, 0.6289, 0.3044, 0.6084, 0.3359, 0.3509, 0.4086,\n",
        "#         0.3281, 0.4347, 0.2080, 0.2665, 0.3921, 0.4583, 0.2833, 0.4716, 0.1986,\n",
        "#         0.8519, 0.2374, 0.4891, 0.3759, 0.5366, 0.2983, 0.1438, 0.4705, 0.2080,\n",
        "#         0.3375, 0.7551, 0.6978, 0.4066, 0.5279, 0.5771, 0.1845, 0.3678, 0.3259,\n",
        "#         0.3678, 0.5667, 0.4188, 0.3600, 0.2092, 0.1839, 0.3777, 0.4884, 0.3750,\n",
        "#         0.2550, 0.1124, 0.5827, 0.3344, 0.5152, 0.2674, 0.3826, 0.2437, 0.2454,\n",
        "#         0.4080, 0.5506, 0.3877, 0.4804, 0.3189, 0.5685, 0.2249, 0.4660, 0.2988,\n",
        "#         0.6129, 0.5870, 0.2916, 0.6732, 0.4578, 0.6597, 0.3098, 0.5236, 0.3821,\n",
        "#         0.4044, 0.2389, 0.5632, 0.5674, 0.6477, 0.6750, 0.2220, 0.5916, 0.5613,\n",
        "#         0.2781, 0.2813, 0.4886, 0.3720, 0.4638, 0.3268, 0.4190, 0.2757, 0.5542,\n",
        "#         0.4846, 0.2732, 0.6278, 0.3644, 0.5544, 0.3397, 0.5387, 0.1307, 0.4701,\n",
        "#         0.3537, 0.6017, 0.6172, 0.4602, 0.5736, 0.2729, 0.3987, 0.1734, 0.3378,\n",
        "#         0.4532, 0.4848, 0.6180, 0.3999, 0.2453, 0.4765, 0.2346, 0.3375, 0.2310,\n",
        "#         0.3223, 0.2671], device='cuda:0', requires_grad=True)\n",
        "\n",
        " #after some training\n",
        "# bn2.weight Parameter containing:\n",
        "# tensor([4.9099, 5.0962, 5.5888, 4.3900, 4.5788, 4.9550, 4.9641, 5.0355, 4.5730,\n",
        "#         4.9639, 4.4276, 5.3666, 5.2169, 5.1509, 4.9650, 5.1409, 4.9874, 5.3097,\n",
        "#         4.7464, 5.5738, 5.2571, 4.5298, 4.9318, 5.7625, 4.4019, 5.1249, 5.2300,\n",
        "#         5.1870, 4.4773, 4.3227, 4.5650, 5.7379, 6.0565, 5.0293, 4.8255, 4.7608,\n",
        "#         4.9786, 5.3047, 4.7430, 5.6802, 5.2579, 4.1642, 5.4533, 4.4647, 4.4885,\n",
        "#         5.7688, 5.6993, 5.0135, 4.8656, 5.1775, 5.5288, 4.8154, 4.0566, 4.9560,\n",
        "#         4.9754, 4.7329, 4.7370, 4.6920, 4.9248, 4.5154, 5.0307, 4.6023, 5.2160,\n",
        "#         4.7534, 4.2167, 4.3286, 4.5850, 6.0492, 4.4160, 4.6090, 4.8223, 5.0231,\n",
        "#         4.9384, 5.3263, 5.1866, 4.2420, 4.2443, 4.4652, 5.2192, 4.4080, 4.8573,\n",
        "#         4.8766, 6.3240, 5.5299, 5.2524, 5.3087, 4.4232, 5.0209, 5.1746, 4.4969,\n",
        "#         4.3094, 4.1203, 5.6062, 5.0312, 4.7815, 4.6387, 5.1705, 4.7637, 4.6557,\n",
        "#         5.3890, 5.1286, 4.8427, 4.1010, 5.2434, 4.7017, 5.3186, 5.0355, 4.7969,\n",
        "#         5.3591, 5.7636, 4.5301, 4.6631, 5.0067, 5.4933, 4.2991, 4.8119, 4.8928,\n",
        "#         5.4807, 5.7014, 4.8133, 5.0600, 4.0004, 4.8092, 4.6587, 5.4306, 5.6190,\n",
        "#         4.8491, 5.6172], device='cuda:0', requires_grad=True)\n",
        "# bn2.bias Parameter containing:\n",
        "# tensor([ 0.9776,  0.0102, -0.3288, -0.1998,  0.1523,  0.0830,  0.0378,  0.4451,\n",
        "#          0.4870, -0.1236,  0.0895,  0.3199,  1.1172, -0.0968,  0.3388,  0.6111,\n",
        "#          0.3126,  0.1159, -0.2493, -0.0373, -0.1575,  0.0478,  0.3408,  0.2455,\n",
        "#          0.3665,  0.2020, -0.6231,  0.5952, -0.1245,  0.2104,  0.0185,  0.6761,\n",
        "#         -0.2078,  0.2258,  0.4542, -0.1995,  0.4682,  0.8909,  0.6613,  0.4796,\n",
        "#          0.2698,  0.5167, -0.0313,  0.7828,  0.2144, -0.0322,  0.9767,  0.2942,\n",
        "#          0.1549, -0.2499, -0.2597,  0.0301,  0.5320,  0.2145,  0.0200,  0.0041,\n",
        "#          1.0410, -0.0218,  0.5749, -0.3903,  0.5178, -0.0901,  0.4260, -0.1015,\n",
        "#          0.2714,  0.6575,  0.5759,  0.4551,  0.4005, -0.0026,  0.3148,  0.1734,\n",
        "#          0.5360,  0.0206, -0.2983,  0.6451,  0.5158,  1.1592, -0.0688,  0.5039,\n",
        "#          0.3904, -0.0353, -0.0839,  0.2897,  0.3479,  0.5520,  0.1291,  0.0839,\n",
        "#          0.1894,  0.1085,  0.3328, -0.0146,  0.3279,  0.0778,  0.2786,  0.1371,\n",
        "#          0.4661, -0.0742,  1.1181,  0.6645, -0.1441,  0.3702,  0.2503,  0.2494,\n",
        "#          0.4468,  0.2508, -0.2617, -0.0379,  0.2208,  0.4755,  0.5585,  0.5917,\n",
        "#          0.5314,  0.2416,  0.3132, -0.1299,  0.1267,  0.3698,  0.6280,  0.7991,\n",
        "#         -0.1371,  0.1800,  0.2070, -0.0646,  0.2283,  0.1236,  0.2599,  0.3201],\n",
        "#        device='cuda:0', requires_grad=True)"
      ],
      "metadata": {
        "id": "Z1sjIxiWOQax"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "cfg = config.copy()\n",
        "cfg['model'] = 'NetFC'\n",
        "cfg['num_steps'] = 10\n",
        "cfg['enable_batch_norm'] = False\n",
        "cfg['enable_bias'] = False\n",
        "run(cfg)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 812
        },
        "id": "yEKbdQD3Z16M",
        "outputId": "2c71882f-b432-41d0-f07a-45a2fb3cc571"
      },
      "execution_count": 47,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{'model': 'NetFC', 'exp_name': 'mnist_tha', 'num_trials': 5, 'num_epochs': 10, 'binarize': True, 'binarize_input': True, 'enable_bias': False, 'enable_batchnorm': True, 'enable_dropout': True, 'enable_threshold': True, 'enable_slope': True, 'data_dir': '~/data/mnist', 'batch_size': 128, 'seed': 0, 'num_workers': 0, 'save_csv': True, 'save_model': True, 'early_stopping': True, 'patience': 100, 'grad_clip': False, 'weight_clip': False, 'batch_norm': False, 'dropout1': 0.02856, 'beta': 0.99, 'lr': 0.00997, 'slope': 10.22, 'threshold1': 11.666, 'alpha_thr1': 0.024, 'thr_final1': 4.317, 'threshold2': 14.105, 'alpha_thr2': 0.119, 'thr_final2': 16.29, 'threshold3': 0.6656, 'alpha_thr3': 0.0011, 'thr_final3': 3.496, 'num_steps': 10, 'correct_rate': 0.8, 'incorrect_rate': 0.2, 'betas': (0.9, 0.999), 't_0': 4688, 'eta_min': 0, 'df_lr': True}\n",
            "tensor(1.)\n",
            "tensor(1.)\n",
            "NetFC(\n",
            "  (bfc1): SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "  (fc1): Linear(in_features=256, out_features=256, bias=False)\n",
            "  (bn1): BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "  (lif1): Leaky()\n",
            "  (bfc2): SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "  (fc2): Linear(in_features=256, out_features=128, bias=False)\n",
            "  (bn2): BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "  (lif2): Leaky()\n",
            "  (bfc3): BinaryLinear(in_features=128, out_features=10, bias=False)\n",
            "  (fc3): Linear(in_features=128, out_features=10, bias=False)\n",
            "  (lif3): Leaky()\n",
            "  (dropout): Dropout(p=0.02856, inplace=False)\n",
            ")\n",
            "=======Trial: 0=======\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "loss: 0.5350: 100%|██████████| 469/469 [00:31<00:00, 14.96it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch: 0 \tTest Accuracy: 63.7\n",
            "Test acc increased (0.000000 --> 63.700000).  Saving model ...\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "loss: 0.2998: 100%|██████████| 469/469 [00:31<00:00, 14.86it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch: 1 \tTest Accuracy: 59.04\n",
            "EarlyStopping counter: 1 out of 100\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "loss: 0.3431: 100%|██████████| 469/469 [00:32<00:00, 14.45it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch: 2 \tTest Accuracy: 63.63\n",
            "EarlyStopping counter: 2 out of 100\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "loss: 0.3111:   4%|▍         | 18/469 [00:01<00:32, 13.88it/s]\n"
          ]
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-47-8d668780d4c6>\u001b[0m in \u001b[0;36m<cell line: 6>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mcfg\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'batch_norm'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0mcfg\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'enable_bias'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 6\u001b[0;31m \u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcfg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m<ipython-input-45-12228137bcd4>\u001b[0m in \u001b[0;36mrun\u001b[0;34m(config)\u001b[0m\n\u001b[1;32m     74\u001b[0m             \u001b[0;31m# train\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     75\u001b[0m             \u001b[0mstart_time\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 76\u001b[0;31m             \u001b[0mloss_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlr_list\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnet\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepoch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrainloader\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtestloader\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcriterion\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mscheduler\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     77\u001b[0m             \u001b[0mepoch_time\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mstart_time\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     78\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-44-e3ed6f056ebd>\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(config, net, epoch, trainloader, testloader, criterion, optimizer, scheduler, device)\u001b[0m\n\u001b[1;32m     34\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     35\u001b[0m     \u001b[0;31m#for data, labels in trainloader:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 36\u001b[0;31m     \u001b[0;32mfor\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabels\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mprogress_bar\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     37\u001b[0m         \u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabels\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabels\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     38\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/tqdm/std.py\u001b[0m in \u001b[0;36m__iter__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1176\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1177\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1178\u001b[0;31m             \u001b[0;32mfor\u001b[0m \u001b[0mobj\u001b[0m \u001b[0;32min\u001b[0m \u001b[0miterable\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1179\u001b[0m                 \u001b[0;32myield\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1180\u001b[0m                 \u001b[0;31m# Update and possibly print the progressbar.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/torch/utils/data/dataloader.py\u001b[0m in \u001b[0;36m__next__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    631\u001b[0m                 \u001b[0;31m# TODO(https://github.com/pytorch/pytorch/issues/76750)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    632\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_reset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# type: ignore[call-arg]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 633\u001b[0;31m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_next_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    634\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_num_yielded\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    635\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_dataset_kind\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0m_DatasetKind\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mIterable\u001b[0m \u001b[0;32mand\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m\\\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/torch/utils/data/dataloader.py\u001b[0m in \u001b[0;36m_next_data\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    675\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_next_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    676\u001b[0m         \u001b[0mindex\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_next_index\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# may raise StopIteration\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 677\u001b[0;31m         \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_dataset_fetcher\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfetch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mindex\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# may raise StopIteration\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    678\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_pin_memory\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    679\u001b[0m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_utils\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpin_memory\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpin_memory\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_pin_memory_device\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/torch/utils/data/_utils/fetch.py\u001b[0m in \u001b[0;36mfetch\u001b[0;34m(self, possibly_batched_index)\u001b[0m\n\u001b[1;32m     49\u001b[0m                 \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__getitems__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpossibly_batched_index\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     50\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 51\u001b[0;31m                 \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0midx\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0midx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mpossibly_batched_index\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     52\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     53\u001b[0m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mpossibly_batched_index\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/torch/utils/data/_utils/fetch.py\u001b[0m in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m     49\u001b[0m                 \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__getitems__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpossibly_batched_index\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     50\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 51\u001b[0;31m                 \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0midx\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0midx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mpossibly_batched_index\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     52\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     53\u001b[0m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mpossibly_batched_index\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/torchvision/datasets/mnist.py\u001b[0m in \u001b[0;36m__getitem__\u001b[0;34m(self, index)\u001b[0m\n\u001b[1;32m    140\u001b[0m         \u001b[0;31m# doing this so that it is consistent with all other datasets\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    141\u001b[0m         \u001b[0;31m# to return a PIL Image\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 142\u001b[0;31m         \u001b[0mimg\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mImage\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfromarray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mimg\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnumpy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmode\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"L\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    143\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    144\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtransform\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "cfg = config.copy()\n",
        "cfg['model'] = 'NetFC'\n",
        "cfg['num_steps'] = 10\n",
        "cfg['enable_batch_norm'] = False\n",
        "cfg['enable_bias'] = True\n",
        "run(cfg)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "avOm47mIdErS",
        "outputId": "5e7af48d-e037-41c5-adda-380406d78d93"
      },
      "execution_count": 62,
      "outputs": [
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'model': 'NetFC', 'exp_name': 'mnist_tha', 'num_trials': 5, 'num_epochs': 10, 'binarize': True, 'binarize_input': True, 'enable_bias': True, 'enable_batchnorm': True, 'enable_dropout': True, 'enable_threshold': True, 'enable_slope': True, 'data_dir': '~/data/mnist', 'batch_size': 128, 'seed': 0, 'num_workers': 0, 'save_csv': True, 'save_model': True, 'early_stopping': True, 'patience': 100, 'grad_clip': False, 'weight_clip': False, 'dropout1': 0.02856, 'beta': 0.99, 'lr': 0.00997, 'slope': 10.22, 'threshold1': 11.666, 'alpha_thr1': 0.024, 'thr_final1': 4.317, 'threshold2': 14.105, 'alpha_thr2': 0.119, 'thr_final2': 16.29, 'threshold3': 0.6656, 'alpha_thr3': 0.0011, 'thr_final3': 3.496, 'num_steps': 10, 'correct_rate': 0.8, 'incorrect_rate': 0.2, 'betas': (0.9, 0.999), 't_0': 4688, 'eta_min': 0, 'df_lr': True, 'enable_batch_norm': False}\n",
            "tensor(1.)\n",
            "tensor(1.)\n",
            "NetFC(\n",
            "  (bfc1): SparseBinaryLinear(in_features=256, out_features=256, bias=True)\n",
            "  (fc1): Linear(in_features=256, out_features=256, bias=True)\n",
            "  (bn1): BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "  (lif1): Leaky()\n",
            "  (bfc2): SparseBinaryLinear(in_features=256, out_features=128, bias=True)\n",
            "  (fc2): Linear(in_features=256, out_features=128, bias=True)\n",
            "  (bn2): BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "  (lif2): Leaky()\n",
            "  (bfc3): BinaryLinear(in_features=128, out_features=10, bias=True)\n",
            "  (fc3): Linear(in_features=128, out_features=10, bias=True)\n",
            "  (lif3): Leaky()\n",
            "  (dropout): Dropout(p=0.02856, inplace=False)\n",
            ")\n",
            "=======Trial: 0=======\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 0.2818: 100%|██████████| 469/469 [00:31<00:00, 14.69it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 0 \tTest Accuracy: 56.2\n",
            "Test acc increased (0.000000 --> 56.200000).  Saving model ...\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 0.2309: 100%|██████████| 469/469 [00:32<00:00, 14.60it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 1 \tTest Accuracy: 68.83\n",
            "Test acc increased (56.200000 --> 68.830000).  Saving model ...\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 0.2095: 100%|██████████| 469/469 [00:33<00:00, 14.07it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 2 \tTest Accuracy: 72.61\n",
            "Test acc increased (68.830000 --> 72.610000).  Saving model ...\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 0.1916: 100%|██████████| 469/469 [00:32<00:00, 14.49it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 3 \tTest Accuracy: 74.09\n",
            "Test acc increased (72.610000 --> 74.090000).  Saving model ...\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 0.1828: 100%|██████████| 469/469 [00:32<00:00, 14.47it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 4 \tTest Accuracy: 76.61\n",
            "Test acc increased (74.090000 --> 76.610000).  Saving model ...\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 0.1858: 100%|██████████| 469/469 [00:32<00:00, 14.42it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 5 \tTest Accuracy: 80.09\n",
            "Test acc increased (76.610000 --> 80.090000).  Saving model ...\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 0.1777: 100%|██████████| 469/469 [00:32<00:00, 14.32it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 6 \tTest Accuracy: 81.8\n",
            "Test acc increased (80.090000 --> 81.800000).  Saving model ...\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 0.1673: 100%|██████████| 469/469 [00:32<00:00, 14.65it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 7 \tTest Accuracy: 82.58\n",
            "Test acc increased (81.800000 --> 82.580000).  Saving model ...\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 0.1676: 100%|██████████| 469/469 [00:31<00:00, 14.99it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 8 \tTest Accuracy: 83.35\n",
            "Test acc increased (82.580000 --> 83.350000).  Saving model ...\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 0.1630: 100%|██████████| 469/469 [00:31<00:00, 14.87it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 9 \tTest Accuracy: 83.68\n",
            "Test acc increased (83.350000 --> 83.680000).  Saving model ...\n",
            "tensor(1.)\n",
            "tensor(1.)\n",
            "=======Trial: 1=======\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 0.2818: 100%|██████████| 469/469 [00:31<00:00, 14.94it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 0 \tTest Accuracy: 56.2\n",
            "Test acc increased (0.000000 --> 56.200000).  Saving model ...\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 0.2309: 100%|██████████| 469/469 [00:31<00:00, 14.85it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 1 \tTest Accuracy: 68.83\n",
            "Test acc increased (56.200000 --> 68.830000).  Saving model ...\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 0.2095: 100%|██████████| 469/469 [00:32<00:00, 14.59it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 2 \tTest Accuracy: 72.61\n",
            "Test acc increased (68.830000 --> 72.610000).  Saving model ...\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 0.1916: 100%|██████████| 469/469 [00:32<00:00, 14.43it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 3 \tTest Accuracy: 74.09\n",
            "Test acc increased (72.610000 --> 74.090000).  Saving model ...\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 0.1828: 100%|██████████| 469/469 [00:32<00:00, 14.25it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 4 \tTest Accuracy: 76.61\n",
            "Test acc increased (74.090000 --> 76.610000).  Saving model ...\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 0.1858: 100%|██████████| 469/469 [00:32<00:00, 14.37it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 5 \tTest Accuracy: 80.09\n",
            "Test acc increased (76.610000 --> 80.090000).  Saving model ...\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 0.1777: 100%|██████████| 469/469 [00:32<00:00, 14.59it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 6 \tTest Accuracy: 81.8\n",
            "Test acc increased (80.090000 --> 81.800000).  Saving model ...\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 0.1673: 100%|██████████| 469/469 [00:31<00:00, 14.78it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 7 \tTest Accuracy: 82.58\n",
            "Test acc increased (81.800000 --> 82.580000).  Saving model ...\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 0.1676: 100%|██████████| 469/469 [00:31<00:00, 14.73it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 8 \tTest Accuracy: 83.35\n",
            "Test acc increased (82.580000 --> 83.350000).  Saving model ...\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 0.1630: 100%|██████████| 469/469 [00:31<00:00, 14.76it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 9 \tTest Accuracy: 83.68\n",
            "Test acc increased (83.350000 --> 83.680000).  Saving model ...\n",
            "tensor(1.)\n",
            "tensor(1.)\n",
            "=======Trial: 2=======\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 0.2818: 100%|██████████| 469/469 [00:31<00:00, 14.68it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 0 \tTest Accuracy: 56.2\n",
            "Test acc increased (0.000000 --> 56.200000).  Saving model ...\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 0.2309: 100%|██████████| 469/469 [00:31<00:00, 14.87it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 1 \tTest Accuracy: 68.83\n",
            "Test acc increased (56.200000 --> 68.830000).  Saving model ...\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 0.2095: 100%|██████████| 469/469 [00:31<00:00, 14.96it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 2 \tTest Accuracy: 72.61\n",
            "Test acc increased (68.830000 --> 72.610000).  Saving model ...\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 0.1916: 100%|██████████| 469/469 [00:31<00:00, 15.02it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 3 \tTest Accuracy: 74.09\n",
            "Test acc increased (72.610000 --> 74.090000).  Saving model ...\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 0.1828: 100%|██████████| 469/469 [00:31<00:00, 15.11it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 4 \tTest Accuracy: 76.61\n",
            "Test acc increased (74.090000 --> 76.610000).  Saving model ...\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 0.1858: 100%|██████████| 469/469 [00:30<00:00, 15.21it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 5 \tTest Accuracy: 80.09\n",
            "Test acc increased (76.610000 --> 80.090000).  Saving model ...\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 0.1777: 100%|██████████| 469/469 [00:31<00:00, 15.04it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 6 \tTest Accuracy: 81.8\n",
            "Test acc increased (80.090000 --> 81.800000).  Saving model ...\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 0.1673: 100%|██████████| 469/469 [00:31<00:00, 14.99it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 7 \tTest Accuracy: 82.58\n",
            "Test acc increased (81.800000 --> 82.580000).  Saving model ...\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 0.1676: 100%|██████████| 469/469 [00:30<00:00, 15.23it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 8 \tTest Accuracy: 83.35\n",
            "Test acc increased (82.580000 --> 83.350000).  Saving model ...\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 0.1630: 100%|██████████| 469/469 [00:30<00:00, 15.19it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 9 \tTest Accuracy: 83.68\n",
            "Test acc increased (83.350000 --> 83.680000).  Saving model ...\n",
            "tensor(1.)\n",
            "tensor(1.)\n",
            "=======Trial: 3=======\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 0.2818: 100%|██████████| 469/469 [00:30<00:00, 15.27it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 0 \tTest Accuracy: 56.2\n",
            "Test acc increased (0.000000 --> 56.200000).  Saving model ...\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 0.2309: 100%|██████████| 469/469 [00:30<00:00, 15.23it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 1 \tTest Accuracy: 68.83\n",
            "Test acc increased (56.200000 --> 68.830000).  Saving model ...\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 0.2095: 100%|██████████| 469/469 [00:30<00:00, 15.17it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 2 \tTest Accuracy: 72.61\n",
            "Test acc increased (68.830000 --> 72.610000).  Saving model ...\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 0.1916: 100%|██████████| 469/469 [00:30<00:00, 15.24it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 3 \tTest Accuracy: 74.09\n",
            "Test acc increased (72.610000 --> 74.090000).  Saving model ...\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 0.1828: 100%|██████████| 469/469 [00:31<00:00, 15.12it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 4 \tTest Accuracy: 76.61\n",
            "Test acc increased (74.090000 --> 76.610000).  Saving model ...\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 0.1858: 100%|██████████| 469/469 [00:30<00:00, 15.29it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 5 \tTest Accuracy: 80.09\n",
            "Test acc increased (76.610000 --> 80.090000).  Saving model ...\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 0.1777: 100%|██████████| 469/469 [00:31<00:00, 15.12it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 6 \tTest Accuracy: 81.8\n",
            "Test acc increased (80.090000 --> 81.800000).  Saving model ...\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "loss: 0.1673: 100%|██████████| 469/469 [00:31<00:00, 14.87it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch: 7 \tTest Accuracy: 82.58\n",
            "Test acc increased (81.800000 --> 82.580000).  Saving model ...\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "loss: 0.1676: 100%|██████████| 469/469 [00:31<00:00, 14.69it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch: 8 \tTest Accuracy: 83.35\n",
            "Test acc increased (82.580000 --> 83.350000).  Saving model ...\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "loss: 0.1630: 100%|██████████| 469/469 [00:31<00:00, 14.77it/s]\n"
          ]
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-62-8771fe417d0f>\u001b[0m in \u001b[0;36m<cell line: 6>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mcfg\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'enable_batch_norm'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0mcfg\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'enable_bias'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 6\u001b[0;31m \u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcfg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m<ipython-input-60-12228137bcd4>\u001b[0m in \u001b[0;36mrun\u001b[0;34m(config)\u001b[0m\n\u001b[1;32m     78\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     79\u001b[0m             \u001b[0;31m# test\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 80\u001b[0;31m             \u001b[0mtest_acc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtest_accuracy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnet\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtestloader\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     81\u001b[0m             \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf'Epoch: {epoch} \\tTest Accuracy: {test_acc}'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     82\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-57-98d5f6844a42>\u001b[0m in \u001b[0;36mtest_accuracy\u001b[0;34m(config, net, testloader, device)\u001b[0m\n\u001b[1;32m     12\u001b[0m             \u001b[0mimages\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabels\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mimages\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabels\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     13\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 14\u001b[0;31m             \u001b[0moutputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnet\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mimages\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     15\u001b[0m             \u001b[0maccuracy\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mSF\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maccuracy_rate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabels\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     16\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1499\u001b[0m                 \u001b[0;32mor\u001b[0m \u001b[0m_global_backward_pre_hooks\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0m_global_backward_hooks\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1500\u001b[0m                 or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[0;32m-> 1501\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1502\u001b[0m         \u001b[0;31m# Do not call functions when jit is used\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1503\u001b[0m         \u001b[0mfull_backward_hooks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnon_full_backward_hooks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-61-50b56aaee373>\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, x)\u001b[0m\n\u001b[1;32m    149\u001b[0m                     \u001b[0mcur1\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbn1\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcur1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    150\u001b[0m                 \u001b[0mspk1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmem1\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlif1\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcur1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmem1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 151\u001b[0;31m                 \u001b[0mcur2\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbfc2\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mspk1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    152\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbatch_norm\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    153\u001b[0m                     \u001b[0mcur2\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbn2\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcur2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1499\u001b[0m                 \u001b[0;32mor\u001b[0m \u001b[0m_global_backward_pre_hooks\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0m_global_backward_hooks\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1500\u001b[0m                 or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[0;32m-> 1501\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1502\u001b[0m         \u001b[0;31m# Do not call functions when jit is used\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1503\u001b[0m         \u001b[0mfull_backward_hooks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnon_full_backward_hooks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-54-92e216b258ff>\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m     49\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mF\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlinear\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbinary_weight\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     50\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 51\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mF\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlinear\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbinary_weight\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbias\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     52\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     53\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mreset_parameters\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "cfg = config.copy()\n",
        "cfg['model'] = 'NetFC'\n",
        "cfg['num_steps'] = 10\n",
        "cfg['enable_batch_norm'] = True\n",
        "cfg['enable_bias'] = False\n",
        "run(cfg)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "shiE_uQOd6NQ",
        "outputId": "168e746c-4fd9-4c99-91f7-d1194b181c22"
      },
      "execution_count": 49,
      "outputs": [
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'model': 'NetFC', 'exp_name': 'mnist_tha', 'num_trials': 5, 'num_epochs': 10, 'binarize': True, 'binarize_input': True, 'enable_bias': False, 'enable_batchnorm': True, 'enable_dropout': True, 'enable_threshold': True, 'enable_slope': True, 'data_dir': '~/data/mnist', 'batch_size': 128, 'seed': 0, 'num_workers': 0, 'save_csv': True, 'save_model': True, 'early_stopping': True, 'patience': 100, 'grad_clip': False, 'weight_clip': False, 'batch_norm': True, 'dropout1': 0.02856, 'beta': 0.99, 'lr': 0.00997, 'slope': 10.22, 'threshold1': 11.666, 'alpha_thr1': 0.024, 'thr_final1': 4.317, 'threshold2': 14.105, 'alpha_thr2': 0.119, 'thr_final2': 16.29, 'threshold3': 0.6656, 'alpha_thr3': 0.0011, 'thr_final3': 3.496, 'num_steps': 10, 'correct_rate': 0.8, 'incorrect_rate': 0.2, 'betas': (0.9, 0.999), 't_0': 4688, 'eta_min': 0, 'df_lr': True, 'enable_batch_norm': True}\n",
            "tensor(1.)\n",
            "tensor(1.)\n",
            "NetFC(\n",
            "  (bfc1): SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "  (fc1): Linear(in_features=256, out_features=256, bias=False)\n",
            "  (bn1): BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "  (lif1): Leaky()\n",
            "  (bfc2): SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "  (fc2): Linear(in_features=256, out_features=128, bias=False)\n",
            "  (bn2): BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "  (lif2): Leaky()\n",
            "  (bfc3): BinaryLinear(in_features=128, out_features=10, bias=False)\n",
            "  (fc3): Linear(in_features=128, out_features=10, bias=False)\n",
            "  (lif3): Leaky()\n",
            "  (dropout): Dropout(p=0.02856, inplace=False)\n",
            ")\n",
            "=======Trial: 0=======\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 0.4230: 100%|██████████| 469/469 [00:32<00:00, 14.61it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 0 \tTest Accuracy: 11.37\n",
            "Test acc increased (0.000000 --> 11.370000).  Saving model ...\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 0.3242: 100%|██████████| 469/469 [00:31<00:00, 14.68it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 1 \tTest Accuracy: 20.87\n",
            "Test acc increased (11.370000 --> 20.870000).  Saving model ...\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 0.3067: 100%|██████████| 469/469 [00:32<00:00, 14.65it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 2 \tTest Accuracy: 34.23\n",
            "Test acc increased (20.870000 --> 34.230000).  Saving model ...\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 0.2928: 100%|██████████| 469/469 [00:31<00:00, 14.88it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 3 \tTest Accuracy: 49.65\n",
            "Test acc increased (34.230000 --> 49.650000).  Saving model ...\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 0.2761: 100%|██████████| 469/469 [00:31<00:00, 15.09it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 4 \tTest Accuracy: 47.5\n",
            "EarlyStopping counter: 1 out of 100\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 0.2742: 100%|██████████| 469/469 [00:31<00:00, 14.94it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 5 \tTest Accuracy: 45.11\n",
            "EarlyStopping counter: 2 out of 100\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 0.2703: 100%|██████████| 469/469 [00:30<00:00, 15.23it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 6 \tTest Accuracy: 48.17\n",
            "EarlyStopping counter: 3 out of 100\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 0.2730: 100%|██████████| 469/469 [00:31<00:00, 15.08it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 7 \tTest Accuracy: 52.54\n",
            "Test acc increased (49.650000 --> 52.540000).  Saving model ...\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 0.2699: 100%|██████████| 469/469 [00:30<00:00, 15.20it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 8 \tTest Accuracy: 51.32\n",
            "EarlyStopping counter: 1 out of 100\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 0.2684: 100%|██████████| 469/469 [00:31<00:00, 15.11it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 9 \tTest Accuracy: 52.0\n",
            "EarlyStopping counter: 2 out of 100\n",
            "tensor(1.)\n",
            "tensor(1.)\n",
            "=======Trial: 1=======\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 0.4230: 100%|██████████| 469/469 [00:31<00:00, 15.06it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 0 \tTest Accuracy: 11.37\n",
            "Test acc increased (0.000000 --> 11.370000).  Saving model ...\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 0.3242: 100%|██████████| 469/469 [00:31<00:00, 15.07it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 1 \tTest Accuracy: 20.87\n",
            "Test acc increased (11.370000 --> 20.870000).  Saving model ...\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 0.3067: 100%|██████████| 469/469 [00:31<00:00, 15.03it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 2 \tTest Accuracy: 34.23\n",
            "Test acc increased (20.870000 --> 34.230000).  Saving model ...\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 0.2928: 100%|██████████| 469/469 [00:30<00:00, 15.18it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 3 \tTest Accuracy: 49.65\n",
            "Test acc increased (34.230000 --> 49.650000).  Saving model ...\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 0.2761: 100%|██████████| 469/469 [00:31<00:00, 15.09it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 4 \tTest Accuracy: 47.5\n",
            "EarlyStopping counter: 1 out of 100\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 0.2742: 100%|██████████| 469/469 [00:31<00:00, 14.95it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 5 \tTest Accuracy: 45.11\n",
            "EarlyStopping counter: 2 out of 100\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 0.2703: 100%|██████████| 469/469 [00:30<00:00, 15.17it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 6 \tTest Accuracy: 48.17\n",
            "EarlyStopping counter: 3 out of 100\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 0.2730: 100%|██████████| 469/469 [00:30<00:00, 15.14it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 7 \tTest Accuracy: 52.54\n",
            "Test acc increased (49.650000 --> 52.540000).  Saving model ...\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 0.2699: 100%|██████████| 469/469 [00:30<00:00, 15.17it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 8 \tTest Accuracy: 51.32\n",
            "EarlyStopping counter: 1 out of 100\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "loss: 0.2684: 100%|██████████| 469/469 [00:30<00:00, 15.14it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch: 9 \tTest Accuracy: 52.0\n",
            "EarlyStopping counter: 2 out of 100\n",
            "tensor(1.)\n",
            "tensor(1.)\n",
            "=======Trial: 2=======\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "loss: 0.4230: 100%|██████████| 469/469 [00:31<00:00, 14.83it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch: 0 \tTest Accuracy: 11.37\n",
            "Test acc increased (0.000000 --> 11.370000).  Saving model ...\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "loss: 0.3482:  54%|█████▍    | 254/469 [00:17<00:14, 14.50it/s]\n"
          ]
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-49-9a7f7012451d>\u001b[0m in \u001b[0;36m<cell line: 6>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mcfg\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'enable_batch_norm'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0mcfg\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'enable_bias'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 6\u001b[0;31m \u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcfg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m<ipython-input-45-12228137bcd4>\u001b[0m in \u001b[0;36mrun\u001b[0;34m(config)\u001b[0m\n\u001b[1;32m     74\u001b[0m             \u001b[0;31m# train\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     75\u001b[0m             \u001b[0mstart_time\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 76\u001b[0;31m             \u001b[0mloss_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlr_list\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnet\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepoch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrainloader\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtestloader\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcriterion\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mscheduler\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     77\u001b[0m             \u001b[0mepoch_time\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mstart_time\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     78\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-44-e3ed6f056ebd>\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(config, net, epoch, trainloader, testloader, criterion, optimizer, scheduler, device)\u001b[0m\n\u001b[1;32m     34\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     35\u001b[0m     \u001b[0;31m#for data, labels in trainloader:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 36\u001b[0;31m     \u001b[0;32mfor\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabels\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mprogress_bar\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     37\u001b[0m         \u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabels\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabels\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     38\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/tqdm/std.py\u001b[0m in \u001b[0;36m__iter__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1176\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1177\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1178\u001b[0;31m             \u001b[0;32mfor\u001b[0m \u001b[0mobj\u001b[0m \u001b[0;32min\u001b[0m \u001b[0miterable\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1179\u001b[0m                 \u001b[0;32myield\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1180\u001b[0m                 \u001b[0;31m# Update and possibly print the progressbar.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/torch/utils/data/dataloader.py\u001b[0m in \u001b[0;36m__next__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    631\u001b[0m                 \u001b[0;31m# TODO(https://github.com/pytorch/pytorch/issues/76750)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    632\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_reset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# type: ignore[call-arg]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 633\u001b[0;31m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_next_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    634\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_num_yielded\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    635\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_dataset_kind\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0m_DatasetKind\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mIterable\u001b[0m \u001b[0;32mand\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m\\\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/torch/utils/data/dataloader.py\u001b[0m in \u001b[0;36m_next_data\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    675\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_next_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    676\u001b[0m         \u001b[0mindex\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_next_index\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# may raise StopIteration\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 677\u001b[0;31m         \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_dataset_fetcher\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfetch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mindex\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# may raise StopIteration\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    678\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_pin_memory\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    679\u001b[0m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_utils\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpin_memory\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpin_memory\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_pin_memory_device\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/torch/utils/data/_utils/fetch.py\u001b[0m in \u001b[0;36mfetch\u001b[0;34m(self, possibly_batched_index)\u001b[0m\n\u001b[1;32m     49\u001b[0m                 \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__getitems__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpossibly_batched_index\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     50\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 51\u001b[0;31m                 \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0midx\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0midx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mpossibly_batched_index\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     52\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     53\u001b[0m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mpossibly_batched_index\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/torch/utils/data/_utils/fetch.py\u001b[0m in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m     49\u001b[0m                 \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__getitems__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpossibly_batched_index\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     50\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 51\u001b[0;31m                 \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0midx\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0midx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mpossibly_batched_index\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     52\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     53\u001b[0m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mpossibly_batched_index\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/torchvision/datasets/mnist.py\u001b[0m in \u001b[0;36m__getitem__\u001b[0;34m(self, index)\u001b[0m\n\u001b[1;32m    143\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    144\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtransform\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 145\u001b[0;31m             \u001b[0mimg\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtransform\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mimg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    146\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    147\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtarget_transform\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/torchvision/transforms/transforms.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, img)\u001b[0m\n\u001b[1;32m     93\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__call__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mimg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     94\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mt\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtransforms\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 95\u001b[0;31m             \u001b[0mimg\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mimg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     96\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mimg\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     97\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1499\u001b[0m                 \u001b[0;32mor\u001b[0m \u001b[0m_global_backward_pre_hooks\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0m_global_backward_hooks\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1500\u001b[0m                 or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[0;32m-> 1501\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1502\u001b[0m         \u001b[0;31m# Do not call functions when jit is used\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1503\u001b[0m         \u001b[0mfull_backward_hooks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnon_full_backward_hooks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/torchvision/transforms/transforms.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, tensor)\u001b[0m\n\u001b[1;32m    275\u001b[0m             \u001b[0mTensor\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mNormalized\u001b[0m \u001b[0mTensor\u001b[0m \u001b[0mimage\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    276\u001b[0m         \"\"\"\n\u001b[0;32m--> 277\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mF\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnormalize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtensor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmean\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstd\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minplace\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    278\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    279\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__repr__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/torchvision/transforms/functional.py\u001b[0m in \u001b[0;36mnormalize\u001b[0;34m(tensor, mean, std, inplace)\u001b[0m\n\u001b[1;32m    361\u001b[0m         \u001b[0;32mraise\u001b[0m \u001b[0mTypeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf\"img should be Tensor Image. Got {type(tensor)}\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    362\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 363\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mF_t\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnormalize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtensor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmean\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmean\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstd\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mstd\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minplace\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minplace\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    364\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    365\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/torchvision/transforms/_functional_tensor.py\u001b[0m in \u001b[0;36mnormalize\u001b[0;34m(tensor, mean, std, inplace)\u001b[0m\n\u001b[1;32m    923\u001b[0m         \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf\"std evaluated to zero after conversion to {dtype}, leading to division by zero.\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    924\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mmean\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mndim\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 925\u001b[0;31m         \u001b[0mmean\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmean\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mview\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    926\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mstd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mndim\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    927\u001b[0m         \u001b[0mstd\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mstd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mview\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#binarized inputs\n",
        "cfg = config\n",
        "cfg['model'] = 'NetFC'\n",
        "run(cfg)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 730
        },
        "id": "INPErVUdnw5s",
        "outputId": "9f60ea13-72ca-449d-c750-fbc90ad10d0a"
      },
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{'model': 'NetFC', 'exp_name': 'mnist_tha', 'num_trials': 5, 'num_epochs': 500, 'binarize': True, 'data_dir': '~/data/mnist', 'batch_size': 128, 'seed': 0, 'num_workers': 0, 'save_csv': True, 'save_model': True, 'early_stopping': True, 'patience': 100, 'grad_clip': False, 'weight_clip': False, 'batch_norm': True, 'dropout1': 0.02856, 'beta': 0.99, 'lr': 0.00997, 'slope': 10.22, 'threshold1': 11.666, 'alpha_thr1': 0.024, 'thr_final1': 4.317, 'threshold2': 14.105, 'alpha_thr2': 0.119, 'thr_final2': 16.29, 'threshold3': 0.6656, 'alpha_thr3': 0.0011, 'thr_final3': 3.496, 'num_steps': 100, 'correct_rate': 0.8, 'incorrect_rate': 0.2, 'betas': (0.9, 0.999), 't_0': 4688, 'eta_min': 0, 'df_lr': True}\n",
            "NetFC(\n",
            "  (bfc1): SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "  (fc1): Linear(in_features=256, out_features=256, bias=False)\n",
            "  (bn1): BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "  (lif1): Leaky()\n",
            "  (bfc2): SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "  (fc2): Linear(in_features=256, out_features=128, bias=False)\n",
            "  (bn2): BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "  (lif2): Leaky()\n",
            "  (bfc3): BinaryLinear(in_features=128, out_features=10, bias=False)\n",
            "  (fc3): Linear(in_features=128, out_features=10, bias=False)\n",
            "  (lif3): Leaky()\n",
            "  (dropout): Dropout(p=0.02856, inplace=False)\n",
            ")\n",
            "=======Trial: 0=======\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "loss: 0.8944: 100%|██████████| 469/469 [02:55<00:00,  2.67it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch: 0 \tTest Accuracy: 91.56\n",
            "Test acc increased (0.000000 --> 91.560000).  Saving model ...\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "loss: 0.8133: 100%|██████████| 469/469 [02:55<00:00,  2.67it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch: 1 \tTest Accuracy: 93.5\n",
            "Test acc increased (91.560000 --> 93.500000).  Saving model ...\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "loss: 0.7441:  27%|██▋       | 125/469 [00:46<02:08,  2.69it/s]\n"
          ]
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-13-62141dcf62a4>\u001b[0m in \u001b[0;36m<cell line: 4>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0mcfg\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mconfig\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mcfg\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'model'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m'NetFC'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m \u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcfg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m<ipython-input-10-24fc7e61737e>\u001b[0m in \u001b[0;36mrun\u001b[0;34m(config)\u001b[0m\n\u001b[1;32m     87\u001b[0m             \u001b[0;31m# train\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     88\u001b[0m             \u001b[0mstart_time\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 89\u001b[0;31m             \u001b[0mloss_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlr_list\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnet\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepoch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrainloader\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtestloader\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcriterion\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mscheduler\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     90\u001b[0m             \u001b[0mepoch_time\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mstart_time\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     91\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-9-e3ed6f056ebd>\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(config, net, epoch, trainloader, testloader, criterion, optimizer, scheduler, device)\u001b[0m\n\u001b[1;32m     37\u001b[0m         \u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabels\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabels\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     38\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 39\u001b[0;31m         \u001b[0mspk_rec2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnet\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     40\u001b[0m         \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcriterion\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mspk_rec2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabels\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     41\u001b[0m         \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mzero_grad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1499\u001b[0m                 \u001b[0;32mor\u001b[0m \u001b[0m_global_backward_pre_hooks\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0m_global_backward_hooks\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1500\u001b[0m                 or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[0;32m-> 1501\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1502\u001b[0m         \u001b[0;31m# Do not call functions when jit is used\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1503\u001b[0m         \u001b[0mfull_backward_hooks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnon_full_backward_hooks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-11-1db99282087d>\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, x)\u001b[0m\n\u001b[1;32m    144\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbatch_norm\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    145\u001b[0m                     \u001b[0mcur2\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbn2\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcur2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 146\u001b[0;31m                 \u001b[0mspk2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmem2\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlif2\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcur2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmem2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    147\u001b[0m                 \u001b[0mcur3\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdropout\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbfc3\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mspk2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    148\u001b[0m                 \u001b[0mspk3\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmem3\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlif3\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcur3\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmem3\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1499\u001b[0m                 \u001b[0;32mor\u001b[0m \u001b[0m_global_backward_pre_hooks\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0m_global_backward_hooks\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1500\u001b[0m                 or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[0;32m-> 1501\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1502\u001b[0m         \u001b[0;31m# Do not call functions when jit is used\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1503\u001b[0m         \u001b[0mfull_backward_hooks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnon_full_backward_hooks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/snntorch/_neurons/leaky.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input_, mem)\u001b[0m\n\u001b[1;32m    175\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minit_hidden\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    176\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreset\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmem_reset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmem\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 177\u001b[0;31m             \u001b[0mmem\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_build_state_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmem\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    178\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    179\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstate_quant\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/snntorch/_neurons/leaky.py\u001b[0m in \u001b[0;36m_build_state_function\u001b[0;34m(self, input_, mem)\u001b[0m\n\u001b[1;32m    213\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_build_state_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmem\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    214\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreset_mechanism_val\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m  \u001b[0;31m# reset by subtraction\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 215\u001b[0;31m             state_fn = self._base_state_function(\n\u001b[0m\u001b[1;32m    216\u001b[0m                 \u001b[0minput_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmem\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreset\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mthreshold\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    217\u001b[0m             )\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/snntorch/_neurons/leaky.py\u001b[0m in \u001b[0;36m_base_state_function\u001b[0;34m(self, input_, mem)\u001b[0m\n\u001b[1;32m    208\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    209\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_base_state_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmem\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 210\u001b[0;31m         \u001b[0mbase_fn\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbeta\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclamp\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0mmem\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0minput_\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    211\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mbase_fn\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    212\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "cfg = config\n",
        "cfg['model'] = 'NetFC(config, sparsity=[0.5, 0.5])'\n",
        "run(cfg)"
      ],
      "metadata": {
        "id": "Bav7tGbu-lI7",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "d8a4169e-5621-4e33-b7a9-00a5d219f3fc"
      },
      "execution_count": 14,
      "outputs": [
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'model': 'NetFC(config, sparsity=0.5)', 'exp_name': 'mnist_tha', 'num_trials': 5, 'num_epochs': 500, 'binarize': True, 'data_dir': '~/data/mnist', 'batch_size': 128, 'seed': 0, 'num_workers': 0, 'save_csv': True, 'save_model': True, 'early_stopping': True, 'patience': 100, 'grad_clip': False, 'weight_clip': False, 'batch_norm': True, 'dropout1': 0.02856, 'beta': 0.99, 'lr': 0.00997, 'slope': 10.22, 'threshold1': 11.666, 'alpha_thr1': 0.024, 'thr_final1': 15.983, 'threshold2': 14.105, 'alpha_thr2': 0.119, 'thr_final2': 30.395, 'threshold3': 0.6656, 'alpha_thr3': 0.0011, 'thr_final3': 4.1616, 'num_steps': 100, 'correct_rate': 0.8, 'incorrect_rate': 0.2, 'betas': (0.9, 0.999), 't_0': 4688, 'eta_min': 0, 'df_lr': True, 'dataset_length': 60000}\n",
            "NetFC(\n",
            "  (bfc1): SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "  (fc1): Linear(in_features=256, out_features=256, bias=False)\n",
            "  (bn1): BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "  (lif1): Leaky()\n",
            "  (bfc2): SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "  (fc2): Linear(in_features=256, out_features=128, bias=False)\n",
            "  (bn2): BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "  (lif2): Leaky()\n",
            "  (bfc3): BinaryLinear(in_features=128, out_features=10, bias=False)\n",
            "  (fc3): Linear(in_features=128, out_features=10, bias=False)\n",
            "  (lif3): Leaky()\n",
            "  (dropout): Dropout(p=0.02856, inplace=False)\n",
            ")\n",
            "=======Trial: 0=======\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 1.0576: 100%|██████████| 469/469 [02:57<00:00,  2.64it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 0 \tTest Accuracy: 89.58\n",
            "Test acc increased (0.000000 --> 89.580000).  Saving model ...\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 0.9438: 100%|██████████| 469/469 [02:57<00:00,  2.64it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 1 \tTest Accuracy: 90.51\n",
            "Test acc increased (89.580000 --> 90.510000).  Saving model ...\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 0.9071: 100%|██████████| 469/469 [02:57<00:00,  2.64it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 2 \tTest Accuracy: 91.0\n",
            "Test acc increased (90.510000 --> 91.000000).  Saving model ...\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 0.8651: 100%|██████████| 469/469 [02:57<00:00,  2.64it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 3 \tTest Accuracy: 91.85\n",
            "Test acc increased (91.000000 --> 91.850000).  Saving model ...\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 0.8304: 100%|██████████| 469/469 [02:53<00:00,  2.71it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 4 \tTest Accuracy: 92.0\n",
            "Test acc increased (91.850000 --> 92.000000).  Saving model ...\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 0.8051: 100%|██████████| 469/469 [02:52<00:00,  2.72it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 5 \tTest Accuracy: 92.3\n",
            "Test acc increased (92.000000 --> 92.300000).  Saving model ...\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 0.7615: 100%|██████████| 469/469 [02:50<00:00,  2.74it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 6 \tTest Accuracy: 92.17\n",
            "EarlyStopping counter: 1 out of 100\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 0.7586: 100%|██████████| 469/469 [02:49<00:00,  2.77it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 7 \tTest Accuracy: 92.73\n",
            "Test acc increased (92.300000 --> 92.730000).  Saving model ...\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 0.7067: 100%|██████████| 469/469 [02:47<00:00,  2.80it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 8 \tTest Accuracy: 92.95\n",
            "Test acc increased (92.730000 --> 92.950000).  Saving model ...\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss: 0.6712: 100%|██████████| 469/469 [02:44<00:00,  2.85it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch: 9 \tTest Accuracy: 92.91\n",
            "EarlyStopping counter: 1 out of 100\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "loss: 0.7154:  30%|███       | 141/469 [00:50<01:58,  2.78it/s]\n"
          ]
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-14-b909bb6cfd8a>\u001b[0m in \u001b[0;36m<cell line: 3>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0mcfg\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mconfig\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0mcfg\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'model'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m'NetFC(config, sparsity=0.5)'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcfg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m<ipython-input-10-24fc7e61737e>\u001b[0m in \u001b[0;36mrun\u001b[0;34m(config)\u001b[0m\n\u001b[1;32m     87\u001b[0m             \u001b[0;31m# train\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     88\u001b[0m             \u001b[0mstart_time\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 89\u001b[0;31m             \u001b[0mloss_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlr_list\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnet\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepoch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrainloader\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtestloader\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcriterion\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mscheduler\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     90\u001b[0m             \u001b[0mepoch_time\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mstart_time\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     91\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-9-e3ed6f056ebd>\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(config, net, epoch, trainloader, testloader, criterion, optimizer, scheduler, device)\u001b[0m\n\u001b[1;32m     37\u001b[0m         \u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabels\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabels\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     38\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 39\u001b[0;31m         \u001b[0mspk_rec2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnet\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     40\u001b[0m         \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcriterion\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mspk_rec2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabels\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     41\u001b[0m         \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mzero_grad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1499\u001b[0m                 \u001b[0;32mor\u001b[0m \u001b[0m_global_backward_pre_hooks\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0m_global_backward_hooks\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1500\u001b[0m                 or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[0;32m-> 1501\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1502\u001b[0m         \u001b[0;31m# Do not call functions when jit is used\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1503\u001b[0m         \u001b[0mfull_backward_hooks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnon_full_backward_hooks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-11-1db99282087d>\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, x)\u001b[0m\n\u001b[1;32m    146\u001b[0m                 \u001b[0mspk2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmem2\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlif2\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcur2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmem2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    147\u001b[0m                 \u001b[0mcur3\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdropout\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbfc3\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mspk2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 148\u001b[0;31m                 \u001b[0mspk3\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmem3\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlif3\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcur3\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmem3\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    149\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    150\u001b[0m                 \u001b[0mspk3_rec\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mspk3\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1499\u001b[0m                 \u001b[0;32mor\u001b[0m \u001b[0m_global_backward_pre_hooks\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0m_global_backward_hooks\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1500\u001b[0m                 or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[0;32m-> 1501\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1502\u001b[0m         \u001b[0;31m# Do not call functions when jit is used\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1503\u001b[0m         \u001b[0mfull_backward_hooks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnon_full_backward_hooks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/snntorch/_neurons/leaky.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input_, mem)\u001b[0m\n\u001b[1;32m    175\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minit_hidden\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    176\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreset\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmem_reset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmem\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 177\u001b[0;31m             \u001b[0mmem\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_build_state_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmem\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    178\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    179\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstate_quant\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/snntorch/_neurons/leaky.py\u001b[0m in \u001b[0;36m_build_state_function\u001b[0;34m(self, input_, mem)\u001b[0m\n\u001b[1;32m    213\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_build_state_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmem\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    214\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreset_mechanism_val\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m  \u001b[0;31m# reset by subtraction\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 215\u001b[0;31m             state_fn = self._base_state_function(\n\u001b[0m\u001b[1;32m    216\u001b[0m                 \u001b[0minput_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmem\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreset\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mthreshold\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    217\u001b[0m             )\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/snntorch/_neurons/leaky.py\u001b[0m in \u001b[0;36m_base_state_function\u001b[0;34m(self, input_, mem)\u001b[0m\n\u001b[1;32m    207\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mspk\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    208\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 209\u001b[0;31m     \u001b[0;32mdef\u001b[0m \u001b[0m_base_state_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmem\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    210\u001b[0m         \u001b[0mbase_fn\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbeta\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclamp\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0mmem\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0minput_\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    211\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mbase_fn\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "cfg = config\n",
        "cfg['model'] = 'NetFC(config, sparsity=[0.9, 0.9])'\n",
        "run(cfg)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pAdG8QcKymlc",
        "outputId": "7328ec2f-48b4-48f0-92c1-36eabd81818d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{'model': 'NetFC(config, sparsity=0.9)', 'exp_name': 'mnist_tha', 'num_trials': 5, 'num_epochs': 10, 'binarize': True, 'binarize_input': True, 'data_dir': '~/data/mnist', 'batch_size': 128, 'seed': 0, 'num_workers': 0, 'save_csv': True, 'save_model': True, 'early_stopping': True, 'patience': 100, 'grad_clip': False, 'weight_clip': False, 'batch_norm': True, 'dropout1': 0.02856, 'beta': 0.99, 'lr': 0.00997, 'slope': 10.22, 'threshold1': 11.666, 'alpha_thr1': 0.024, 'thr_final1': 4.317, 'threshold2': 14.105, 'alpha_thr2': 0.119, 'thr_final2': 16.29, 'threshold3': 0.6656, 'alpha_thr3': 0.0011, 'thr_final3': 3.496, 'num_steps': 100, 'correct_rate': 0.8, 'incorrect_rate': 0.2, 'betas': (0.9, 0.999), 't_0': 4688, 'eta_min': 0, 'df_lr': True}\n",
            "tensor(0.1001)\n",
            "tensor(0.1024)\n",
            "NetFC(\n",
            "  (bfc1): SparseBinaryLinear(in_features=256, out_features=256, bias=False)\n",
            "  (fc1): Linear(in_features=256, out_features=256, bias=False)\n",
            "  (bn1): BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "  (lif1): Leaky()\n",
            "  (bfc2): SparseBinaryLinear(in_features=256, out_features=128, bias=False)\n",
            "  (fc2): Linear(in_features=256, out_features=128, bias=False)\n",
            "  (bn2): BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "  (lif2): Leaky()\n",
            "  (bfc3): BinaryLinear(in_features=128, out_features=10, bias=False)\n",
            "  (fc3): Linear(in_features=128, out_features=10, bias=False)\n",
            "  (lif3): Leaky()\n",
            "  (dropout): Dropout(p=0.02856, inplace=False)\n",
            ")\n",
            "=======Trial: 0=======\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "loss: 1.6109: 100%|██████████| 469/469 [02:47<00:00,  2.80it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch: 0 \tTest Accuracy: 85.25\n",
            "Test acc increased (0.000000 --> 85.250000).  Saving model ...\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "loss: 1.4953: 100%|██████████| 469/469 [02:46<00:00,  2.82it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch: 1 \tTest Accuracy: 84.37\n",
            "EarlyStopping counter: 1 out of 100\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "loss: 1.4063: 100%|██████████| 469/469 [02:48<00:00,  2.79it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch: 2 \tTest Accuracy: 85.62\n",
            "Test acc increased (85.250000 --> 85.620000).  Saving model ...\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "loss: 1.3945: 100%|██████████| 469/469 [02:48<00:00,  2.78it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch: 3 \tTest Accuracy: 86.7\n",
            "Test acc increased (85.620000 --> 86.700000).  Saving model ...\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "loss: 1.4162: 100%|██████████| 469/469 [02:46<00:00,  2.82it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch: 4 \tTest Accuracy: 87.12\n",
            "Test acc increased (86.700000 --> 87.120000).  Saving model ...\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "loss: 1.3976: 100%|██████████| 469/469 [02:46<00:00,  2.82it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch: 5 \tTest Accuracy: 87.33\n",
            "Test acc increased (87.120000 --> 87.330000).  Saving model ...\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "loss: 1.2937: 100%|██████████| 469/469 [02:48<00:00,  2.79it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch: 6 \tTest Accuracy: 87.65\n",
            "Test acc increased (87.330000 --> 87.650000).  Saving model ...\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "loss: 1.3561:  95%|█████████▍| 445/469 [02:38<00:08,  2.76it/s]"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "cfg = config\n",
        "cfg['model'] = 'NetFC(config, [128, 128])'\n",
        "run(cfg)"
      ],
      "metadata": {
        "id": "15U4jQ5WAj2O"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}