Capsule Networks MNIST.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import torch\n",
    "import torch.nn.functional as F\n",
    "import torch.utils.data as data_utils\n",
    "import torchvision\n",
    "\n",
    "from datetime import datetime\n",
    "from torch import nn"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Define constants and dataset\n",
    "- TODO All these definitions should be done using CL args\n",
    "- Set important constant such as CUDA use and batch size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "epochs = 5\n",
    "batch_size = 32\n",
    "validation_split = 0.2\n",
    "dataset = torchvision.datasets.MNIST"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Create DataSet and DataLoader objects"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Only data augmentation for CapsNet is translation of up to 2px\n",
    "# As described in Section 5.\n",
    "data_transforms = {\n",
    "    'train': torchvision.transforms.Compose([\n",
    "        torchvision.transforms.RandomAffine(0, (0.08, 0.08)),\n",
    "        torchvision.transforms.ToTensor(),\n",
    "    ]),\n",
    "    'test': torchvision.transforms.Compose([\n",
    "        torchvision.transforms.ToTensor()\n",
    "    ])\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data = dataset('/datasets', train=True, download=False, transform=data_transforms['train'])\n",
    "test_data = dataset('/datasets', train=False, download=False, transform=data_transforms['test'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_samples = len(train_data)\n",
    "split_idx = int(validation_split * train_samples)\n",
    "indices = np.arange(train_samples)\n",
    "np.random.shuffle(indices)\n",
    "\n",
    "train_idx, val_idx = indices[split_idx:], indices[:split_idx]\n",
    "\n",
    "train_sampler = data_utils.sampler.SubsetRandomSampler(train_idx)\n",
    "val_sampler = data_utils.sampler.SubsetRandomSampler(val_idx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_loader = data_utils.DataLoader(train_data, batch_size=batch_size, num_workers=4, sampler=train_sampler)\n",
    "val_loader  = data_utils.DataLoader(train_data, batch_size=batch_size, num_workers=4, sampler=val_sampler)\n",
    "test_loader = data_utils.DataLoader(test_data, batch_size=batch_size, num_workers=4)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### TODO List\n",
    "- TEST routing algorithm\n",
    "- TEST capsule layer architecture\n",
    "- TEST capsule network architecture\n",
    "- TEST Squash function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def squash(tensor):\n",
    "    '''\n",
    "    TODO test\n",
    "    Squash function, defined in [1]. Works as a nonlinearity for CapsNets.\n",
    "    Input tensor will be of format (bs, units, C, H, W) or (bs, units, C)\n",
    "    Norm should be computed on the axis representing the number of units.\n",
    "    params:\n",
    "        tensor:    torch Variable containing n-dimensional tensor\n",
    "    output:\n",
    "        (||tensor||^2 / (1+ ||tensor||^2)) * tensor/||tensor||\n",
    "    '''\n",
    "    norm = torch.norm(tensor, p=2, dim=1, keepdim=True)\n",
    "    sq_norm = norm ** 2 # Avoid computing square twice\n",
    "        \n",
    "    return tensor.div(norm) * sq_norm/(1 + sq_norm)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Prototype of the capsule architecture"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "class CapsuleLayer(nn.Module):\n",
    "    \"\"\"\n",
    "    TODO add very long doc\n",
    "    \"\"\"\n",
    "    def __init__(self, input_units, input_channels, num_units, channels_per_unit, kernel_size, stride, routing, routing_iterations):\n",
    "        super(CapsuleLayer, self).__init__()\n",
    "        self.input_units = input_units\n",
    "        self.input_channels = input_channels\n",
    "        self.num_units = num_units\n",
    "        self.channels_per_unit = channels_per_unit\n",
    "        self.kernel_size = kernel_size\n",
    "        self.stride = stride\n",
    "        self.routing = routing\n",
    "        self.routing_iterations = routing_iterations\n",
    "        \n",
    "        if self.routing:\n",
    "            \"\"\"\n",
    "            'W_ij is a weight matrix between each u_i, for i in (1, 32x6x6) in PrimaryCapsules and v_j, for j in (1, 10)'\n",
    "            Additionally, W_ij is an (8, 16) matrix.\n",
    "            This means the layer will have a parameter matrix of size (input_units * H_in * W_in, num_classes, input_channels, channels_per_unit).\n",
    "            To make it easier for us to define this matrix, let us assumme `input_units == original_input_units * H_in * W_in` when routing is active.\n",
    "            \"\"\"\n",
    "            self.weights = nn.Parameter(torch.randn(input_units, num_units, input_channels, channels_per_unit))     \n",
    "        else:\n",
    "            \"\"\"\n",
    "            For the PrimaryCaps layer (if the previous layer is not capsular too), the output should be the same as using multiple small \n",
    "            convolutional layers. Using a ModuleList facilitates interaction with all the units in a pythonic way.\n",
    "            Section 4,  3rd paragraph, describes the PrimaryCaps layer as having 32 units, each with 8 channels, with 9x9 kernel and stride 2.\n",
    "            \"\"\"\n",
    "            self.units = nn.ModuleList([nn.Conv2d(input_channels, channels_per_unit, kernel_size, stride) for unit in range(self.num_units)])\n",
    "            \n",
    "        \n",
    "    def forward(self, x):\n",
    "        \"\"\"\n",
    "        Decide between applying routing or plain convolutions.\n",
    "        Routing is only used if between 2 consecutive layers\n",
    "        TODO try to implement routing as a method of the network and not the layers\n",
    "        \"\"\"\n",
    "        if self.routing:\n",
    "            return self._routing(x)\n",
    "        else:\n",
    "            return self._apply_conv_units(x)\n",
    "\n",
    "\n",
    "    def _routing(self, inputs):\n",
    "        \"\"\"\n",
    "        TODO add doc\n",
    "        This function is probably rather heavy. Should try profiling.\n",
    "        \"\"\"\n",
    "        batch_size = inputs.data.shape[0]\n",
    "        weights = torch.stack([self.weights] * batch_size, dim=0)\n",
    "        \n",
    "        current_votes = inputs.permute([0, 2, 1])\n",
    "        current_votes = torch.stack([current_votes] * self.num_units, dim=2)\n",
    "        current_votes = torch.stack([current_votes] * self.channels_per_unit, dim=-1)\n",
    "        \n",
    "        logits = torch.zeros(current_votes.data.shape, requires_grad=True)\n",
    "        logits = logits.to(device)\n",
    "        \n",
    "        pondered_votes = weights * current_votes  # Uji \n",
    "        \n",
    "        for iteration in range(self.routing_iterations):\n",
    "            couplings = F.softmax(logits, dim=-1)\n",
    "            out = couplings * pondered_votes\n",
    "            out = squash(out)\n",
    "            agreement = pondered_votes * out\n",
    "            logits = logits + agreement\n",
    "        \n",
    "        out = out.permute([0, 2, 1, 3, 4])\n",
    "        return out\n",
    "    \n",
    "    \n",
    "    def _apply_conv_units(self, x):\n",
    "        \"\"\"\n",
    "        Shape: (batch_size, input_channels, H, W) -> (batch_size, units, channels_per_unit, H', W')\n",
    "        H' and W' can be calculated using standard formulae for convolutional outputs\n",
    "        \"\"\"\n",
    "        caps_output = [unit(x) for unit in self.units]\n",
    "        caps_output = torch.stack(caps_output, dim=1)  # New dimension 1 will have size `units`\n",
    "        return caps_output        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "class CapsNet(nn.Module):\n",
    "    def __init__(self, conv_in_channels=1, conv_out_channels=256, conv_kernel_size=9, conv_stride=1, \n",
    "                 primary_units=32, primary_dim=8, primary_kernel_size=9, primary_stride=2,\n",
    "                 num_classes=10, digits_dim=16, dense_units_1=512, dense_units_2=1024, dense_units_3=784,\n",
    "                 routing_iterations=1):\n",
    "        \"\"\"\n",
    "        TODO Add very long doc for this...\n",
    "        dense_units_3 : int, number of pixels in an input image\n",
    "        \"\"\"\n",
    "        super(CapsNet, self).__init__()\n",
    "        self.conv0 = nn.Conv2d(in_channels=conv_in_channels,\n",
    "                               out_channels=conv_out_channels,\n",
    "                               kernel_size=conv_kernel_size,\n",
    "                               stride=conv_stride)\n",
    "        self.primary_caps = CapsuleLayer(input_units=None, \n",
    "                                         input_channels=conv_out_channels,\n",
    "                                         num_units=primary_units,\n",
    "                                         channels_per_unit=primary_dim,\n",
    "                                         kernel_size=primary_kernel_size,\n",
    "                                         stride=primary_stride,\n",
    "                                         routing=False,\n",
    "                                         routing_iterations=routing_iterations)\n",
    "        self.digits_caps = CapsuleLayer(input_units=6*6*primary_units,\n",
    "                                        input_channels=primary_dim,\n",
    "                                        num_units=num_classes,\n",
    "                                        channels_per_unit=digits_dim,\n",
    "                                        kernel_size=0,\n",
    "                                        stride=0,\n",
    "                                        routing=True,\n",
    "                                        routing_iterations=routing_iterations)\n",
    "        self.decoder = nn.Sequential(nn.Linear(num_classes * digits_dim, dense_units_1),\n",
    "                                     nn.ReLU(),\n",
    "                                     nn.Linear(dense_units_1, dense_units_2),\n",
    "                                     nn.ReLU(),\n",
    "                                     nn.Linear(dense_units_2, dense_units_3),\n",
    "                                     nn.Sigmoid())\n",
    "        \n",
    "    \n",
    "    def forward(self, x):\n",
    "        \"\"\"\n",
    "        TODO add doc\n",
    "        \"\"\"\n",
    "        batch_size = x.shape[0]\n",
    "        \n",
    "        conv_out = self.conv0(x)\n",
    "        conv_out = F.relu(conv_out, inplace=False)\n",
    "        \n",
    "        primary_caps_out = self.primary_caps(conv_out)\n",
    "        squashed_primary_out = squash(primary_caps_out)\n",
    "        \n",
    "        digit_in = squashed_primary_out.view(batch_size, self.primary_caps.channels_per_unit, -1)  # -> (batch_size, primary_units, )\n",
    "        digit_out = self.digits_caps(digit_in)\n",
    "        \n",
    "        out = digit_out\n",
    "        while len(out.shape) > 2:\n",
    "            out = torch.norm(out, dim=-1)\n",
    "        \n",
    "        return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "#TODO\n",
    "def margin_loss(votes, targets):\n",
    "    \"\"\"\n",
    "    TODO add doc\n",
    "    \"\"\"\n",
    "    pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "capsnet = CapsNet()\n",
    "capsnet = capsnet.to(device)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Optimizer definition according to default Tensorflow initiation\n",
    "From Tensorflow [AdamOptimizer docs](https://www.tensorflow.org/api_docs/python/tf/train/AdamOptimizer):\n",
    "```\n",
    "__init__(\n",
    "    learning_rate=0.001,\n",
    "    beta1=0.9,\n",
    "    beta2=0.999,\n",
    "    epsilon=1e-08,\n",
    "    use_locking=False,\n",
    "    name='Adam'\n",
    ")```\n",
    "\n",
    "These are also the default values for torch.optim.Adam"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = torch.optim.Adam(capsnet.parameters())\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "criterion = criterion.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(model, train_loader, epochs, loss_fn, optimizer, validation_loader=None, patience=None):\n",
    "    model.train()\n",
    "    loss_history = torch.zeros(epochs)\n",
    "    acc_history = torch.zeros(epochs)\n",
    "    best_val_acc = 0\n",
    "    patience_counter = 0\n",
    "    \n",
    "    for epoch in range(epochs):\n",
    "        loss_sum = 0\n",
    "        for i, data in enumerate(train_loader):\n",
    "            print('starting batch #{:5.0f}'.format(i))\n",
    "            input, target = data\n",
    "            input, target = input.to(device), target.to(device)\n",
    "            \n",
    "            optimizer.zero_grad()\n",
    "            log_probs = model(input)\n",
    "            \n",
    "            loss = loss_fn(log_probs, target)\n",
    "            loss_sum += loss.item()\n",
    "\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "\n",
    "        loss_history[epoch] = loss_sum / len(train_loader)\n",
    "        print('Loss in epoch {}: {}'.format(epoch+1, loss_history[epoch]))\n",
    "        torch.save(model, './caps_epoch{}.pth'.format(epoch))\n",
    "        if patience:\n",
    "            acc_history[epoch] = evaluate_model(model, validation_loader, len(validation_loader) * validation_loader.batch_size)\n",
    "            if acc_history[epoch] > best_val_acc:\n",
    "                best_val_acc = acc_history[epoch]\n",
    "                patience_counter = 0\n",
    "            else:\n",
    "                patience_counter += 1\n",
    "                if patience_counter > patience:\n",
    "                    print(\"Early Stopping in epoch {}.\".format(epoch))\n",
    "                    return loss_history, acc_history\n",
    "        \n",
    "        \n",
    "    return loss_history, acc_history"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate_model(model, data_loader, num_samples):\n",
    "    hits = 0.0\n",
    "    model.eval()\n",
    "\n",
    "    for i, data in enumerate(data_loader):\n",
    "        images, targets = data\n",
    "        with torch.no_grad():\n",
    "            images = images.to(device)\n",
    "            targets = targets.to(device)\n",
    "            \n",
    "            log_probs = model(images)\n",
    "            predictions = F.softmax(log_probs, dim=-1)\n",
    "            predictions = predictions.max(dim=-1)[1]\n",
    "            hits += (predictions == targets).sum().item()\n",
    "        \n",
    "    model.train()\n",
    "    return hits/num_samples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "starting batch #    0\n",
      "starting batch #    1\n",
      "starting batch #    2\n",
      "starting batch #    3\n",
      "starting batch #    4\n",
      "starting batch #    5\n",
      "starting batch #    6\n",
      "starting batch #    7\n",
      "starting batch #    8\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-15-f3cae2d819c1>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mloss_history\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0macc_history\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtrain\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcapsnet\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtrain_loader\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mepochs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcriterion\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mval_loader\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mpatience\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m2\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m<ipython-input-13-d19b54568ba9>\u001b[0m in \u001b[0;36mtrain\u001b[1;34m(model, train_loader, epochs, loss_fn, optimizer, validation_loader, patience)\u001b[0m\n\u001b[0;32m     19\u001b[0m             \u001b[0mloss_sum\u001b[0m \u001b[1;33m+=\u001b[0m \u001b[0mloss\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mitem\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     20\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 21\u001b[1;33m             \u001b[0mloss\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     22\u001b[0m             \u001b[0moptimizer\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     23\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Programs\\Anaconda3\\envs\\torch\\lib\\site-packages\\torch\\tensor.py\u001b[0m in \u001b[0;36mbackward\u001b[1;34m(self, gradient, retain_graph, create_graph)\u001b[0m\n\u001b[0;32m     91\u001b[0m                 \u001b[0mproducts\u001b[0m\u001b[1;33m.\u001b[0m \u001b[0mDefaults\u001b[0m \u001b[0mto\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;31m`\u001b[0m\u001b[1;32mFalse\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;31m`\u001b[0m\u001b[1;33m.\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     92\u001b[0m         \"\"\"\n\u001b[1;32m---> 93\u001b[1;33m         \u001b[0mtorch\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mautograd\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mgradient\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     94\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     95\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mregister_hook\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mhook\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Programs\\Anaconda3\\envs\\torch\\lib\\site-packages\\torch\\autograd\\__init__.py\u001b[0m in \u001b[0;36mbackward\u001b[1;34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables)\u001b[0m\n\u001b[0;32m     87\u001b[0m     Variable._execution_engine.run_backward(\n\u001b[0;32m     88\u001b[0m         \u001b[0mtensors\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mgrad_tensors\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 89\u001b[1;33m         allow_unreachable=True)  # allow_unreachable flag\n\u001b[0m\u001b[0;32m     90\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     91\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "loss_history, acc_history = train(capsnet, train_loader, epochs, criterion, optimizer, val_loader, patience=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-18-f67b5ab555c3>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mevals\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mevaluate_model\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcapsnet\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtest_loader\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtest_data\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m<ipython-input-17-0620895ea6e6>\u001b[0m in \u001b[0;36mevaluate_model\u001b[1;34m(model, data_loader, num_samples)\u001b[0m\n\u001b[0;32m      9\u001b[0m             \u001b[0mtargets\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtargets\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mto\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     10\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 11\u001b[1;33m             \u001b[0mlog_probs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mimages\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     12\u001b[0m             \u001b[0mpredictions\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mF\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msoftmax\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlog_probs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdim\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m-\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     13\u001b[0m             \u001b[0mpredictions\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mpredictions\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmax\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdim\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m-\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Programs\\Anaconda3\\envs\\torch\\lib\\site-packages\\torch\\nn\\modules\\module.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, *input, **kwargs)\u001b[0m\n\u001b[0;32m    489\u001b[0m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    490\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 491\u001b[1;33m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    492\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    493\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<ipython-input-9-f0f2147408d6>\u001b[0m in \u001b[0;36mforward\u001b[1;34m(self, x)\u001b[0m\n\u001b[0;32m     50\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     51\u001b[0m         \u001b[0mdigit_in\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0msquashed_primary_out\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mview\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mbatch_size\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mprimary_caps\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mchannels_per_unit\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m-\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m)\u001b[0m  \u001b[1;31m# -> (batch_size, primary_units, )\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 52\u001b[1;33m         \u001b[0mdigit_out\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdigits_caps\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdigit_in\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     53\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     54\u001b[0m         \u001b[0mout\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mdigit_out\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Programs\\Anaconda3\\envs\\torch\\lib\\site-packages\\torch\\nn\\modules\\module.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, *input, **kwargs)\u001b[0m\n\u001b[0;32m    489\u001b[0m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    490\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 491\u001b[1;33m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    492\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    493\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<ipython-input-8-14b4388dbafb>\u001b[0m in \u001b[0;36mforward\u001b[1;34m(self, x)\u001b[0m\n\u001b[0;32m     38\u001b[0m         \"\"\"\n\u001b[0;32m     39\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrouting\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 40\u001b[1;33m             \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_routing\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     41\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     42\u001b[0m             \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_apply_conv_units\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<ipython-input-8-14b4388dbafb>\u001b[0m in \u001b[0;36m_routing\u001b[1;34m(self, inputs)\u001b[0m\n\u001b[0;32m     56\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     57\u001b[0m         \u001b[0mlogits\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mzeros\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcurrent_votes\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mrequires_grad\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mTrue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 58\u001b[1;33m         \u001b[0mlogits\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mlogits\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mto\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     59\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     60\u001b[0m         \u001b[0mpondered_votes\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mweights\u001b[0m \u001b[1;33m*\u001b[0m \u001b[0mcurrent_votes\u001b[0m  \u001b[1;31m# Uji\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "evals = evaluate_model(capsnet, test_loader, len(test_data))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "evals"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(evals*100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}