MiniProject_1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import matplotlib.pyplot as plt\n",
    "import dlc_practical_prologue as prologue\n",
    "from torch import nn\n",
    "from torch.nn import functional as F"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Generating the dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_input size = torch.Size([1000, 2, 14, 14])\n",
      "train_target size = torch.Size([1000])\n",
      "train_classes size = torch.Size([1000, 2])\n",
      "test_input size = torch.Size([1000, 2, 14, 14])\n",
      "test_target size = torch.Size([1000])\n",
      "test_classes size = torch.Size([1000, 2])\n"
     ]
    }
   ],
   "source": [
    "nbr_pairs = 1000\n",
    "train_input, train_target, train_classes, test_input, test_target, test_classes = prologue.generate_pair_sets(nbr_pairs)\n",
    "\n",
    "print('train_input size =', train_input.size())\n",
    "print('train_target size =', train_target.size()) #The boolean telling if the two pairs are the same or not \n",
    "print('train_classes size =', train_classes.size())\n",
    "print('test_input size =', test_input.size())\n",
    "print('test_target size =', test_target.size())\n",
    "print('test_classes size =', test_classes.size())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Utilities"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_model_NOaux(model, train_input, train_target, nb_epochs, batch_size, criterion, eta): \n",
    "    optimizer = torch.optim.Adam(model.parameters(), lr = eta)\n",
    "    for e in range(nb_epochs):\n",
    "        if (e % 10 == 0 and e > 0):\n",
    "            eta = eta/10\n",
    "            optimizer = torch.optim.Adam(model.parameters(), lr = eta)\n",
    "        for step_ in range(0,train_input.size(0),batch_size):                              \n",
    "            output = model(train_input[step_:step_+batch_size])\n",
    "            loss = criterion(output, train_target[step_:step_+batch_size])\n",
    "            optimizer.zero_grad()\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "\n",
    "def train_model_aux(model, train_input, train_target, nb_epochs, batch_size, criterion, eta, lambda_):   \n",
    "    optimizer = torch.optim.Adam(model.parameters(), lr = eta)\n",
    "    for e in range(nb_epochs):\n",
    "        if (e % 10 == 0 and e > 0):\n",
    "            eta = eta/10\n",
    "            optimizer = torch.optim.Adam(model.parameters(), lr = eta)\n",
    "        for step_ in range(0,train_input.size(0),batch_size):                              \n",
    "            output_target, output_im1, output_im2 = model(train_input[step_:step_+batch_size])\n",
    "            loss_target = criterion(output_target, train_target[step_:step_+batch_size])\n",
    "            loss_im1 = criterion(output_im1, train_classes[step_:step_+batch_size,0])\n",
    "            loss_im2 = criterion(output_im2, train_classes[step_:step_+batch_size,1])\n",
    "            loss = loss_target + lambda_*(loss_im1 + loss_im2)\n",
    "            optimizer.zero_grad()\n",
    "            loss.backward()\n",
    "            optimizer.step()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Number of error"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_nb_errors_NOaux(model, data_input, data_target, mini_batch_size): \n",
    "    nb_errors = 0\n",
    "    for b in range(0, data_input.size(0), mini_batch_size):\n",
    "        output = model(data_input.narrow(0, b, mini_batch_size))\n",
    "        _, predicted_classes = output.data.max(1)\n",
    "        for k in range(mini_batch_size):\n",
    "            if data_target[b + k] != predicted_classes[k]:\n",
    "                nb_errors = nb_errors + 1\n",
    "    return nb_errors\n",
    "\n",
    "def compute_nb_errors_aux(model, data_input, data_target, mini_batch_size): \n",
    "    nb_errors = 0\n",
    "    for b in range(0, data_input.size(0), mini_batch_size):\n",
    "        output,_,_ = model(data_input.narrow(0, b, mini_batch_size))\n",
    "        _, predicted_classes = output.data.max(1)\n",
    "        for k in range(mini_batch_size):\n",
    "            if data_target[b + k] != predicted_classes[k]:\n",
    "                nb_errors = nb_errors + 1\n",
    "    return nb_errors"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Architectures"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Shallow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Shallow_NOsharing_NOaux(nn.Module):\n",
    "    def __init__(self, hidden, act_fun):\n",
    "        super(Shallow_NOsharing_NOaux, self).__init__()\n",
    "        self.act_fun = act_fun\n",
    "        self.fc1_1 = nn.Linear(196, hidden)\n",
    "        self.fc1_2 = nn.Linear(196, hidden)\n",
    "        # After concatenation of the features from image 1 and image 2\n",
    "        self.fc2 = nn.Linear(hidden*2,2)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x_1 = self.act_fun(self.fc1_1(x[:,0,:,:].view(-1,196)))\n",
    "        x_2 = self.act_fun(self.fc1_2(x[:,1,:,:].view(-1,196)))\n",
    "        x = torch.cat([x_1, x_2],1)\n",
    "        x = self.fc2(x)       \n",
    "        return x\n",
    "    \n",
    "class Shallow_sharing_NOaux(nn.Module):\n",
    "    def __init__(self, hidden, act_fun):\n",
    "        super(Shallow_sharing_NOaux, self).__init__()\n",
    "        self.act_fun = act_fun\n",
    "        self.fc1 = nn.Linear(196, hidden)\n",
    "        # After concatenation of the features from image 1 and image 2\n",
    "        self.fc2 = nn.Linear(hidden*2,2)\n",
    "\n",
    "    def forward(self, x):\n",
    "        fc_image = []\n",
    "        for image in range(2):\n",
    "            x1 = self.act_fun(self.fc1(x[:,image,:,:].view(-1,196)))\n",
    "            fc_image.append(x1)\n",
    "        x = torch.cat([fc_image[0],fc_image[1]],1)\n",
    "        x = self.fc2(x)       \n",
    "        return x\n",
    "    \n",
    "class Shallow_NOsharing_aux(nn.Module):\n",
    "    def __init__(self, hidden, act_fun):\n",
    "        super(Shallow_NOsharing_aux, self).__init__()\n",
    "        self.act_fun = act_fun\n",
    "        self.fc1_1 = nn.Linear(196, hidden)\n",
    "        self.fc1_2 = nn.Linear(196, hidden)\n",
    "        \n",
    "        # For classification with classes\n",
    "        self.fc_aux1 = nn.Linear(hidden, 10)\n",
    "        self.fc_aux2 = nn.Linear(hidden, 10)\n",
    "        \n",
    "        # After concatenation of the features from image 1 and image 2\n",
    "        self.fc2 = nn.Linear(hidden*2,2)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x1 = self.act_fun(self.fc1_1(x[:,0,:,:].view(-1,196)))\n",
    "        x2 = self.act_fun(self.fc1_2(x[:,1,:,:].view(-1,196)))\n",
    "        \n",
    "        aux1 = F.softmax(self.fc_aux1(x1),1)\n",
    "        aux2 = F.softmax(self.fc_aux2(x2),1)\n",
    "        \n",
    "        x = torch.cat([x1, x2],1)\n",
    "        x = self.fc2(x)       \n",
    "        return x, aux1, aux2\n",
    "    \n",
    "class Shallow_sharing_aux(nn.Module):\n",
    "    def __init__(self, hidden, act_fun):\n",
    "        super(Shallow_sharing_aux, self).__init__()\n",
    "        self.act_fun = act_fun\n",
    "        self.fc1 = nn.Linear(196, hidden)\n",
    "        \n",
    "        # For classification with classes\n",
    "        self.fc_aux1 = nn.Linear(hidden, 10)\n",
    "        self.fc_aux2 = nn.Linear(hidden, 10)\n",
    "        \n",
    "        # After concatenation of the features from image 1 and image 2\n",
    "        self.fc2 = nn.Linear(hidden*2,2)\n",
    "\n",
    "    def forward(self, x):\n",
    "        fc_image = []\n",
    "        for image in range(2):\n",
    "            x1 = self.act_fun(self.fc1(x[:,image,:,:].view(-1,196)))\n",
    "            fc_image.append(x1)\n",
    "        \n",
    "        aux1 = F.softmax(self.fc_aux1(fc_image[0]),1)\n",
    "        aux2 = F.softmax(self.fc_aux2(fc_image[1]),1)\n",
    "        \n",
    "        x = torch.cat([fc_image[0],fc_image[1]],1)\n",
    "        x = self.fc2(x)       \n",
    "        return x, aux1, aux2      "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### MLP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MLP_NOsharing_NOaux(nn.Module):\n",
    "    def __init__(self, hidden, act_fun):\n",
    "        super(MLP_NOsharing_NOaux, self).__init__()\n",
    "        self.act_fun = act_fun\n",
    "        self.fc1_1 = nn.Linear(196, hidden)\n",
    "        self.fc1_2 = nn.Linear(196, hidden)\n",
    "        self.fc2_1 = nn.Linear(hidden,hidden)\n",
    "        self.fc2_2 = nn.Linear(hidden,hidden)\n",
    "        # After concatenation of the features from image 1 and image 2\n",
    "        self.fc3 = nn.Linear(hidden*2,2)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x1_1 = self.act_fun(self.fc1_1(x[:,0,:,:].view(-1,196)))\n",
    "        x1_2 = self.act_fun(self.fc1_2(x[:,1,:,:].view(-1,196)))\n",
    "        x2_1 = self.act_fun(self.fc2_1(x1_1))\n",
    "        x2_2 = self.act_fun(self.fc2_2(x1_2))\n",
    "        x = torch.cat([x2_1, x2_2],1)\n",
    "        x = self.fc3(x)       \n",
    "        return x\n",
    "\n",
    "class MLP_sharing_NOaux(nn.Module):\n",
    "    def __init__(self, hidden, act_fun):\n",
    "        super(MLP_sharing_NOaux, self).__init__()\n",
    "        self.act_fun = act_fun\n",
    "        self.fc1 = nn.Linear(196, hidden)\n",
    "        self.fc2 = nn.Linear(hidden,hidden)\n",
    "        # After concatenation of the features from image 1 and image 2\n",
    "        self.fc3 = nn.Linear(hidden*2,2)\n",
    "\n",
    "    def forward(self, x):\n",
    "        fc_image = []\n",
    "        for image in range(2):\n",
    "            x1 = self.act_fun(self.fc1(x[:,image,:,:].view(-1,196)))\n",
    "            x2 = self.act_fun(self.fc2(x1))\n",
    "            fc_image.append(x2)\n",
    "        x = torch.cat([fc_image[0],fc_image[1]],1)\n",
    "        x = self.fc3(x)       \n",
    "        return x\n",
    "    \n",
    "class MLP_NOsharing_aux(nn.Module):\n",
    "    def __init__(self, hidden, act_fun):\n",
    "        super(MLP_NOsharing_aux, self).__init__()\n",
    "        self.act_fun = act_fun\n",
    "        self.fc1_1 = nn.Linear(196, hidden)\n",
    "        self.fc1_2 = nn.Linear(196, hidden)\n",
    "        self.fc2_1 = nn.Linear(hidden,hidden)\n",
    "        self.fc2_2 = nn.Linear(hidden,hidden)\n",
    "        \n",
    "        # For classification with classes\n",
    "        self.fc_aux1 = nn.Linear(hidden, 10)\n",
    "        self.fc_aux2 = nn.Linear(hidden, 10)\n",
    "        \n",
    "        # After concatenation of the features from image 1 and image 2\n",
    "        self.fc3 = nn.Linear(hidden*2,2)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x1_1 = self.act_fun(self.fc1_1(x[:,0,:,:].view(-1,196)))\n",
    "        x1_2 = self.act_fun(self.fc1_2(x[:,1,:,:].view(-1,196)))\n",
    "        x2_1 = self.act_fun(self.fc2_1(x1_1))\n",
    "        x2_2 = self.act_fun(self.fc2_2(x1_2))\n",
    "        \n",
    "        aux1 = F.softmax(self.fc_aux1(x2_1),1)\n",
    "        aux2 = F.softmax(self.fc_aux2(x2_2),1)\n",
    "        \n",
    "        x = torch.cat([x2_1, x2_2],1)\n",
    "        x = self.fc3(x)       \n",
    "        return x, aux1, aux2\n",
    "    \n",
    "class MLP_sharing_aux(nn.Module):\n",
    "    def __init__(self, hidden, act_fun):\n",
    "        super(MLP_sharing_aux, self).__init__()\n",
    "        self.act_fun = act_fun\n",
    "        self.fc1 = nn.Linear(196, hidden)\n",
    "        self.fc2 = nn.Linear(hidden,hidden)\n",
    "        \n",
    "        # For classification with classes\n",
    "        self.fc_aux1 = nn.Linear(hidden, 10)\n",
    "        self.fc_aux2 = nn.Linear(hidden, 10)\n",
    "        \n",
    "        # After concatenation of the features from image 1 and image 2\n",
    "        self.fc3 = nn.Linear(hidden*2,2)\n",
    "\n",
    "    def forward(self, x):\n",
    "        fc_image = []\n",
    "        for image in range(2):\n",
    "            x1 = self.act_fun(self.fc1(x[:,image,:,:].view(-1,196)))\n",
    "            x2 = self.act_fun(self.fc2(x1))\n",
    "            fc_image.append(x2)\n",
    "            \n",
    "        aux1 = F.softmax(self.fc_aux1(fc_image[0]),1)\n",
    "        aux2 = F.softmax(self.fc_aux2(fc_image[1]),1)\n",
    "        \n",
    "        x = torch.cat([fc_image[0],fc_image[1]],1)\n",
    "        x = self.fc3(x)       \n",
    "        return x, aux1, aux2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Deep 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Deep_NOsharing_NOaux(nn.Module):\n",
    "    def __init__(self, hidden, act_fun):\n",
    "        super(Deep_NOsharing_NOaux, self).__init__()\n",
    "        self.act_fun = act_fun\n",
    "        self.conv1_1 = nn.Conv2d(1, 32, kernel_size=3)\n",
    "        self.conv1_2 = nn.Conv2d(1, 32, kernel_size=3)\n",
    "        self.conv2_1 = nn.Conv2d(32, 64, kernel_size=3)\n",
    "        self.conv2_2 = nn.Conv2d(32, 64, kernel_size=3)\n",
    "        \n",
    "        # After concatenation of the features from image 1 and image 2\n",
    "        self.fc1 = nn.Linear(512, hidden)\n",
    "        self.fc2 = nn.Linear(hidden,2)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x1_1 = self.act_fun(F.max_pool2d(self.conv1_1(x[:,0,:,:].view(100,1,14,14)), kernel_size=2, stride=2))\n",
    "        x2_1 = self.act_fun(F.max_pool2d(self.conv2_1(x1_1), kernel_size=2, stride=2))\n",
    "        \n",
    "        x1_2 = self.act_fun(F.max_pool2d(self.conv1_2(x[:,1,:,:].view(100,1,14,14)), kernel_size=2, stride=2))\n",
    "        x2_2 = self.act_fun(F.max_pool2d(self.conv2_2(x1_2), kernel_size=2, stride=2))\n",
    "        \n",
    "        x = torch.cat([x2_1, x2_2],1)\n",
    "        x = self.act_fun(self.fc1(x.view(-1, 512)))\n",
    "        x = self.fc2(x)\n",
    "        return x\n",
    "\n",
    "class Deep_sharing_NOaux(nn.Module):\n",
    "    def __init__(self, hidden, act_fun):\n",
    "        super(Deep_sharing_NOaux, self).__init__()\n",
    "        self.act_fun = act_fun\n",
    "        self.conv1 = nn.Conv2d(1, 32, kernel_size=3)\n",
    "        self.conv2 = nn.Conv2d(32, 64, kernel_size=3)\n",
    "        \n",
    "        # After concatenation of the features from image 1 and image 2\n",
    "        self.fc1 = nn.Linear(512, hidden)\n",
    "        self.fc2 = nn.Linear(hidden,2)\n",
    "\n",
    "    def forward(self, x):\n",
    "        conv_images = []\n",
    "        for image in range(2):\n",
    "            first_conv = self.act_fun(F.max_pool2d(self.conv1(x[:,image,:,:].view(100,1,14,14)), kernel_size=2, stride=2))\n",
    "            conv_images.append(self.act_fun(F.max_pool2d(self.conv2(first_conv), kernel_size=2, stride=2)))\n",
    "        \n",
    "        x = torch.cat([conv_images[0], conv_images[1]],1)\n",
    "        x = self.act_fun(self.fc1(x.view(-1, 512)))\n",
    "        x = self.fc2(x)\n",
    "        return x\n",
    "    \n",
    "class Deep_NOsharing_aux(nn.Module):\n",
    "    def __init__(self, hidden, act_fun):\n",
    "        super(Deep_NOsharing_aux, self).__init__()\n",
    "        self.act_fun = act_fun\n",
    "        self.conv1_1 = nn.Conv2d(1, 32, kernel_size=3)\n",
    "        self.conv1_2 = nn.Conv2d(1, 32, kernel_size=3)\n",
    "        self.conv2_1 = nn.Conv2d(32, 64, kernel_size=3)\n",
    "        self.conv2_2 = nn.Conv2d(32, 64, kernel_size=3)\n",
    "        \n",
    "        # For classification with classes\n",
    "        self.fc_aux1 = nn.Linear(256, 10)\n",
    "        self.fc_aux2 = nn.Linear(256, 10)\n",
    "        \n",
    "        # After concatenation of the features from image 1 and image 2\n",
    "        self.fc1 = nn.Linear(512, hidden)\n",
    "        self.fc2 = nn.Linear(hidden,2)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x1_1 = self.act_fun(F.max_pool2d(self.conv1_1(x[:,0,:,:].view(100,1,14,14)), kernel_size=2, stride=2))\n",
    "        x2_1 = self.act_fun(F.max_pool2d(self.conv2_1(x1_1), kernel_size=2, stride=2))\n",
    "        \n",
    "        x1_2 = self.act_fun(F.max_pool2d(self.conv1_2(x[:,1,:,:].view(100,1,14,14)), kernel_size=2, stride=2))\n",
    "        x2_2 = self.act_fun(F.max_pool2d(self.conv2_2(x1_2), kernel_size=2, stride=2))\n",
    "\n",
    "        aux1 = F.softmax(self.fc_aux1(x2_1.view(-1,256)),1)\n",
    "        aux2 = F.softmax(self.fc_aux2(x2_2.view(-1,256)),1)\n",
    "        \n",
    "        x = torch.cat([x2_1, x2_2],1)\n",
    "        x = self.act_fun(self.fc1(x.view(-1, 512)))\n",
    "        x = self.fc2(x)\n",
    "        return x, aux1, aux2\n",
    "    \n",
    "class Deep_sharing_aux(nn.Module):\n",
    "    def __init__(self, hidden, act_fun):\n",
    "        super(Deep_sharing_aux, self).__init__()\n",
    "        self.act_fun = act_fun\n",
    "        self.conv1 = nn.Conv2d(1, 32, kernel_size=3)\n",
    "        self.conv2 = nn.Conv2d(32, 64, kernel_size=3)\n",
    "        \n",
    "        # For classification with classes\n",
    "        self.fc_aux1 = nn.Linear(256, 10)\n",
    "        self.fc_aux2 = nn.Linear(256, 10)\n",
    "        \n",
    "        # After concatenation of the features from image 1 and image 2\n",
    "        self.fc1 = nn.Linear(512, hidden)\n",
    "        self.fc2 = nn.Linear(hidden,2)\n",
    "\n",
    "    def forward(self, x):\n",
    "        conv_images = []\n",
    "        for image in range(2):\n",
    "            first_conv = self.act_fun(F.max_pool2d(self.conv1(x[:,image,:,:].view(100,1,14,14)), kernel_size=2, stride=2))\n",
    "            conv_images.append(self.act_fun(F.max_pool2d(self.conv2(first_conv), kernel_size=2, stride=2)))\n",
    "            \n",
    "        aux1 = F.softmax(self.fc_aux1(conv_images[0].view(-1,256)),1)\n",
    "        aux2 = F.softmax(self.fc_aux2(conv_images[1].view(-1,256)),1)\n",
    "        \n",
    "        x = torch.cat([conv_images[0], conv_images[1]],1)\n",
    "        x = self.act_fun(self.fc1(x.view(-1, 512)))\n",
    "        x = self.fc2(x)\n",
    "        return x, aux1, aux2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Deep 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Deep_NOsharing_NOaux2(nn.Module):\n",
    "    def __init__(self, hidden, act_fun):\n",
    "        super(Deep_NOsharing_NOaux2, self).__init__()\n",
    "        self.act_fun = act_fun\n",
    "        self.conv1_1 = nn.Conv2d(1, 16, kernel_size=5)\n",
    "        self.conv1_2 = nn.Conv2d(1, 16, kernel_size=5)\n",
    "        self.conv2_1 = nn.Conv2d(16, 32, kernel_size=3)\n",
    "        self.conv2_2 = nn.Conv2d(16, 32, kernel_size=3)\n",
    "        self.conv3_1 = nn.Conv2d(32, 64, kernel_size=3)\n",
    "        self.conv3_2 = nn.Conv2d(32, 64, kernel_size=3)\n",
    "        \n",
    "        self.conv4_1 = nn.Conv2d(64, 128, kernel_size=2)\n",
    "        self.conv4_2 = nn.Conv2d(64, 128, kernel_size=2)\n",
    "        \n",
    "        # After concatenation of the features from image 1 and image 2\n",
    "        self.fc1 = nn.Linear(1024, hidden)\n",
    "        self.fc2 = nn.Linear(hidden,2)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x1_1 = self.act_fun(self.conv1_1(x[:,0,:,:].view(100,1,14,14)))\n",
    "        x2_1 = self.act_fun(self.conv2_1(x1_1))\n",
    "        x3_1 = self.act_fun(self.conv3_2(x2_1))\n",
    "        x4_1 = self.act_fun(F.max_pool2d(self.conv4_1(x3_1), kernel_size=2, stride=2))\n",
    "        \n",
    "        x1_2 = self.act_fun(self.conv1_2(x[:,1,:,:].view(100,1,14,14)))\n",
    "        x2_2 = self.act_fun(self.conv2_2(x1_2))\n",
    "        x3_2 = self.act_fun(self.conv3_2(x2_2))\n",
    "        x4_2 = self.act_fun(F.max_pool2d(self.conv4_2(x3_2), kernel_size=2, stride=2))\n",
    "\n",
    "        x = torch.cat([x4_1, x4_2],1)\n",
    "        x = self.act_fun(self.fc1(x.view(-1, 1024)))\n",
    "        x = self.fc2(x)\n",
    "        return x\n",
    "\n",
    "class Deep_sharing_NOaux2(nn.Module):\n",
    "    def __init__(self, hidden, act_fun):\n",
    "        super(Deep_sharing_NOaux2, self).__init__()\n",
    "        self.act_fun = act_fun\n",
    "        self.conv1 = nn.Conv2d(1, 16, kernel_size=5)\n",
    "        self.conv2 = nn.Conv2d(16, 32, kernel_size=3)\n",
    "        self.conv3 = nn.Conv2d(32, 64, kernel_size=3)\n",
    "        \n",
    "        self.conv4 = nn.Conv2d(64, 128, kernel_size=2)\n",
    "        \n",
    "        # After concatenation of the features from image 1 and image 2\n",
    "        self.fc1 = nn.Linear(1024, hidden)\n",
    "        self.fc2 = nn.Linear(hidden,2)\n",
    "\n",
    "    def forward(self, x):\n",
    "        conv_images = []\n",
    "        for image in range(2):\n",
    "            x1 = self.act_fun(self.conv1(x[:,image,:,:].view(100,1,14,14)))\n",
    "            x2 = self.act_fun(self.conv2(x1))\n",
    "            x3 = self.act_fun(self.conv3(x2))\n",
    "            x4 = self.act_fun(F.max_pool2d(self.conv4(x3), kernel_size=2, stride=2))\n",
    "            conv_images.append(x4)\n",
    "        \n",
    "        x = torch.cat([conv_images[0], conv_images[1]],1)\n",
    "        x = self.act_fun(self.fc1(x.view(-1, 1024)))\n",
    "        x = self.fc2(x)\n",
    "        return x\n",
    "    \n",
    "class Deep_NOsharing_aux2(nn.Module):\n",
    "    def __init__(self, hidden, act_fun):\n",
    "        super(Deep_NOsharing_aux2, self).__init__()\n",
    "        self.act_fun = act_fun\n",
    "        self.conv1_1 = nn.Conv2d(1, 16, kernel_size=5)\n",
    "        self.conv1_2 = nn.Conv2d(1, 16, kernel_size=5)\n",
    "        self.conv2_1 = nn.Conv2d(16, 32, kernel_size=3)\n",
    "        self.conv2_2 = nn.Conv2d(16, 32, kernel_size=3)\n",
    "        self.conv3_1 = nn.Conv2d(32, 64, kernel_size=3)\n",
    "        self.conv3_2 = nn.Conv2d(32, 64, kernel_size=3)\n",
    "        \n",
    "        self.conv4_1 = nn.Conv2d(64, 128, kernel_size=2)\n",
    "        self.conv4_2 = nn.Conv2d(64, 128, kernel_size=2)\n",
    "        \n",
    "        # For classification with classes\n",
    "        self.fc_aux1 = nn.Linear(512, 10)\n",
    "        self.fc_aux2 = nn.Linear(512, 10)\n",
    "        \n",
    "        # After concatenation of the features from image 1 and image 2\n",
    "        self.fc1 = nn.Linear(1024, hidden)\n",
    "        self.fc2 = nn.Linear(hidden,2)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x1_1 = self.act_fun(self.conv1_1(x[:,0,:,:].view(100,1,14,14)))\n",
    "        x2_1 = self.act_fun(self.conv2_1(x1_1))\n",
    "        x3_1 = self.act_fun(self.conv3_2(x2_1))\n",
    "        x4_1 = self.act_fun(F.max_pool2d(self.conv4_1(x3_1), kernel_size=2, stride=2))\n",
    "        \n",
    "        x1_2 = self.act_fun(self.conv1_2(x[:,1,:,:].view(100,1,14,14)))\n",
    "        x2_2 = self.act_fun(self.conv2_2(x1_2))\n",
    "        x3_2 = self.act_fun(self.conv3_2(x2_2))\n",
    "        x4_2 = self.act_fun(F.max_pool2d(self.conv4_2(x3_2), kernel_size=2, stride=2))\n",
    "\n",
    "        aux1 = F.softmax(self.fc_aux1(x4_1.view(-1,512)),1)\n",
    "        aux2 = F.softmax(self.fc_aux2(x4_2.view(-1,512)),1)\n",
    "        \n",
    "        x = torch.cat([x4_1, x4_2],1)\n",
    "        x = self.act_fun(self.fc1(x.view(-1, 1024)))\n",
    "        x = self.fc2(x)\n",
    "        return x, aux1, aux2\n",
    "    \n",
    "class Deep_sharing_aux2(nn.Module):\n",
    "    def __init__(self, hidden, act_fun):\n",
    "        super(Deep_sharing_aux2, self).__init__()\n",
    "        self.act_fun = act_fun\n",
    "        self.conv1 = nn.Conv2d(1, 16, kernel_size=5)\n",
    "        self.conv2 = nn.Conv2d(16, 32, kernel_size=3)\n",
    "        self.conv3 = nn.Conv2d(32, 64, kernel_size=3)\n",
    "        \n",
    "        self.conv4 = nn.Conv2d(64, 128, kernel_size=2)\n",
    "        \n",
    "        # For classification with classes\n",
    "        self.fc_aux1 = nn.Linear(512, 10)\n",
    "        self.fc_aux2 = nn.Linear(512, 10)\n",
    "        \n",
    "        # After concatenation of the features from image 1 and image 2\n",
    "        self.fc1 = nn.Linear(1024, hidden)\n",
    "        self.fc2 = nn.Linear(hidden,2)\n",
    "\n",
    "    def forward(self, x):\n",
    "        conv_images = []\n",
    "        for image in range(2):\n",
    "            x1 = self.act_fun(self.conv1(x[:,image,:,:].view(100,1,14,14)))\n",
    "            x2 = self.act_fun(self.conv2(x1))\n",
    "            x3 = self.act_fun(self.conv3(x2))\n",
    "            x4 = self.act_fun(F.max_pool2d(self.conv4(x3), kernel_size=2, stride=2))\n",
    "            conv_images.append(x4)\n",
    "            \n",
    "        aux1 = F.softmax(self.fc_aux1(conv_images[0].view(-1,512)),1)\n",
    "        aux2 = F.softmax(self.fc_aux2(conv_images[1].view(-1,512)),1)\n",
    "        \n",
    "        x = torch.cat([conv_images[0], conv_images[1]],1)\n",
    "        x = self.act_fun(self.fc1(x.view(-1, 1024)))\n",
    "        x = self.fc2(x)\n",
    "        return x, aux1, aux2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Models generation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def model_training(input_, target, hidden_units, eta, lambda_, model_type = 'Shallow', sub_model = 'NOsharing_NOaux', \n",
    "                   nb_epochs = 25, mini_batch_size = 100, criterion = nn.CrossEntropyLoss()):\n",
    "    if(model_type == 'Shallow'):\n",
    "        if(sub_model == 'NOsharing_NOaux'):\n",
    "            model = Shallow_NOsharing_NOaux(hidden = hidden_units, act_fun = F.relu)\n",
    "        elif(sub_model == 'sharing_NOaux'):\n",
    "            model = Shallow_sharing_NOaux(hidden = hidden_units, act_fun = F.relu)\n",
    "        elif(sub_model == 'NOsharing_aux'):\n",
    "            model = Shallow_NOsharing_aux(hidden = hidden_units, act_fun = F.relu)\n",
    "        elif(sub_model == 'sharing_aux'):\n",
    "            model = Shallow_sharing_aux(hidden = hidden_units, act_fun = F.relu)\n",
    "    elif(model_type == 'MLP'):\n",
    "        if(sub_model == 'NOsharing_NOaux'):\n",
    "            model = MLP_NOsharing_NOaux(hidden = hidden_units, act_fun = F.relu)\n",
    "        elif(sub_model == 'sharing_NOaux'):\n",
    "            model = MLP_sharing_NOaux(hidden = hidden_units, act_fun = F.relu)\n",
    "        elif(sub_model == 'NOsharing_aux'):\n",
    "            model = MLP_NOsharing_aux(hidden = hidden_units, act_fun = F.relu)\n",
    "        elif(sub_model == 'sharing_aux'):\n",
    "            model = MLP_sharing_aux(hidden = hidden_units, act_fun = F.relu)\n",
    "    elif(model_type == 'Deep1'):\n",
    "        if(sub_model == 'NOsharing_NOaux'):\n",
    "            model = Deep_NOsharing_NOaux(hidden = hidden_units, act_fun = F.relu)\n",
    "        elif(sub_model == 'sharing_NOaux'):\n",
    "            model = Deep_sharing_NOaux(hidden = hidden_units, act_fun = F.relu)\n",
    "        elif(sub_model == 'NOsharing_aux'):\n",
    "            model = Deep_NOsharing_aux(hidden = hidden_units, act_fun = F.relu)\n",
    "        elif(sub_model == 'sharing_aux'):\n",
    "            model = Deep_sharing_aux(hidden = hidden_units, act_fun = F.relu)\n",
    "    elif(model_type == 'Deep2'):\n",
    "        if(sub_model == 'NOsharing_NOaux'):\n",
    "            model = Deep_NOsharing_NOaux2(hidden = hidden_units, act_fun = F.relu)\n",
    "        elif(sub_model == 'sharing_NOaux'):\n",
    "            model = Deep_sharing_NOaux2(hidden = hidden_units, act_fun = F.relu)\n",
    "        elif(sub_model == 'NOsharing_aux'):\n",
    "            model = Deep_NOsharing_aux2(hidden = hidden_units, act_fun = F.relu)\n",
    "        elif(sub_model == 'sharing_aux'):\n",
    "            model = Deep_sharing_aux2(hidden = hidden_units, act_fun = F.relu)\n",
    "                \n",
    "    if(sub_model == 'NOsharing_aux' or sub_model == 'sharing_aux'): \n",
    "        train_model_aux(model, input_[:700], target[:700], nb_epochs, mini_batch_size, criterion, eta, lambda_)\n",
    "        accuracy = 1 - compute_nb_errors_aux(model, input_[700:], target[700:], mini_batch_size)/len(target[700:])\n",
    "    else: \n",
    "        train_model_NOaux(model, input_[:700], target[:700], nb_epochs, mini_batch_size, criterion, eta)\n",
    "        accuracy = 1 - compute_nb_errors_NOaux(model, input_[700:], target[700:], mini_batch_size)/len(target[700:])\n",
    "    \n",
    "    return accuracy, model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_dict():\n",
    "    results = {'Shallow':{'NOsharing_NOaux':{}, 'sharing_NOaux':{}, 'NOsharing_aux':{}, 'sharing_aux':{}},\n",
    "               'MLP':{'NOsharing_NOaux':{}, 'sharing_NOaux':{}, 'NOsharing_aux':{}, 'sharing_aux':{}},  \n",
    "               'Deep1':{'NOsharing_NOaux':{}, 'sharing_NOaux':{}, 'NOsharing_aux':{}, 'sharing_aux':{}},\n",
    "               'Deep2':{'NOsharing_NOaux':{}, 'sharing_NOaux':{}, 'NOsharing_aux':{}, 'sharing_aux':{}}  \n",
    "              }\n",
    "    return results\n",
    "\n",
    "def fill_results(results, type_model, sub_model, acc, eta, hidden, lambda_):\n",
    "    results[type_model][sub_model]['Acc'] = acc\n",
    "    results[type_model][sub_model]['eta'] = eta\n",
    "    results[type_model][sub_model]['hidden'] = hidden\n",
    "    if(sub_model == 'NOsharing_aux' or sub_model == 'sharing_aux'): \n",
    "        results[type_model][sub_model]['lambda'] = lambda_\n",
    "    return results\n",
    "\n",
    "def grid_search_(lambdas, etas, hidden_units, train_input, train_target, test_input, test_target):\n",
    "    type_models = ['Shallow', 'MLP', 'Deep1', 'Deep2']\n",
    "    sub_models = ['NOsharing_NOaux', 'sharing_NOaux', 'NOsharing_aux', 'sharing_aux']\n",
    "    acc_test = torch.zeros(len(type_models),len(sub_models))\n",
    "    results = create_dict()\n",
    "    \n",
    "    i = 0\n",
    "\n",
    "    for t, type_model in enumerate(type_models):\n",
    "        for s, sub_model in enumerate(sub_models):\n",
    "            i += 1\n",
    "            print('Getting hyper-parameters for architecture', i, '/ 16...')\n",
    "            performances = torch.zeros(len(lambdas),len(hidden_units),len(etas))\n",
    "            for l, lambda_ in enumerate(lambdas):\n",
    "                for h, hidden in enumerate(hidden_units):\n",
    "                    for e, eta in enumerate(etas):\n",
    "                        acc, _ = model_training(train_input, train_target, hidden.item(), eta.item(), \n",
    "                                              lambda_.item(), model_type = type_model, \n",
    "                                              sub_model = sub_model)\n",
    "                        performances[l,h,e] = acc\n",
    "            best_performance = torch.max(performances)\n",
    "            best_idx = (performances == best_performance).nonzero();\n",
    "            \n",
    "            best_eta = etas[best_idx[0,2]].item()\n",
    "            best_hidden = hidden_units[best_idx[0,1]].item()\n",
    "            best_lambda = lambdas[best_idx[0,0]].item()\n",
    "                \n",
    "            results = fill_results(results, type_model, sub_model, best_performance.item(), \n",
    "                                       best_eta, best_hidden, best_lambda)\n",
    "    return results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Getting hyper-parameters for architecture 1 / 16...\n",
      "Getting hyper-parameters for architecture 2 / 16...\n",
      "Getting hyper-parameters for architecture 3 / 16...\n",
      "Getting hyper-parameters for architecture 4 / 16...\n",
      "Getting hyper-parameters for architecture 5 / 16...\n",
      "Getting hyper-parameters for architecture 6 / 16...\n",
      "Getting hyper-parameters for architecture 7 / 16...\n",
      "Getting hyper-parameters for architecture 8 / 16...\n",
      "Getting hyper-parameters for architecture 9 / 16...\n",
      "Getting hyper-parameters for architecture 10 / 16...\n",
      "Getting hyper-parameters for architecture 11 / 16...\n",
      "Getting hyper-parameters for architecture 12 / 16...\n",
      "Getting hyper-parameters for architecture 13 / 16...\n",
      "Getting hyper-parameters for architecture 14 / 16...\n",
      "Getting hyper-parameters for architecture 15 / 16...\n",
      "Getting hyper-parameters for architecture 16 / 16...\n"
     ]
    }
   ],
   "source": [
    "lambdas = torch.tensor([0.25, 0.5, 0.75, 1])\n",
    "etas = torch.tensor([0.1, 0.01, 0.001])\n",
    "hidden_units = torch.tensor([50, 100, 200, 300])\n",
    "HP = grid_search_(lambdas, etas, hidden_units, train_input, train_target, test_input, test_target)\n",
    "\n",
    "f = open(\"HP.txt\",\"w\")\n",
    "f.write( str(HP) )\n",
    "f.close()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration 1 / 30...\n",
      "    Architecture 1 / 16...\n",
      "    Architecture 2 / 16...\n",
      "    Architecture 3 / 16...\n",
      "    Architecture 4 / 16...\n",
      "    Architecture 5 / 16...\n",
      "    Architecture 6 / 16...\n",
      "    Architecture 7 / 16...\n",
      "    Architecture 8 / 16...\n",
      "    Architecture 9 / 16...\n",
      "    Architecture 10 / 16...\n",
      "    Architecture 11 / 16...\n",
      "    Architecture 12 / 16...\n",
      "    Architecture 13 / 16...\n",
      "    Architecture 14 / 16...\n",
      "    Architecture 15 / 16...\n",
      "    Architecture 16 / 16...\n",
      "Iteration 2 / 30...\n",
      "    Architecture 1 / 16...\n",
      "    Architecture 2 / 16...\n",
      "    Architecture 3 / 16...\n",
      "    Architecture 4 / 16...\n",
      "    Architecture 5 / 16...\n",
      "    Architecture 6 / 16...\n",
      "    Architecture 7 / 16...\n",
      "    Architecture 8 / 16...\n",
      "    Architecture 9 / 16...\n",
      "    Architecture 10 / 16...\n",
      "    Architecture 11 / 16...\n",
      "    Architecture 12 / 16...\n",
      "    Architecture 13 / 16...\n",
      "    Architecture 14 / 16...\n",
      "    Architecture 15 / 16...\n",
      "    Architecture 16 / 16...\n",
      "Iteration 3 / 30...\n",
      "    Architecture 1 / 16...\n",
      "    Architecture 2 / 16...\n",
      "    Architecture 3 / 16...\n",
      "    Architecture 4 / 16...\n",
      "    Architecture 5 / 16...\n",
      "    Architecture 6 / 16...\n",
      "    Architecture 7 / 16...\n",
      "    Architecture 8 / 16...\n",
      "    Architecture 9 / 16...\n",
      "    Architecture 10 / 16...\n",
      "    Architecture 11 / 16...\n",
      "    Architecture 12 / 16...\n",
      "    Architecture 13 / 16...\n",
      "    Architecture 14 / 16...\n",
      "    Architecture 15 / 16...\n",
      "    Architecture 16 / 16...\n",
      "Iteration 4 / 30...\n",
      "    Architecture 1 / 16...\n",
      "    Architecture 2 / 16...\n",
      "    Architecture 3 / 16...\n",
      "    Architecture 4 / 16...\n",
      "    Architecture 5 / 16...\n",
      "    Architecture 6 / 16...\n",
      "    Architecture 7 / 16...\n",
      "    Architecture 8 / 16...\n",
      "    Architecture 9 / 16...\n",
      "    Architecture 10 / 16...\n",
      "    Architecture 11 / 16...\n",
      "    Architecture 12 / 16...\n",
      "    Architecture 13 / 16...\n",
      "    Architecture 14 / 16...\n",
      "    Architecture 15 / 16...\n",
      "    Architecture 16 / 16...\n",
      "Iteration 5 / 30...\n",
      "    Architecture 1 / 16...\n",
      "    Architecture 2 / 16...\n",
      "    Architecture 3 / 16...\n",
      "    Architecture 4 / 16...\n",
      "    Architecture 5 / 16...\n",
      "    Architecture 6 / 16...\n",
      "    Architecture 7 / 16...\n",
      "    Architecture 8 / 16...\n",
      "    Architecture 9 / 16...\n",
      "    Architecture 10 / 16...\n",
      "    Architecture 11 / 16...\n",
      "    Architecture 12 / 16...\n",
      "    Architecture 13 / 16...\n",
      "    Architecture 14 / 16...\n",
      "    Architecture 15 / 16...\n",
      "    Architecture 16 / 16...\n",
      "Iteration 6 / 30...\n",
      "    Architecture 1 / 16...\n",
      "    Architecture 2 / 16...\n",
      "    Architecture 3 / 16...\n",
      "    Architecture 4 / 16...\n",
      "    Architecture 5 / 16...\n",
      "    Architecture 6 / 16...\n",
      "    Architecture 7 / 16...\n",
      "    Architecture 8 / 16...\n",
      "    Architecture 9 / 16...\n",
      "    Architecture 10 / 16...\n",
      "    Architecture 11 / 16...\n",
      "    Architecture 12 / 16...\n",
      "    Architecture 13 / 16...\n",
      "    Architecture 14 / 16...\n",
      "    Architecture 15 / 16...\n",
      "    Architecture 16 / 16...\n",
      "Iteration 7 / 30...\n",
      "    Architecture 1 / 16...\n",
      "    Architecture 2 / 16...\n",
      "    Architecture 3 / 16...\n",
      "    Architecture 4 / 16...\n",
      "    Architecture 5 / 16...\n",
      "    Architecture 6 / 16...\n",
      "    Architecture 7 / 16...\n",
      "    Architecture 8 / 16...\n",
      "    Architecture 9 / 16...\n",
      "    Architecture 10 / 16...\n",
      "    Architecture 11 / 16...\n",
      "    Architecture 12 / 16...\n",
      "    Architecture 13 / 16...\n",
      "    Architecture 14 / 16...\n",
      "    Architecture 15 / 16...\n",
      "    Architecture 16 / 16...\n",
      "Iteration 8 / 30...\n",
      "    Architecture 1 / 16...\n",
      "    Architecture 2 / 16...\n",
      "    Architecture 3 / 16...\n",
      "    Architecture 4 / 16...\n",
      "    Architecture 5 / 16...\n",
      "    Architecture 6 / 16...\n",
      "    Architecture 7 / 16...\n",
      "    Architecture 8 / 16...\n",
      "    Architecture 9 / 16...\n",
      "    Architecture 10 / 16...\n",
      "    Architecture 11 / 16...\n",
      "    Architecture 12 / 16...\n",
      "    Architecture 13 / 16...\n",
      "    Architecture 14 / 16...\n",
      "    Architecture 15 / 16...\n",
      "    Architecture 16 / 16...\n",
      "Iteration 9 / 30...\n",
      "    Architecture 1 / 16...\n",
      "    Architecture 2 / 16...\n",
      "    Architecture 3 / 16...\n",
      "    Architecture 4 / 16...\n",
      "    Architecture 5 / 16...\n",
      "    Architecture 6 / 16...\n",
      "    Architecture 7 / 16...\n",
      "    Architecture 8 / 16...\n",
      "    Architecture 9 / 16...\n",
      "    Architecture 10 / 16...\n",
      "    Architecture 11 / 16...\n",
      "    Architecture 12 / 16...\n",
      "    Architecture 13 / 16...\n",
      "    Architecture 14 / 16...\n",
      "    Architecture 15 / 16...\n",
      "    Architecture 16 / 16...\n",
      "Iteration 10 / 30...\n",
      "    Architecture 1 / 16...\n",
      "    Architecture 2 / 16...\n",
      "    Architecture 3 / 16...\n",
      "    Architecture 4 / 16...\n",
      "    Architecture 5 / 16...\n",
      "    Architecture 6 / 16...\n",
      "    Architecture 7 / 16...\n",
      "    Architecture 8 / 16...\n",
      "    Architecture 9 / 16...\n",
      "    Architecture 10 / 16...\n",
      "    Architecture 11 / 16...\n",
      "    Architecture 12 / 16...\n",
      "    Architecture 13 / 16...\n",
      "    Architecture 14 / 16...\n",
      "    Architecture 15 / 16...\n",
      "    Architecture 16 / 16...\n",
      "Iteration 11 / 30...\n",
      "    Architecture 1 / 16...\n",
      "    Architecture 2 / 16...\n",
      "    Architecture 3 / 16...\n",
      "    Architecture 4 / 16...\n",
      "    Architecture 5 / 16...\n",
      "    Architecture 6 / 16...\n",
      "    Architecture 7 / 16...\n",
      "    Architecture 8 / 16...\n",
      "    Architecture 9 / 16...\n",
      "    Architecture 10 / 16...\n",
      "    Architecture 11 / 16...\n",
      "    Architecture 12 / 16...\n",
      "    Architecture 13 / 16...\n",
      "    Architecture 14 / 16...\n",
      "    Architecture 15 / 16...\n",
      "    Architecture 16 / 16...\n",
      "Iteration 12 / 30...\n",
      "    Architecture 1 / 16...\n",
      "    Architecture 2 / 16...\n",
      "    Architecture 3 / 16...\n",
      "    Architecture 4 / 16...\n",
      "    Architecture 5 / 16...\n",
      "    Architecture 6 / 16...\n",
      "    Architecture 7 / 16...\n",
      "    Architecture 8 / 16...\n",
      "    Architecture 9 / 16...\n",
      "    Architecture 10 / 16...\n",
      "    Architecture 11 / 16...\n",
      "    Architecture 12 / 16...\n",
      "    Architecture 13 / 16...\n",
      "    Architecture 14 / 16...\n",
      "    Architecture 15 / 16...\n",
      "    Architecture 16 / 16...\n",
      "Iteration 13 / 30...\n",
      "    Architecture 1 / 16...\n",
      "    Architecture 2 / 16...\n",
      "    Architecture 3 / 16...\n",
      "    Architecture 4 / 16...\n",
      "    Architecture 5 / 16...\n",
      "    Architecture 6 / 16...\n",
      "    Architecture 7 / 16...\n",
      "    Architecture 8 / 16...\n",
      "    Architecture 9 / 16...\n",
      "    Architecture 10 / 16...\n",
      "    Architecture 11 / 16...\n",
      "    Architecture 12 / 16...\n",
      "    Architecture 13 / 16...\n",
      "    Architecture 14 / 16...\n",
      "    Architecture 15 / 16...\n",
      "    Architecture 16 / 16...\n",
      "Iteration 14 / 30...\n",
      "    Architecture 1 / 16...\n",
      "    Architecture 2 / 16...\n",
      "    Architecture 3 / 16...\n",
      "    Architecture 4 / 16...\n",
      "    Architecture 5 / 16...\n",
      "    Architecture 6 / 16...\n",
      "    Architecture 7 / 16...\n",
      "    Architecture 8 / 16...\n",
      "    Architecture 9 / 16...\n",
      "    Architecture 10 / 16...\n",
      "    Architecture 11 / 16...\n",
      "    Architecture 12 / 16...\n",
      "    Architecture 13 / 16...\n",
      "    Architecture 14 / 16...\n",
      "    Architecture 15 / 16...\n",
      "    Architecture 16 / 16...\n",
      "Iteration 15 / 30...\n",
      "    Architecture 1 / 16...\n",
      "    Architecture 2 / 16...\n",
      "    Architecture 3 / 16...\n",
      "    Architecture 4 / 16...\n",
      "    Architecture 5 / 16...\n",
      "    Architecture 6 / 16...\n",
      "    Architecture 7 / 16...\n",
      "    Architecture 8 / 16...\n",
      "    Architecture 9 / 16...\n",
      "    Architecture 10 / 16...\n",
      "    Architecture 11 / 16...\n",
      "    Architecture 12 / 16...\n",
      "    Architecture 13 / 16...\n",
      "    Architecture 14 / 16...\n",
      "    Architecture 15 / 16...\n",
      "    Architecture 16 / 16...\n",
      "Iteration 16 / 30...\n",
      "    Architecture 1 / 16...\n",
      "    Architecture 2 / 16...\n",
      "    Architecture 3 / 16...\n",
      "    Architecture 4 / 16...\n",
      "    Architecture 5 / 16...\n",
      "    Architecture 6 / 16...\n",
      "    Architecture 7 / 16...\n",
      "    Architecture 8 / 16...\n",
      "    Architecture 9 / 16...\n",
      "    Architecture 10 / 16...\n",
      "    Architecture 11 / 16...\n",
      "    Architecture 12 / 16...\n",
      "    Architecture 13 / 16...\n",
      "    Architecture 14 / 16...\n",
      "    Architecture 15 / 16...\n",
      "    Architecture 16 / 16...\n",
      "Iteration 17 / 30...\n",
      "    Architecture 1 / 16...\n",
      "    Architecture 2 / 16...\n",
      "    Architecture 3 / 16...\n",
      "    Architecture 4 / 16...\n",
      "    Architecture 5 / 16...\n",
      "    Architecture 6 / 16...\n",
      "    Architecture 7 / 16...\n",
      "    Architecture 8 / 16...\n",
      "    Architecture 9 / 16...\n",
      "    Architecture 10 / 16...\n",
      "    Architecture 11 / 16...\n",
      "    Architecture 12 / 16...\n",
      "    Architecture 13 / 16...\n",
      "    Architecture 14 / 16...\n",
      "    Architecture 15 / 16...\n",
      "    Architecture 16 / 16...\n",
      "Iteration 18 / 30...\n",
      "    Architecture 1 / 16...\n",
      "    Architecture 2 / 16...\n",
      "    Architecture 3 / 16...\n",
      "    Architecture 4 / 16...\n",
      "    Architecture 5 / 16...\n",
      "    Architecture 6 / 16...\n",
      "    Architecture 7 / 16...\n",
      "    Architecture 8 / 16...\n",
      "    Architecture 9 / 16...\n",
      "    Architecture 10 / 16...\n",
      "    Architecture 11 / 16...\n",
      "    Architecture 12 / 16...\n",
      "    Architecture 13 / 16...\n",
      "    Architecture 14 / 16...\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    Architecture 15 / 16...\n",
      "    Architecture 16 / 16...\n",
      "Iteration 19 / 30...\n",
      "    Architecture 1 / 16...\n",
      "    Architecture 2 / 16...\n",
      "    Architecture 3 / 16...\n",
      "    Architecture 4 / 16...\n",
      "    Architecture 5 / 16...\n",
      "    Architecture 6 / 16...\n",
      "    Architecture 7 / 16...\n",
      "    Architecture 8 / 16...\n",
      "    Architecture 9 / 16...\n",
      "    Architecture 10 / 16...\n",
      "    Architecture 11 / 16...\n",
      "    Architecture 12 / 16...\n",
      "    Architecture 13 / 16...\n",
      "    Architecture 14 / 16...\n",
      "    Architecture 15 / 16...\n",
      "    Architecture 16 / 16...\n",
      "Iteration 20 / 30...\n",
      "    Architecture 1 / 16...\n",
      "    Architecture 2 / 16...\n",
      "    Architecture 3 / 16...\n",
      "    Architecture 4 / 16...\n",
      "    Architecture 5 / 16...\n",
      "    Architecture 6 / 16...\n",
      "    Architecture 7 / 16...\n",
      "    Architecture 8 / 16...\n",
      "    Architecture 9 / 16...\n",
      "    Architecture 10 / 16...\n",
      "    Architecture 11 / 16...\n",
      "    Architecture 12 / 16...\n",
      "    Architecture 13 / 16...\n",
      "    Architecture 14 / 16...\n",
      "    Architecture 15 / 16...\n",
      "    Architecture 16 / 16...\n",
      "Iteration 21 / 30...\n",
      "    Architecture 1 / 16...\n",
      "    Architecture 2 / 16...\n",
      "    Architecture 3 / 16...\n",
      "    Architecture 4 / 16...\n",
      "    Architecture 5 / 16...\n",
      "    Architecture 6 / 16...\n",
      "    Architecture 7 / 16...\n",
      "    Architecture 8 / 16...\n",
      "    Architecture 9 / 16...\n",
      "    Architecture 10 / 16...\n",
      "    Architecture 11 / 16...\n",
      "    Architecture 12 / 16...\n",
      "    Architecture 13 / 16...\n",
      "    Architecture 14 / 16...\n",
      "    Architecture 15 / 16...\n",
      "    Architecture 16 / 16...\n",
      "Iteration 22 / 30...\n",
      "    Architecture 1 / 16...\n",
      "    Architecture 2 / 16...\n",
      "    Architecture 3 / 16...\n",
      "    Architecture 4 / 16...\n",
      "    Architecture 5 / 16...\n",
      "    Architecture 6 / 16...\n",
      "    Architecture 7 / 16...\n",
      "    Architecture 8 / 16...\n",
      "    Architecture 9 / 16...\n",
      "    Architecture 10 / 16...\n",
      "    Architecture 11 / 16...\n",
      "    Architecture 12 / 16...\n",
      "    Architecture 13 / 16...\n",
      "    Architecture 14 / 16...\n",
      "    Architecture 15 / 16...\n",
      "    Architecture 16 / 16...\n",
      "Iteration 23 / 30...\n",
      "    Architecture 1 / 16...\n",
      "    Architecture 2 / 16...\n",
      "    Architecture 3 / 16...\n",
      "    Architecture 4 / 16...\n",
      "    Architecture 5 / 16...\n",
      "    Architecture 6 / 16...\n",
      "    Architecture 7 / 16...\n",
      "    Architecture 8 / 16...\n",
      "    Architecture 9 / 16...\n",
      "    Architecture 10 / 16...\n",
      "    Architecture 11 / 16...\n",
      "    Architecture 12 / 16...\n",
      "    Architecture 13 / 16...\n",
      "    Architecture 14 / 16...\n",
      "    Architecture 15 / 16...\n",
      "    Architecture 16 / 16...\n",
      "Iteration 24 / 30...\n",
      "    Architecture 1 / 16...\n",
      "    Architecture 2 / 16...\n",
      "    Architecture 3 / 16...\n",
      "    Architecture 4 / 16...\n",
      "    Architecture 5 / 16...\n",
      "    Architecture 6 / 16...\n",
      "    Architecture 7 / 16...\n",
      "    Architecture 8 / 16...\n",
      "    Architecture 9 / 16...\n",
      "    Architecture 10 / 16...\n",
      "    Architecture 11 / 16...\n",
      "    Architecture 12 / 16...\n",
      "    Architecture 13 / 16...\n",
      "    Architecture 14 / 16...\n",
      "    Architecture 15 / 16...\n",
      "    Architecture 16 / 16...\n",
      "Iteration 25 / 30...\n",
      "    Architecture 1 / 16...\n",
      "    Architecture 2 / 16...\n",
      "    Architecture 3 / 16...\n",
      "    Architecture 4 / 16...\n",
      "    Architecture 5 / 16...\n",
      "    Architecture 6 / 16...\n",
      "    Architecture 7 / 16...\n",
      "    Architecture 8 / 16...\n",
      "    Architecture 9 / 16...\n",
      "    Architecture 10 / 16...\n",
      "    Architecture 11 / 16...\n",
      "    Architecture 12 / 16...\n",
      "    Architecture 13 / 16...\n",
      "    Architecture 14 / 16...\n",
      "    Architecture 15 / 16...\n",
      "    Architecture 16 / 16...\n",
      "Iteration 26 / 30...\n",
      "    Architecture 1 / 16...\n",
      "    Architecture 2 / 16...\n",
      "    Architecture 3 / 16...\n",
      "    Architecture 4 / 16...\n",
      "    Architecture 5 / 16...\n",
      "    Architecture 6 / 16...\n",
      "    Architecture 7 / 16...\n",
      "    Architecture 8 / 16...\n",
      "    Architecture 9 / 16...\n",
      "    Architecture 10 / 16...\n",
      "    Architecture 11 / 16...\n",
      "    Architecture 12 / 16...\n",
      "    Architecture 13 / 16...\n",
      "    Architecture 14 / 16...\n",
      "    Architecture 15 / 16...\n",
      "    Architecture 16 / 16...\n",
      "Iteration 27 / 30...\n",
      "    Architecture 1 / 16...\n",
      "    Architecture 2 / 16...\n",
      "    Architecture 3 / 16...\n",
      "    Architecture 4 / 16...\n",
      "    Architecture 5 / 16...\n",
      "    Architecture 6 / 16...\n",
      "    Architecture 7 / 16...\n",
      "    Architecture 8 / 16...\n",
      "    Architecture 9 / 16...\n",
      "    Architecture 10 / 16...\n",
      "    Architecture 11 / 16...\n",
      "    Architecture 12 / 16...\n",
      "    Architecture 13 / 16...\n",
      "    Architecture 14 / 16...\n",
      "    Architecture 15 / 16...\n",
      "    Architecture 16 / 16...\n",
      "Iteration 28 / 30...\n",
      "    Architecture 1 / 16...\n",
      "    Architecture 2 / 16...\n",
      "    Architecture 3 / 16...\n",
      "    Architecture 4 / 16...\n",
      "    Architecture 5 / 16...\n",
      "    Architecture 6 / 16...\n",
      "    Architecture 7 / 16...\n",
      "    Architecture 8 / 16...\n",
      "    Architecture 9 / 16...\n",
      "    Architecture 10 / 16...\n",
      "    Architecture 11 / 16...\n",
      "    Architecture 12 / 16...\n",
      "    Architecture 13 / 16...\n",
      "    Architecture 14 / 16...\n",
      "    Architecture 15 / 16...\n",
      "    Architecture 16 / 16...\n",
      "Iteration 29 / 30...\n",
      "    Architecture 1 / 16...\n",
      "    Architecture 2 / 16...\n",
      "    Architecture 3 / 16...\n",
      "    Architecture 4 / 16...\n",
      "    Architecture 5 / 16...\n",
      "    Architecture 6 / 16...\n",
      "    Architecture 7 / 16...\n",
      "    Architecture 8 / 16...\n",
      "    Architecture 9 / 16...\n",
      "    Architecture 10 / 16...\n",
      "    Architecture 11 / 16...\n",
      "    Architecture 12 / 16...\n",
      "    Architecture 13 / 16...\n",
      "    Architecture 14 / 16...\n",
      "    Architecture 15 / 16...\n",
      "    Architecture 16 / 16...\n",
      "Iteration 30 / 30...\n",
      "    Architecture 1 / 16...\n",
      "    Architecture 2 / 16...\n",
      "    Architecture 3 / 16...\n",
      "    Architecture 4 / 16...\n",
      "    Architecture 5 / 16...\n",
      "    Architecture 6 / 16...\n",
      "    Architecture 7 / 16...\n",
      "    Architecture 8 / 16...\n",
      "    Architecture 9 / 16...\n",
      "    Architecture 10 / 16...\n",
      "    Architecture 11 / 16...\n",
      "    Architecture 12 / 16...\n",
      "    Architecture 13 / 16...\n",
      "    Architecture 14 / 16...\n",
      "    Architecture 15 / 16...\n",
      "    Architecture 16 / 16...\n"
     ]
    }
   ],
   "source": [
    "type_models = ['Shallow', 'MLP', 'Deep1', 'Deep2']\n",
    "sub_models = ['NOsharing_NOaux', 'sharing_NOaux', 'NOsharing_aux', 'sharing_aux']\n",
    "n_iter = 30\n",
    "all_values = torch.zeros(n_iter,len(type_models),len(sub_models))\n",
    "\n",
    "for j in range(n_iter):\n",
    "    print('Iteration', j+1, '/ 30...')\n",
    "    train_input, train_target, train_classes, test_input, test_target, test_classes = prologue.generate_pair_sets(nbr_pairs)\n",
    "    acc_test = torch.zeros(len(type_models),len(sub_models))\n",
    "    i = 0\n",
    "    for t, type_model in enumerate(type_models):\n",
    "        for s, sub_model in enumerate(sub_models):\n",
    "            i += 1\n",
    "            print('    Architecture', i, '/ 16...')\n",
    "            best_hidden = HP[type_model][sub_model]['hidden']\n",
    "            best_eta = HP[type_model][sub_model]['eta']\n",
    "            if(sub_model == 'NOsharing_aux' or sub_model == 'sharing_aux'): \n",
    "                best_lambda = HP[type_model][sub_model]['lambda']\n",
    "            else: best_lambda = 0\n",
    "            \n",
    "            _, model = model_training(train_input, train_target, best_hidden, best_eta, best_lambda, \n",
    "                                      model_type = type_model, sub_model = sub_model)\n",
    "                \n",
    "            if(sub_model == 'NOsharing_aux' or sub_model == 'sharing_aux'): \n",
    "                acc_test[t,s] = 1 - compute_nb_errors_aux(model, test_input, test_target, 100)/len(test_target)\n",
    "            else: \n",
    "                acc_test[t,s] = 1 - compute_nb_errors_NOaux(model, test_input, test_target, 100)/len(test_target)\n",
    "    all_values[j] = acc_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean :\n",
      "tensor([[0.7188, 0.7755, 0.7604, 0.7751],\n",
      "        [0.8028, 0.7963, 0.8050, 0.8122],\n",
      "        [0.7978, 0.8043, 0.8043, 0.8154],\n",
      "        [0.8148, 0.8300, 0.8205, 0.8319]])\n",
      "Std :\n",
      "tensor([[0.0584, 0.0114, 0.0403, 0.0096],\n",
      "        [0.0130, 0.0190, 0.0094, 0.0168],\n",
      "        [0.0136, 0.0174, 0.0117, 0.0143],\n",
      "        [0.0147, 0.0120, 0.0127, 0.0111]])\n"
     ]
    }
   ],
   "source": [
    "torch.save(all_values, 'results.pt')\n",
    "print('Mean :')\n",
    "print(torch.mean(all_values, 0))\n",
    "print('Std :')\n",
    "print(torch.std(all_values, 0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[0.6560, 0.7800, 0.6670, 0.7750],\n",
       "         [0.7960, 0.7980, 0.8060, 0.8120],\n",
       "         [0.7920, 0.7700, 0.7880, 0.8090],\n",
       "         [0.8110, 0.8210, 0.8120, 0.8360]],\n",
       "\n",
       "        [[0.7810, 0.7730, 0.7820, 0.7750],\n",
       "         [0.7800, 0.7930, 0.7910, 0.7950],\n",
       "         [0.8160, 0.8130, 0.7890, 0.8020],\n",
       "         [0.8120, 0.8280, 0.8150, 0.8440]],\n",
       "\n",
       "        [[0.7650, 0.7780, 0.8150, 0.7630],\n",
       "         [0.8290, 0.8050, 0.8070, 0.7830],\n",
       "         [0.8210, 0.8000, 0.8120, 0.8160],\n",
       "         [0.8110, 0.8250, 0.8080, 0.8260]],\n",
       "\n",
       "        [[0.7400, 0.7680, 0.7980, 0.7870],\n",
       "         [0.7960, 0.7730, 0.8090, 0.8240],\n",
       "         [0.8040, 0.8030, 0.8050, 0.8160],\n",
       "         [0.8000, 0.8390, 0.8340, 0.8430]],\n",
       "\n",
       "        [[0.7100, 0.7680, 0.7250, 0.7760],\n",
       "         [0.8150, 0.8080, 0.8050, 0.7990],\n",
       "         [0.7810, 0.8260, 0.8230, 0.8180],\n",
       "         [0.8060, 0.8420, 0.8110, 0.8340]],\n",
       "\n",
       "        [[0.7470, 0.7670, 0.8000, 0.7700],\n",
       "         [0.8090, 0.8020, 0.8140, 0.8220],\n",
       "         [0.8100, 0.8230, 0.8080, 0.8160],\n",
       "         [0.8320, 0.8520, 0.8390, 0.8460]],\n",
       "\n",
       "        [[0.7800, 0.7780, 0.7300, 0.7610],\n",
       "         [0.8070, 0.7940, 0.8010, 0.7890],\n",
       "         [0.8100, 0.8110, 0.8080, 0.8230],\n",
       "         [0.8290, 0.8330, 0.8230, 0.8400]],\n",
       "\n",
       "        [[0.6820, 0.7940, 0.7420, 0.7730],\n",
       "         [0.7870, 0.7610, 0.7900, 0.8160],\n",
       "         [0.7980, 0.7960, 0.8130, 0.8270],\n",
       "         [0.8120, 0.8110, 0.8250, 0.8230]],\n",
       "\n",
       "        [[0.6760, 0.7940, 0.7990, 0.8010],\n",
       "         [0.8230, 0.8210, 0.8180, 0.8380],\n",
       "         [0.8040, 0.8100, 0.8130, 0.8300],\n",
       "         [0.8280, 0.8410, 0.8430, 0.8400]],\n",
       "\n",
       "        [[0.6750, 0.7710, 0.7560, 0.7760],\n",
       "         [0.8110, 0.8120, 0.8030, 0.8330],\n",
       "         [0.7940, 0.8230, 0.8050, 0.8330],\n",
       "         [0.8160, 0.8170, 0.8200, 0.8140]],\n",
       "\n",
       "        [[0.7710, 0.7680, 0.7290, 0.7750],\n",
       "         [0.7860, 0.8210, 0.7980, 0.8230],\n",
       "         [0.8020, 0.7820, 0.8010, 0.8110],\n",
       "         [0.8140, 0.8330, 0.8210, 0.8340]],\n",
       "\n",
       "        [[0.7550, 0.7730, 0.7960, 0.7890],\n",
       "         [0.7990, 0.7690, 0.7990, 0.8210],\n",
       "         [0.7970, 0.8170, 0.8010, 0.8220],\n",
       "         [0.8260, 0.8270, 0.8090, 0.8190]],\n",
       "\n",
       "        [[0.6190, 0.7670, 0.6970, 0.7710],\n",
       "         [0.7880, 0.7990, 0.8030, 0.7930],\n",
       "         [0.7920, 0.8030, 0.7980, 0.8060],\n",
       "         [0.8140, 0.8300, 0.8170, 0.8180]],\n",
       "\n",
       "        [[0.7580, 0.7920, 0.7930, 0.7840],\n",
       "         [0.8110, 0.7890, 0.8160, 0.8090],\n",
       "         [0.8060, 0.7970, 0.8180, 0.8350],\n",
       "         [0.8430, 0.8440, 0.8330, 0.8280]],\n",
       "\n",
       "        [[0.5880, 0.7700, 0.7770, 0.7710],\n",
       "         [0.7940, 0.7500, 0.7970, 0.8080],\n",
       "         [0.7800, 0.7800, 0.7790, 0.7940],\n",
       "         [0.7860, 0.8180, 0.8040, 0.8320]],\n",
       "\n",
       "        [[0.7300, 0.7660, 0.7550, 0.7740],\n",
       "         [0.7910, 0.7820, 0.7910, 0.7890],\n",
       "         [0.7780, 0.7870, 0.7900, 0.8000],\n",
       "         [0.8160, 0.8300, 0.8130, 0.8370]],\n",
       "\n",
       "        [[0.7670, 0.7770, 0.7290, 0.7760],\n",
       "         [0.8150, 0.7760, 0.8010, 0.7980],\n",
       "         [0.8080, 0.7930, 0.8130, 0.8210],\n",
       "         [0.8230, 0.8350, 0.8300, 0.8210]],\n",
       "\n",
       "        [[0.7210, 0.7790, 0.8030, 0.7590],\n",
       "         [0.7900, 0.7770, 0.8030, 0.7910],\n",
       "         [0.7870, 0.7870, 0.7830, 0.8060],\n",
       "         [0.7940, 0.8100, 0.7970, 0.8130]],\n",
       "\n",
       "        [[0.7060, 0.7470, 0.7430, 0.7710],\n",
       "         [0.7980, 0.7970, 0.8060, 0.7990],\n",
       "         [0.7600, 0.7990, 0.8020, 0.8160],\n",
       "         [0.7900, 0.8280, 0.8010, 0.8200]],\n",
       "\n",
       "        [[0.7780, 0.7690, 0.7450, 0.7680],\n",
       "         [0.7760, 0.8130, 0.7970, 0.8100],\n",
       "         [0.7930, 0.7690, 0.7940, 0.8070],\n",
       "         [0.8220, 0.8390, 0.8140, 0.8260]],\n",
       "\n",
       "        [[0.7370, 0.7820, 0.7970, 0.7670],\n",
       "         [0.7930, 0.7830, 0.8130, 0.8210],\n",
       "         [0.8030, 0.8240, 0.8120, 0.8390],\n",
       "         [0.8080, 0.8320, 0.8120, 0.8330]],\n",
       "\n",
       "        [[0.7350, 0.7520, 0.7600, 0.7590],\n",
       "         [0.8000, 0.7920, 0.7960, 0.8070],\n",
       "         [0.8000, 0.7980, 0.7860, 0.7850],\n",
       "         [0.7880, 0.8070, 0.8080, 0.8150]],\n",
       "\n",
       "        [[0.7540, 0.7810, 0.7650, 0.7880],\n",
       "         [0.8170, 0.8080, 0.8300, 0.8100],\n",
       "         [0.7870, 0.8290, 0.8050, 0.8310],\n",
       "         [0.8270, 0.8460, 0.8260, 0.8550]],\n",
       "\n",
       "        [[0.6640, 0.7870, 0.7980, 0.7820],\n",
       "         [0.8040, 0.8150, 0.8010, 0.8160],\n",
       "         [0.8080, 0.8160, 0.8200, 0.8050],\n",
       "         [0.8370, 0.8320, 0.8240, 0.8290]],\n",
       "\n",
       "        [[0.7250, 0.7690, 0.7670, 0.7810],\n",
       "         [0.8040, 0.7940, 0.7980, 0.8180],\n",
       "         [0.7780, 0.7930, 0.8140, 0.7890],\n",
       "         [0.8010, 0.8170, 0.8210, 0.8290]],\n",
       "\n",
       "        [[0.5500, 0.7700, 0.6910, 0.7820],\n",
       "         [0.8130, 0.8040, 0.8050, 0.7950],\n",
       "         [0.7940, 0.8300, 0.7990, 0.7970],\n",
       "         [0.8040, 0.8150, 0.8430, 0.8380]],\n",
       "\n",
       "        [[0.7350, 0.7800, 0.6860, 0.7760],\n",
       "         [0.8110, 0.8120, 0.8150, 0.8330],\n",
       "         [0.7960, 0.8000, 0.8100, 0.8170],\n",
       "         [0.8100, 0.8320, 0.8100, 0.8300]],\n",
       "\n",
       "        [[0.7720, 0.7960, 0.8180, 0.7750],\n",
       "         [0.8170, 0.8160, 0.8200, 0.8470],\n",
       "         [0.8140, 0.8330, 0.8190, 0.8280],\n",
       "         [0.8280, 0.8490, 0.8250, 0.8410]],\n",
       "\n",
       "        [[0.7770, 0.7810, 0.7730, 0.7840],\n",
       "         [0.8130, 0.7910, 0.8000, 0.8270],\n",
       "         [0.8050, 0.8040, 0.8150, 0.8280],\n",
       "         [0.8240, 0.8230, 0.8360, 0.8420]],\n",
       "\n",
       "        [[0.7090, 0.7870, 0.7760, 0.7640],\n",
       "         [0.8100, 0.8330, 0.8160, 0.8390],\n",
       "         [0.8150, 0.8130, 0.8060, 0.8340],\n",
       "         [0.8320, 0.8440, 0.8420, 0.8520]]])"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.load('results.pt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}