training.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training a model from auvlib data\n",
    "\n",
    "Let's start by importing some useful functions. In this tutorial we are going to use [pytorch](https://pytorch.org/) for training a neural network. Let's import the necessary modules..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.optim as optim\n",
    "import torch.nn as nn\n",
    "from data.aligned_dataset import AlignedDataset\n",
    "from util import visualizer as vs\n",
    "from util import net_util\n",
    "import os\n",
    "import functools\n",
    "from util import html\n",
    "from IPython.display import IFrame"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Defining the neural network\n",
    "\n",
    "We are going to train a convolutional network with residual blocks, a so called [*resnet*](https://arxiv.org/abs/1512.03385), to map sidescan patches to corresponding sea floor depths. A resnet is a popular for of neural network that uses skip connections to facilitate training, see sketch below. To begin with, let us define the network structure. ![resnet](ResNets.svg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ResnetBlock(nn.Module):\n",
    "    \"\"\"Define a Resnet block\"\"\"\n",
    "\n",
    "    def __init__(self, dim, padding_type, norm_layer, use_dropout, use_bias):\n",
    "        \"\"\"Initialize the Resnet block\n",
    "\n",
    "        A resnet block is a conv block with skip connections\n",
    "        We construct a conv block with build_conv_block function,\n",
    "        and implement skip connections in <forward> function.\n",
    "        Original Resnet paper: https://arxiv.org/pdf/1512.03385.pdf\n",
    "        \"\"\"\n",
    "        super(ResnetBlock, self).__init__()\n",
    "        self.conv_block = self.build_conv_block(dim, padding_type, norm_layer, use_dropout, use_bias)\n",
    "\n",
    "    def build_conv_block(self, dim, padding_type, norm_layer, use_dropout, use_bias):\n",
    "        \"\"\"Construct a convolutional block.\n",
    "\n",
    "        Parameters:\n",
    "            dim (int)           -- the number of channels in the conv layer.\n",
    "            padding_type (str)  -- the name of padding layer: reflect | replicate | zero\n",
    "            norm_layer          -- normalization layer\n",
    "            use_dropout (bool)  -- if use dropout layers.\n",
    "            use_bias (bool)     -- if the conv layer uses bias or not\n",
    "\n",
    "        Returns a conv block (with a conv layer, a normalization layer, and a non-linearity layer (ReLU))\n",
    "        \"\"\"\n",
    "        conv_block = []\n",
    "        p = 0\n",
    "        if padding_type == 'reflect':\n",
    "            conv_block += [nn.ReflectionPad2d(1)]\n",
    "        elif padding_type == 'replicate':\n",
    "            conv_block += [nn.ReplicationPad2d(1)]\n",
    "        elif padding_type == 'zero':\n",
    "            p = 1\n",
    "        else:\n",
    "            raise NotImplementedError('padding [%s] is not implemented' % padding_type)\n",
    "\n",
    "        conv_block += [nn.Conv2d(dim, dim, kernel_size=3, padding=p, bias=use_bias), norm_layer(dim), nn.ReLU(True)]\n",
    "        if use_dropout:\n",
    "            conv_block += [nn.Dropout(0.5)]\n",
    "\n",
    "        p = 0\n",
    "        if padding_type == 'reflect':\n",
    "            conv_block += [nn.ReflectionPad2d(1)]\n",
    "        elif padding_type == 'replicate':\n",
    "            conv_block += [nn.ReplicationPad2d(1)]\n",
    "        elif padding_type == 'zero':\n",
    "            p = 1\n",
    "        else:\n",
    "            raise NotImplementedError('padding [%s] is not implemented' % padding_type)\n",
    "        conv_block += [nn.Conv2d(dim, dim, kernel_size=3, padding=p, bias=use_bias), norm_layer(dim)]\n",
    "\n",
    "        return nn.Sequential(*conv_block)\n",
    "\n",
    "    def forward(self, x):\n",
    "        \"\"\"Forward function (with skip connections)\"\"\"\n",
    "        out = x + self.conv_block(x)  # add skip connections\n",
    "        return out\n",
    "\n",
    "class ResnetGenerator(nn.Module):\n",
    "    \"\"\"Resnet-based generator that consists of Resnet blocks between a few downsampling/upsampling operations.\n",
    "\n",
    "    We adapt Torch code and idea from Justin Johnson's neural style transfer project(https://github.com/jcjohnson/fast-neural-style)\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self, input_nc, output_nc, ngf=64, norm_layer=nn.BatchNorm2d, use_dropout=False, n_blocks=6, padding_type='reflect'):\n",
    "        \"\"\"Construct a Resnet-based generator\n",
    "\n",
    "        Parameters:\n",
    "            input_nc (int)      -- the number of channels in input images\n",
    "            output_nc (int)     -- the number of channels in output images\n",
    "            ngf (int)           -- the number of filters in the last conv layer\n",
    "            norm_layer          -- normalization layer\n",
    "            use_dropout (bool)  -- if use dropout layers\n",
    "            n_blocks (int)      -- the number of ResNet blocks\n",
    "            padding_type (str)  -- the name of padding layer in conv layers: reflect | replicate | zero\n",
    "        \"\"\"\n",
    "        assert(n_blocks >= 0)\n",
    "        super(ResnetGenerator, self).__init__()\n",
    "        if type(norm_layer) == functools.partial:\n",
    "            use_bias = norm_layer.func == nn.InstanceNorm2d\n",
    "        else:\n",
    "            use_bias = norm_layer == nn.InstanceNorm2d\n",
    "\n",
    "        model = [nn.ReflectionPad2d(3),\n",
    "                 nn.Conv2d(input_nc, ngf, kernel_size=7, padding=0, bias=use_bias),\n",
    "                 norm_layer(ngf),\n",
    "                 nn.ReLU(True)]\n",
    "\n",
    "        n_downsampling = 2\n",
    "        for i in range(n_downsampling):  # add downsampling layers\n",
    "            mult = 2 ** i\n",
    "            model += [nn.Conv2d(ngf * mult, ngf * mult * 2, kernel_size=3, stride=2, padding=1, bias=use_bias),\n",
    "                      norm_layer(ngf * mult * 2),\n",
    "                      nn.ReLU(True)]\n",
    "\n",
    "        mult = 2 ** n_downsampling\n",
    "        for i in range(n_blocks):       # add ResNet blocks\n",
    "\n",
    "            model += [ResnetBlock(ngf * mult, padding_type=padding_type, norm_layer=norm_layer, use_dropout=use_dropout, use_bias=use_bias)]\n",
    "\n",
    "        for i in range(n_downsampling):  # add upsampling layers\n",
    "            mult = 2 ** (n_downsampling - i)\n",
    "            model += [nn.ConvTranspose2d(ngf * mult, int(ngf * mult / 2),\n",
    "                                         kernel_size=3, stride=2,\n",
    "                                         padding=1, output_padding=1,\n",
    "                                         bias=use_bias),\n",
    "                      norm_layer(int(ngf * mult / 2)),\n",
    "                      nn.ReLU(True)]\n",
    "        model += [nn.ReflectionPad2d(3)]\n",
    "        model += [nn.Conv2d(ngf, output_nc, kernel_size=7, padding=0)]\n",
    "        model += [nn.Tanh()]\n",
    "\n",
    "        self.model = nn.Sequential(*model)\n",
    "\n",
    "    def forward(self, input):\n",
    "        \"\"\"Standard forward\"\"\"\n",
    "        return self.model(input)\n",
    "    \n",
    "def save_network(save_dir, epoch):\n",
    "    \"\"\"Save the network to the disk.\n",
    "    Parameters:\n",
    "        epoch (int) -- current epoch; used in the file name '%s_net_%s.pth' % (epoch, name)\n",
    "    \"\"\"\n",
    "    save_filename = '%s_resnet.pth' % (epoch,)\n",
    "    save_path = os.path.join(save_dir, save_filename)\n",
    "\n",
    "    torch.save(net.state_dict(), save_path)\n",
    "    \n",
    "def load_network(save_dir, epoch, device):\n",
    "    \"\"\"Load the network from disk.\n",
    "    Parameters:\n",
    "        epoch (int) -- current epoch; used in the file name '%s_net_%s.pth' % (epoch, name)\n",
    "        device (str) -- the name of the torch device (e.g. \"cpu\")\n",
    "    \"\"\"\n",
    "    save_filename = '%s_resnet.pth' % (epoch,)\n",
    "    save_path = os.path.join(save_dir, save_filename)\n",
    "    \n",
    "    norm_layer = net_util.get_norm_layer(norm_type='batch')\n",
    "    model = ResnetGenerator(1, 1, 64, norm_layer=norm_layer, use_dropout=True, n_blocks=6).to(device)\n",
    "    model.load_state_dict(torch.load(save_path, map_location=device))\n",
    "    return model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Setting up the training\n",
    "\n",
    "The network will get loaded on the GPU if there is one available. Note that training requires a GPU, while testing could be done on a CPU."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "initialize network with normal\n"
     ]
    }
   ],
   "source": [
    "norm_layer = net_util.get_norm_layer(norm_type='batch')\n",
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "net = ResnetGenerator(1, 1, 64, norm_layer=norm_layer, use_dropout=True, n_blocks=6).to(device)\n",
    "net_util.init_weights(net, init_type='normal', init_gain=0.02)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We are going to use a L1 loss to force the predicted depths to be close to the ground truth depths of the training examples. We will use the Adam optmizer because we found that it works good while being fast. However, one could also use the standard stochastic gradient descent method."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "criterion = torch.nn.L1Loss()\n",
    "#optimizer = optim.SGD(net.parameters(), lr=0.001, momentum=0.9)\n",
    "optimizer = optim.Adam(net.parameters(), lr=0.0001, betas=(0.5, 0.999))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we are going to feed in the images that we stored in the previous part of the tutorial. For the training part, we will use the images from the `train` subdirectory."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataroot = 'sss2depth' # the name of the dataset folder\n",
    "batch_size = 1 # number of examples to feed in at a time\n",
    "epochs = 100 # number of epochs to train the network\n",
    "\n",
    "opt = net_util.get_default_train_options() # e.g. display config\n",
    "opt.batch_size = batch_size\n",
    "opt.dataroot = dataroot\n",
    "opt.phase = 'train'\n",
    "# this iterator will feed data to our network\n",
    "dataset = torch.utils.data.DataLoader(\n",
    "                AlignedDataset(opt),\n",
    "                batch_size=opt.batch_size,\n",
    "                shuffle=not opt.serial_batches,\n",
    "                num_workers=int(opt.num_threads))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training the network\n",
    "\n",
    "Now we are ready to start training our network. Every 5 epochs we will save the current model in `checkpoints/XX_resnet.pth`. You can inspect the process of the training in [visdom](http://localhost:8097/). The website should look something like this: ![visdom](visdom.png)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Setting up a new session...\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "create web directory ./checkpoints/regress/web...\n",
      "[1,    20] loss: 0.385\n",
      "Epoch 1 / 100, example 20 / 104\n",
      "X len:  1 Y len:  1\n",
      "X:  (1, 1)\n",
      "Y:  (1, 1)\n",
      "[1,    40] loss: 0.387\n",
      "Epoch 1 / 100, example 40 / 104\n",
      "X len:  2 Y len:  2\n",
      "X:  (2, 1)\n",
      "Y:  (2, 1)\n",
      "[1,    60] loss: 0.241\n",
      "Epoch 1 / 100, example 60 / 104\n",
      "X len:  3 Y len:  3\n",
      "X:  (3, 1)\n",
      "Y:  (3, 1)\n",
      "[1,    80] loss: 0.274\n",
      "Epoch 1 / 100, example 80 / 104\n",
      "X len:  4 Y len:  4\n",
      "X:  (4, 1)\n",
      "Y:  (4, 1)\n",
      "[1,   100] loss: 0.296\n",
      "Epoch 1 / 100, example 100 / 104\n",
      "X len:  5 Y len:  5\n",
      "X:  (5, 1)\n",
      "Y:  (5, 1)\n",
      "saving the model at the end of epoch 0, iters 103\n",
      "[2,    16] loss: 0.256\n",
      "Epoch 2 / 100, example 16 / 104\n",
      "X len:  6 Y len:  6\n",
      "X:  (6, 1)\n",
      "Y:  (6, 1)\n",
      "[2,    36] loss: 0.332\n",
      "Epoch 2 / 100, example 36 / 104\n",
      "X len:  7 Y len:  7\n",
      "X:  (7, 1)\n",
      "Y:  (7, 1)\n",
      "[2,    56] loss: 0.243\n",
      "Epoch 2 / 100, example 56 / 104\n",
      "X len:  8 Y len:  8\n",
      "X:  (8, 1)\n",
      "Y:  (8, 1)\n",
      "[2,    76] loss: 0.247\n",
      "Epoch 2 / 100, example 76 / 104\n",
      "X len:  9 Y len:  9\n",
      "X:  (9, 1)\n",
      "Y:  (9, 1)\n",
      "[2,    96] loss: 0.275\n",
      "Epoch 2 / 100, example 96 / 104\n",
      "X len:  10 Y len:  10\n",
      "X:  (10, 1)\n",
      "Y:  (10, 1)\n",
      "[3,    12] loss: 0.180\n",
      "Epoch 3 / 100, example 12 / 104\n",
      "X len:  11 Y len:  11\n",
      "X:  (11, 1)\n",
      "Y:  (11, 1)\n",
      "[3,    32] loss: 0.335\n",
      "Epoch 3 / 100, example 32 / 104\n",
      "X len:  12 Y len:  12\n",
      "X:  (12, 1)\n",
      "Y:  (12, 1)\n",
      "[3,    52] loss: 0.241\n",
      "Epoch 3 / 100, example 52 / 104\n",
      "X len:  13 Y len:  13\n",
      "X:  (13, 1)\n",
      "Y:  (13, 1)\n",
      "[3,    72] loss: 0.261\n",
      "Epoch 3 / 100, example 72 / 104\n",
      "X len:  14 Y len:  14\n",
      "X:  (14, 1)\n",
      "Y:  (14, 1)\n",
      "[3,    92] loss: 0.228\n",
      "Epoch 3 / 100, example 92 / 104\n",
      "X len:  15 Y len:  15\n",
      "X:  (15, 1)\n",
      "Y:  (15, 1)\n",
      "[4,     8] loss: 0.123\n",
      "Epoch 4 / 100, example 8 / 104\n",
      "X len:  16 Y len:  16\n",
      "X:  (16, 1)\n",
      "Y:  (16, 1)\n",
      "[4,    28] loss: 0.313\n",
      "Epoch 4 / 100, example 28 / 104\n",
      "X len:  17 Y len:  17\n",
      "X:  (17, 1)\n",
      "Y:  (17, 1)\n",
      "[4,    48] loss: 0.267\n",
      "Epoch 4 / 100, example 48 / 104\n",
      "X len:  18 Y len:  18\n",
      "X:  (18, 1)\n",
      "Y:  (18, 1)\n",
      "[4,    68] loss: 0.232\n",
      "Epoch 4 / 100, example 68 / 104\n",
      "X len:  19 Y len:  19\n",
      "X:  (19, 1)\n",
      "Y:  (19, 1)\n",
      "[4,    88] loss: 0.262\n",
      "Epoch 4 / 100, example 88 / 104\n",
      "X len:  20 Y len:  20\n",
      "X:  (20, 1)\n",
      "Y:  (20, 1)\n",
      "[5,     4] loss: 0.064\n",
      "Epoch 5 / 100, example 4 / 104\n",
      "X len:  21 Y len:  21\n",
      "X:  (21, 1)\n",
      "Y:  (21, 1)\n",
      "[5,    24] loss: 0.289\n",
      "Epoch 5 / 100, example 24 / 104\n",
      "X len:  22 Y len:  22\n",
      "X:  (22, 1)\n",
      "Y:  (22, 1)\n",
      "[5,    44] loss: 0.308\n",
      "Epoch 5 / 100, example 44 / 104\n",
      "X len:  23 Y len:  23\n",
      "X:  (23, 1)\n",
      "Y:  (23, 1)\n",
      "[5,    64] loss: 0.221\n",
      "Epoch 5 / 100, example 64 / 104\n",
      "X len:  24 Y len:  24\n",
      "X:  (24, 1)\n",
      "Y:  (24, 1)\n",
      "[5,    84] loss: 0.263\n",
      "Epoch 5 / 100, example 84 / 104\n",
      "X len:  25 Y len:  25\n",
      "X:  (25, 1)\n",
      "Y:  (25, 1)\n",
      "[5,   104] loss: 0.295\n",
      "Epoch 5 / 100, example 104 / 104\n",
      "X len:  26 Y len:  26\n",
      "X:  (26, 1)\n",
      "Y:  (26, 1)\n",
      "[6,    20] loss: 0.296\n",
      "Epoch 6 / 100, example 20 / 104\n",
      "X len:  27 Y len:  27\n",
      "X:  (27, 1)\n",
      "Y:  (27, 1)\n",
      "[6,    40] loss: 0.347\n",
      "Epoch 6 / 100, example 40 / 104\n",
      "X len:  28 Y len:  28\n",
      "X:  (28, 1)\n",
      "Y:  (28, 1)\n",
      "[6,    60] loss: 0.213\n",
      "Epoch 6 / 100, example 60 / 104\n",
      "X len:  29 Y len:  29\n",
      "X:  (29, 1)\n",
      "Y:  (29, 1)\n",
      "[6,    80] loss: 0.242\n",
      "Epoch 6 / 100, example 80 / 104\n",
      "X len:  30 Y len:  30\n",
      "X:  (30, 1)\n",
      "Y:  (30, 1)\n",
      "[6,   100] loss: 0.282\n",
      "Epoch 6 / 100, example 100 / 104\n",
      "X len:  31 Y len:  31\n",
      "X:  (31, 1)\n",
      "Y:  (31, 1)\n",
      "saving the model at the end of epoch 5, iters 103\n",
      "[7,    16] loss: 0.243\n",
      "Epoch 7 / 100, example 16 / 104\n",
      "X len:  32 Y len:  32\n",
      "X:  (32, 1)\n",
      "Y:  (32, 1)\n",
      "[7,    36] loss: 0.320\n",
      "Epoch 7 / 100, example 36 / 104\n",
      "X len:  33 Y len:  33\n",
      "X:  (33, 1)\n",
      "Y:  (33, 1)\n",
      "[7,    56] loss: 0.233\n",
      "Epoch 7 / 100, example 56 / 104\n",
      "X len:  34 Y len:  34\n",
      "X:  (34, 1)\n",
      "Y:  (34, 1)\n",
      "[7,    76] loss: 0.237\n",
      "Epoch 7 / 100, example 76 / 104\n",
      "X len:  35 Y len:  35\n",
      "X:  (35, 1)\n",
      "Y:  (35, 1)\n",
      "[7,    96] loss: 0.268\n",
      "Epoch 7 / 100, example 96 / 104\n",
      "X len:  36 Y len:  36\n",
      "X:  (36, 1)\n",
      "Y:  (36, 1)\n",
      "[8,    12] loss: 0.175\n",
      "Epoch 8 / 100, example 12 / 104\n",
      "X len:  37 Y len:  37\n",
      "X:  (37, 1)\n",
      "Y:  (37, 1)\n",
      "[8,    32] loss: 0.329\n",
      "Epoch 8 / 100, example 32 / 104\n",
      "X len:  38 Y len:  38\n",
      "X:  (38, 1)\n",
      "Y:  (38, 1)\n",
      "[8,    52] loss: 0.235\n",
      "Epoch 8 / 100, example 52 / 104\n",
      "X len:  39 Y len:  39\n",
      "X:  (39, 1)\n",
      "Y:  (39, 1)\n",
      "[8,    72] loss: 0.255\n",
      "Epoch 8 / 100, example 72 / 104\n",
      "X len:  40 Y len:  40\n",
      "X:  (40, 1)\n",
      "Y:  (40, 1)\n",
      "[8,    92] loss: 0.223\n",
      "Epoch 8 / 100, example 92 / 104\n",
      "X len:  41 Y len:  41\n",
      "X:  (41, 1)\n",
      "Y:  (41, 1)\n",
      "[9,     8] loss: 0.121\n",
      "Epoch 9 / 100, example 8 / 104\n",
      "X len:  42 Y len:  42\n",
      "X:  (42, 1)\n",
      "Y:  (42, 1)\n",
      "[9,    28] loss: 0.309\n",
      "Epoch 9 / 100, example 28 / 104\n",
      "X len:  43 Y len:  43\n",
      "X:  (43, 1)\n",
      "Y:  (43, 1)\n",
      "[9,    48] loss: 0.261\n",
      "Epoch 9 / 100, example 48 / 104\n",
      "X len:  44 Y len:  44\n",
      "X:  (44, 1)\n",
      "Y:  (44, 1)\n",
      "[9,    68] loss: 0.228\n",
      "Epoch 9 / 100, example 68 / 104\n",
      "X len:  45 Y len:  45\n",
      "X:  (45, 1)\n",
      "Y:  (45, 1)\n",
      "[9,    88] loss: 0.256\n",
      "Epoch 9 / 100, example 88 / 104\n",
      "X len:  46 Y len:  46\n",
      "X:  (46, 1)\n",
      "Y:  (46, 1)\n",
      "[10,     4] loss: 0.063\n",
      "Epoch 10 / 100, example 4 / 104\n",
      "X len:  47 Y len:  47\n",
      "X:  (47, 1)\n",
      "Y:  (47, 1)\n",
      "[10,    24] loss: 0.292\n",
      "Epoch 10 / 100, example 24 / 104\n",
      "X len:  48 Y len:  48\n",
      "X:  (48, 1)\n",
      "Y:  (48, 1)\n",
      "[10,    44] loss: 0.310\n",
      "Epoch 10 / 100, example 44 / 104\n",
      "X len:  49 Y len:  49\n",
      "X:  (49, 1)\n",
      "Y:  (49, 1)\n",
      "[10,    64] loss: 0.218\n",
      "Epoch 10 / 100, example 64 / 104\n",
      "X len:  50 Y len:  50\n",
      "X:  (50, 1)\n",
      "Y:  (50, 1)\n",
      "[10,    84] loss: 0.260\n",
      "Epoch 10 / 100, example 84 / 104\n",
      "X len:  51 Y len:  51\n",
      "X:  (51, 1)\n",
      "Y:  (51, 1)\n",
      "[10,   104] loss: 0.287\n",
      "Epoch 10 / 100, example 104 / 104\n",
      "X len:  52 Y len:  52\n",
      "X:  (52, 1)\n",
      "Y:  (52, 1)\n",
      "[11,    20] loss: 0.292\n",
      "Epoch 11 / 100, example 20 / 104\n",
      "X len:  53 Y len:  53\n",
      "X:  (53, 1)\n",
      "Y:  (53, 1)\n",
      "[11,    40] loss: 0.344\n",
      "Epoch 11 / 100, example 40 / 104\n",
      "X len:  54 Y len:  54\n",
      "X:  (54, 1)\n",
      "Y:  (54, 1)\n",
      "[11,    60] loss: 0.210\n",
      "Epoch 11 / 100, example 60 / 104\n",
      "X len:  55 Y len:  55\n",
      "X:  (55, 1)\n",
      "Y:  (55, 1)\n",
      "[11,    80] loss: 0.238\n",
      "Epoch 11 / 100, example 80 / 104\n",
      "X len:  56 Y len:  56\n",
      "X:  (56, 1)\n",
      "Y:  (56, 1)\n",
      "[11,   100] loss: 0.274\n",
      "Epoch 11 / 100, example 100 / 104\n",
      "X len:  57 Y len:  57\n",
      "X:  (57, 1)\n",
      "Y:  (57, 1)\n",
      "saving the model at the end of epoch 10, iters 103\n",
      "[12,    16] loss: 0.241\n",
      "Epoch 12 / 100, example 16 / 104\n",
      "X len:  58 Y len:  58\n",
      "X:  (58, 1)\n",
      "Y:  (58, 1)\n",
      "[12,    36] loss: 0.316\n",
      "Epoch 12 / 100, example 36 / 104\n",
      "X len:  59 Y len:  59\n",
      "X:  (59, 1)\n",
      "Y:  (59, 1)\n",
      "[12,    56] loss: 0.229\n",
      "Epoch 12 / 100, example 56 / 104\n",
      "X len:  60 Y len:  60\n",
      "X:  (60, 1)\n",
      "Y:  (60, 1)\n",
      "[12,    76] loss: 0.234\n",
      "Epoch 12 / 100, example 76 / 104\n",
      "X len:  61 Y len:  61\n",
      "X:  (61, 1)\n",
      "Y:  (61, 1)\n",
      "[12,    96] loss: 0.264\n",
      "Epoch 12 / 100, example 96 / 104\n",
      "X len:  62 Y len:  62\n",
      "X:  (62, 1)\n",
      "Y:  (62, 1)\n",
      "[13,    12] loss: 0.174\n",
      "Epoch 13 / 100, example 12 / 104\n",
      "X len:  63 Y len:  63\n",
      "X:  (63, 1)\n",
      "Y:  (63, 1)\n",
      "[13,    32] loss: 0.328\n",
      "Epoch 13 / 100, example 32 / 104\n",
      "X len:  64 Y len:  64\n",
      "X:  (64, 1)\n",
      "Y:  (64, 1)\n",
      "[13,    52] loss: 0.233\n",
      "Epoch 13 / 100, example 52 / 104\n",
      "X len:  65 Y len:  65\n",
      "X:  (65, 1)\n",
      "Y:  (65, 1)\n",
      "[13,    72] loss: 0.253\n",
      "Epoch 13 / 100, example 72 / 104\n",
      "X len:  66 Y len:  66\n",
      "X:  (66, 1)\n",
      "Y:  (66, 1)\n",
      "[13,    92] loss: 0.220\n",
      "Epoch 13 / 100, example 92 / 104\n",
      "X len:  67 Y len:  67\n",
      "X:  (67, 1)\n",
      "Y:  (67, 1)\n",
      "[14,     8] loss: 0.120\n",
      "Epoch 14 / 100, example 8 / 104\n",
      "X len:  68 Y len:  68\n",
      "X:  (68, 1)\n",
      "Y:  (68, 1)\n",
      "[14,    28] loss: 0.307\n",
      "Epoch 14 / 100, example 28 / 104\n",
      "X len:  69 Y len:  69\n",
      "X:  (69, 1)\n",
      "Y:  (69, 1)\n",
      "[14,    48] loss: 0.258\n",
      "Epoch 14 / 100, example 48 / 104\n",
      "X len:  70 Y len:  70\n",
      "X:  (70, 1)\n",
      "Y:  (70, 1)\n",
      "[14,    68] loss: 0.225\n",
      "Epoch 14 / 100, example 68 / 104\n",
      "X len:  71 Y len:  71\n",
      "X:  (71, 1)\n",
      "Y:  (71, 1)\n",
      "[14,    88] loss: 0.252\n",
      "Epoch 14 / 100, example 88 / 104\n",
      "X len:  72 Y len:  72\n",
      "X:  (72, 1)\n",
      "Y:  (72, 1)\n",
      "[15,     4] loss: 0.063\n",
      "Epoch 15 / 100, example 4 / 104\n",
      "X len:  73 Y len:  73\n",
      "X:  (73, 1)\n",
      "Y:  (73, 1)\n",
      "[15,    24] loss: 0.284\n",
      "Epoch 15 / 100, example 24 / 104\n",
      "X len:  74 Y len:  74\n",
      "X:  (74, 1)\n",
      "Y:  (74, 1)\n",
      "[15,    44] loss: 0.301\n",
      "Epoch 15 / 100, example 44 / 104\n",
      "X len:  75 Y len:  75\n",
      "X:  (75, 1)\n",
      "Y:  (75, 1)\n",
      "[15,    64] loss: 0.213\n",
      "Epoch 15 / 100, example 64 / 104\n",
      "X len:  76 Y len:  76\n",
      "X:  (76, 1)\n",
      "Y:  (76, 1)\n",
      "[15,    84] loss: 0.257\n",
      "Epoch 15 / 100, example 84 / 104\n",
      "X len:  77 Y len:  77\n",
      "X:  (77, 1)\n",
      "Y:  (77, 1)\n",
      "[15,   104] loss: 0.279\n",
      "Epoch 15 / 100, example 104 / 104\n",
      "X len:  78 Y len:  78\n",
      "X:  (78, 1)\n",
      "Y:  (78, 1)\n",
      "[16,    20] loss: 0.287\n",
      "Epoch 16 / 100, example 20 / 104\n",
      "X len:  79 Y len:  79\n",
      "X:  (79, 1)\n",
      "Y:  (79, 1)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[16,    40] loss: 0.352\n",
      "Epoch 16 / 100, example 40 / 104\n",
      "X len:  80 Y len:  80\n",
      "X:  (80, 1)\n",
      "Y:  (80, 1)\n",
      "[16,    60] loss: 0.208\n",
      "Epoch 16 / 100, example 60 / 104\n",
      "X len:  81 Y len:  81\n",
      "X:  (81, 1)\n",
      "Y:  (81, 1)\n",
      "[16,    80] loss: 0.236\n",
      "Epoch 16 / 100, example 80 / 104\n",
      "X len:  82 Y len:  82\n",
      "X:  (82, 1)\n",
      "Y:  (82, 1)\n",
      "[16,   100] loss: 0.267\n",
      "Epoch 16 / 100, example 100 / 104\n",
      "X len:  83 Y len:  83\n",
      "X:  (83, 1)\n",
      "Y:  (83, 1)\n",
      "saving the model at the end of epoch 15, iters 103\n",
      "[17,    16] loss: 0.237\n",
      "Epoch 17 / 100, example 16 / 104\n",
      "X len:  84 Y len:  84\n",
      "X:  (84, 1)\n",
      "Y:  (84, 1)\n",
      "[17,    36] loss: 0.321\n",
      "Epoch 17 / 100, example 36 / 104\n",
      "X len:  85 Y len:  85\n",
      "X:  (85, 1)\n",
      "Y:  (85, 1)\n",
      "[17,    56] loss: 0.228\n",
      "Epoch 17 / 100, example 56 / 104\n",
      "X len:  86 Y len:  86\n",
      "X:  (86, 1)\n",
      "Y:  (86, 1)\n",
      "[17,    76] loss: 0.233\n",
      "Epoch 17 / 100, example 76 / 104\n",
      "X len:  87 Y len:  87\n",
      "X:  (87, 1)\n",
      "Y:  (87, 1)\n",
      "[17,    96] loss: 0.258\n",
      "Epoch 17 / 100, example 96 / 104\n",
      "X len:  88 Y len:  88\n",
      "X:  (88, 1)\n",
      "Y:  (88, 1)\n",
      "[18,    12] loss: 0.172\n",
      "Epoch 18 / 100, example 12 / 104\n",
      "X len:  89 Y len:  89\n",
      "X:  (89, 1)\n",
      "Y:  (89, 1)\n",
      "[18,    32] loss: 0.325\n",
      "Epoch 18 / 100, example 32 / 104\n",
      "X len:  90 Y len:  90\n",
      "X:  (90, 1)\n",
      "Y:  (90, 1)\n",
      "[18,    52] loss: 0.232\n",
      "Epoch 18 / 100, example 52 / 104\n",
      "X len:  91 Y len:  91\n",
      "X:  (91, 1)\n",
      "Y:  (91, 1)\n",
      "[18,    72] loss: 0.252\n",
      "Epoch 18 / 100, example 72 / 104\n",
      "X len:  92 Y len:  92\n",
      "X:  (92, 1)\n",
      "Y:  (92, 1)\n",
      "[18,    92] loss: 0.213\n",
      "Epoch 18 / 100, example 92 / 104\n",
      "X len:  93 Y len:  93\n",
      "X:  (93, 1)\n",
      "Y:  (93, 1)\n",
      "[19,     8] loss: 0.119\n",
      "Epoch 19 / 100, example 8 / 104\n",
      "X len:  94 Y len:  94\n",
      "X:  (94, 1)\n",
      "Y:  (94, 1)\n",
      "[19,    28] loss: 0.304\n",
      "Epoch 19 / 100, example 28 / 104\n",
      "X len:  95 Y len:  95\n",
      "X:  (95, 1)\n",
      "Y:  (95, 1)\n",
      "[19,    48] loss: 0.259\n",
      "Epoch 19 / 100, example 48 / 104\n",
      "X len:  96 Y len:  96\n",
      "X:  (96, 1)\n",
      "Y:  (96, 1)\n",
      "[19,    68] loss: 0.222\n",
      "Epoch 19 / 100, example 68 / 104\n",
      "X len:  97 Y len:  97\n",
      "X:  (97, 1)\n",
      "Y:  (97, 1)\n",
      "[19,    88] loss: 0.248\n",
      "Epoch 19 / 100, example 88 / 104\n",
      "X len:  98 Y len:  98\n",
      "X:  (98, 1)\n",
      "Y:  (98, 1)\n",
      "[20,     4] loss: 0.063\n",
      "Epoch 20 / 100, example 4 / 104\n",
      "X len:  99 Y len:  99\n",
      "X:  (99, 1)\n",
      "Y:  (99, 1)\n",
      "[20,    24] loss: 0.280\n",
      "Epoch 20 / 100, example 24 / 104\n",
      "X len:  100 Y len:  100\n",
      "X:  (100, 1)\n",
      "Y:  (100, 1)\n",
      "[20,    44] loss: 0.302\n",
      "Epoch 20 / 100, example 44 / 104\n",
      "X len:  101 Y len:  101\n",
      "X:  (101, 1)\n",
      "Y:  (101, 1)\n",
      "[20,    64] loss: 0.209\n",
      "Epoch 20 / 100, example 64 / 104\n",
      "X len:  102 Y len:  102\n",
      "X:  (102, 1)\n",
      "Y:  (102, 1)\n",
      "[20,    84] loss: 0.251\n",
      "Epoch 20 / 100, example 84 / 104\n",
      "X len:  103 Y len:  103\n",
      "X:  (103, 1)\n",
      "Y:  (103, 1)\n",
      "[20,   104] loss: 0.271\n",
      "Epoch 20 / 100, example 104 / 104\n",
      "X len:  104 Y len:  104\n",
      "X:  (104, 1)\n",
      "Y:  (104, 1)\n",
      "[21,    20] loss: 0.284\n",
      "Epoch 21 / 100, example 20 / 104\n",
      "X len:  105 Y len:  105\n",
      "X:  (105, 1)\n",
      "Y:  (105, 1)\n",
      "[21,    40] loss: 0.341\n",
      "Epoch 21 / 100, example 40 / 104\n",
      "X len:  106 Y len:  106\n",
      "X:  (106, 1)\n",
      "Y:  (106, 1)\n",
      "[21,    60] loss: 0.200\n",
      "Epoch 21 / 100, example 60 / 104\n",
      "X len:  107 Y len:  107\n",
      "X:  (107, 1)\n",
      "Y:  (107, 1)\n",
      "[21,    80] loss: 0.234\n",
      "Epoch 21 / 100, example 80 / 104\n",
      "X len:  108 Y len:  108\n",
      "X:  (108, 1)\n",
      "Y:  (108, 1)\n",
      "[21,   100] loss: 0.256\n",
      "Epoch 21 / 100, example 100 / 104\n",
      "X len:  109 Y len:  109\n",
      "X:  (109, 1)\n",
      "Y:  (109, 1)\n",
      "saving the model at the end of epoch 20, iters 103\n",
      "[22,    16] loss: 0.233\n",
      "Epoch 22 / 100, example 16 / 104\n",
      "X len:  110 Y len:  110\n",
      "X:  (110, 1)\n",
      "Y:  (110, 1)\n",
      "[22,    36] loss: 0.311\n",
      "Epoch 22 / 100, example 36 / 104\n",
      "X len:  111 Y len:  111\n",
      "X:  (111, 1)\n",
      "Y:  (111, 1)\n",
      "[22,    56] loss: 0.219\n",
      "Epoch 22 / 100, example 56 / 104\n",
      "X len:  112 Y len:  112\n",
      "X:  (112, 1)\n",
      "Y:  (112, 1)\n",
      "[22,    76] loss: 0.234\n",
      "Epoch 22 / 100, example 76 / 104\n",
      "X len:  113 Y len:  113\n",
      "X:  (113, 1)\n",
      "Y:  (113, 1)\n",
      "[22,    96] loss: 0.250\n",
      "Epoch 22 / 100, example 96 / 104\n",
      "X len:  114 Y len:  114\n",
      "X:  (114, 1)\n",
      "Y:  (114, 1)\n",
      "[23,    12] loss: 0.170\n",
      "Epoch 23 / 100, example 12 / 104\n",
      "X len:  115 Y len:  115\n",
      "X:  (115, 1)\n",
      "Y:  (115, 1)\n",
      "[23,    32] loss: 0.317\n",
      "Epoch 23 / 100, example 32 / 104\n",
      "X len:  116 Y len:  116\n",
      "X:  (116, 1)\n",
      "Y:  (116, 1)\n",
      "[23,    52] loss: 0.227\n",
      "Epoch 23 / 100, example 52 / 104\n",
      "X len:  117 Y len:  117\n",
      "X:  (117, 1)\n",
      "Y:  (117, 1)\n",
      "[23,    72] loss: 0.245\n",
      "Epoch 23 / 100, example 72 / 104\n",
      "X len:  118 Y len:  118\n",
      "X:  (118, 1)\n",
      "Y:  (118, 1)\n",
      "[23,    92] loss: 0.205\n",
      "Epoch 23 / 100, example 92 / 104\n",
      "X len:  119 Y len:  119\n",
      "X:  (119, 1)\n",
      "Y:  (119, 1)\n",
      "[24,     8] loss: 0.118\n",
      "Epoch 24 / 100, example 8 / 104\n",
      "X len:  120 Y len:  120\n",
      "X:  (120, 1)\n",
      "Y:  (120, 1)\n",
      "[24,    28] loss: 0.300\n",
      "Epoch 24 / 100, example 28 / 104\n",
      "X len:  121 Y len:  121\n",
      "X:  (121, 1)\n",
      "Y:  (121, 1)\n",
      "[24,    48] loss: 0.252\n",
      "Epoch 24 / 100, example 48 / 104\n",
      "X len:  122 Y len:  122\n",
      "X:  (122, 1)\n",
      "Y:  (122, 1)\n",
      "[24,    68] loss: 0.220\n",
      "Epoch 24 / 100, example 68 / 104\n",
      "X len:  123 Y len:  123\n",
      "X:  (123, 1)\n",
      "Y:  (123, 1)\n",
      "[24,    88] loss: 0.241\n",
      "Epoch 24 / 100, example 88 / 104\n",
      "X len:  124 Y len:  124\n",
      "X:  (124, 1)\n",
      "Y:  (124, 1)\n",
      "[25,     4] loss: 0.064\n",
      "Epoch 25 / 100, example 4 / 104\n",
      "X len:  125 Y len:  125\n",
      "X:  (125, 1)\n",
      "Y:  (125, 1)\n",
      "[25,    24] loss: 0.274\n",
      "Epoch 25 / 100, example 24 / 104\n",
      "X len:  126 Y len:  126\n",
      "X:  (126, 1)\n",
      "Y:  (126, 1)\n",
      "[25,    44] loss: 0.296\n",
      "Epoch 25 / 100, example 44 / 104\n",
      "X len:  127 Y len:  127\n",
      "X:  (127, 1)\n",
      "Y:  (127, 1)\n",
      "[25,    64] loss: 0.193\n",
      "Epoch 25 / 100, example 64 / 104\n",
      "X len:  128 Y len:  128\n",
      "X:  (128, 1)\n",
      "Y:  (128, 1)\n",
      "[25,    84] loss: 0.246\n",
      "Epoch 25 / 100, example 84 / 104\n",
      "X len:  129 Y len:  129\n",
      "X:  (129, 1)\n",
      "Y:  (129, 1)\n",
      "[25,   104] loss: 0.260\n",
      "Epoch 25 / 100, example 104 / 104\n",
      "X len:  130 Y len:  130\n",
      "X:  (130, 1)\n",
      "Y:  (130, 1)\n",
      "[26,    20] loss: 0.280\n",
      "Epoch 26 / 100, example 20 / 104\n",
      "X len:  131 Y len:  131\n",
      "X:  (131, 1)\n",
      "Y:  (131, 1)\n",
      "[26,    40] loss: 0.337\n",
      "Epoch 26 / 100, example 40 / 104\n",
      "X len:  132 Y len:  132\n",
      "X:  (132, 1)\n",
      "Y:  (132, 1)\n",
      "[26,    60] loss: 0.186\n",
      "Epoch 26 / 100, example 60 / 104\n",
      "X len:  133 Y len:  133\n",
      "X:  (133, 1)\n",
      "Y:  (133, 1)\n",
      "[26,    80] loss: 0.230\n",
      "Epoch 26 / 100, example 80 / 104\n",
      "X len:  134 Y len:  134\n",
      "X:  (134, 1)\n",
      "Y:  (134, 1)\n",
      "[26,   100] loss: 0.239\n",
      "Epoch 26 / 100, example 100 / 104\n",
      "X len:  135 Y len:  135\n",
      "X:  (135, 1)\n",
      "Y:  (135, 1)\n",
      "saving the model at the end of epoch 25, iters 103\n",
      "[27,    16] loss: 0.228\n",
      "Epoch 27 / 100, example 16 / 104\n",
      "X len:  136 Y len:  136\n",
      "X:  (136, 1)\n",
      "Y:  (136, 1)\n",
      "[27,    36] loss: 0.304\n",
      "Epoch 27 / 100, example 36 / 104\n",
      "X len:  137 Y len:  137\n",
      "X:  (137, 1)\n",
      "Y:  (137, 1)\n",
      "[27,    56] loss: 0.214\n",
      "Epoch 27 / 100, example 56 / 104\n",
      "X len:  138 Y len:  138\n",
      "X:  (138, 1)\n",
      "Y:  (138, 1)\n",
      "[27,    76] loss: 0.214\n",
      "Epoch 27 / 100, example 76 / 104\n",
      "X len:  139 Y len:  139\n",
      "X:  (139, 1)\n",
      "Y:  (139, 1)\n",
      "[27,    96] loss: 0.237\n",
      "Epoch 27 / 100, example 96 / 104\n",
      "X len:  140 Y len:  140\n",
      "X:  (140, 1)\n",
      "Y:  (140, 1)\n",
      "[28,    12] loss: 0.167\n",
      "Epoch 28 / 100, example 12 / 104\n",
      "X len:  141 Y len:  141\n",
      "X:  (141, 1)\n",
      "Y:  (141, 1)\n",
      "[28,    32] loss: 0.309\n",
      "Epoch 28 / 100, example 32 / 104\n",
      "X len:  142 Y len:  142\n",
      "X:  (142, 1)\n",
      "Y:  (142, 1)\n",
      "[28,    52] loss: 0.219\n",
      "Epoch 28 / 100, example 52 / 104\n",
      "X len:  143 Y len:  143\n",
      "X:  (143, 1)\n",
      "Y:  (143, 1)\n",
      "[28,    72] loss: 0.232\n",
      "Epoch 28 / 100, example 72 / 104\n",
      "X len:  144 Y len:  144\n",
      "X:  (144, 1)\n",
      "Y:  (144, 1)\n",
      "[28,    92] loss: 0.195\n",
      "Epoch 28 / 100, example 92 / 104\n",
      "X len:  145 Y len:  145\n",
      "X:  (145, 1)\n",
      "Y:  (145, 1)\n",
      "[29,     8] loss: 0.116\n",
      "Epoch 29 / 100, example 8 / 104\n",
      "X len:  146 Y len:  146\n",
      "X:  (146, 1)\n",
      "Y:  (146, 1)\n",
      "[29,    28] loss: 0.292\n",
      "Epoch 29 / 100, example 28 / 104\n",
      "X len:  147 Y len:  147\n",
      "X:  (147, 1)\n",
      "Y:  (147, 1)\n",
      "[29,    48] loss: 0.241\n",
      "Epoch 29 / 100, example 48 / 104\n",
      "X len:  148 Y len:  148\n",
      "X:  (148, 1)\n",
      "Y:  (148, 1)\n",
      "[29,    68] loss: 0.195\n",
      "Epoch 29 / 100, example 68 / 104\n",
      "X len:  149 Y len:  149\n",
      "X:  (149, 1)\n",
      "Y:  (149, 1)\n",
      "[29,    88] loss: 0.239\n",
      "Epoch 29 / 100, example 88 / 104\n",
      "X len:  150 Y len:  150\n",
      "X:  (150, 1)\n",
      "Y:  (150, 1)\n",
      "[30,     4] loss: 0.066\n",
      "Epoch 30 / 100, example 4 / 104\n",
      "X len:  151 Y len:  151\n",
      "X:  (151, 1)\n",
      "Y:  (151, 1)\n",
      "[30,    24] loss: 0.261\n",
      "Epoch 30 / 100, example 24 / 104\n",
      "X len:  152 Y len:  152\n",
      "X:  (152, 1)\n",
      "Y:  (152, 1)\n",
      "[30,    44] loss: 0.291\n",
      "Epoch 30 / 100, example 44 / 104\n",
      "X len:  153 Y len:  153\n",
      "X:  (153, 1)\n",
      "Y:  (153, 1)\n",
      "[30,    64] loss: 0.177\n",
      "Epoch 30 / 100, example 64 / 104\n",
      "X len:  154 Y len:  154\n",
      "X:  (154, 1)\n",
      "Y:  (154, 1)\n",
      "[30,    84] loss: 0.240\n",
      "Epoch 30 / 100, example 84 / 104\n",
      "X len:  155 Y len:  155\n",
      "X:  (155, 1)\n",
      "Y:  (155, 1)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[30,   104] loss: 0.253\n",
      "Epoch 30 / 100, example 104 / 104\n",
      "X len:  156 Y len:  156\n",
      "X:  (156, 1)\n",
      "Y:  (156, 1)\n",
      "[31,    20] loss: 0.268\n",
      "Epoch 31 / 100, example 20 / 104\n",
      "X len:  157 Y len:  157\n",
      "X:  (157, 1)\n",
      "Y:  (157, 1)\n",
      "[31,    40] loss: 0.329\n",
      "Epoch 31 / 100, example 40 / 104\n",
      "X len:  158 Y len:  158\n",
      "X:  (158, 1)\n",
      "Y:  (158, 1)\n",
      "[31,    60] loss: 0.172\n",
      "Epoch 31 / 100, example 60 / 104\n",
      "X len:  159 Y len:  159\n",
      "X:  (159, 1)\n",
      "Y:  (159, 1)\n",
      "[31,    80] loss: 0.211\n",
      "Epoch 31 / 100, example 80 / 104\n",
      "X len:  160 Y len:  160\n",
      "X:  (160, 1)\n",
      "Y:  (160, 1)\n",
      "[31,   100] loss: 0.234\n",
      "Epoch 31 / 100, example 100 / 104\n",
      "X len:  161 Y len:  161\n",
      "X:  (161, 1)\n",
      "Y:  (161, 1)\n",
      "saving the model at the end of epoch 30, iters 103\n",
      "[32,    16] loss: 0.220\n",
      "Epoch 32 / 100, example 16 / 104\n",
      "X len:  162 Y len:  162\n",
      "X:  (162, 1)\n",
      "Y:  (162, 1)\n",
      "[32,    36] loss: 0.295\n",
      "Epoch 32 / 100, example 36 / 104\n",
      "X len:  163 Y len:  163\n",
      "X:  (163, 1)\n",
      "Y:  (163, 1)\n",
      "[32,    56] loss: 0.208\n",
      "Epoch 32 / 100, example 56 / 104\n",
      "X len:  164 Y len:  164\n",
      "X:  (164, 1)\n",
      "Y:  (164, 1)\n",
      "[32,    76] loss: 0.197\n",
      "Epoch 32 / 100, example 76 / 104\n",
      "X len:  165 Y len:  165\n",
      "X:  (165, 1)\n",
      "Y:  (165, 1)\n",
      "[32,    96] loss: 0.230\n",
      "Epoch 32 / 100, example 96 / 104\n",
      "X len:  166 Y len:  166\n",
      "X:  (166, 1)\n",
      "Y:  (166, 1)\n",
      "[33,    12] loss: 0.165\n",
      "Epoch 33 / 100, example 12 / 104\n",
      "X len:  167 Y len:  167\n",
      "X:  (167, 1)\n",
      "Y:  (167, 1)\n",
      "[33,    32] loss: 0.307\n",
      "Epoch 33 / 100, example 32 / 104\n",
      "X len:  168 Y len:  168\n",
      "X:  (168, 1)\n",
      "Y:  (168, 1)\n",
      "[33,    52] loss: 0.224\n",
      "Epoch 33 / 100, example 52 / 104\n",
      "X len:  169 Y len:  169\n",
      "X:  (169, 1)\n",
      "Y:  (169, 1)\n",
      "[33,    72] loss: 0.231\n",
      "Epoch 33 / 100, example 72 / 104\n",
      "X len:  170 Y len:  170\n",
      "X:  (170, 1)\n",
      "Y:  (170, 1)\n",
      "[33,    92] loss: 0.187\n",
      "Epoch 33 / 100, example 92 / 104\n",
      "X len:  171 Y len:  171\n",
      "X:  (171, 1)\n",
      "Y:  (171, 1)\n",
      "[34,     8] loss: 0.116\n",
      "Epoch 34 / 100, example 8 / 104\n",
      "X len:  172 Y len:  172\n",
      "X:  (172, 1)\n",
      "Y:  (172, 1)\n",
      "[34,    28] loss: 0.278\n",
      "Epoch 34 / 100, example 28 / 104\n",
      "X len:  173 Y len:  173\n",
      "X:  (173, 1)\n",
      "Y:  (173, 1)\n",
      "[34,    48] loss: 0.235\n",
      "Epoch 34 / 100, example 48 / 104\n",
      "X len:  174 Y len:  174\n",
      "X:  (174, 1)\n",
      "Y:  (174, 1)\n",
      "[34,    68] loss: 0.184\n",
      "Epoch 34 / 100, example 68 / 104\n",
      "X len:  175 Y len:  175\n",
      "X:  (175, 1)\n",
      "Y:  (175, 1)\n",
      "[34,    88] loss: 0.211\n",
      "Epoch 34 / 100, example 88 / 104\n",
      "X len:  176 Y len:  176\n",
      "X:  (176, 1)\n",
      "Y:  (176, 1)\n",
      "[35,     4] loss: 0.069\n",
      "Epoch 35 / 100, example 4 / 104\n",
      "X len:  177 Y len:  177\n",
      "X:  (177, 1)\n",
      "Y:  (177, 1)\n",
      "[35,    24] loss: 0.247\n",
      "Epoch 35 / 100, example 24 / 104\n",
      "X len:  178 Y len:  178\n",
      "X:  (178, 1)\n",
      "Y:  (178, 1)\n",
      "[35,    44] loss: 0.287\n",
      "Epoch 35 / 100, example 44 / 104\n",
      "X len:  179 Y len:  179\n",
      "X:  (179, 1)\n",
      "Y:  (179, 1)\n",
      "[35,    64] loss: 0.163\n",
      "Epoch 35 / 100, example 64 / 104\n",
      "X len:  180 Y len:  180\n",
      "X:  (180, 1)\n",
      "Y:  (180, 1)\n",
      "[35,    84] loss: 0.217\n",
      "Epoch 35 / 100, example 84 / 104\n",
      "X len:  181 Y len:  181\n",
      "X:  (181, 1)\n",
      "Y:  (181, 1)\n",
      "[35,   104] loss: 0.232\n",
      "Epoch 35 / 100, example 104 / 104\n",
      "X len:  182 Y len:  182\n",
      "X:  (182, 1)\n",
      "Y:  (182, 1)\n",
      "[36,    20] loss: 0.263\n",
      "Epoch 36 / 100, example 20 / 104\n",
      "X len:  183 Y len:  183\n",
      "X:  (183, 1)\n",
      "Y:  (183, 1)\n",
      "[36,    40] loss: 0.327\n",
      "Epoch 36 / 100, example 40 / 104\n",
      "X len:  184 Y len:  184\n",
      "X:  (184, 1)\n",
      "Y:  (184, 1)\n",
      "[36,    60] loss: 0.164\n",
      "Epoch 36 / 100, example 60 / 104\n",
      "X len:  185 Y len:  185\n",
      "X:  (185, 1)\n",
      "Y:  (185, 1)\n",
      "[36,    80] loss: 0.197\n",
      "Epoch 36 / 100, example 80 / 104\n",
      "X len:  186 Y len:  186\n",
      "X:  (186, 1)\n",
      "Y:  (186, 1)\n",
      "[36,   100] loss: 0.226\n",
      "Epoch 36 / 100, example 100 / 104\n",
      "X len:  187 Y len:  187\n",
      "X:  (187, 1)\n",
      "Y:  (187, 1)\n",
      "saving the model at the end of epoch 35, iters 103\n",
      "[37,    16] loss: 0.218\n",
      "Epoch 37 / 100, example 16 / 104\n",
      "X len:  188 Y len:  188\n",
      "X:  (188, 1)\n",
      "Y:  (188, 1)\n",
      "[37,    36] loss: 0.293\n",
      "Epoch 37 / 100, example 36 / 104\n",
      "X len:  189 Y len:  189\n",
      "X:  (189, 1)\n",
      "Y:  (189, 1)\n",
      "[37,    56] loss: 0.205\n",
      "Epoch 37 / 100, example 56 / 104\n",
      "X len:  190 Y len:  190\n",
      "X:  (190, 1)\n",
      "Y:  (190, 1)\n",
      "[37,    76] loss: 0.183\n",
      "Epoch 37 / 100, example 76 / 104\n",
      "X len:  191 Y len:  191\n",
      "X:  (191, 1)\n",
      "Y:  (191, 1)\n",
      "[37,    96] loss: 0.205\n",
      "Epoch 37 / 100, example 96 / 104\n",
      "X len:  192 Y len:  192\n",
      "X:  (192, 1)\n",
      "Y:  (192, 1)\n",
      "[38,    12] loss: 0.167\n",
      "Epoch 38 / 100, example 12 / 104\n",
      "X len:  193 Y len:  193\n",
      "X:  (193, 1)\n",
      "Y:  (193, 1)\n",
      "[38,    32] loss: 0.289\n",
      "Epoch 38 / 100, example 32 / 104\n",
      "X len:  194 Y len:  194\n",
      "X:  (194, 1)\n",
      "Y:  (194, 1)\n",
      "[38,    52] loss: 0.205\n",
      "Epoch 38 / 100, example 52 / 104\n",
      "X len:  195 Y len:  195\n",
      "X:  (195, 1)\n",
      "Y:  (195, 1)\n",
      "[38,    72] loss: 0.203\n",
      "Epoch 38 / 100, example 72 / 104\n",
      "X len:  196 Y len:  196\n",
      "X:  (196, 1)\n",
      "Y:  (196, 1)\n",
      "[38,    92] loss: 0.163\n",
      "Epoch 38 / 100, example 92 / 104\n",
      "X len:  197 Y len:  197\n",
      "X:  (197, 1)\n",
      "Y:  (197, 1)\n",
      "[39,     8] loss: 0.118\n",
      "Epoch 39 / 100, example 8 / 104\n",
      "X len:  198 Y len:  198\n",
      "X:  (198, 1)\n",
      "Y:  (198, 1)\n",
      "[39,    28] loss: 0.270\n",
      "Epoch 39 / 100, example 28 / 104\n",
      "X len:  199 Y len:  199\n",
      "X:  (199, 1)\n",
      "Y:  (199, 1)\n",
      "[39,    48] loss: 0.232\n",
      "Epoch 39 / 100, example 48 / 104\n",
      "X len:  200 Y len:  200\n",
      "X:  (200, 1)\n",
      "Y:  (200, 1)\n",
      "[39,    68] loss: 0.174\n",
      "Epoch 39 / 100, example 68 / 104\n",
      "X len:  201 Y len:  201\n",
      "X:  (201, 1)\n",
      "Y:  (201, 1)\n",
      "[39,    88] loss: 0.204\n",
      "Epoch 39 / 100, example 88 / 104\n",
      "X len:  202 Y len:  202\n",
      "X:  (202, 1)\n",
      "Y:  (202, 1)\n",
      "[40,     4] loss: 0.070\n",
      "Epoch 40 / 100, example 4 / 104\n",
      "X len:  203 Y len:  203\n",
      "X:  (203, 1)\n",
      "Y:  (203, 1)\n",
      "[40,    24] loss: 0.238\n",
      "Epoch 40 / 100, example 24 / 104\n",
      "X len:  204 Y len:  204\n",
      "X:  (204, 1)\n",
      "Y:  (204, 1)\n",
      "[40,    44] loss: 0.282\n",
      "Epoch 40 / 100, example 44 / 104\n",
      "X len:  205 Y len:  205\n",
      "X:  (205, 1)\n",
      "Y:  (205, 1)\n",
      "[40,    64] loss: 0.143\n",
      "Epoch 40 / 100, example 64 / 104\n",
      "X len:  206 Y len:  206\n",
      "X:  (206, 1)\n",
      "Y:  (206, 1)\n",
      "[40,    84] loss: 0.212\n",
      "Epoch 40 / 100, example 84 / 104\n",
      "X len:  207 Y len:  207\n",
      "X:  (207, 1)\n",
      "Y:  (207, 1)\n",
      "[40,   104] loss: 0.217\n",
      "Epoch 40 / 100, example 104 / 104\n",
      "X len:  208 Y len:  208\n",
      "X:  (208, 1)\n",
      "Y:  (208, 1)\n",
      "[41,    20] loss: 0.251\n",
      "Epoch 41 / 100, example 20 / 104\n",
      "X len:  209 Y len:  209\n",
      "X:  (209, 1)\n",
      "Y:  (209, 1)\n",
      "[41,    40] loss: 0.317\n",
      "Epoch 41 / 100, example 40 / 104\n",
      "X len:  210 Y len:  210\n",
      "X:  (210, 1)\n",
      "Y:  (210, 1)\n",
      "[41,    60] loss: 0.144\n",
      "Epoch 41 / 100, example 60 / 104\n",
      "X len:  211 Y len:  211\n",
      "X:  (211, 1)\n",
      "Y:  (211, 1)\n",
      "[41,    80] loss: 0.189\n",
      "Epoch 41 / 100, example 80 / 104\n",
      "X len:  212 Y len:  212\n",
      "X:  (212, 1)\n",
      "Y:  (212, 1)\n",
      "[41,   100] loss: 0.193\n",
      "Epoch 41 / 100, example 100 / 104\n",
      "X len:  213 Y len:  213\n",
      "X:  (213, 1)\n",
      "Y:  (213, 1)\n",
      "saving the model at the end of epoch 40, iters 103\n",
      "[42,    16] loss: 0.215\n",
      "Epoch 42 / 100, example 16 / 104\n",
      "X len:  214 Y len:  214\n",
      "X:  (214, 1)\n",
      "Y:  (214, 1)\n",
      "[42,    36] loss: 0.274\n",
      "Epoch 42 / 100, example 36 / 104\n",
      "X len:  215 Y len:  215\n",
      "X:  (215, 1)\n",
      "Y:  (215, 1)\n",
      "[42,    56] loss: 0.184\n",
      "Epoch 42 / 100, example 56 / 104\n",
      "X len:  216 Y len:  216\n",
      "X:  (216, 1)\n",
      "Y:  (216, 1)\n",
      "[42,    76] loss: 0.185\n",
      "Epoch 42 / 100, example 76 / 104\n",
      "X len:  217 Y len:  217\n",
      "X:  (217, 1)\n",
      "Y:  (217, 1)\n",
      "[42,    96] loss: 0.201\n",
      "Epoch 42 / 100, example 96 / 104\n",
      "X len:  218 Y len:  218\n",
      "X:  (218, 1)\n",
      "Y:  (218, 1)\n",
      "[43,    12] loss: 0.164\n",
      "Epoch 43 / 100, example 12 / 104\n",
      "X len:  219 Y len:  219\n",
      "X:  (219, 1)\n",
      "Y:  (219, 1)\n",
      "[43,    32] loss: 0.265\n",
      "Epoch 43 / 100, example 32 / 104\n",
      "X len:  220 Y len:  220\n",
      "X:  (220, 1)\n",
      "Y:  (220, 1)\n",
      "[43,    52] loss: 0.192\n",
      "Epoch 43 / 100, example 52 / 104\n",
      "X len:  221 Y len:  221\n",
      "X:  (221, 1)\n",
      "Y:  (221, 1)\n",
      "[43,    72] loss: 0.191\n",
      "Epoch 43 / 100, example 72 / 104\n",
      "X len:  222 Y len:  222\n",
      "X:  (222, 1)\n",
      "Y:  (222, 1)\n",
      "[43,    92] loss: 0.150\n",
      "Epoch 43 / 100, example 92 / 104\n",
      "X len:  223 Y len:  223\n",
      "X:  (223, 1)\n",
      "Y:  (223, 1)\n",
      "[44,     8] loss: 0.118\n",
      "Epoch 44 / 100, example 8 / 104\n",
      "X len:  224 Y len:  224\n",
      "X:  (224, 1)\n",
      "Y:  (224, 1)\n",
      "[44,    28] loss: 0.261\n",
      "Epoch 44 / 100, example 28 / 104\n",
      "X len:  225 Y len:  225\n",
      "X:  (225, 1)\n",
      "Y:  (225, 1)\n",
      "[44,    48] loss: 0.212\n",
      "Epoch 44 / 100, example 48 / 104\n",
      "X len:  226 Y len:  226\n",
      "X:  (226, 1)\n",
      "Y:  (226, 1)\n",
      "[44,    68] loss: 0.164\n",
      "Epoch 44 / 100, example 68 / 104\n",
      "X len:  227 Y len:  227\n",
      "X:  (227, 1)\n",
      "Y:  (227, 1)\n",
      "[44,    88] loss: 0.187\n",
      "Epoch 44 / 100, example 88 / 104\n",
      "X len:  228 Y len:  228\n",
      "X:  (228, 1)\n",
      "Y:  (228, 1)\n",
      "[45,     4] loss: 0.062\n",
      "Epoch 45 / 100, example 4 / 104\n",
      "X len:  229 Y len:  229\n",
      "X:  (229, 1)\n",
      "Y:  (229, 1)\n",
      "[45,    24] loss: 0.241\n",
      "Epoch 45 / 100, example 24 / 104\n",
      "X len:  230 Y len:  230\n",
      "X:  (230, 1)\n",
      "Y:  (230, 1)\n",
      "[45,    44] loss: 0.271\n",
      "Epoch 45 / 100, example 44 / 104\n",
      "X len:  231 Y len:  231\n",
      "X:  (231, 1)\n",
      "Y:  (231, 1)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[45,    64] loss: 0.125\n",
      "Epoch 45 / 100, example 64 / 104\n",
      "X len:  232 Y len:  232\n",
      "X:  (232, 1)\n",
      "Y:  (232, 1)\n",
      "[45,    84] loss: 0.191\n",
      "Epoch 45 / 100, example 84 / 104\n",
      "X len:  233 Y len:  233\n",
      "X:  (233, 1)\n",
      "Y:  (233, 1)\n",
      "[45,   104] loss: 0.199\n",
      "Epoch 45 / 100, example 104 / 104\n",
      "X len:  234 Y len:  234\n",
      "X:  (234, 1)\n",
      "Y:  (234, 1)\n",
      "[46,    20] loss: 0.244\n",
      "Epoch 46 / 100, example 20 / 104\n",
      "X len:  235 Y len:  235\n",
      "X:  (235, 1)\n",
      "Y:  (235, 1)\n",
      "[46,    40] loss: 0.290\n",
      "Epoch 46 / 100, example 40 / 104\n",
      "X len:  236 Y len:  236\n",
      "X:  (236, 1)\n",
      "Y:  (236, 1)\n",
      "[46,    60] loss: 0.126\n",
      "Epoch 46 / 100, example 60 / 104\n",
      "X len:  237 Y len:  237\n",
      "X:  (237, 1)\n",
      "Y:  (237, 1)\n",
      "[46,    80] loss: 0.196\n",
      "Epoch 46 / 100, example 80 / 104\n",
      "X len:  238 Y len:  238\n",
      "X:  (238, 1)\n",
      "Y:  (238, 1)\n",
      "[46,   100] loss: 0.178\n",
      "Epoch 46 / 100, example 100 / 104\n",
      "X len:  239 Y len:  239\n",
      "X:  (239, 1)\n",
      "Y:  (239, 1)\n",
      "saving the model at the end of epoch 45, iters 103\n",
      "[47,    16] loss: 0.209\n",
      "Epoch 47 / 100, example 16 / 104\n",
      "X len:  240 Y len:  240\n",
      "X:  (240, 1)\n",
      "Y:  (240, 1)\n",
      "[47,    36] loss: 0.253\n",
      "Epoch 47 / 100, example 36 / 104\n",
      "X len:  241 Y len:  241\n",
      "X:  (241, 1)\n",
      "Y:  (241, 1)\n",
      "[47,    56] loss: 0.157\n",
      "Epoch 47 / 100, example 56 / 104\n",
      "X len:  242 Y len:  242\n",
      "X:  (242, 1)\n",
      "Y:  (242, 1)\n",
      "[47,    76] loss: 0.191\n",
      "Epoch 47 / 100, example 76 / 104\n",
      "X len:  243 Y len:  243\n",
      "X:  (243, 1)\n",
      "Y:  (243, 1)\n",
      "[47,    96] loss: 0.191\n",
      "Epoch 47 / 100, example 96 / 104\n",
      "X len:  244 Y len:  244\n",
      "X:  (244, 1)\n",
      "Y:  (244, 1)\n",
      "[48,    12] loss: 0.151\n",
      "Epoch 48 / 100, example 12 / 104\n",
      "X len:  245 Y len:  245\n",
      "X:  (245, 1)\n",
      "Y:  (245, 1)\n",
      "[48,    32] loss: 0.241\n",
      "Epoch 48 / 100, example 32 / 104\n",
      "X len:  246 Y len:  246\n",
      "X:  (246, 1)\n",
      "Y:  (246, 1)\n",
      "[48,    52] loss: 0.164\n",
      "Epoch 48 / 100, example 52 / 104\n",
      "X len:  247 Y len:  247\n",
      "X:  (247, 1)\n",
      "Y:  (247, 1)\n",
      "[48,    72] loss: 0.173\n",
      "Epoch 48 / 100, example 72 / 104\n",
      "X len:  248 Y len:  248\n",
      "X:  (248, 1)\n",
      "Y:  (248, 1)\n",
      "[48,    92] loss: 0.139\n",
      "Epoch 48 / 100, example 92 / 104\n",
      "X len:  249 Y len:  249\n",
      "X:  (249, 1)\n",
      "Y:  (249, 1)\n",
      "[49,     8] loss: 0.094\n",
      "Epoch 49 / 100, example 8 / 104\n",
      "X len:  250 Y len:  250\n",
      "X:  (250, 1)\n",
      "Y:  (250, 1)\n",
      "[49,    28] loss: 0.241\n",
      "Epoch 49 / 100, example 28 / 104\n",
      "X len:  251 Y len:  251\n",
      "X:  (251, 1)\n",
      "Y:  (251, 1)\n",
      "[49,    48] loss: 0.172\n",
      "Epoch 49 / 100, example 48 / 104\n",
      "X len:  252 Y len:  252\n",
      "X:  (252, 1)\n",
      "Y:  (252, 1)\n",
      "[49,    68] loss: 0.135\n",
      "Epoch 49 / 100, example 68 / 104\n",
      "X len:  253 Y len:  253\n",
      "X:  (253, 1)\n",
      "Y:  (253, 1)\n",
      "[49,    88] loss: 0.170\n",
      "Epoch 49 / 100, example 88 / 104\n",
      "X len:  254 Y len:  254\n",
      "X:  (254, 1)\n",
      "Y:  (254, 1)\n",
      "[50,     4] loss: 0.041\n",
      "Epoch 50 / 100, example 4 / 104\n",
      "X len:  255 Y len:  255\n",
      "X:  (255, 1)\n",
      "Y:  (255, 1)\n",
      "[50,    24] loss: 0.241\n",
      "Epoch 50 / 100, example 24 / 104\n",
      "X len:  256 Y len:  256\n",
      "X:  (256, 1)\n",
      "Y:  (256, 1)\n",
      "[50,    44] loss: 0.213\n",
      "Epoch 50 / 100, example 44 / 104\n",
      "X len:  257 Y len:  257\n",
      "X:  (257, 1)\n",
      "Y:  (257, 1)\n",
      "[50,    64] loss: 0.087\n",
      "Epoch 50 / 100, example 64 / 104\n",
      "X len:  258 Y len:  258\n",
      "X:  (258, 1)\n",
      "Y:  (258, 1)\n",
      "[50,    84] loss: 0.192\n",
      "Epoch 50 / 100, example 84 / 104\n",
      "X len:  259 Y len:  259\n",
      "X:  (259, 1)\n",
      "Y:  (259, 1)\n",
      "[50,   104] loss: 0.200\n",
      "Epoch 50 / 100, example 104 / 104\n",
      "X len:  260 Y len:  260\n",
      "X:  (260, 1)\n",
      "Y:  (260, 1)\n",
      "[51,    20] loss: 0.225\n",
      "Epoch 51 / 100, example 20 / 104\n",
      "X len:  261 Y len:  261\n",
      "X:  (261, 1)\n",
      "Y:  (261, 1)\n",
      "[51,    40] loss: 0.234\n",
      "Epoch 51 / 100, example 40 / 104\n",
      "X len:  262 Y len:  262\n",
      "X:  (262, 1)\n",
      "Y:  (262, 1)\n",
      "[51,    60] loss: 0.093\n",
      "Epoch 51 / 100, example 60 / 104\n",
      "X len:  263 Y len:  263\n",
      "X:  (263, 1)\n",
      "Y:  (263, 1)\n",
      "[51,    80] loss: 0.172\n",
      "Epoch 51 / 100, example 80 / 104\n",
      "X len:  264 Y len:  264\n",
      "X:  (264, 1)\n",
      "Y:  (264, 1)\n",
      "[51,   100] loss: 0.190\n",
      "Epoch 51 / 100, example 100 / 104\n",
      "X len:  265 Y len:  265\n",
      "X:  (265, 1)\n",
      "Y:  (265, 1)\n",
      "saving the model at the end of epoch 50, iters 103\n",
      "[52,    16] loss: 0.165\n",
      "Epoch 52 / 100, example 16 / 104\n",
      "X len:  266 Y len:  266\n",
      "X:  (266, 1)\n",
      "Y:  (266, 1)\n",
      "[52,    36] loss: 0.234\n",
      "Epoch 52 / 100, example 36 / 104\n",
      "X len:  267 Y len:  267\n",
      "X:  (267, 1)\n",
      "Y:  (267, 1)\n",
      "[52,    56] loss: 0.109\n",
      "Epoch 52 / 100, example 56 / 104\n",
      "X len:  268 Y len:  268\n",
      "X:  (268, 1)\n",
      "Y:  (268, 1)\n",
      "[52,    76] loss: 0.158\n",
      "Epoch 52 / 100, example 76 / 104\n",
      "X len:  269 Y len:  269\n",
      "X:  (269, 1)\n",
      "Y:  (269, 1)\n",
      "[52,    96] loss: 0.165\n",
      "Epoch 52 / 100, example 96 / 104\n",
      "X len:  270 Y len:  270\n",
      "X:  (270, 1)\n",
      "Y:  (270, 1)\n",
      "[53,    12] loss: 0.122\n",
      "Epoch 53 / 100, example 12 / 104\n",
      "X len:  271 Y len:  271\n",
      "X:  (271, 1)\n",
      "Y:  (271, 1)\n",
      "[53,    32] loss: 0.214\n",
      "Epoch 53 / 100, example 32 / 104\n",
      "X len:  272 Y len:  272\n",
      "X:  (272, 1)\n",
      "Y:  (272, 1)\n",
      "[53,    52] loss: 0.127\n",
      "Epoch 53 / 100, example 52 / 104\n",
      "X len:  273 Y len:  273\n",
      "X:  (273, 1)\n",
      "Y:  (273, 1)\n",
      "[53,    72] loss: 0.147\n",
      "Epoch 53 / 100, example 72 / 104\n",
      "X len:  274 Y len:  274\n",
      "X:  (274, 1)\n",
      "Y:  (274, 1)\n",
      "[53,    92] loss: 0.134\n",
      "Epoch 53 / 100, example 92 / 104\n",
      "X len:  275 Y len:  275\n",
      "X:  (275, 1)\n",
      "Y:  (275, 1)\n",
      "[54,     8] loss: 0.071\n",
      "Epoch 54 / 100, example 8 / 104\n",
      "X len:  276 Y len:  276\n",
      "X:  (276, 1)\n",
      "Y:  (276, 1)\n",
      "[54,    28] loss: 0.222\n",
      "Epoch 54 / 100, example 28 / 104\n",
      "X len:  277 Y len:  277\n",
      "X:  (277, 1)\n",
      "Y:  (277, 1)\n",
      "[54,    48] loss: 0.134\n",
      "Epoch 54 / 100, example 48 / 104\n",
      "X len:  278 Y len:  278\n",
      "X:  (278, 1)\n",
      "Y:  (278, 1)\n",
      "[54,    68] loss: 0.101\n",
      "Epoch 54 / 100, example 68 / 104\n",
      "X len:  279 Y len:  279\n",
      "X:  (279, 1)\n",
      "Y:  (279, 1)\n",
      "[54,    88] loss: 0.146\n",
      "Epoch 54 / 100, example 88 / 104\n",
      "X len:  280 Y len:  280\n",
      "X:  (280, 1)\n",
      "Y:  (280, 1)\n",
      "[55,     4] loss: 0.016\n",
      "Epoch 55 / 100, example 4 / 104\n",
      "X len:  281 Y len:  281\n",
      "X:  (281, 1)\n",
      "Y:  (281, 1)\n",
      "[55,    24] loss: 0.183\n",
      "Epoch 55 / 100, example 24 / 104\n",
      "X len:  282 Y len:  282\n",
      "X:  (282, 1)\n",
      "Y:  (282, 1)\n",
      "[55,    44] loss: 0.156\n",
      "Epoch 55 / 100, example 44 / 104\n",
      "X len:  283 Y len:  283\n",
      "X:  (283, 1)\n",
      "Y:  (283, 1)\n",
      "[55,    64] loss: 0.073\n",
      "Epoch 55 / 100, example 64 / 104\n",
      "X len:  284 Y len:  284\n",
      "X:  (284, 1)\n",
      "Y:  (284, 1)\n",
      "[55,    84] loss: 0.209\n",
      "Epoch 55 / 100, example 84 / 104\n",
      "X len:  285 Y len:  285\n",
      "X:  (285, 1)\n",
      "Y:  (285, 1)\n",
      "[55,   104] loss: 0.193\n",
      "Epoch 55 / 100, example 104 / 104\n",
      "X len:  286 Y len:  286\n",
      "X:  (286, 1)\n",
      "Y:  (286, 1)\n",
      "[56,    20] loss: 0.162\n",
      "Epoch 56 / 100, example 20 / 104\n",
      "X len:  287 Y len:  287\n",
      "X:  (287, 1)\n",
      "Y:  (287, 1)\n",
      "[56,    40] loss: 0.217\n",
      "Epoch 56 / 100, example 40 / 104\n",
      "X len:  288 Y len:  288\n",
      "X:  (288, 1)\n",
      "Y:  (288, 1)\n",
      "[56,    60] loss: 0.087\n",
      "Epoch 56 / 100, example 60 / 104\n",
      "X len:  289 Y len:  289\n",
      "X:  (289, 1)\n",
      "Y:  (289, 1)\n",
      "[56,    80] loss: 0.168\n",
      "Epoch 56 / 100, example 80 / 104\n",
      "X len:  290 Y len:  290\n",
      "X:  (290, 1)\n",
      "Y:  (290, 1)\n",
      "[56,   100] loss: 0.156\n",
      "Epoch 56 / 100, example 100 / 104\n",
      "X len:  291 Y len:  291\n",
      "X:  (291, 1)\n",
      "Y:  (291, 1)\n",
      "saving the model at the end of epoch 55, iters 103\n",
      "[57,    16] loss: 0.103\n",
      "Epoch 57 / 100, example 16 / 104\n",
      "X len:  292 Y len:  292\n",
      "X:  (292, 1)\n",
      "Y:  (292, 1)\n",
      "[57,    36] loss: 0.197\n",
      "Epoch 57 / 100, example 36 / 104\n",
      "X len:  293 Y len:  293\n",
      "X:  (293, 1)\n",
      "Y:  (293, 1)\n",
      "[57,    56] loss: 0.093\n",
      "Epoch 57 / 100, example 56 / 104\n",
      "X len:  294 Y len:  294\n",
      "X:  (294, 1)\n",
      "Y:  (294, 1)\n",
      "[57,    76] loss: 0.174\n",
      "Epoch 57 / 100, example 76 / 104\n",
      "X len:  295 Y len:  295\n",
      "X:  (295, 1)\n",
      "Y:  (295, 1)\n",
      "[57,    96] loss: 0.175\n",
      "Epoch 57 / 100, example 96 / 104\n",
      "X len:  296 Y len:  296\n",
      "X:  (296, 1)\n",
      "Y:  (296, 1)\n",
      "[58,    12] loss: 0.058\n",
      "Epoch 58 / 100, example 12 / 104\n",
      "X len:  297 Y len:  297\n",
      "X:  (297, 1)\n",
      "Y:  (297, 1)\n",
      "[58,    32] loss: 0.130\n",
      "Epoch 58 / 100, example 32 / 104\n",
      "X len:  298 Y len:  298\n",
      "X:  (298, 1)\n",
      "Y:  (298, 1)\n",
      "[58,    52] loss: 0.128\n",
      "Epoch 58 / 100, example 52 / 104\n",
      "X len:  299 Y len:  299\n",
      "X:  (299, 1)\n",
      "Y:  (299, 1)\n",
      "[58,    72] loss: 0.148\n",
      "Epoch 58 / 100, example 72 / 104\n",
      "X len:  300 Y len:  300\n",
      "X:  (300, 1)\n",
      "Y:  (300, 1)\n",
      "[58,    92] loss: 0.129\n",
      "Epoch 58 / 100, example 92 / 104\n",
      "X len:  301 Y len:  301\n",
      "X:  (301, 1)\n",
      "Y:  (301, 1)\n",
      "[59,     8] loss: 0.028\n",
      "Epoch 59 / 100, example 8 / 104\n",
      "X len:  302 Y len:  302\n",
      "X:  (302, 1)\n",
      "Y:  (302, 1)\n",
      "[59,    28] loss: 0.174\n",
      "Epoch 59 / 100, example 28 / 104\n",
      "X len:  303 Y len:  303\n",
      "X:  (303, 1)\n",
      "Y:  (303, 1)\n",
      "[59,    48] loss: 0.127\n",
      "Epoch 59 / 100, example 48 / 104\n",
      "X len:  304 Y len:  304\n",
      "X:  (304, 1)\n",
      "Y:  (304, 1)\n",
      "[59,    68] loss: 0.074\n",
      "Epoch 59 / 100, example 68 / 104\n",
      "X len:  305 Y len:  305\n",
      "X:  (305, 1)\n",
      "Y:  (305, 1)\n",
      "[59,    88] loss: 0.158\n",
      "Epoch 59 / 100, example 88 / 104\n",
      "X len:  306 Y len:  306\n",
      "X:  (306, 1)\n",
      "Y:  (306, 1)\n",
      "[60,     4] loss: 0.012\n",
      "Epoch 60 / 100, example 4 / 104\n",
      "X len:  307 Y len:  307\n",
      "X:  (307, 1)\n",
      "Y:  (307, 1)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[60,    24] loss: 0.102\n",
      "Epoch 60 / 100, example 24 / 104\n",
      "X len:  308 Y len:  308\n",
      "X:  (308, 1)\n",
      "Y:  (308, 1)\n",
      "[60,    44] loss: 0.142\n",
      "Epoch 60 / 100, example 44 / 104\n",
      "X len:  309 Y len:  309\n",
      "X:  (309, 1)\n",
      "Y:  (309, 1)\n",
      "[60,    64] loss: 0.074\n",
      "Epoch 60 / 100, example 64 / 104\n",
      "X len:  310 Y len:  310\n",
      "X:  (310, 1)\n",
      "Y:  (310, 1)\n",
      "[60,    84] loss: 0.117\n",
      "Epoch 60 / 100, example 84 / 104\n",
      "X len:  311 Y len:  311\n",
      "X:  (311, 1)\n",
      "Y:  (311, 1)\n",
      "[60,   104] loss: 0.166\n",
      "Epoch 60 / 100, example 104 / 104\n",
      "X len:  312 Y len:  312\n",
      "X:  (312, 1)\n",
      "Y:  (312, 1)\n",
      "[61,    20] loss: 0.109\n",
      "Epoch 61 / 100, example 20 / 104\n",
      "X len:  313 Y len:  313\n",
      "X:  (313, 1)\n",
      "Y:  (313, 1)\n",
      "[61,    40] loss: 0.103\n",
      "Epoch 61 / 100, example 40 / 104\n",
      "X len:  314 Y len:  314\n",
      "X:  (314, 1)\n",
      "Y:  (314, 1)\n",
      "[61,    60] loss: 0.067\n",
      "Epoch 61 / 100, example 60 / 104\n",
      "X len:  315 Y len:  315\n",
      "X:  (315, 1)\n",
      "Y:  (315, 1)\n",
      "[61,    80] loss: 0.098\n",
      "Epoch 61 / 100, example 80 / 104\n",
      "X len:  316 Y len:  316\n",
      "X:  (316, 1)\n",
      "Y:  (316, 1)\n",
      "[61,   100] loss: 0.151\n",
      "Epoch 61 / 100, example 100 / 104\n",
      "X len:  317 Y len:  317\n",
      "X:  (317, 1)\n",
      "Y:  (317, 1)\n",
      "saving the model at the end of epoch 60, iters 103\n",
      "[62,    16] loss: 0.084\n",
      "Epoch 62 / 100, example 16 / 104\n",
      "X len:  318 Y len:  318\n",
      "X:  (318, 1)\n",
      "Y:  (318, 1)\n",
      "[62,    36] loss: 0.066\n",
      "Epoch 62 / 100, example 36 / 104\n",
      "X len:  319 Y len:  319\n",
      "X:  (319, 1)\n",
      "Y:  (319, 1)\n",
      "[62,    56] loss: 0.069\n",
      "Epoch 62 / 100, example 56 / 104\n",
      "X len:  320 Y len:  320\n",
      "X:  (320, 1)\n",
      "Y:  (320, 1)\n",
      "[62,    76] loss: 0.105\n",
      "Epoch 62 / 100, example 76 / 104\n",
      "X len:  321 Y len:  321\n",
      "X:  (321, 1)\n",
      "Y:  (321, 1)\n",
      "[62,    96] loss: 0.115\n",
      "Epoch 62 / 100, example 96 / 104\n",
      "X len:  322 Y len:  322\n",
      "X:  (322, 1)\n",
      "Y:  (322, 1)\n",
      "[63,    12] loss: 0.062\n",
      "Epoch 63 / 100, example 12 / 104\n",
      "X len:  323 Y len:  323\n",
      "X:  (323, 1)\n",
      "Y:  (323, 1)\n",
      "[63,    32] loss: 0.082\n",
      "Epoch 63 / 100, example 32 / 104\n",
      "X len:  324 Y len:  324\n",
      "X:  (324, 1)\n",
      "Y:  (324, 1)\n",
      "[63,    52] loss: 0.065\n",
      "Epoch 63 / 100, example 52 / 104\n",
      "X len:  325 Y len:  325\n",
      "X:  (325, 1)\n",
      "Y:  (325, 1)\n",
      "[63,    72] loss: 0.111\n",
      "Epoch 63 / 100, example 72 / 104\n",
      "X len:  326 Y len:  326\n",
      "X:  (326, 1)\n",
      "Y:  (326, 1)\n",
      "[63,    92] loss: 0.078\n",
      "Epoch 63 / 100, example 92 / 104\n",
      "X len:  327 Y len:  327\n",
      "X:  (327, 1)\n",
      "Y:  (327, 1)\n",
      "[64,     8] loss: 0.028\n",
      "Epoch 64 / 100, example 8 / 104\n",
      "X len:  328 Y len:  328\n",
      "X:  (328, 1)\n",
      "Y:  (328, 1)\n",
      "[64,    28] loss: 0.080\n",
      "Epoch 64 / 100, example 28 / 104\n",
      "X len:  329 Y len:  329\n",
      "X:  (329, 1)\n",
      "Y:  (329, 1)\n",
      "[64,    48] loss: 0.063\n",
      "Epoch 64 / 100, example 48 / 104\n",
      "X len:  330 Y len:  330\n",
      "X:  (330, 1)\n",
      "Y:  (330, 1)\n",
      "[64,    68] loss: 0.066\n",
      "Epoch 64 / 100, example 68 / 104\n",
      "X len:  331 Y len:  331\n",
      "X:  (331, 1)\n",
      "Y:  (331, 1)\n",
      "[64,    88] loss: 0.111\n",
      "Epoch 64 / 100, example 88 / 104\n",
      "X len:  332 Y len:  332\n",
      "X:  (332, 1)\n",
      "Y:  (332, 1)\n",
      "[65,     4] loss: 0.011\n",
      "Epoch 65 / 100, example 4 / 104\n",
      "X len:  333 Y len:  333\n",
      "X:  (333, 1)\n",
      "Y:  (333, 1)\n",
      "[65,    24] loss: 0.091\n",
      "Epoch 65 / 100, example 24 / 104\n",
      "X len:  334 Y len:  334\n",
      "X:  (334, 1)\n",
      "Y:  (334, 1)\n",
      "[65,    44] loss: 0.067\n",
      "Epoch 65 / 100, example 44 / 104\n",
      "X len:  335 Y len:  335\n",
      "X:  (335, 1)\n",
      "Y:  (335, 1)\n",
      "[65,    64] loss: 0.059\n",
      "Epoch 65 / 100, example 64 / 104\n",
      "X len:  336 Y len:  336\n",
      "X:  (336, 1)\n",
      "Y:  (336, 1)\n",
      "[65,    84] loss: 0.091\n",
      "Epoch 65 / 100, example 84 / 104\n",
      "X len:  337 Y len:  337\n",
      "X:  (337, 1)\n",
      "Y:  (337, 1)\n",
      "[65,   104] loss: 0.142\n",
      "Epoch 65 / 100, example 104 / 104\n",
      "X len:  338 Y len:  338\n",
      "X:  (338, 1)\n",
      "Y:  (338, 1)\n",
      "[66,    20] loss: 0.104\n",
      "Epoch 66 / 100, example 20 / 104\n",
      "X len:  339 Y len:  339\n",
      "X:  (339, 1)\n",
      "Y:  (339, 1)\n",
      "[66,    40] loss: 0.057\n",
      "Epoch 66 / 100, example 40 / 104\n",
      "X len:  340 Y len:  340\n",
      "X:  (340, 1)\n",
      "Y:  (340, 1)\n",
      "[66,    60] loss: 0.059\n",
      "Epoch 66 / 100, example 60 / 104\n",
      "X len:  341 Y len:  341\n",
      "X:  (341, 1)\n",
      "Y:  (341, 1)\n",
      "[66,    80] loss: 0.095\n",
      "Epoch 66 / 100, example 80 / 104\n",
      "X len:  342 Y len:  342\n",
      "X:  (342, 1)\n",
      "Y:  (342, 1)\n",
      "[66,   100] loss: 0.140\n",
      "Epoch 66 / 100, example 100 / 104\n",
      "X len:  343 Y len:  343\n",
      "X:  (343, 1)\n",
      "Y:  (343, 1)\n",
      "saving the model at the end of epoch 65, iters 103\n",
      "[67,    16] loss: 0.070\n",
      "Epoch 67 / 100, example 16 / 104\n",
      "X len:  344 Y len:  344\n",
      "X:  (344, 1)\n",
      "Y:  (344, 1)\n",
      "[67,    36] loss: 0.064\n",
      "Epoch 67 / 100, example 36 / 104\n",
      "X len:  345 Y len:  345\n",
      "X:  (345, 1)\n",
      "Y:  (345, 1)\n",
      "[67,    56] loss: 0.063\n",
      "Epoch 67 / 100, example 56 / 104\n",
      "X len:  346 Y len:  346\n",
      "X:  (346, 1)\n",
      "Y:  (346, 1)\n",
      "[67,    76] loss: 0.093\n",
      "Epoch 67 / 100, example 76 / 104\n",
      "X len:  347 Y len:  347\n",
      "X:  (347, 1)\n",
      "Y:  (347, 1)\n",
      "[67,    96] loss: 0.117\n",
      "Epoch 67 / 100, example 96 / 104\n",
      "X len:  348 Y len:  348\n",
      "X:  (348, 1)\n",
      "Y:  (348, 1)\n",
      "[68,    12] loss: 0.035\n",
      "Epoch 68 / 100, example 12 / 104\n",
      "X len:  349 Y len:  349\n",
      "X:  (349, 1)\n",
      "Y:  (349, 1)\n",
      "[68,    32] loss: 0.082\n",
      "Epoch 68 / 100, example 32 / 104\n",
      "X len:  350 Y len:  350\n",
      "X:  (350, 1)\n",
      "Y:  (350, 1)\n",
      "[68,    52] loss: 0.058\n",
      "Epoch 68 / 100, example 52 / 104\n",
      "X len:  351 Y len:  351\n",
      "X:  (351, 1)\n",
      "Y:  (351, 1)\n",
      "[68,    72] loss: 0.094\n",
      "Epoch 68 / 100, example 72 / 104\n",
      "X len:  352 Y len:  352\n",
      "X:  (352, 1)\n",
      "Y:  (352, 1)\n",
      "[68,    92] loss: 0.086\n",
      "Epoch 68 / 100, example 92 / 104\n",
      "X len:  353 Y len:  353\n",
      "X:  (353, 1)\n",
      "Y:  (353, 1)\n",
      "[69,     8] loss: 0.026\n",
      "Epoch 69 / 100, example 8 / 104\n",
      "X len:  354 Y len:  354\n",
      "X:  (354, 1)\n",
      "Y:  (354, 1)\n",
      "[69,    28] loss: 0.075\n",
      "Epoch 69 / 100, example 28 / 104\n",
      "X len:  355 Y len:  355\n",
      "X:  (355, 1)\n",
      "Y:  (355, 1)\n",
      "[69,    48] loss: 0.064\n",
      "Epoch 69 / 100, example 48 / 104\n",
      "X len:  356 Y len:  356\n",
      "X:  (356, 1)\n",
      "Y:  (356, 1)\n",
      "[69,    68] loss: 0.064\n",
      "Epoch 69 / 100, example 68 / 104\n",
      "X len:  357 Y len:  357\n",
      "X:  (357, 1)\n",
      "Y:  (357, 1)\n",
      "[69,    88] loss: 0.104\n",
      "Epoch 69 / 100, example 88 / 104\n",
      "X len:  358 Y len:  358\n",
      "X:  (358, 1)\n",
      "Y:  (358, 1)\n",
      "[70,     4] loss: 0.011\n",
      "Epoch 70 / 100, example 4 / 104\n",
      "X len:  359 Y len:  359\n",
      "X:  (359, 1)\n",
      "Y:  (359, 1)\n",
      "[70,    24] loss: 0.089\n",
      "Epoch 70 / 100, example 24 / 104\n",
      "X len:  360 Y len:  360\n",
      "X:  (360, 1)\n",
      "Y:  (360, 1)\n",
      "[70,    44] loss: 0.065\n",
      "Epoch 70 / 100, example 44 / 104\n",
      "X len:  361 Y len:  361\n",
      "X:  (361, 1)\n",
      "Y:  (361, 1)\n",
      "[70,    64] loss: 0.057\n",
      "Epoch 70 / 100, example 64 / 104\n",
      "X len:  362 Y len:  362\n",
      "X:  (362, 1)\n",
      "Y:  (362, 1)\n",
      "[70,    84] loss: 0.096\n",
      "Epoch 70 / 100, example 84 / 104\n",
      "X len:  363 Y len:  363\n",
      "X:  (363, 1)\n",
      "Y:  (363, 1)\n",
      "[70,   104] loss: 0.128\n",
      "Epoch 70 / 100, example 104 / 104\n",
      "X len:  364 Y len:  364\n",
      "X:  (364, 1)\n",
      "Y:  (364, 1)\n",
      "[71,    20] loss: 0.083\n",
      "Epoch 71 / 100, example 20 / 104\n",
      "X len:  365 Y len:  365\n",
      "X:  (365, 1)\n",
      "Y:  (365, 1)\n",
      "[71,    40] loss: 0.066\n",
      "Epoch 71 / 100, example 40 / 104\n",
      "X len:  366 Y len:  366\n",
      "X:  (366, 1)\n",
      "Y:  (366, 1)\n",
      "[71,    60] loss: 0.055\n",
      "Epoch 71 / 100, example 60 / 104\n",
      "X len:  367 Y len:  367\n",
      "X:  (367, 1)\n",
      "Y:  (367, 1)\n",
      "[71,    80] loss: 0.103\n",
      "Epoch 71 / 100, example 80 / 104\n",
      "X len:  368 Y len:  368\n",
      "X:  (368, 1)\n",
      "Y:  (368, 1)\n",
      "[71,   100] loss: 0.135\n",
      "Epoch 71 / 100, example 100 / 104\n",
      "X len:  369 Y len:  369\n",
      "X:  (369, 1)\n",
      "Y:  (369, 1)\n",
      "saving the model at the end of epoch 70, iters 103\n",
      "[72,    16] loss: 0.086\n",
      "Epoch 72 / 100, example 16 / 104\n",
      "X len:  370 Y len:  370\n",
      "X:  (370, 1)\n",
      "Y:  (370, 1)\n",
      "[72,    36] loss: 0.060\n",
      "Epoch 72 / 100, example 36 / 104\n",
      "X len:  371 Y len:  371\n",
      "X:  (371, 1)\n",
      "Y:  (371, 1)\n",
      "[72,    56] loss: 0.058\n",
      "Epoch 72 / 100, example 56 / 104\n",
      "X len:  372 Y len:  372\n",
      "X:  (372, 1)\n",
      "Y:  (372, 1)\n",
      "[72,    76] loss: 0.093\n",
      "Epoch 72 / 100, example 76 / 104\n",
      "X len:  373 Y len:  373\n",
      "X:  (373, 1)\n",
      "Y:  (373, 1)\n",
      "[72,    96] loss: 0.126\n",
      "Epoch 72 / 100, example 96 / 104\n",
      "X len:  374 Y len:  374\n",
      "X:  (374, 1)\n",
      "Y:  (374, 1)\n",
      "[73,    12] loss: 0.035\n",
      "Epoch 73 / 100, example 12 / 104\n",
      "X len:  375 Y len:  375\n",
      "X:  (375, 1)\n",
      "Y:  (375, 1)\n",
      "[73,    32] loss: 0.078\n",
      "Epoch 73 / 100, example 32 / 104\n",
      "X len:  376 Y len:  376\n",
      "X:  (376, 1)\n",
      "Y:  (376, 1)\n",
      "[73,    52] loss: 0.048\n",
      "Epoch 73 / 100, example 52 / 104\n",
      "X len:  377 Y len:  377\n",
      "X:  (377, 1)\n",
      "Y:  (377, 1)\n",
      "[73,    72] loss: 0.091\n",
      "Epoch 73 / 100, example 72 / 104\n",
      "X len:  378 Y len:  378\n",
      "X:  (378, 1)\n",
      "Y:  (378, 1)\n",
      "[73,    92] loss: 0.085\n",
      "Epoch 73 / 100, example 92 / 104\n",
      "X len:  379 Y len:  379\n",
      "X:  (379, 1)\n",
      "Y:  (379, 1)\n",
      "[74,     8] loss: 0.024\n",
      "Epoch 74 / 100, example 8 / 104\n",
      "X len:  380 Y len:  380\n",
      "X:  (380, 1)\n",
      "Y:  (380, 1)\n",
      "[74,    28] loss: 0.071\n",
      "Epoch 74 / 100, example 28 / 104\n",
      "X len:  381 Y len:  381\n",
      "X:  (381, 1)\n",
      "Y:  (381, 1)\n",
      "[74,    48] loss: 0.055\n",
      "Epoch 74 / 100, example 48 / 104\n",
      "X len:  382 Y len:  382\n",
      "X:  (382, 1)\n",
      "Y:  (382, 1)\n",
      "[74,    68] loss: 0.068\n",
      "Epoch 74 / 100, example 68 / 104\n",
      "X len:  383 Y len:  383\n",
      "X:  (383, 1)\n",
      "Y:  (383, 1)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[74,    88] loss: 0.118\n",
      "Epoch 74 / 100, example 88 / 104\n",
      "X len:  384 Y len:  384\n",
      "X:  (384, 1)\n",
      "Y:  (384, 1)\n",
      "[75,     4] loss: 0.009\n",
      "Epoch 75 / 100, example 4 / 104\n",
      "X len:  385 Y len:  385\n",
      "X:  (385, 1)\n",
      "Y:  (385, 1)\n",
      "[75,    24] loss: 0.080\n",
      "Epoch 75 / 100, example 24 / 104\n",
      "X len:  386 Y len:  386\n",
      "X:  (386, 1)\n",
      "Y:  (386, 1)\n",
      "[75,    44] loss: 0.056\n",
      "Epoch 75 / 100, example 44 / 104\n",
      "X len:  387 Y len:  387\n",
      "X:  (387, 1)\n",
      "Y:  (387, 1)\n",
      "[75,    64] loss: 0.055\n",
      "Epoch 75 / 100, example 64 / 104\n",
      "X len:  388 Y len:  388\n",
      "X:  (388, 1)\n",
      "Y:  (388, 1)\n",
      "[75,    84] loss: 0.096\n",
      "Epoch 75 / 100, example 84 / 104\n",
      "X len:  389 Y len:  389\n",
      "X:  (389, 1)\n",
      "Y:  (389, 1)\n",
      "[75,   104] loss: 0.139\n",
      "Epoch 75 / 100, example 104 / 104\n",
      "X len:  390 Y len:  390\n",
      "X:  (390, 1)\n",
      "Y:  (390, 1)\n",
      "[76,    20] loss: 0.192\n",
      "Epoch 76 / 100, example 20 / 104\n",
      "X len:  391 Y len:  391\n",
      "X:  (391, 1)\n",
      "Y:  (391, 1)\n",
      "[76,    40] loss: 0.133\n",
      "Epoch 76 / 100, example 40 / 104\n",
      "X len:  392 Y len:  392\n",
      "X:  (392, 1)\n",
      "Y:  (392, 1)\n",
      "[76,    60] loss: 0.058\n",
      "Epoch 76 / 100, example 60 / 104\n",
      "X len:  393 Y len:  393\n",
      "X:  (393, 1)\n",
      "Y:  (393, 1)\n",
      "[76,    80] loss: 0.088\n",
      "Epoch 76 / 100, example 80 / 104\n",
      "X len:  394 Y len:  394\n",
      "X:  (394, 1)\n",
      "Y:  (394, 1)\n",
      "[76,   100] loss: 0.131\n",
      "Epoch 76 / 100, example 100 / 104\n",
      "X len:  395 Y len:  395\n",
      "X:  (395, 1)\n",
      "Y:  (395, 1)\n",
      "saving the model at the end of epoch 75, iters 103\n",
      "[77,    16] loss: 0.057\n",
      "Epoch 77 / 100, example 16 / 104\n",
      "X len:  396 Y len:  396\n",
      "X:  (396, 1)\n",
      "Y:  (396, 1)\n",
      "[77,    36] loss: 0.056\n",
      "Epoch 77 / 100, example 36 / 104\n",
      "X len:  397 Y len:  397\n",
      "X:  (397, 1)\n",
      "Y:  (397, 1)\n",
      "[77,    56] loss: 0.052\n",
      "Epoch 77 / 100, example 56 / 104\n",
      "X len:  398 Y len:  398\n",
      "X:  (398, 1)\n",
      "Y:  (398, 1)\n",
      "[77,    76] loss: 0.087\n",
      "Epoch 77 / 100, example 76 / 104\n",
      "X len:  399 Y len:  399\n",
      "X:  (399, 1)\n",
      "Y:  (399, 1)\n",
      "[77,    96] loss: 0.105\n",
      "Epoch 77 / 100, example 96 / 104\n",
      "X len:  400 Y len:  400\n",
      "X:  (400, 1)\n",
      "Y:  (400, 1)\n",
      "[78,    12] loss: 0.038\n",
      "Epoch 78 / 100, example 12 / 104\n",
      "X len:  401 Y len:  401\n",
      "X:  (401, 1)\n",
      "Y:  (401, 1)\n",
      "[78,    32] loss: 0.072\n",
      "Epoch 78 / 100, example 32 / 104\n",
      "X len:  402 Y len:  402\n",
      "X:  (402, 1)\n",
      "Y:  (402, 1)\n",
      "[78,    52] loss: 0.050\n",
      "Epoch 78 / 100, example 52 / 104\n",
      "X len:  403 Y len:  403\n",
      "X:  (403, 1)\n",
      "Y:  (403, 1)\n",
      "[78,    72] loss: 0.085\n",
      "Epoch 78 / 100, example 72 / 104\n",
      "X len:  404 Y len:  404\n",
      "X:  (404, 1)\n",
      "Y:  (404, 1)\n",
      "[78,    92] loss: 0.091\n",
      "Epoch 78 / 100, example 92 / 104\n",
      "X len:  405 Y len:  405\n",
      "X:  (405, 1)\n",
      "Y:  (405, 1)\n",
      "[79,     8] loss: 0.025\n",
      "Epoch 79 / 100, example 8 / 104\n",
      "X len:  406 Y len:  406\n",
      "X:  (406, 1)\n",
      "Y:  (406, 1)\n",
      "[79,    28] loss: 0.069\n",
      "Epoch 79 / 100, example 28 / 104\n",
      "X len:  407 Y len:  407\n",
      "X:  (407, 1)\n",
      "Y:  (407, 1)\n",
      "[79,    48] loss: 0.059\n",
      "Epoch 79 / 100, example 48 / 104\n",
      "X len:  408 Y len:  408\n",
      "X:  (408, 1)\n",
      "Y:  (408, 1)\n",
      "[79,    68] loss: 0.059\n",
      "Epoch 79 / 100, example 68 / 104\n",
      "X len:  409 Y len:  409\n",
      "X:  (409, 1)\n",
      "Y:  (409, 1)\n",
      "[79,    88] loss: 0.103\n",
      "Epoch 79 / 100, example 88 / 104\n",
      "X len:  410 Y len:  410\n",
      "X:  (410, 1)\n",
      "Y:  (410, 1)\n"
     ]
    }
   ],
   "source": [
    "# create a visualizer that display/save images and plots\n",
    "visualizer = vs.Visualizer(opt)\n",
    "\n",
    "# show a colorbar between the min and max depths as defined before\n",
    "visualizer.display_colorbar_jet(-19., -11.)\n",
    "\n",
    "dataset_size = len(dataset) # the number of examples in dataset\n",
    "display_freq = 100 # how often to update web page images\n",
    "update_html_freq = 1000 # how often to save html on disk\n",
    "\n",
    "total_iters = 0\n",
    "for epoch in range(epochs):  # loop over the dataset multiple times\n",
    "\n",
    "    running_loss = 0.0\n",
    "    for i, data in enumerate(dataset):\n",
    "        visualizer.reset()\n",
    "\n",
    "        # get the inputs; data is a list of [inputs, labels]\n",
    "        inputs = data['A'].to(device)\n",
    "        target = data['B'].to(device)\n",
    "\n",
    "        # zero the parameter gradients\n",
    "        optimizer.zero_grad()\n",
    "\n",
    "        mask = (target > -.99).float()\n",
    "        # forward + backward + optimize\n",
    "        outputs = net(inputs)\n",
    "        \n",
    "        loss = criterion(mask*outputs, mask*target)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        \n",
    "        if i % display_freq == 0: # display images on visdom and save images to a HTML file\n",
    "            save_result = i % update_html_freq == 0\n",
    "            visual_ret = {'Input': inputs, 'Output_Jet': mask*(outputs+1.)-1., 'Real_Jet': target}\n",
    "            visualizer.display_current_results(visual_ret, epoch, save_result)\n",
    "\n",
    "        # print statistics\n",
    "        running_loss += loss.item()\n",
    "        total_iters += 1\n",
    "        if total_iters % 20 == 0: # print every 20 mini-batches\n",
    "            print('[%d, %5d] loss: %.3f' %\n",
    "                  (epoch + 1, i + 1, running_loss / 20.))\n",
    "            print('Epoch %d / %d, example %d / %d' % (epoch + 1, epochs, i+1, dataset_size))\n",
    "            losses = {'L1_loss': running_loss/20.}\n",
    "            #print(float(i)/dataset_size)\n",
    "            visualizer.plot_current_losses(epoch, float(i) / dataset_size, losses)\n",
    "            running_loss = 0.0\n",
    "            \n",
    "    if epoch % 5 == 0: # cache our model every 5 epochs\n",
    "        print('saving the model at the end of epoch %d, iters %d' % (epoch, i))\n",
    "        save_network('checkpoints', 'latest')\n",
    "        save_network('checkpoints', epoch)\n",
    "\n",
    "print('Finished Training')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluating the network\n",
    "\n",
    "We can now load any of the saved epochs and test the trained model on our test dataset. The results will be stored in `results/regress/test_XX`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "opt.phase = \"test\"\n",
    "opt.isTrain = False\n",
    "opt.dataroot = \"sss2depth\"\n",
    "epoch = \"latest\"\n",
    "\n",
    "dataset = torch.utils.data.DataLoader(\n",
    "                AlignedDataset(opt),\n",
    "                batch_size=opt.batch_size,\n",
    "                shuffle=not opt.serial_batches,\n",
    "                num_workers=int(opt.num_threads))\n",
    "\n",
    "net = load_network('checkpoints', epoch, device)      # create a model given opt.model and other options\n",
    "\n",
    "# create a website\n",
    "web_dir = os.path.join(\"results\", opt.name, '%s_%s' % (\"test\", epoch))  # define the website directory\n",
    "webpage = html.HTML(web_dir, 'Experiment = %s, Phase = %s, Epoch = %s' % (opt.name, opt.phase, epoch))\n",
    "\n",
    "num_test = 25\n",
    "for i, data in enumerate(dataset):\n",
    "    if i >= num_test:  # only apply our model to opt.num_test images.\n",
    "        break\n",
    "    inputs = data['A'].to(device)\n",
    "    target = data['B'].to(device)\n",
    "    \n",
    "    mask = (target > -.99).float()\n",
    "    outputs = net(inputs)\n",
    "    \n",
    "    visuals = {'Input': inputs, 'Output_Jet': mask*(outputs+1.)-1., 'Real_Jet': target}\n",
    "    img_path = data['A_paths'] # get image paths\n",
    "    if i % 5 == 0:  # save images to an HTML file\n",
    "        print('processing (%04d)-th image... %s' % (i, img_path))\n",
    "    vs.save_images(webpage, visuals, img_path, aspect_ratio=opt.aspect_ratio, width=opt.display_winsize)\n",
    "webpage.save()  # save the HTML\n",
    "\n",
    "# show the saved web page with results\n",
    "IFrame(src=os.path.join(web_dir, \"index.html\"), width=850, height=600)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}