R) RoadMap 18 - Appendix 1 - Replicating Classification 4 with Monk.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Appendix 1 - Transfer Learning with MONK"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Steps to take\n",
    "\n",
    "1. Create a network\n",
    "    - Creating base network\n",
    "    - Visualize layers\n",
    "    - Creating loss function module\n",
    "    - Creating optimizer module [Set learning rates here]\n",
    "    - Creating learning rate scheduler\n",
    "    \n",
    "    \n",
    "2. Data prepraration\n",
    "    - Creating a data transformer\n",
    "    - Downloading and storing dataset\n",
    "    - Applying transformation\n",
    "    - Understanding dataset\n",
    "    - Loading the transformed dataset [Set batch size and number of parallel processors here]\n",
    "    \n",
    "    \n",
    "3. Setting up data - plotters\n",
    "\n",
    "\n",
    "4. Training\n",
    "    - Set Epoch\n",
    "    - Train model\n",
    "    \n",
    "    \n",
    "5. Testing\n",
    "    - Just after training\n",
    "    - After loading saved weights\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cloning into 'monk_v1'...\n",
      "remote: Enumerating objects: 773, done.\u001b[K\n",
      "remote: Counting objects: 100% (773/773), done.\u001b[K\n",
      "remote: Compressing objects: 100% (482/482), done.\u001b[K\n",
      "remote: Total 773 (delta 349), reused 672 (delta 265), pack-reused 0\n",
      "Receiving objects: 100% (773/773), 4.74 MiB | 3.68 MiB/s, done.\n",
      "Resolving deltas: 100% (349/349), done.\n"
     ]
    }
   ],
   "source": [
    "!git clone https://github.com/Tessellate-Imaging/monk_v1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#!pip install -r monk_v1/installation/requirements_cu9.txt    \n",
    "#Select Installation file as per the system"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Pytorch Version: 1.2.0\n",
      "\n",
      "Experiment Details\n",
      "    Project: exp-1\n",
      "    Experiment: proj-1\n",
      "    Dir: /home/abhi/Desktop/Work/tess_tool/gui/v0.3/finetune_models/Resources/v0.1/workspace/exp-1/proj-1/\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import sys\n",
    "sys.path.append(\"monk_v1/monk/\")\n",
    "from pytorch_prototype import prototype\n",
    "\n",
    "# Step 1 - Create experiment\n",
    "ptf = prototype(verbose=1);\n",
    "ptf.Prototype(\"exp-1\", \"proj-1\");"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''\n",
    "original code - snippet\n",
    "\n",
    "data_transforms = {\n",
    "    'train': transforms.Compose([\n",
    "        transforms.RandomHorizontalFlip(),\n",
    "        transforms.ToTensor(),\n",
    "        transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n",
    "    ]),\n",
    "    'val': transforms.Compose([\n",
    "        transforms.ToTensor(),\n",
    "        transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n",
    "    ]),\n",
    "}\n",
    "\n",
    "data_dir = 'cat_dog/'\n",
    "image_datasets = {x: datasets.ImageFolder(os.path.join(data_dir, x),\n",
    "                                          data_transforms[x])\n",
    "                  for x in ['train', 'val']}\n",
    "                  \n",
    "                  \n",
    "\n",
    "\n",
    "batch = 4\n",
    "parallel_processors = 3\n",
    "\n",
    "dataloaders = {x: torch.utils.data.DataLoader(image_datasets[x], batch_size=batch,\n",
    "                                             shuffle=True, num_workers=parallel_processors)\n",
    "              for x in ['train', 'val']}\n",
    "dataset_sizes = {x: len(image_datasets[x]) for x in ['train', 'val']}\n",
    "class_names = image_datasets['train'].classes\n",
    "\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset Details\n",
      "    Train path:     cat_dog/train\n",
      "    Val path:       cat_dog/val\n",
      "    CSV train path: None\n",
      "    CSV val path:  None\n",
      "\n",
      "Dataset Params\n",
      "    Input Size:  224\n",
      "    Batch Size:  4\n",
      "    Data Shuffle: True\n",
      "    Processors:   3\n",
      "\n",
      "Pre-Composed Train Transforms\n",
      "[{'RandomHorizontalFlip': {'p': 0.5}}, {'Normalize': {'mean': [0.485, 0.456, 0.406], 'std': [0.229, 0.224, 0.225]}}]\n",
      "\n",
      "Pre-Composed Val Transforms\n",
      "[{'Normalize': {'mean': [0.485, 0.456, 0.406], 'std': [0.229, 0.224, 0.225]}}]\n",
      "\n",
      "Dataset Numbers\n",
      "    Num train images: 200\n",
      "    Num val images:   50\n",
      "    Num classes:      2\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Replicating same using Monk\n",
    "\n",
    "\n",
    "ptf.Dataset_Params(dataset_path=[\"cat_dog/train\", \"cat_dog/val\"],\n",
    "                   input_size=224, \n",
    "                   batch_size=4, shuffle_data=True, num_processors=3);\n",
    "\n",
    "# Transform\n",
    "ptf.apply_random_horizontal_flip(train=True);\n",
    "ptf.apply_normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], train=True, val=True, test=True);\n",
    "\n",
    "\n",
    "ptf.Dataset();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''\n",
    "Original Code Snippet\n",
    "\n",
    "model_ft = models.resnet18(pretrained=True) # Using pretrained models\n",
    "num_ftrs = model_ft.fc.in_features # Adding a new layer to next final layer\n",
    "model_ft.fc = nn.Linear(num_ftrs, 2)\n",
    "model_ft.cuda()\n",
    "\n",
    "\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model Params\n",
      "    Model name:           resnet18\n",
      "    Use Gpu:              True\n",
      "    Use pretrained:       True\n",
      "    Freeze base network:  True\n",
      "\n",
      "Model Details\n",
      "    Loading pretrained model\n",
      "    Model Loaded on device\n",
      "        Model name:           resnet18\n",
      "        Num layers in model:  31\n",
      "        Num trainable layers: 1\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Replicating same using Monk\n",
    "\n",
    "ptf.Model_Params(model_name=\"resnet18\", freeze_base_network=True, \n",
    "                 use_gpu=True, use_pretrained=True);\n",
    "ptf.Model();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''\n",
    "Original Code Snippet\n",
    "\n",
    "cross_entropy_loss = nn.CrossEntropyLoss()\n",
    "optimizer_ft = optim.SGD(model_ft.parameters(), lr=0.001, momentum=0.9)\n",
    "\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Optimizer\n",
      "    Name:          sgd\n",
      "    Learning rate: 0.001\n",
      "    Params:        {'lr': 0.001, 'dampening': 0, 'nesterov': False, 'weight_decay': 0, 'momentum': 0}\n",
      "\n",
      "Learning rate scheduler\n",
      "    Name:   fixed\n",
      "    Params: {}\n",
      "\n",
      "Loss\n",
      "    Name:          softmaxcrossentropy\n",
      "    Params:        {'weight': None, 'size_average': None, 'ignore_index': -100, 'reduce': None, 'reduction': 'mean'}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Replicating same using Monk\n",
    "ptf.optimizer_sgd(0.001);\n",
    "ptf.lr_fixed();\n",
    "ptf.loss_softmax_crossentropy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''\n",
    "Original Code Snippet\n",
    "\n",
    "def train_model(model, criterion, optimizer, num_epochs=25):\n",
    "    since = time.time()\n",
    "\n",
    "    # A shallow copy constructs a new compound object and then (to the extent possible) inserts \n",
    "    #    references into it to the objects found in the original. \n",
    "    # A deep copy constructs a new compound object and then, recursively, inserts copies into it of \n",
    "    #    the objects found in the original.\n",
    "    best_model_wts = copy.deepcopy(model.state_dict())\n",
    "    best_acc = 0.0\n",
    "\n",
    "    for epoch in range(num_epochs):\n",
    "        print('Epoch {}/{}'.format(epoch, num_epochs - 1))\n",
    "        print('-' * 10)\n",
    "\n",
    "        # Each epoch has a training and validation phase\n",
    "        for phase in ['train', 'val']:\n",
    "            if phase == 'train':\n",
    "                model.train()  # Set model to training mode\n",
    "            else:\n",
    "                model.eval()   # Set model to evaluate mode\n",
    "\n",
    "            running_loss = 0.0\n",
    "            running_corrects = 0\n",
    "\n",
    "            # Iterate over data.\n",
    "            for inputs, labels in dataloaders[phase]:\n",
    "                inputs = inputs.cuda()\n",
    "                labels = labels.cuda()\n",
    "\n",
    "                # zero the parameter gradients\n",
    "                optimizer.zero_grad()\n",
    "\n",
    "                # forward\n",
    "                # track history if only in train\n",
    "                with torch.set_grad_enabled(phase == 'train'):\n",
    "                    outputs = model(inputs)\n",
    "                    _, preds = torch.max(outputs, 1)\n",
    "                    loss = criterion(outputs, labels)\n",
    "\n",
    "                    # backward + optimize only if in training phase\n",
    "                    if phase == 'train':\n",
    "                        loss.backward()\n",
    "                        optimizer.step()\n",
    "\n",
    "                # statistics\n",
    "                running_loss += loss.item() * inputs.size(0)\n",
    "                running_corrects += torch.sum(preds == labels.data)\n",
    "\n",
    "            epoch_loss = running_loss / dataset_sizes[phase]\n",
    "            epoch_acc = running_corrects.double() / dataset_sizes[phase]\n",
    "\n",
    "            print('{} Loss: {:.4f} Acc: {:.4f}'.format(\n",
    "                phase, epoch_loss, epoch_acc))\n",
    "\n",
    "            # deep copy the model\n",
    "            if phase == 'val' and epoch_acc > best_acc:\n",
    "                best_acc = epoch_acc\n",
    "                best_model_wts = copy.deepcopy(model.state_dict())\n",
    "\n",
    "        print()\n",
    "\n",
    "    time_elapsed = time.time() - since\n",
    "    print('Training complete in {:.0f}m {:.0f}s'.format(\n",
    "        time_elapsed // 60, time_elapsed % 60))\n",
    "    print('Best val Acc: {:4f}'.format(best_acc))\n",
    "\n",
    "    torch.save(model.state_dict(), \"cat_dog_model\")\n",
    "    \n",
    "    # load best model weights\n",
    "    model.load_state_dict(best_model_wts)\n",
    "    return model\n",
    "\n",
    "model_ft = train_model(model_ft, cross_entropy_loss, optimizer_ft,\n",
    "                       num_epochs=5)\n",
    "\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training params\n",
      "    Num Epochs: 4\n",
      "\n",
      "Display params\n",
      "    Display progress:          True\n",
      "    Display progress realtime: True\n",
      "    Save Training logs:        True\n",
      "    Save Intermediate models:  True\n",
      "    Intermediate model prefix: intermediate_model_\n",
      "\n",
      "Training Start\n",
      "    Epoch 1/4\n",
      "    ----------\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4db62c227c2247899ffed468e0cdcc4b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=50), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5674f74fee064e1e84149ea521d3bd41",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=13), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "    curr_lr - 0.001\n",
      "    [Epoch 1] Train-acc: 0.570, Train-loss: 0.702 | Val-acc: 0.820000, Val-loss: 0.532, | time: 1.3 sec\n",
      "\n",
      "    Epoch 2/4\n",
      "    ----------\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cc73ada75ee74da8be64f227bdb7410e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=50), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e048fce6802d442a919ce2f6eae7bc0c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=13), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "    curr_lr - 0.001\n",
      "    [Epoch 2] Train-acc: 0.825, Train-loss: 0.471 | Val-acc: 0.880000, Val-loss: 0.396, | time: 1.4 sec\n",
      "\n",
      "    Epoch 3/4\n",
      "    ----------\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3827f55a70fc4ed7bb19bc3a44fc11e2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=50), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "04db03ced7b14a589fb1a155fd9cc235",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=13), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "    curr_lr - 0.001\n",
      "    [Epoch 3] Train-acc: 0.830, Train-loss: 0.425 | Val-acc: 0.880000, Val-loss: 0.329, | time: 1.3 sec\n",
      "\n",
      "    Epoch 4/4\n",
      "    ----------\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9878cb040ae54a1f8199fd9016220abd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=50), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a8294ccbb19c4d3a8fc28124e3de3f11",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=13), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "    curr_lr - 0.001\n",
      "    [Epoch 4] Train-acc: 0.850, Train-loss: 0.386 | Val-acc: 0.960000, Val-loss: 0.265, | time: 1.5 sec\n",
      "\n",
      "    Training completed in: 0m 4s\n",
      "    Best val Acc:          0.960000\n",
      "\n",
      "Training End\n",
      "\n",
      "Training Outputs\n",
      "    Model Dir:   /home/abhi/Desktop/Work/tess_tool/gui/v0.3/finetune_models/Resources/v0.1/workspace/exp-1/proj-1/output/models/\n",
      "    Log Dir:     /home/abhi/Desktop/Work/tess_tool/gui/v0.3/finetune_models/Resources/v0.1/workspace/exp-1/proj-1/output/logs/\n",
      "    Final model: final\n",
      "    Best model:  best_model\n",
      "    Log 1 - Validation accuracy history log: val_acc_history.npy\n",
      "    Log 2 - Validation loss history log:     val_loss_history.npy\n",
      "    Log 3 - Training accuracy history log:   train_acc_history.npy\n",
      "    Log 4 - Training loss history log:       train_loss_history.npy\n",
      "    Log 5 - Training curve:                  train_loss_history.npy\n",
      "    Log 6 - Validation curve:                train_loss_history.npy\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Replicating using Monk\n",
    "\n",
    "ptf.Training_Params(num_epochs=4, display_progress=True, display_progress_realtime=True, \n",
    "        save_intermediate_models=True, intermediate_model_prefix=\"intermediate_model_\", save_training_logs=True);\n",
    "ptf.Train();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}