swin_qa.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torchvision"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torchvision.models import swin_b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import os\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torchvision import datasets, transforms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "device(type='cuda')"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "device"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_dir = '/kaggle/input/data/'\n",
    "transform = transforms.Compose([\n",
    "    transforms.Resize((224, 224)),\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize(mean=[0.485, 0.456, 0.406],\n",
    "                         std=[0.229, 0.224, 0.225])\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_path = os.path.join(os.getcwd(),\"QA-Polyp/train/\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = datasets.ImageFolder(data_path, transform=transform)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1062"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataloader = torch.utils.data.DataLoader(dataset, batch_size=10, shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch.nn as nn\n",
    "import torch.optim as optim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SwinTransformer(\n",
      "  (features): Sequential(\n",
      "    (0): Sequential(\n",
      "      (0): Conv2d(3, 128, kernel_size=(4, 4), stride=(4, 4))\n",
      "      (1): Permute()\n",
      "      (2): LayerNorm((128,), eps=1e-05, elementwise_affine=True)\n",
      "    )\n",
      "    (1): Sequential(\n",
      "      (0): SwinTransformerBlock(\n",
      "        (norm1): LayerNorm((128,), eps=1e-05, elementwise_affine=True)\n",
      "        (attn): ShiftedWindowAttention(\n",
      "          (qkv): Linear(in_features=128, out_features=384, bias=True)\n",
      "          (proj): Linear(in_features=128, out_features=128, bias=True)\n",
      "        )\n",
      "        (stochastic_depth): StochasticDepth(p=0.0, mode=row)\n",
      "        (norm2): LayerNorm((128,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): MLP(\n",
      "          (0): Linear(in_features=128, out_features=512, bias=True)\n",
      "          (1): GELU(approximate='none')\n",
      "          (2): Dropout(p=0.0, inplace=False)\n",
      "          (3): Linear(in_features=512, out_features=128, bias=True)\n",
      "          (4): Dropout(p=0.0, inplace=False)\n",
      "        )\n",
      "      )\n",
      "      (1): SwinTransformerBlock(\n",
      "        (norm1): LayerNorm((128,), eps=1e-05, elementwise_affine=True)\n",
      "        (attn): ShiftedWindowAttention(\n",
      "          (qkv): Linear(in_features=128, out_features=384, bias=True)\n",
      "          (proj): Linear(in_features=128, out_features=128, bias=True)\n",
      "        )\n",
      "        (stochastic_depth): StochasticDepth(p=0.021739130434782608, mode=row)\n",
      "        (norm2): LayerNorm((128,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): MLP(\n",
      "          (0): Linear(in_features=128, out_features=512, bias=True)\n",
      "          (1): GELU(approximate='none')\n",
      "          (2): Dropout(p=0.0, inplace=False)\n",
      "          (3): Linear(in_features=512, out_features=128, bias=True)\n",
      "          (4): Dropout(p=0.0, inplace=False)\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "    (2): PatchMerging(\n",
      "      (reduction): Linear(in_features=512, out_features=256, bias=False)\n",
      "      (norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
      "    )\n",
      "    (3): Sequential(\n",
      "      (0): SwinTransformerBlock(\n",
      "        (norm1): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
      "        (attn): ShiftedWindowAttention(\n",
      "          (qkv): Linear(in_features=256, out_features=768, bias=True)\n",
      "          (proj): Linear(in_features=256, out_features=256, bias=True)\n",
      "        )\n",
      "        (stochastic_depth): StochasticDepth(p=0.043478260869565216, mode=row)\n",
      "        (norm2): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): MLP(\n",
      "          (0): Linear(in_features=256, out_features=1024, bias=True)\n",
      "          (1): GELU(approximate='none')\n",
      "          (2): Dropout(p=0.0, inplace=False)\n",
      "          (3): Linear(in_features=1024, out_features=256, bias=True)\n",
      "          (4): Dropout(p=0.0, inplace=False)\n",
      "        )\n",
      "      )\n",
      "      (1): SwinTransformerBlock(\n",
      "        (norm1): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
      "        (attn): ShiftedWindowAttention(\n",
      "          (qkv): Linear(in_features=256, out_features=768, bias=True)\n",
      "          (proj): Linear(in_features=256, out_features=256, bias=True)\n",
      "        )\n",
      "        (stochastic_depth): StochasticDepth(p=0.06521739130434782, mode=row)\n",
      "        (norm2): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): MLP(\n",
      "          (0): Linear(in_features=256, out_features=1024, bias=True)\n",
      "          (1): GELU(approximate='none')\n",
      "          (2): Dropout(p=0.0, inplace=False)\n",
      "          (3): Linear(in_features=1024, out_features=256, bias=True)\n",
      "          (4): Dropout(p=0.0, inplace=False)\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "    (4): PatchMerging(\n",
      "      (reduction): Linear(in_features=1024, out_features=512, bias=False)\n",
      "      (norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "    )\n",
      "    (5): Sequential(\n",
      "      (0): SwinTransformerBlock(\n",
      "        (norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
      "        (attn): ShiftedWindowAttention(\n",
      "          (qkv): Linear(in_features=512, out_features=1536, bias=True)\n",
      "          (proj): Linear(in_features=512, out_features=512, bias=True)\n",
      "        )\n",
      "        (stochastic_depth): StochasticDepth(p=0.08695652173913043, mode=row)\n",
      "        (norm2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): MLP(\n",
      "          (0): Linear(in_features=512, out_features=2048, bias=True)\n",
      "          (1): GELU(approximate='none')\n",
      "          (2): Dropout(p=0.0, inplace=False)\n",
      "          (3): Linear(in_features=2048, out_features=512, bias=True)\n",
      "          (4): Dropout(p=0.0, inplace=False)\n",
      "        )\n",
      "      )\n",
      "      (1): SwinTransformerBlock(\n",
      "        (norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
      "        (attn): ShiftedWindowAttention(\n",
      "          (qkv): Linear(in_features=512, out_features=1536, bias=True)\n",
      "          (proj): Linear(in_features=512, out_features=512, bias=True)\n",
      "        )\n",
      "        (stochastic_depth): StochasticDepth(p=0.10869565217391304, mode=row)\n",
      "        (norm2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): MLP(\n",
      "          (0): Linear(in_features=512, out_features=2048, bias=True)\n",
      "          (1): GELU(approximate='none')\n",
      "          (2): Dropout(p=0.0, inplace=False)\n",
      "          (3): Linear(in_features=2048, out_features=512, bias=True)\n",
      "          (4): Dropout(p=0.0, inplace=False)\n",
      "        )\n",
      "      )\n",
      "      (2): SwinTransformerBlock(\n",
      "        (norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
      "        (attn): ShiftedWindowAttention(\n",
      "          (qkv): Linear(in_features=512, out_features=1536, bias=True)\n",
      "          (proj): Linear(in_features=512, out_features=512, bias=True)\n",
      "        )\n",
      "        (stochastic_depth): StochasticDepth(p=0.13043478260869565, mode=row)\n",
      "        (norm2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): MLP(\n",
      "          (0): Linear(in_features=512, out_features=2048, bias=True)\n",
      "          (1): GELU(approximate='none')\n",
      "          (2): Dropout(p=0.0, inplace=False)\n",
      "          (3): Linear(in_features=2048, out_features=512, bias=True)\n",
      "          (4): Dropout(p=0.0, inplace=False)\n",
      "        )\n",
      "      )\n",
      "      (3): SwinTransformerBlock(\n",
      "        (norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
      "        (attn): ShiftedWindowAttention(\n",
      "          (qkv): Linear(in_features=512, out_features=1536, bias=True)\n",
      "          (proj): Linear(in_features=512, out_features=512, bias=True)\n",
      "        )\n",
      "        (stochastic_depth): StochasticDepth(p=0.15217391304347827, mode=row)\n",
      "        (norm2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): MLP(\n",
      "          (0): Linear(in_features=512, out_features=2048, bias=True)\n",
      "          (1): GELU(approximate='none')\n",
      "          (2): Dropout(p=0.0, inplace=False)\n",
      "          (3): Linear(in_features=2048, out_features=512, bias=True)\n",
      "          (4): Dropout(p=0.0, inplace=False)\n",
      "        )\n",
      "      )\n",
      "      (4): SwinTransformerBlock(\n",
      "        (norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
      "        (attn): ShiftedWindowAttention(\n",
      "          (qkv): Linear(in_features=512, out_features=1536, bias=True)\n",
      "          (proj): Linear(in_features=512, out_features=512, bias=True)\n",
      "        )\n",
      "        (stochastic_depth): StochasticDepth(p=0.17391304347826086, mode=row)\n",
      "        (norm2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): MLP(\n",
      "          (0): Linear(in_features=512, out_features=2048, bias=True)\n",
      "          (1): GELU(approximate='none')\n",
      "          (2): Dropout(p=0.0, inplace=False)\n",
      "          (3): Linear(in_features=2048, out_features=512, bias=True)\n",
      "          (4): Dropout(p=0.0, inplace=False)\n",
      "        )\n",
      "      )\n",
      "      (5): SwinTransformerBlock(\n",
      "        (norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
      "        (attn): ShiftedWindowAttention(\n",
      "          (qkv): Linear(in_features=512, out_features=1536, bias=True)\n",
      "          (proj): Linear(in_features=512, out_features=512, bias=True)\n",
      "        )\n",
      "        (stochastic_depth): StochasticDepth(p=0.1956521739130435, mode=row)\n",
      "        (norm2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): MLP(\n",
      "          (0): Linear(in_features=512, out_features=2048, bias=True)\n",
      "          (1): GELU(approximate='none')\n",
      "          (2): Dropout(p=0.0, inplace=False)\n",
      "          (3): Linear(in_features=2048, out_features=512, bias=True)\n",
      "          (4): Dropout(p=0.0, inplace=False)\n",
      "        )\n",
      "      )\n",
      "      (6): SwinTransformerBlock(\n",
      "        (norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
      "        (attn): ShiftedWindowAttention(\n",
      "          (qkv): Linear(in_features=512, out_features=1536, bias=True)\n",
      "          (proj): Linear(in_features=512, out_features=512, bias=True)\n",
      "        )\n",
      "        (stochastic_depth): StochasticDepth(p=0.21739130434782608, mode=row)\n",
      "        (norm2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): MLP(\n",
      "          (0): Linear(in_features=512, out_features=2048, bias=True)\n",
      "          (1): GELU(approximate='none')\n",
      "          (2): Dropout(p=0.0, inplace=False)\n",
      "          (3): Linear(in_features=2048, out_features=512, bias=True)\n",
      "          (4): Dropout(p=0.0, inplace=False)\n",
      "        )\n",
      "      )\n",
      "      (7): SwinTransformerBlock(\n",
      "        (norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
      "        (attn): ShiftedWindowAttention(\n",
      "          (qkv): Linear(in_features=512, out_features=1536, bias=True)\n",
      "          (proj): Linear(in_features=512, out_features=512, bias=True)\n",
      "        )\n",
      "        (stochastic_depth): StochasticDepth(p=0.2391304347826087, mode=row)\n",
      "        (norm2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): MLP(\n",
      "          (0): Linear(in_features=512, out_features=2048, bias=True)\n",
      "          (1): GELU(approximate='none')\n",
      "          (2): Dropout(p=0.0, inplace=False)\n",
      "          (3): Linear(in_features=2048, out_features=512, bias=True)\n",
      "          (4): Dropout(p=0.0, inplace=False)\n",
      "        )\n",
      "      )\n",
      "      (8): SwinTransformerBlock(\n",
      "        (norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
      "        (attn): ShiftedWindowAttention(\n",
      "          (qkv): Linear(in_features=512, out_features=1536, bias=True)\n",
      "          (proj): Linear(in_features=512, out_features=512, bias=True)\n",
      "        )\n",
      "        (stochastic_depth): StochasticDepth(p=0.2608695652173913, mode=row)\n",
      "        (norm2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): MLP(\n",
      "          (0): Linear(in_features=512, out_features=2048, bias=True)\n",
      "          (1): GELU(approximate='none')\n",
      "          (2): Dropout(p=0.0, inplace=False)\n",
      "          (3): Linear(in_features=2048, out_features=512, bias=True)\n",
      "          (4): Dropout(p=0.0, inplace=False)\n",
      "        )\n",
      "      )\n",
      "      (9): SwinTransformerBlock(\n",
      "        (norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
      "        (attn): ShiftedWindowAttention(\n",
      "          (qkv): Linear(in_features=512, out_features=1536, bias=True)\n",
      "          (proj): Linear(in_features=512, out_features=512, bias=True)\n",
      "        )\n",
      "        (stochastic_depth): StochasticDepth(p=0.2826086956521739, mode=row)\n",
      "        (norm2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): MLP(\n",
      "          (0): Linear(in_features=512, out_features=2048, bias=True)\n",
      "          (1): GELU(approximate='none')\n",
      "          (2): Dropout(p=0.0, inplace=False)\n",
      "          (3): Linear(in_features=2048, out_features=512, bias=True)\n",
      "          (4): Dropout(p=0.0, inplace=False)\n",
      "        )\n",
      "      )\n",
      "      (10): SwinTransformerBlock(\n",
      "        (norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
      "        (attn): ShiftedWindowAttention(\n",
      "          (qkv): Linear(in_features=512, out_features=1536, bias=True)\n",
      "          (proj): Linear(in_features=512, out_features=512, bias=True)\n",
      "        )\n",
      "        (stochastic_depth): StochasticDepth(p=0.30434782608695654, mode=row)\n",
      "        (norm2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): MLP(\n",
      "          (0): Linear(in_features=512, out_features=2048, bias=True)\n",
      "          (1): GELU(approximate='none')\n",
      "          (2): Dropout(p=0.0, inplace=False)\n",
      "          (3): Linear(in_features=2048, out_features=512, bias=True)\n",
      "          (4): Dropout(p=0.0, inplace=False)\n",
      "        )\n",
      "      )\n",
      "      (11): SwinTransformerBlock(\n",
      "        (norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
      "        (attn): ShiftedWindowAttention(\n",
      "          (qkv): Linear(in_features=512, out_features=1536, bias=True)\n",
      "          (proj): Linear(in_features=512, out_features=512, bias=True)\n",
      "        )\n",
      "        (stochastic_depth): StochasticDepth(p=0.32608695652173914, mode=row)\n",
      "        (norm2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): MLP(\n",
      "          (0): Linear(in_features=512, out_features=2048, bias=True)\n",
      "          (1): GELU(approximate='none')\n",
      "          (2): Dropout(p=0.0, inplace=False)\n",
      "          (3): Linear(in_features=2048, out_features=512, bias=True)\n",
      "          (4): Dropout(p=0.0, inplace=False)\n",
      "        )\n",
      "      )\n",
      "      (12): SwinTransformerBlock(\n",
      "        (norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
      "        (attn): ShiftedWindowAttention(\n",
      "          (qkv): Linear(in_features=512, out_features=1536, bias=True)\n",
      "          (proj): Linear(in_features=512, out_features=512, bias=True)\n",
      "        )\n",
      "        (stochastic_depth): StochasticDepth(p=0.34782608695652173, mode=row)\n",
      "        (norm2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): MLP(\n",
      "          (0): Linear(in_features=512, out_features=2048, bias=True)\n",
      "          (1): GELU(approximate='none')\n",
      "          (2): Dropout(p=0.0, inplace=False)\n",
      "          (3): Linear(in_features=2048, out_features=512, bias=True)\n",
      "          (4): Dropout(p=0.0, inplace=False)\n",
      "        )\n",
      "      )\n",
      "      (13): SwinTransformerBlock(\n",
      "        (norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
      "        (attn): ShiftedWindowAttention(\n",
      "          (qkv): Linear(in_features=512, out_features=1536, bias=True)\n",
      "          (proj): Linear(in_features=512, out_features=512, bias=True)\n",
      "        )\n",
      "        (stochastic_depth): StochasticDepth(p=0.3695652173913043, mode=row)\n",
      "        (norm2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): MLP(\n",
      "          (0): Linear(in_features=512, out_features=2048, bias=True)\n",
      "          (1): GELU(approximate='none')\n",
      "          (2): Dropout(p=0.0, inplace=False)\n",
      "          (3): Linear(in_features=2048, out_features=512, bias=True)\n",
      "          (4): Dropout(p=0.0, inplace=False)\n",
      "        )\n",
      "      )\n",
      "      (14): SwinTransformerBlock(\n",
      "        (norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
      "        (attn): ShiftedWindowAttention(\n",
      "          (qkv): Linear(in_features=512, out_features=1536, bias=True)\n",
      "          (proj): Linear(in_features=512, out_features=512, bias=True)\n",
      "        )\n",
      "        (stochastic_depth): StochasticDepth(p=0.391304347826087, mode=row)\n",
      "        (norm2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): MLP(\n",
      "          (0): Linear(in_features=512, out_features=2048, bias=True)\n",
      "          (1): GELU(approximate='none')\n",
      "          (2): Dropout(p=0.0, inplace=False)\n",
      "          (3): Linear(in_features=2048, out_features=512, bias=True)\n",
      "          (4): Dropout(p=0.0, inplace=False)\n",
      "        )\n",
      "      )\n",
      "      (15): SwinTransformerBlock(\n",
      "        (norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
      "        (attn): ShiftedWindowAttention(\n",
      "          (qkv): Linear(in_features=512, out_features=1536, bias=True)\n",
      "          (proj): Linear(in_features=512, out_features=512, bias=True)\n",
      "        )\n",
      "        (stochastic_depth): StochasticDepth(p=0.41304347826086957, mode=row)\n",
      "        (norm2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): MLP(\n",
      "          (0): Linear(in_features=512, out_features=2048, bias=True)\n",
      "          (1): GELU(approximate='none')\n",
      "          (2): Dropout(p=0.0, inplace=False)\n",
      "          (3): Linear(in_features=2048, out_features=512, bias=True)\n",
      "          (4): Dropout(p=0.0, inplace=False)\n",
      "        )\n",
      "      )\n",
      "      (16): SwinTransformerBlock(\n",
      "        (norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
      "        (attn): ShiftedWindowAttention(\n",
      "          (qkv): Linear(in_features=512, out_features=1536, bias=True)\n",
      "          (proj): Linear(in_features=512, out_features=512, bias=True)\n",
      "        )\n",
      "        (stochastic_depth): StochasticDepth(p=0.43478260869565216, mode=row)\n",
      "        (norm2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): MLP(\n",
      "          (0): Linear(in_features=512, out_features=2048, bias=True)\n",
      "          (1): GELU(approximate='none')\n",
      "          (2): Dropout(p=0.0, inplace=False)\n",
      "          (3): Linear(in_features=2048, out_features=512, bias=True)\n",
      "          (4): Dropout(p=0.0, inplace=False)\n",
      "        )\n",
      "      )\n",
      "      (17): SwinTransformerBlock(\n",
      "        (norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
      "        (attn): ShiftedWindowAttention(\n",
      "          (qkv): Linear(in_features=512, out_features=1536, bias=True)\n",
      "          (proj): Linear(in_features=512, out_features=512, bias=True)\n",
      "        )\n",
      "        (stochastic_depth): StochasticDepth(p=0.45652173913043476, mode=row)\n",
      "        (norm2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): MLP(\n",
      "          (0): Linear(in_features=512, out_features=2048, bias=True)\n",
      "          (1): GELU(approximate='none')\n",
      "          (2): Dropout(p=0.0, inplace=False)\n",
      "          (3): Linear(in_features=2048, out_features=512, bias=True)\n",
      "          (4): Dropout(p=0.0, inplace=False)\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "    (6): PatchMerging(\n",
      "      (reduction): Linear(in_features=2048, out_features=1024, bias=False)\n",
      "      (norm): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
      "    )\n",
      "    (7): Sequential(\n",
      "      (0): SwinTransformerBlock(\n",
      "        (norm1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "        (attn): ShiftedWindowAttention(\n",
      "          (qkv): Linear(in_features=1024, out_features=3072, bias=True)\n",
      "          (proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "        )\n",
      "        (stochastic_depth): StochasticDepth(p=0.4782608695652174, mode=row)\n",
      "        (norm2): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): MLP(\n",
      "          (0): Linear(in_features=1024, out_features=4096, bias=True)\n",
      "          (1): GELU(approximate='none')\n",
      "          (2): Dropout(p=0.0, inplace=False)\n",
      "          (3): Linear(in_features=4096, out_features=1024, bias=True)\n",
      "          (4): Dropout(p=0.0, inplace=False)\n",
      "        )\n",
      "      )\n",
      "      (1): SwinTransformerBlock(\n",
      "        (norm1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "        (attn): ShiftedWindowAttention(\n",
      "          (qkv): Linear(in_features=1024, out_features=3072, bias=True)\n",
      "          (proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "        )\n",
      "        (stochastic_depth): StochasticDepth(p=0.5, mode=row)\n",
      "        (norm2): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): MLP(\n",
      "          (0): Linear(in_features=1024, out_features=4096, bias=True)\n",
      "          (1): GELU(approximate='none')\n",
      "          (2): Dropout(p=0.0, inplace=False)\n",
      "          (3): Linear(in_features=4096, out_features=1024, bias=True)\n",
      "          (4): Dropout(p=0.0, inplace=False)\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "  (permute): Permute()\n",
      "  (avgpool): AdaptiveAvgPool2d(output_size=1)\n",
      "  (flatten): Flatten(start_dim=1, end_dim=-1)\n",
      "  (head): Linear(in_features=1024, out_features=1, bias=True)\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "from torchvision.models import swin_t\n",
    "# Use a pre-trained imagenet\n",
    "#model = swin_t(weights='DEFAULT')\n",
    "# Use no weights\n",
    "model = swin_b()\n",
    "\n",
    "# Update the fully connected layer based on the number of classes in the dataset\n",
    "#model.fc = torch.nn.Linear(model.fc.in_features, len(ds_train.labels.info.class_names))\n",
    "model.head = torch.nn.Linear(out_features=1,in_features=1024)\n",
    "\n",
    "model.to(device)\n",
    "\n",
    "print(model)\n",
    "# Specity the loss function and optimizer\n",
    "criterion = nn.BCEWithLogitsLoss()\n",
    "optimizer = optim.Adam(model.parameters(), lr=1e-4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import roc_auc_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 107/107 [01:02<00:00,  1.70it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [1/5], Loss: 0.5444\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 107/107 [00:48<00:00,  2.22it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [2/5], Loss: 0.4355\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 107/107 [00:48<00:00,  2.20it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [3/5], Loss: 0.3689\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 107/107 [00:49<00:00,  2.16it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [4/5], Loss: 0.3150\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 107/107 [00:50<00:00,  2.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [5/5], Loss: 0.2963\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "\n",
    "#train\n",
    "model.train()\n",
    "num_epochs = 5\n",
    "for epoch in range(num_epochs):\n",
    "    \n",
    "    running_loss = 0.0\n",
    "    running_auc=0.0\n",
    "    for images, labels in tqdm(dataloader):\n",
    "        images = images.to(device)\n",
    "        labels = labels.to(device)\n",
    "        optimizer.zero_grad()\n",
    "        outputs = model(images)\n",
    "        labels = labels.unsqueeze(1)\n",
    "        labels = labels.float()\n",
    "        loss = criterion(outputs, labels)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        running_loss += loss.item() * images.size(0)\n",
    "        # labels = labels.detach().cpu()\n",
    "        # outputs = outputs.detach().cpu()\n",
    "        # auc = roc_auc_score(labels,outputs)\n",
    "        # running_auc += auc\n",
    "    auc_score = running_auc/len(dataset)\n",
    "    epoch_loss = running_loss / len(dataset)\n",
    "    print('Epoch [{}/{}], Loss: {:.4f}'.format(epoch+1, num_epochs, epoch_loss))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Downloading: \"https://download.pytorch.org/models/swin_b-68c6b09e.pth\" to C:\\Users\\anish/.cache\\torch\\hub\\checkpoints\\swin_b-68c6b09e.pth\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "351b51fce3c2461382093d89095bc172",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0.00/335M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SwinTransformer(\n",
      "  (features): Sequential(\n",
      "    (0): Sequential(\n",
      "      (0): Conv2d(3, 128, kernel_size=(4, 4), stride=(4, 4))\n",
      "      (1): Permute()\n",
      "      (2): LayerNorm((128,), eps=1e-05, elementwise_affine=True)\n",
      "    )\n",
      "    (1): Sequential(\n",
      "      (0): SwinTransformerBlock(\n",
      "        (norm1): LayerNorm((128,), eps=1e-05, elementwise_affine=True)\n",
      "        (attn): ShiftedWindowAttention(\n",
      "          (qkv): Linear(in_features=128, out_features=384, bias=True)\n",
      "          (proj): Linear(in_features=128, out_features=128, bias=True)\n",
      "        )\n",
      "        (stochastic_depth): StochasticDepth(p=0.0, mode=row)\n",
      "        (norm2): LayerNorm((128,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): MLP(\n",
      "          (0): Linear(in_features=128, out_features=512, bias=True)\n",
      "          (1): GELU(approximate='none')\n",
      "          (2): Dropout(p=0.0, inplace=False)\n",
      "          (3): Linear(in_features=512, out_features=128, bias=True)\n",
      "          (4): Dropout(p=0.0, inplace=False)\n",
      "        )\n",
      "      )\n",
      "      (1): SwinTransformerBlock(\n",
      "        (norm1): LayerNorm((128,), eps=1e-05, elementwise_affine=True)\n",
      "        (attn): ShiftedWindowAttention(\n",
      "          (qkv): Linear(in_features=128, out_features=384, bias=True)\n",
      "          (proj): Linear(in_features=128, out_features=128, bias=True)\n",
      "        )\n",
      "        (stochastic_depth): StochasticDepth(p=0.021739130434782608, mode=row)\n",
      "        (norm2): LayerNorm((128,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): MLP(\n",
      "          (0): Linear(in_features=128, out_features=512, bias=True)\n",
      "          (1): GELU(approximate='none')\n",
      "          (2): Dropout(p=0.0, inplace=False)\n",
      "          (3): Linear(in_features=512, out_features=128, bias=True)\n",
      "          (4): Dropout(p=0.0, inplace=False)\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "    (2): PatchMerging(\n",
      "      (reduction): Linear(in_features=512, out_features=256, bias=False)\n",
      "      (norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
      "    )\n",
      "    (3): Sequential(\n",
      "      (0): SwinTransformerBlock(\n",
      "        (norm1): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
      "        (attn): ShiftedWindowAttention(\n",
      "          (qkv): Linear(in_features=256, out_features=768, bias=True)\n",
      "          (proj): Linear(in_features=256, out_features=256, bias=True)\n",
      "        )\n",
      "        (stochastic_depth): StochasticDepth(p=0.043478260869565216, mode=row)\n",
      "        (norm2): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): MLP(\n",
      "          (0): Linear(in_features=256, out_features=1024, bias=True)\n",
      "          (1): GELU(approximate='none')\n",
      "          (2): Dropout(p=0.0, inplace=False)\n",
      "          (3): Linear(in_features=1024, out_features=256, bias=True)\n",
      "          (4): Dropout(p=0.0, inplace=False)\n",
      "        )\n",
      "      )\n",
      "      (1): SwinTransformerBlock(\n",
      "        (norm1): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
      "        (attn): ShiftedWindowAttention(\n",
      "          (qkv): Linear(in_features=256, out_features=768, bias=True)\n",
      "          (proj): Linear(in_features=256, out_features=256, bias=True)\n",
      "        )\n",
      "        (stochastic_depth): StochasticDepth(p=0.06521739130434782, mode=row)\n",
      "        (norm2): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): MLP(\n",
      "          (0): Linear(in_features=256, out_features=1024, bias=True)\n",
      "          (1): GELU(approximate='none')\n",
      "          (2): Dropout(p=0.0, inplace=False)\n",
      "          (3): Linear(in_features=1024, out_features=256, bias=True)\n",
      "          (4): Dropout(p=0.0, inplace=False)\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "    (4): PatchMerging(\n",
      "      (reduction): Linear(in_features=1024, out_features=512, bias=False)\n",
      "      (norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "    )\n",
      "    (5): Sequential(\n",
      "      (0): SwinTransformerBlock(\n",
      "        (norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
      "        (attn): ShiftedWindowAttention(\n",
      "          (qkv): Linear(in_features=512, out_features=1536, bias=True)\n",
      "          (proj): Linear(in_features=512, out_features=512, bias=True)\n",
      "        )\n",
      "        (stochastic_depth): StochasticDepth(p=0.08695652173913043, mode=row)\n",
      "        (norm2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): MLP(\n",
      "          (0): Linear(in_features=512, out_features=2048, bias=True)\n",
      "          (1): GELU(approximate='none')\n",
      "          (2): Dropout(p=0.0, inplace=False)\n",
      "          (3): Linear(in_features=2048, out_features=512, bias=True)\n",
      "          (4): Dropout(p=0.0, inplace=False)\n",
      "        )\n",
      "      )\n",
      "      (1): SwinTransformerBlock(\n",
      "        (norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
      "        (attn): ShiftedWindowAttention(\n",
      "          (qkv): Linear(in_features=512, out_features=1536, bias=True)\n",
      "          (proj): Linear(in_features=512, out_features=512, bias=True)\n",
      "        )\n",
      "        (stochastic_depth): StochasticDepth(p=0.10869565217391304, mode=row)\n",
      "        (norm2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): MLP(\n",
      "          (0): Linear(in_features=512, out_features=2048, bias=True)\n",
      "          (1): GELU(approximate='none')\n",
      "          (2): Dropout(p=0.0, inplace=False)\n",
      "          (3): Linear(in_features=2048, out_features=512, bias=True)\n",
      "          (4): Dropout(p=0.0, inplace=False)\n",
      "        )\n",
      "      )\n",
      "      (2): SwinTransformerBlock(\n",
      "        (norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
      "        (attn): ShiftedWindowAttention(\n",
      "          (qkv): Linear(in_features=512, out_features=1536, bias=True)\n",
      "          (proj): Linear(in_features=512, out_features=512, bias=True)\n",
      "        )\n",
      "        (stochastic_depth): StochasticDepth(p=0.13043478260869565, mode=row)\n",
      "        (norm2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): MLP(\n",
      "          (0): Linear(in_features=512, out_features=2048, bias=True)\n",
      "          (1): GELU(approximate='none')\n",
      "          (2): Dropout(p=0.0, inplace=False)\n",
      "          (3): Linear(in_features=2048, out_features=512, bias=True)\n",
      "          (4): Dropout(p=0.0, inplace=False)\n",
      "        )\n",
      "      )\n",
      "      (3): SwinTransformerBlock(\n",
      "        (norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
      "        (attn): ShiftedWindowAttention(\n",
      "          (qkv): Linear(in_features=512, out_features=1536, bias=True)\n",
      "          (proj): Linear(in_features=512, out_features=512, bias=True)\n",
      "        )\n",
      "        (stochastic_depth): StochasticDepth(p=0.15217391304347827, mode=row)\n",
      "        (norm2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): MLP(\n",
      "          (0): Linear(in_features=512, out_features=2048, bias=True)\n",
      "          (1): GELU(approximate='none')\n",
      "          (2): Dropout(p=0.0, inplace=False)\n",
      "          (3): Linear(in_features=2048, out_features=512, bias=True)\n",
      "          (4): Dropout(p=0.0, inplace=False)\n",
      "        )\n",
      "      )\n",
      "      (4): SwinTransformerBlock(\n",
      "        (norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
      "        (attn): ShiftedWindowAttention(\n",
      "          (qkv): Linear(in_features=512, out_features=1536, bias=True)\n",
      "          (proj): Linear(in_features=512, out_features=512, bias=True)\n",
      "        )\n",
      "        (stochastic_depth): StochasticDepth(p=0.17391304347826086, mode=row)\n",
      "        (norm2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): MLP(\n",
      "          (0): Linear(in_features=512, out_features=2048, bias=True)\n",
      "          (1): GELU(approximate='none')\n",
      "          (2): Dropout(p=0.0, inplace=False)\n",
      "          (3): Linear(in_features=2048, out_features=512, bias=True)\n",
      "          (4): Dropout(p=0.0, inplace=False)\n",
      "        )\n",
      "      )\n",
      "      (5): SwinTransformerBlock(\n",
      "        (norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
      "        (attn): ShiftedWindowAttention(\n",
      "          (qkv): Linear(in_features=512, out_features=1536, bias=True)\n",
      "          (proj): Linear(in_features=512, out_features=512, bias=True)\n",
      "        )\n",
      "        (stochastic_depth): StochasticDepth(p=0.1956521739130435, mode=row)\n",
      "        (norm2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): MLP(\n",
      "          (0): Linear(in_features=512, out_features=2048, bias=True)\n",
      "          (1): GELU(approximate='none')\n",
      "          (2): Dropout(p=0.0, inplace=False)\n",
      "          (3): Linear(in_features=2048, out_features=512, bias=True)\n",
      "          (4): Dropout(p=0.0, inplace=False)\n",
      "        )\n",
      "      )\n",
      "      (6): SwinTransformerBlock(\n",
      "        (norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
      "        (attn): ShiftedWindowAttention(\n",
      "          (qkv): Linear(in_features=512, out_features=1536, bias=True)\n",
      "          (proj): Linear(in_features=512, out_features=512, bias=True)\n",
      "        )\n",
      "        (stochastic_depth): StochasticDepth(p=0.21739130434782608, mode=row)\n",
      "        (norm2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): MLP(\n",
      "          (0): Linear(in_features=512, out_features=2048, bias=True)\n",
      "          (1): GELU(approximate='none')\n",
      "          (2): Dropout(p=0.0, inplace=False)\n",
      "          (3): Linear(in_features=2048, out_features=512, bias=True)\n",
      "          (4): Dropout(p=0.0, inplace=False)\n",
      "        )\n",
      "      )\n",
      "      (7): SwinTransformerBlock(\n",
      "        (norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
      "        (attn): ShiftedWindowAttention(\n",
      "          (qkv): Linear(in_features=512, out_features=1536, bias=True)\n",
      "          (proj): Linear(in_features=512, out_features=512, bias=True)\n",
      "        )\n",
      "        (stochastic_depth): StochasticDepth(p=0.2391304347826087, mode=row)\n",
      "        (norm2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): MLP(\n",
      "          (0): Linear(in_features=512, out_features=2048, bias=True)\n",
      "          (1): GELU(approximate='none')\n",
      "          (2): Dropout(p=0.0, inplace=False)\n",
      "          (3): Linear(in_features=2048, out_features=512, bias=True)\n",
      "          (4): Dropout(p=0.0, inplace=False)\n",
      "        )\n",
      "      )\n",
      "      (8): SwinTransformerBlock(\n",
      "        (norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
      "        (attn): ShiftedWindowAttention(\n",
      "          (qkv): Linear(in_features=512, out_features=1536, bias=True)\n",
      "          (proj): Linear(in_features=512, out_features=512, bias=True)\n",
      "        )\n",
      "        (stochastic_depth): StochasticDepth(p=0.2608695652173913, mode=row)\n",
      "        (norm2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): MLP(\n",
      "          (0): Linear(in_features=512, out_features=2048, bias=True)\n",
      "          (1): GELU(approximate='none')\n",
      "          (2): Dropout(p=0.0, inplace=False)\n",
      "          (3): Linear(in_features=2048, out_features=512, bias=True)\n",
      "          (4): Dropout(p=0.0, inplace=False)\n",
      "        )\n",
      "      )\n",
      "      (9): SwinTransformerBlock(\n",
      "        (norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
      "        (attn): ShiftedWindowAttention(\n",
      "          (qkv): Linear(in_features=512, out_features=1536, bias=True)\n",
      "          (proj): Linear(in_features=512, out_features=512, bias=True)\n",
      "        )\n",
      "        (stochastic_depth): StochasticDepth(p=0.2826086956521739, mode=row)\n",
      "        (norm2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): MLP(\n",
      "          (0): Linear(in_features=512, out_features=2048, bias=True)\n",
      "          (1): GELU(approximate='none')\n",
      "          (2): Dropout(p=0.0, inplace=False)\n",
      "          (3): Linear(in_features=2048, out_features=512, bias=True)\n",
      "          (4): Dropout(p=0.0, inplace=False)\n",
      "        )\n",
      "      )\n",
      "      (10): SwinTransformerBlock(\n",
      "        (norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
      "        (attn): ShiftedWindowAttention(\n",
      "          (qkv): Linear(in_features=512, out_features=1536, bias=True)\n",
      "          (proj): Linear(in_features=512, out_features=512, bias=True)\n",
      "        )\n",
      "        (stochastic_depth): StochasticDepth(p=0.30434782608695654, mode=row)\n",
      "        (norm2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): MLP(\n",
      "          (0): Linear(in_features=512, out_features=2048, bias=True)\n",
      "          (1): GELU(approximate='none')\n",
      "          (2): Dropout(p=0.0, inplace=False)\n",
      "          (3): Linear(in_features=2048, out_features=512, bias=True)\n",
      "          (4): Dropout(p=0.0, inplace=False)\n",
      "        )\n",
      "      )\n",
      "      (11): SwinTransformerBlock(\n",
      "        (norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
      "        (attn): ShiftedWindowAttention(\n",
      "          (qkv): Linear(in_features=512, out_features=1536, bias=True)\n",
      "          (proj): Linear(in_features=512, out_features=512, bias=True)\n",
      "        )\n",
      "        (stochastic_depth): StochasticDepth(p=0.32608695652173914, mode=row)\n",
      "        (norm2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): MLP(\n",
      "          (0): Linear(in_features=512, out_features=2048, bias=True)\n",
      "          (1): GELU(approximate='none')\n",
      "          (2): Dropout(p=0.0, inplace=False)\n",
      "          (3): Linear(in_features=2048, out_features=512, bias=True)\n",
      "          (4): Dropout(p=0.0, inplace=False)\n",
      "        )\n",
      "      )\n",
      "      (12): SwinTransformerBlock(\n",
      "        (norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
      "        (attn): ShiftedWindowAttention(\n",
      "          (qkv): Linear(in_features=512, out_features=1536, bias=True)\n",
      "          (proj): Linear(in_features=512, out_features=512, bias=True)\n",
      "        )\n",
      "        (stochastic_depth): StochasticDepth(p=0.34782608695652173, mode=row)\n",
      "        (norm2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): MLP(\n",
      "          (0): Linear(in_features=512, out_features=2048, bias=True)\n",
      "          (1): GELU(approximate='none')\n",
      "          (2): Dropout(p=0.0, inplace=False)\n",
      "          (3): Linear(in_features=2048, out_features=512, bias=True)\n",
      "          (4): Dropout(p=0.0, inplace=False)\n",
      "        )\n",
      "      )\n",
      "      (13): SwinTransformerBlock(\n",
      "        (norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
      "        (attn): ShiftedWindowAttention(\n",
      "          (qkv): Linear(in_features=512, out_features=1536, bias=True)\n",
      "          (proj): Linear(in_features=512, out_features=512, bias=True)\n",
      "        )\n",
      "        (stochastic_depth): StochasticDepth(p=0.3695652173913043, mode=row)\n",
      "        (norm2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): MLP(\n",
      "          (0): Linear(in_features=512, out_features=2048, bias=True)\n",
      "          (1): GELU(approximate='none')\n",
      "          (2): Dropout(p=0.0, inplace=False)\n",
      "          (3): Linear(in_features=2048, out_features=512, bias=True)\n",
      "          (4): Dropout(p=0.0, inplace=False)\n",
      "        )\n",
      "      )\n",
      "      (14): SwinTransformerBlock(\n",
      "        (norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
      "        (attn): ShiftedWindowAttention(\n",
      "          (qkv): Linear(in_features=512, out_features=1536, bias=True)\n",
      "          (proj): Linear(in_features=512, out_features=512, bias=True)\n",
      "        )\n",
      "        (stochastic_depth): StochasticDepth(p=0.391304347826087, mode=row)\n",
      "        (norm2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): MLP(\n",
      "          (0): Linear(in_features=512, out_features=2048, bias=True)\n",
      "          (1): GELU(approximate='none')\n",
      "          (2): Dropout(p=0.0, inplace=False)\n",
      "          (3): Linear(in_features=2048, out_features=512, bias=True)\n",
      "          (4): Dropout(p=0.0, inplace=False)\n",
      "        )\n",
      "      )\n",
      "      (15): SwinTransformerBlock(\n",
      "        (norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
      "        (attn): ShiftedWindowAttention(\n",
      "          (qkv): Linear(in_features=512, out_features=1536, bias=True)\n",
      "          (proj): Linear(in_features=512, out_features=512, bias=True)\n",
      "        )\n",
      "        (stochastic_depth): StochasticDepth(p=0.41304347826086957, mode=row)\n",
      "        (norm2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): MLP(\n",
      "          (0): Linear(in_features=512, out_features=2048, bias=True)\n",
      "          (1): GELU(approximate='none')\n",
      "          (2): Dropout(p=0.0, inplace=False)\n",
      "          (3): Linear(in_features=2048, out_features=512, bias=True)\n",
      "          (4): Dropout(p=0.0, inplace=False)\n",
      "        )\n",
      "      )\n",
      "      (16): SwinTransformerBlock(\n",
      "        (norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
      "        (attn): ShiftedWindowAttention(\n",
      "          (qkv): Linear(in_features=512, out_features=1536, bias=True)\n",
      "          (proj): Linear(in_features=512, out_features=512, bias=True)\n",
      "        )\n",
      "        (stochastic_depth): StochasticDepth(p=0.43478260869565216, mode=row)\n",
      "        (norm2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): MLP(\n",
      "          (0): Linear(in_features=512, out_features=2048, bias=True)\n",
      "          (1): GELU(approximate='none')\n",
      "          (2): Dropout(p=0.0, inplace=False)\n",
      "          (3): Linear(in_features=2048, out_features=512, bias=True)\n",
      "          (4): Dropout(p=0.0, inplace=False)\n",
      "        )\n",
      "      )\n",
      "      (17): SwinTransformerBlock(\n",
      "        (norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
      "        (attn): ShiftedWindowAttention(\n",
      "          (qkv): Linear(in_features=512, out_features=1536, bias=True)\n",
      "          (proj): Linear(in_features=512, out_features=512, bias=True)\n",
      "        )\n",
      "        (stochastic_depth): StochasticDepth(p=0.45652173913043476, mode=row)\n",
      "        (norm2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): MLP(\n",
      "          (0): Linear(in_features=512, out_features=2048, bias=True)\n",
      "          (1): GELU(approximate='none')\n",
      "          (2): Dropout(p=0.0, inplace=False)\n",
      "          (3): Linear(in_features=2048, out_features=512, bias=True)\n",
      "          (4): Dropout(p=0.0, inplace=False)\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "    (6): PatchMerging(\n",
      "      (reduction): Linear(in_features=2048, out_features=1024, bias=False)\n",
      "      (norm): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
      "    )\n",
      "    (7): Sequential(\n",
      "      (0): SwinTransformerBlock(\n",
      "        (norm1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "        (attn): ShiftedWindowAttention(\n",
      "          (qkv): Linear(in_features=1024, out_features=3072, bias=True)\n",
      "          (proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "        )\n",
      "        (stochastic_depth): StochasticDepth(p=0.4782608695652174, mode=row)\n",
      "        (norm2): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): MLP(\n",
      "          (0): Linear(in_features=1024, out_features=4096, bias=True)\n",
      "          (1): GELU(approximate='none')\n",
      "          (2): Dropout(p=0.0, inplace=False)\n",
      "          (3): Linear(in_features=4096, out_features=1024, bias=True)\n",
      "          (4): Dropout(p=0.0, inplace=False)\n",
      "        )\n",
      "      )\n",
      "      (1): SwinTransformerBlock(\n",
      "        (norm1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "        (attn): ShiftedWindowAttention(\n",
      "          (qkv): Linear(in_features=1024, out_features=3072, bias=True)\n",
      "          (proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "        )\n",
      "        (stochastic_depth): StochasticDepth(p=0.5, mode=row)\n",
      "        (norm2): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): MLP(\n",
      "          (0): Linear(in_features=1024, out_features=4096, bias=True)\n",
      "          (1): GELU(approximate='none')\n",
      "          (2): Dropout(p=0.0, inplace=False)\n",
      "          (3): Linear(in_features=4096, out_features=1024, bias=True)\n",
      "          (4): Dropout(p=0.0, inplace=False)\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "  (permute): Permute()\n",
      "  (avgpool): AdaptiveAvgPool2d(output_size=1)\n",
      "  (flatten): Flatten(start_dim=1, end_dim=-1)\n",
      "  (head): Linear(in_features=1024, out_features=1, bias=True)\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "from torchvision.models import swin_t\n",
    "# Use a pre-trained imagenet\n",
    "#model = swin_t(weights='DEFAULT')\n",
    "# Use no weights\n",
    "model_img = swin_b(weights='DEFAULT')\n",
    "\n",
    "# Update the fully connected layer based on the number of classes in the dataset\n",
    "#model.fc = torch.nn.Linear(model.fc.in_features, len(ds_train.labels.info.class_names))\n",
    "model_img.head = torch.nn.Linear(out_features=1,in_features=1024)\n",
    "\n",
    "model_img.to(device)\n",
    "\n",
    "print(model_img)\n",
    "# Specity the loss function and optimizer\n",
    "criterion = nn.BCEWithLogitsLoss()\n",
    "optimizer = optim.Adam(model_img.parameters(), lr=1e-4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 107/107 [04:06<00:00,  2.30s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [1/5], Loss: 0.1938\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 107/107 [03:39<00:00,  2.05s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [2/5], Loss: 0.1363\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 107/107 [03:38<00:00,  2.05s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [3/5], Loss: 0.0938\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 107/107 [03:37<00:00,  2.04s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [4/5], Loss: 0.0709\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 107/107 [03:40<00:00,  2.06s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [5/5], Loss: 0.0324\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "\n",
    "#train\n",
    "model_img.train()\n",
    "num_epochs = 5\n",
    "for epoch in range(num_epochs):\n",
    "    \n",
    "    running_loss = 0.0\n",
    "    running_auc=0.0\n",
    "    for images, labels in tqdm(dataloader):\n",
    "        images = images.to(device)\n",
    "        labels = labels.to(device)\n",
    "        optimizer.zero_grad()\n",
    "        outputs = model_img(images)\n",
    "        labels = labels.unsqueeze(1)\n",
    "        labels = labels.float()\n",
    "        loss = criterion(outputs, labels)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        running_loss += loss.item() * images.size(0)\n",
    "        # labels = labels.detach().cpu()\n",
    "        # outputs = outputs.detach().cpu()\n",
    "        # auc = roc_auc_score(labels,outputs)\n",
    "        # running_auc += auc\n",
    "    auc_score = running_auc/len(dataset)\n",
    "    epoch_loss = running_loss / len(dataset)\n",
    "    print('Epoch [{}/{}], Loss: {:.4f}'.format(epoch+1, num_epochs, epoch_loss))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "xai",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}