lrcn.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from Modular_code.utils import *\n",
    "from Modular_code.clr import OneCycle\n",
    "from Modular_code.radam import RAdam\n",
    "from Modular_code.opts import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "train_label = pd.read_csv(train_label_path)\n",
    "valid_label = pd.read_csv(valid_label_path)\n",
    "test_label = pd.read_csv(test_label_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_id,train_classes = create_label(train_label,'./DataSet/Train') \n",
    "test_id,test_classes = create_label(test_label,'./DataSet/Test') \n",
    "valid_id,valid_classes = create_label(valid_label,'./DataSet/Validation') "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_label(id,classes):\n",
    "    id_new0 = []\n",
    "    id_new1 = []\n",
    "    for i,j in enumerate(classes):\n",
    "        if(j == 0):\n",
    "            id_new0.append(id[i])\n",
    "        if(j == 1):\n",
    "            id_new1.append(id[i])\n",
    "    return id_new0,id_new1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_id0,train_id1 = extract_label(train_id,train_classes)\n",
    "import numpy as np\n",
    "train_id = train_id + 5*train_id0 + 1*train_id1\n",
    "train_classes = train_classes + np.zeros(len(5*train_id0),dtype = int).tolist() + np.ones(len(1*train_id1) , dtype = int).tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dict_keys([2, 3, 1, 0])\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "dict_values([2615, 2493, 426, 204])"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from collections import Counter\n",
    "\n",
    "print(Counter(train_classes).keys()) # equals to list(set(words))\n",
    "Counter(train_classes).values()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torchvision\n",
    "from torchvision import transforms\n",
    "from torch.utils.data import DataLoader\n",
    "from torch.utils.data.dataset import Dataset\n",
    "import os\n",
    "import numpy as np\n",
    "import cv2\n",
    "im_size = 112\n",
    "class video_dataset(Dataset):\n",
    "    def __init__(self,id,label,sequence_length = 60,transform = None):\n",
    "        self.id =id\n",
    "        self.engagement = label\n",
    "        self.transform = transform\n",
    "        self.count = sequence_length\n",
    "    def __len__(self):\n",
    "        return len(self.id)\n",
    "    def __getitem__(self,idx):\n",
    "        video_path = self.id[idx]\n",
    "        frames = []\n",
    "        a = int(300/self.count)\n",
    "        hp = np.random.randint(0,2)\n",
    "        id1 = np.random.randint(0,a)\n",
    "        label = self.engagement[idx]\n",
    "        for i,frame in enumerate(frame_extract(video_path)):\n",
    "            if(i % a == id1):\n",
    "                if(label < 2):\n",
    "                    if(hp == 1):\n",
    "                        frame = cv2.flip(frame,1)\n",
    "                frames.append(self.transform(frame))\n",
    "        frames = torch.stack(frames)\n",
    "        #frames = frames[::a,:,:,:]\n",
    "        frames = frames[:self.count]\n",
    "#        frames = frames.permute(1,0,2,3)\n",
    "        return frames,label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "im_size = 112\n",
    "train_transforms = transforms.Compose([\n",
    "                                        transforms.ToPILImage(),\n",
    "                                        transforms.Resize((im_size,im_size)),\n",
    "                                        transforms.ToTensor(),\n",
    "                                        transforms.Normalize(mean,std)])\n",
    "test_transforms = transforms.Compose([\n",
    "                                        transforms.ToPILImage(),\n",
    "                                        transforms.Resize((im_size,im_size)),\n",
    "                                        transforms.ToTensor(),\n",
    "                                        transforms.Normalize(mean,std)])\n",
    "train_data = video_dataset(train_id,train_classes,sequence_length = 150,transform = train_transforms)\n",
    "val_data = video_dataset(valid_id,valid_classes,sequence_length = 150,transform = train_transforms)\n",
    "test_data = video_dataset(test_id,test_classes,sequence_length = 150,transform = train_transforms)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQEAAAD7CAYAAABqkiE2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nOy9WawlWXYdtk7EjTu/KeepuoZmNZuUWjIpWhQkwBBEG54I80cSZBkCLdPgjyUPsmFK/pH8IUACDMsEDAtoiLJpQwCpCaBhyTJk2jQkQm6x2Gz2xK7u6uoach7efOcbcfyx1zpxz7nvZWZlVpJPfrGBqsh7bwwnTsQ7e1p7bee9RyONNHJ+JfudHkAjjTTyOyvNItBII+dcmkWgkUbOuTSLQCONnHNpFoFGGjnn0iwCjTRyzuWVLALOuX/DOfeuc+4959yffxXXaKSRRj4dcZ82TsA5lwP4NoB/DcBtAL8G4N/13n/zU71QI4008qlI6xWc8/cDeM97/z4AOOd+AcBPADh1EdjaGPhrl3bC57VlKSxUnv93n3xUn3it8yv/Xx1DKhrLSb/zN2fbirtU/Ee6ADvup2/zjMf7KrqGjh+PxwCA0WQKAFguq+h69Zlc+L8u6ZM5DT+4jBtuXWosJmNPzpNxzC0er3vIeZ4sS86f5+Ga8/kSADCdzwAAZbmMhuhOeez6vtsuAABFzmvzh1ZL1wxHRFu3emLej77KW/Ynkrk8vu/6gJM/PUO56jyn7rXy+9ptJ8f68Az5hYt2w7KyL+7vHjz23l9OT/cqFoGbAD5e+XwbwI+mOznnfhrATwPAlYvb+B/+6z8TfksnuqqW3Fb8PX6Iz/MXHv6OeMyzLCD9Hq5dnyC+tn/K+fjW+awNAJgubN/xxF7y5bK03fjG6Q/EZ3aufteOc9WIW9t/yuN/7Te+CgD48lffBQA83rP90Mo59io6b4Y8/KGVS9v6Sls7d6voAgCK3gAAkBcd3ib/YHjOcjG3Lf9gc9jx/Z5d69KgDwDY6vcAAIOunXfQt/O2uvZ7Z3MLVWH//ujOAwDAex++DwDYP9izay/smkUrXpA8n0m3sLG9/dpVAMC1LTvfZscWhauXhwCAtk0nKnrBrlVwujTPHmW1sGsVNocXdi7ZfXXtHAvNW/hDtDFUnL/wfbKV6JmUZRl9ru8pXugrBzidoqyibelsuyzjMWkFW9glsDu1+/3L/8s/+BAnyKtYBE5ar09Q7v6LAL4IAN//5q3od720kqr6nYtfukwv//MtHieegw8lz3NuWzwXTjxni3/EWUbtw2evvY4nZgHcf/gEAHB4bJ8XfLGCAcHz6rqL5RKl/pj5necL1Oacd/iH1m7xJFyQlnwpWzx5yeMrasqcg9vo2KKx0+cfIheBfte+77V57+DiOj7CxNv4UdqC0uvYPrOu/ZHO/SKan7BY8ppzLqZHIzvP5aEtOK12znvl+5NYJWEd5x+ig0O2Zvn8zoqDXzFcuLBkJy8wa5+5QD3rjX0Vd3wbwGsrn28BuPsKrtNII418CvIqLIFfA/C2c+5NAHcA/AkAf/JFTuSDf0ZzWX5Z8Eufb6X7NMQlDuknMQgcl/KisOlu00yTKZiahLIA6ovpmvaPh4/NTH7A7WxehisBQElzUUNeLEyTLuYeHnbugnM3LEzb7gxNU28NTYPLVF7ynPMlLYKFbRcyRfl7h/tf3rDjL21tAAA2+6aVex0zuXOOaUmrZTIfYzE2C2C0f2z3MzqynWilBMPQxz58lnOMNBQORhYbmfPceSvY0QCAqqIlZofVFpPcTGQoada7SjGRF4g/nSCnWZAhDrQWH9KBLjz/ymfc6pj0vYnHvNB9PeMePvVFwHu/dM79GQD/B4AcwN/03n/j075OI4008unIq7AE4L3/hwD+4Uscn3xx2kr2/IHBtSOfc4U/TePXK/hznCMZX57HK7ZWcPnubWpnl1g687mpvPsPHgIARmPTfAtpfp7Xh+yDHbekqlwuKrQZpBx07FrXty3gdf3SJgDgwpZ9rji3U17zeEbNPbHPU0adfGWafkj//dKOWQAXNiwAuNmzmEChWAK1+4z3VmYeHWrqLhiUozaeUKPLssmzVjRfCrxmnK8ZYwMzZUl4DwqywSm4otC/4j21j12FoNvT/e36/YnjPWnALw0QPuv7VDKs+PYcv2ICup21d5mfS9izLv3T/8zPVhSkkUYa+W2XV2IJfOqiiHaaCF3b1vKyIKigpblOVuHaWsnj/Z56vWS1lyXQo5asUzu2yVysJeTHS/MfHpjPHHzZRItIGwWPuFyE83YZzb+yZZr65kXT3FcuyIe372VVzBamuY9nlhKc9207m7eie+oxnbm9becZMkvQZRwkCzgCO+9iqWxDhh4zEhuFbfvU0Mch/R2n1YRFcIyd6Nkwk4jpTLEWWlglYyz0qed8Zm0GB2QJZM6h3bZxKyyj2MWc86DPeuNCahBxfKfexs9SMQdZN6klkKUYj9KjpNUkgyZYKcHqiFPNAXdRdXiqJMaUSGMJNNLIOZczYwk4507QpvKZYwsg+E9y8ap69az3zOJjpF25R5J6fYrQ71QkmZ/Dyp9kMORuVs68OZMlryWtwNx6lloRuj/529IytpLvHzIGsFCknrEAL3+UQCBqDmk4XTfLgJ2B7Xtj2zT+5U2CeZibbzHhT4MBLWIa2gTklL0Wr9WN7jsntkFZgKIlLISP5kmi45xzaHPfYUfxBbuP3dGc88BjEpXlfcl5IOaAz2bMOMayZOaisDhHZ2hbt2H33CJmYarsxOEhRsxMaLxlSQCWt7kvZTV4aW7eX7ix+B8BlyFt7oS74PEyAvl7rjNpP1egTLIiNRKV76JrRb+3mKnBwn4vnqHqG0ugkUbOuZwZS+CFRD50toLrfslYwPPHEk7JTGjVhocPZkbssztIg6e+P3+nBRDuhVpjb99wAXQFMZ9pP0Xe+Vlmi4Lg/L3fauHyJqP3m6YVuz3T3C367oLLhowFx5QTQVgy1y5knaLikjzA8DT0BAtBjdcKiMMKS8YH2m3GBvo2lu6hfT6ay7xKrblYM8pnntAyvPLm5wAAX/jhHwYAfOeDOwCAw4lp93xhY+gPtgEAA9fB8aGhMLcGNi/XrmzxEm1eQ/cfx6EUn1iL1AuyrfhGZvc/GdsYFtNjTgv9esYpulsX7YD2MJyjJfNMVidjHopT6Fn0umbp3H/0GADw3gerKP51aSyBRho55/IvpCWwnuOvP5+OzFrfF1iHICSB+lq9PLes7C/fPLlmyO+GqsA4UozwvW3nM/NHx/RXVXgkDVofr+tRO4esislmt4sdFvZ0iatv5cpzx5o9aB2KkH6npLNrVOPavSS7pYbTyheqS+h37dqbfdOK+/TZlakouV+b8YcMQi8Sf9E2K+faW98PAHjt+38AAPCDP/QjAIA5C7D0HFq0an7jnV/F/MishXab88ICqt7AComuf+azAICixWokWgBCL6ruQzUn4XdlmVgH8e2vW/HXw9vfsv1kNbbNUtv5zO+yj8NL6LIOQ89GxWzjyQQAMKJV0WFGZtCz2MpkTqsrayyBRhpp5ClypiyBZ+PzPx0c9yeR50cWJrl++JXahthPTi2DtWuGUIL9Y0LegBk12NHRUXTNklFyT+2TybzhRpV713c2sUktoXr/sA31GTECTrGBSpwIVWx9VEne3K9E/eObctHv4essW7NYxAewzTqG/YldY+/Q5iHk2BGP2dNCmkxtnu4+eAQAmC/t85XLn+EgEW+Jfmy3MjhNvmOEnRq+TQtq5/p1fm/zqEyM4jCBhyCPYweqQl0uzKoraInl4XoxF0TB0uXB5ia6fGYLWkKeCNCc+Io+KzXb5FPI8jhG5JidOk0aS6CRRs65nA1LwFFzBIRYIimJiHLz6zvW+ee1n+LIcq0MTvFdwxVTVGIc8VekP1eGIlIz2cqeqNVEyCCkoxTmINa242Pz/WbUcHtEDAp9Jh9RbD25E0LONMYONerV7SGGPfMba/+Su4ZkPHg/ikibdhEUQ3nyMrUAypjA5LS4TahrAP3yzIXHqwh7Tq3Yp497cYsR9ZndzxG3NV8C75uWgGIEIbPBPDoqzUs0pJDFWfgqnFN0Cm3VJyR1CMECyPXM4lMr1qJc/mIpAhCOLcKTAHCKJXAO2qbdkeX165/EGfT+lIu43kGZiLnBLAKz0GlyJhaBTruLz77++fBZYJmYVGtFki8EiqnKMkyY/hhL/iGprLJKbMH65U/SdWJxCX+wKs5R4CsG9ASUpz5jhU4s+UOryni1k3kbACq8c/3R5yQPmZUKCJHVJ1nY6lJb+yhT8/K2FQdtbw4xGMgdILCEf0Ai4AimdYCqChSjYKMuxf30x817TglUgoS1U6AZ7p+5EETTQkrcUviD2WEA8JiQ5qPJof0+V8mwqMrCjduYSSUk5qAaZJMAl8IzX9ZpSNGs6Q9LAb62zV8mUpbEZxWkuci4eHqdR2XRIkSJ3QRBmp3X4qvFoKgBamEN0D8I3c74PnAMCnSGhapxBxpppJGnyZmwBOByZO3N8DFdmU6LzSkgomKPo9E4aNFgQWnlVWBL0MpQIIRofxc0gL6X66EvYm0bIM0+tjxWnZJ1ayy2LkKKj9eSBhyPLPUzntHS+dJv2u+CE7v4uMzLWrHfh12bmKs7BMNsDAPppjR9i1pW6TbnZDmp0IXzlZpl0p6632cEUJ2XdcdiJj6nAh5dadU+td9cVojSavaabg9Mu97fZ5AtFEppy2cqGjeRhIpezMVWYG1RyupbdQ757BJLp033yLXIvSj4eDgsDoDytoN21gu39MvVy6xZt7IKq8oHaySFYMv9GwzMQgoAL16rFayV0/K60ZAaaaSR8ypnwhLwAOarGJsEFpqH37jS0++a03+biXZrPsfcKzikQ6SyuTIrlRPc6DiwEzR9giJa+yz3M6Hsrskh3FoQUvuEFJ7qoESeQe07m7Gg5dggpe/eNYrGDwkDXehaYSxKlVFzsnRYxB47O5ZuQrvA7Uf7AID9Qzt3jyzAmyyq2WFBUZ+OebuSP6l5knaOy3tXC4KieQrK186zKOOS3AUcfN6JzrlgQFMQ5QAiYlBT9TECTZ2my06jSz9Nqqpay0tLoy/Iqjw+2LUrKkWYsgcn74HGuJgS8DU7AAA8fHwfADBdMO4jS5UszlWwNN2pVpa+VvA2kK1UscUQ4OSnSGMJNNLIOZczYQnAAa616o3FfneaSlN0tCCvfCjemI7rJbQOV3Of2CKor6X0WopnjaMFaRRcktJQRadIiCNkwchYCKAXxjQWTPWMqAlZ34LR0rTDIanGZ0uRhOiW5PPyXjk/F3as+KXFoqBHe0f4+KEVIe0d8VwLyzwown7pgsVmrl+yOMKloVkTGwQcdXOlzGItk1KkpVBoafkRi55GTPNNliWOSBA6ZhOV8dhSohVh0RskK20V6hMQ06WHnFilMm9ZLcp0JDGA+sWKpCzLE6jtbHO8Z1bYN379n/FYaV1uREumFGIlGnRL545JCz+Z273N52aJFSwoKhTYogWmLFbl6+yFS+JRknWQHVOeS1lfjSXQSCONPEXOhiXggXJZrUT0uSpKcyZLVVgRuV+xQjMlBV6v5/TLFRNQDOB5IcnpjkksIUHoRpZC0uNiRUPpe67YogQnhVdFeGhB7TphbKCkXykt6wI0lRqwVBcgi/RfuXDBLkRtNT46wg6bgHQ4qieHpqlUpPPdQ/NVb981zXeF8YRbF61N3BXGDkQJJjdUWjoP4Bi7pxm1+YzYiEe7dr37jEkcTKeYqIuRADp8ni1iE+bMc19gGbRKjiviBGQxCCQlrTznebNTNH/9ubZq1qBhysTQGpvPLWMTnj9i604WT4BVT23/dsBd2PkKWTXEZ+gEbRGfrHTKWsOohKK20zIyAj/ZM5gvG0ugkUYaeYqcCUvg6PgY/+RXv4QOySo//5aVa168ZJqsIgA0UDGVXLsCp0YdoQ//TlqZ1fn+k8eQRv8lqUURNEWcFFi5jsZUf6hT6xqbbZa8gfGUfuLItktqHZGB7D62iP5MzeVEVCHfWHENXu8itcmFTRJiUPsMiwKLSr64WRWEEmBIjbQgNdf4yDT1fZYx58SgVmOLEQzIWbXBDESPsQNPq+aIx4+Y8z9iLODeI4tJ3D8mVVq1xCatky02LtkZmLWx0RU6z8YoPd2l3+yOiReYK7ZCpCWzK4fEWfi0HDw8JB/9Xq3sF6zSEO3XF4Jb61idKi7nlvnVY8/FeWZjqjLhClgU5G1eC6H/+IxFztJyLlhIATHI56/GL4qDBbS4RpAiVE+RxhJopJFzLmfCEjg8PMI/+j9/OVBw94j13rl4Id7xFBcoy1Y98gQX/pJjexYR6ann9ytos8RqkFWy+9D87i+/8xUAwPSYmks5eMYGvvveBwCAhboY8/i2IsjK1VObXGStQI/Zkwn9/Wo5x+6R4e7n9N27W2YtzBiRB6+ZIfY7pV2PR/S/GXfYucAMBEtutZ+2k6ndy8HIxng8FmLQpJu10FZGgSW/kzFrBnSNTbMyFtTUwwNRhcsf9+H+7Nq2fUxchbIw6xLn9H1VBXUb4gjEKrS7NoaCBB9O8QdaPrrfsqLFRIsL1MJ7u/ZsZ4wxHNP6mzLj0+H89S+YhSAMTKcqQxZI8YRQQp6RwDZXvKvgXWk+eF+NJdBII408Tc6EJeDhsagqZGqXxdVyQbSa+BbyUKZqK9uc+1dcff0K9ruKMPzr1V6ppFVvn04jypPzt/L7Dp+Yr/+977wPAKExp254xrz50SGr5qjR1PzzKjWk5uFwaufd2rCYQEG68DHzxbPpBLO5/Tvv2z4lkW/3iYSbUbt22YRjSH89zxTVtnOGZiObZnX0h7Y9CNFv03yOGtKrCo8+cpePYzlfYkafPlPl3NKes7AMW7zPDq/dF37fToENlUfzs/LqD+4/AAA82bd5/ixEKiKTTJj81SrO2HqraF0Nr94AANz4zJu8b8NRjNgOfXf3gOew+W2xFrnH7Wsjxkim9iy//ptft+MeGvWXUKQBA6PqTudWaOM4Jlk2zI4sZkJ1El9B/MRicsxzP13XN5ZAI42cczkbloC32vigq7kKCn0VorvK8cvnCaiqWsvXRBs85LnHEFsKsgxOw8R/EklbTZXU3ELIlaHxJiPDTIOUIs9QExGeT40vpGWktWf0qQeduAloK9S4140+Nrcs3jK8chMA8PETy9/nxLJfZWvxPtGGkwOj6lJbsSHjNxe2DT8w3DbNWM1MG+3Ragsajfd09eoVuyfW23905xE8cQADZhpaqgmpYqxBQUuhrcaktJh2Ns26uMhquo9JQzZhnONb334PAPD7fo+RdwarL8EJVFW1/pypmT+4Z1ZF+4LN15VrNj+Pj+waD5kNETbkylWjDN+8bNsu/fLLzAY4Upi//zU+Y857xSzN/m2zDo+LbtDsS1p0c74Xy7ma0TAeQSukogV98PiezVPV4AQaaaSRp8gLWwLOudcA/M8ArsGSol/03v+sc+4CgF8E8AaADwD8ce/93rPOV/oqaHbh/KuQc42r8VS5pcR4qMCqKpRcLVPk1zMKyALPQEi9qk1UgokPaMYgzzgxaktAWyG4luE+YuYYp2iuWo7XQIPomm2h9nh8h2PrdlRXTpouNqO4uLWFrDCtsaDWUE3A91FD33tglNvDlnLw5ttfIA7gMluXbw7YcJTklkNeY0Regq5owVVQT23VYsR/a8csiLsP7mNATf77/qXfDQC48+GHAICDfYvuy892VdxuS7RiGbVr13d533ZJZSze/8DO9+ChWTM3blzDqtTvVbn2ncApnhTkY2rZfTYw6TJe8ebbhm1RrcCAmIcxW6ndZc1Gf4M06eQjaDMT5itpc7NiHr1v3BEZfEAtyhKoqwJjPgn9ucgKno8tTuFeIZ/AEsB/7r3/AQB/AMB/5Jz7QQB/HsAve+/fBvDL/NxII42cUXlhS8B7fw/APf77yDn3WwBuAvgJAH+Yu/08gF8B8DNPPRcAjyzkXlsihtQapTx7FVsEoSW3EHjLJaplzKf2HHqa29j3VyuwQCDK3cTMk6IBT6I/SuMMocacUd1OIc3F+AP3D81FtKSLSpznaXFMbTbAEBlmtxByMCYNHWxY5H7YG6DLWoGP71v14IP33rXfaIbcogZbLM3X3SZ703XiAbbZIqxXyNfn2KitO4w59HumhRfezneFr9rB1DTdfO8hAODGIMNG37Ti8QP6sMTbX2OMoBsK9ug/8746jFf0mQWRIndV0hKMY/zGuxYbuMqaipxWi2f0vCxLLJVbV66B8YeOs2d1eGRj+5CZhx22c+spDsOaAMd3+Hhm+z85Mktg6lgbMFYdiIJXeqFogXL+ozdeN1jFloALsTL9vcREpP4UNKzkU4kJOOfeAPBDAL4E4CoXCC0UV0455qedc+84596ZE3DRSCON/PbLS2cHnHNDAH8PwH/qvT/8BM06vgjgiwCwtbnhvVvhDVAOObDW2GfhuOvKLTlBttnfO8ABI8J1E424pjy1DMrYEFjjJmxR2w7pEw+HMaV2YCrSaVbYeuW71RZByuAba/jAmszdxcJTx0Z0LTYPZZxiyKh5n/69tIuqLFvUUp1+BteShmazT2q2OfEDPVon3a5Fv7e3zIrYJC6goManEkYuLkIy4ohzUfj+SlkCjqHP+gjRh2cdh5x1DNM9O4fapAd8AP3xWeAUtGvs0ILYIKvwaMKquYUoySvei11774n5yLfvmhZ//fVbNhec5+PRCCNWag4158yGXLlw1cZIHEVJTd5hFkVoRfBcajE+I3+AL6n5Z8xssCaj5l6k+Ph9ilgPk0xVreHTt1qxEr1Zr5Bj0DlXwBaAv+W9//v8+oFz7jp/vw7g4ctco5FGGnm18jLZAQfg5wD8lvf+v1356X8F8JMA/gq3v/Rc50Ot8Wt+NkaFhewisUBBHzj4ztQQX/nqN/Fg19Bhl4jo+tEvWMT5EisS1YJ7nw08CFZDl8g49Ty4c98iyb/53jcAAG+9+RoA4LPfZ6izNrWuT9BYMZtNjEirLQPbN8121NaK6tvjefDBQor567o9G0vLmQacEjvP9DraitQXBVpDcjCwsUe54LVCcwwi3piv7vA+nbTrXM0zWLNORGBe6FXi74yd5LRKBmzsqXleEDU5n83X+i4U1PyDLrHwtC4mCUPOJv3vgnP+4MA0/SRhFJqpApLz+p0PPgAA3LxmnqrauM2WU3hlImSt6VqMlWSshixaekZ27gXnZcFrzabS6MzQ0FoZMg4xC+3H9DKkFnSdCQoAxzVLIN43zUKpdqV8hnH+Mu7AHwLwpwB8zTn3FX73X8H++P+2c+6nAHwE4I+9xDUaaaSRVywvkx34pzgdkPdjn+RczjlkWbbGDhui/+LT14qXdguittnfP8bDBxaFXU5saB9eMu3w5JA5Vmp6ocn6bDQ5JNpMeIC9Pfv9zm2Lot+8YdjxoLUSrIJkvc34yv0EFGMS01iZB7tGnBfWVjGEJU+0NzaU2me2zY9vM7c/lZaVpVCqMjAP7at79JOVTBG7keo2xEuYOWnTWXRfwnIEfkdh3xlNFwOun9p5FUMYMNuguIUf9ldafKmfgv0mSMZYNSKaQM6P+AYOyd93INRdK8YTTNjItaAWVsPSx/tWL9GRxeGAXOPmc54dGNb/yfvfsXPzHZ2I9emRzWfWkmVoQ5xynjU/A74mORuSTmfH3F/ZrBTZst7PYU3Th34Vqi2I8TVLzuPyGaWwDWKwkUbOuZyJ2gHANEvg8k9ZgXziCyn/yc/y+cbjKQ72bYVdsJ31/7P/qwBqZl51bVEEXtz2sgCkAUes6Buz3jvkjxOuuEz8b3LtViwARW/rKK60ZdwFSD6d7i9w1S9VUxBrX13jMdl5RtT8wvNPGelXxeC8xdx8BuQcb9FllFojY1674FbzMWXd+4zR/FwVnfTtQ9R/sMUxMhbArEpVaf4YGxhIS2sOPHL1IqyLI3gMef3YR0E4CvV6aBd27XsHZq1NK3U1YgcnWkqBgzGPrba75HO4dcMi/+3OAIeM/gvpNxha9kS9HBAasXI/nUyp/KQjU0YMg+O21FiWdl71UPBVbEmof2DpV2MC3NcLu6IYUxoj0DmfrztUYwk00sg5lzNhCTgY/j9PWIYDj37iL9Xdfu0fI1ZTjY+PsZDmErqsZI+9ZDVlVhcLocv4uWpJi9IvpWZYVHGEOXjzgZcu1uYnxQRqlmTVEAgHYL8Ls1DJEhD7sPoThPbV5O6nBXD3wPTRFaLX9ohfny2sgm3OXoZTj8BI3KWWEA4isDNpLGQYEuOQOOzb5P9zBee1JdZc888dfV59D1oWsmo0LeIlMDbdGPkmKZX9CZ10VJlo83BMC3BvFvc+UBbpaGzzsijFm2jatzOwedp9YjGBi2RS3r56C44VmS1G/bOAwmSGJo/7QLpQ35FE+ym+iq3B+r1It3xf0+99ufL+Zyv/r9GayhbV2TVmUUrhUNJ6l1gaS6CRRs65nAlLADBtn4WOwOITYM452beOkvLzCq9AWElVm+9n4QoA4NRJR/na0Tw6Vy6fmDNTMSoeVlettqFk6/n8rlWRBTOZTKLPytXPib6bEr0mHIQmosUIPBsW4cPHFsHeIqpxztz9lBH/OR35aVWtdEOKfVCx+kidLDgGId4y5v07tASWtBSOx2Z16O6nmi/6rV0yCU2ZJZhTe7d4vSzHSpWcaiZsO1V2RAhKzs+M2zvsYTAT92KosWCsiFkAcRqMyD588YJp/sNju7djPof+zg7yjlkJLWrXfGxYt3JicYc6K6TMVYrGSywBJO/qaRH+9J0+kQkrRp4+S3yoMmxiAo000shT5GxYAsQJqI/8J2bx0QKZufDPljjqwzXsl40dw8JfvnUdAPDOl78JAGAgHS3md+Ujz7LFi43ppGGG1V596+j7U+suA4NM3J1YlkLO+njVQyxYjXjIXn53H5uPe2HbIvUTMhfN2PU3r5bBymq1VLSlEr0YjVclXP0jnmt03zSjLAExDU0GZBmm9TFiLEGVbTktDxk1wj60nFurl59xHibcMjGB0Uw9Ge37B7tmAWmkuoZkTA2v2B9mOk0AACAASURBVIIsL/E4CFk5Fhuzy7Fw6gtgUshC1Lwkz7AOyZ/8ftS9GGMLQJKiAJchdqIY06qeVtwh5hFIzyVrrgzIwafjBM7GIgAPeF+nVbJ4YlITKguEoyvHA8jbxVohUHhB9PIzQFOQ3krpFjXlyBkQcojbPa8H/PSQtI3H8jQjqwYc8b70WXDghG5MojJVtSdbZEyh8dr3+UexSRrx8ZS/9wSvXWlSuYjdIKcFWI1NQgGVvSJHJNR88MAWgRmblFy+eMl+36ALQhP8Eem+e0wlXiKJyBbN7SXJOXzp1mDiCphO+QcxJbRZY7j72ABgIxJs5p2Y8EUL1JhFPiITee2mAb6OWU4tV+7wQE1KXAgizlnKm7OVlxYDNcAJb0Hikq6Z8eHvMl4kTiO+XVM2EZy4SrZ899YpdPjr87kNjTvQSCPnXM6IJWASrBkFeJ5lgidZmU6viw6prx0hpHOabXlYPKnpmD5pUeMXufJzKl+WNcLDTlm5a7PttAKQdetBQCMFlUJqh2beIqQOOXZqZRGKKh0lLT0TcIURtD3CaDdJxV1WfW5dMIVFU67imZxglXZwhwieYgBvoNZgnN8+85lzuiSPWaZbtyqnhUX3oFQKjKp0IdLVsg6OzUWVndCvTRgIfHxIgg4CwkKYTM0/A6SZBWZMod65a2QlnyMFWJduQ48Q5lBOPZtjOlIBkM1hm6CdDj5leboXsSIOWDPnE2t0TZVzPmS9NrDhRhpp5GlyRiwBBzhXE3S45wwQutjzd70BOm/+IABg+j2jzRJhhTI6Bckcqj0rNHJ09gpRMgUSEtu/pI+7YOorkDYKBJLFvqBbGVOdBpKPL983Jr0IFOr8fTqXlrYztVmG2g3lugxaTgRJFaUVU43UdPuMCdz0KkXOQ/pNQUjRaIXgYyuGDSs+s0kLALScjhko3N8XMInwK1kWTCn2GccYsGBJY5Tv7Uqg5P1L86vpzIIToGvdZpxBQC+l/mTt+bmITUR9ZttHjywmcP++tV3f6Fm4uGJrtILze3g8xljpVV5zmNvnbgISCkG5NFaUFpSFlB/3o7U2n9hY2z3q4SxOFSqImbd6KDiX8v1doKBPith0jRALiJv1nCaNJdBII+dczoglAIMKB1htvLpmSRolS30dQlS7eYH5vq36ntF+NX7MWrb6T1lWOmHxyOamfa/U1pjNM6RNhiST6NO/DqvxM6K9cSFRnFaacQxjRruVHSjT0mGOoUfNNaeFEMgsk0i+Yghj+umDjrIPhFtnPsQdlsF3rwt5gHVCF+mJghTZw6Fdu8M2ZmrUOaYP3ZKm5PG9QE7S5vk4FnFmVpWMKtD1D1tZArss591nM9TQolsAJ8UEktJsWWDS6ods53Z4bDGFDt+zLqnT5tNjLGkporT7Kb1iGgKLpdmgZ4hiH0R2jWkBHJDUZrBUAZbN74j0a/sca7ezwIDvnhrMClRXJrRzSD5Jw+fPaMHXWAKNNHLO5UxYAs6ZRlKuOstPLnhYp1eiH8vfrw07mA1txT6Y27nujO3X7UtGC1beMt/2D167DAD44dcsd/y9u7cBAP/0N74MAKioZXp903SKjq+MJvqUYhm8X40J8Dv5vLQ6DvaMCi3gA4hYErhFsYAQPafPG8qe6b+LMm3B/PijJxbvGLAsWucv8wxVFmv8VHsqVuASC0GAmiNaMQtqthbx1cN2n/dIAhAXx1gCIUi1Pk+hIEzz4JUtsO1DNvsUvKkoVKYbuMh5LvnVBMuUMUjo8WOD/l67bLRic5KwXCKd+vToAPt7ZiEWolXbKPA0eVajW9HLHzMDcTxR1opxkEMBmGwsx8dsQlKymex4FshLW5nNsWjd61jAyddmmCFkwk6TxhJopJFzLmfCEgBMu0vD5dnzZQcCMQVtgU6njZ0t09g5jGj0cGTL4faOEY3ObpgW2LxoPu0Xfo9lE47m5gMOSDeW0//qq7lETz6tSWiSql4kT4kRpFaCortCn82JbJtMxlgV+f41zRhpw+ZxnwZZBMo+qNW36LPnIvQosAJzNQmxAZ2LP6so55DtwXcfmw87pqZStqVLAtF2K0ZeemrrFqm77OI1BZqi7JWvgqVUiVaNsYAxLZv9EYvAZPJlymjYx4SbJdyj5jugGFk6PKXff3xo26uX7V2Zjg6wnNh9MqmEZW+bExP/qaSFQZK1d5YT22Fcp0Wsx5hjEvGqI5Cl0zbLU5DvdtEJbc4FF69RsTFMWD9oSAuiYifV0/t6NJZAI42cczkjloCDcw6tkJsOpFen7B5nCah00Gnn2BxSc7OJxsFI7b9NC0zu0Le11gjYo785pe823LSYQXdoxw+GZln0lB1wsY8bGKNPKRBZFR0jzd4lUeaMmklRbGUDtN+ETTXk8zvH5p5JTt8lPqIsAW23u33FuEMm2SXjVuuz3T2LTt/+2HLr3ZyIQWYDNkgrVorCjNfwAetA62WmhiA21osX+9F1y6oKLeiFIpxzezSec/zMlohzhvcbrDGlywOEMM6jK4q+R2xIKCwiGc2ceIrFZI62iFNFVqpce4L69AG3r29PfldzWkie55nzfex3uvyezUeWigXYeTvMqkxnM7Q7NueF2FrLuNWeRI9fVtmlgd139xmqvrEEGmnknMvZsASch8t8aKwpyisRfNT+I1FbYk+SL0nLod3toLUgXRbXt4tD8+mOWUvQ3bMc8W9OTNN9575FbdWC+8qATSLoj2+xIq9LS8CrxkDJ6qSUNDYEaDUoV6wGH8S0t5J256oZUPRXMQD5tFUoPV5wnuLHJ4awgKTjcbvE3F/pdUM7b5WZKues+9G1FFHvkrz0xkUj49yhpbSzZfMq4tJSKEfeg8g/75LW+8HjB9FcbG+RjszX91WVspRsLAckUlUIpKU8uRhfgl3DW+DWJdaYYg7CZTxhyfXGwMZwzLjHfFEhZz1BjxT07TyYF9E81c0+k8BEksFSO7IDYhMe7zPuo2c8VdVqnK3p0ErcPTzENV7ryo61PMuT9nUhSybSGTZvubXNWNFGkx1opJFGniJnwxKAof+kpULY3IleLF5tK0Y7Dw9sdT2m79jpdNAhAqsk5nu4TXQd20L5sd3yPW6/dWzL5y364W9s2bV6PdMEW5tmCcg6WY9TnL6OpjiBlB+gbj8mLWKb/f0DHldFx6UVerUGEFVXHp1Hl7v/2PAINzeHcEmOXTwLIvZY8KAhm31uD037QNmCQzuX2m11GBvIQms4EXTY/lsbxOl7RtmVfdDxrbwmGqElMGXsY491/6GGgtgDnUP3q/sPhBzgbqoXaTPLwvPee2BWyeZn37R7OrLrtLsd5LQOHLMelZ/gZUQWQ6H4Dc21Fi3NtipIOQnBEmjVlaNdoS3De/J0ngDFBlRzoO1p0lgCjTRyzuVMWAIO9HMqRZTZsELtrdUiq6S/Spy6GmPMS5JaDjq41DM8gDSVSDuRa0sto+8n5i9fuGRZhUs3TGNts+V0l/iALNFCPvUJn3Z/Se5YH6WZZClkSfVkWpmYchyk5w+04XFRGZ6QOefR8QhFR3Rf4ejVQ8INdmj5lIylHB8rg8FIOv12HyoYidKrbH8xfXXpx3c6cUS/rlGoMxJzqvxD4iZ2yRsQ+HTEr4AYMbks4hhKgA2IdyGZx91dyxJMbhla9OG+HffWm58J+XxZJ88i6UxlPUtEjAYRp7eYFajCQ4o5JdJ6mYuXtsNzzUKHE/CY+BopC5akaT7SSCONPFXOhCWQuwyb7R621GDysfHY5ZWi4uLFI25drDiimmaFW9HqwhWKIJu26A9YB59N+b1ZD5eJC7h1aJHitzaY/2bVYIsaAaGikR9V8qamqE9BClaJ1lvjHJDvnrauymLsu47XSi+8QFGkLEjCT9BHZhqFHdlw98keNlgxl7FBR7lU5aaNrd1WRsO24j5YerMAJmwuMlEr7koWgJCHdp5+aHNmn7sZfW2h/XhvC19iQWz7hM/k3iN7JodT1e4jmg81Fg2i1mfMo+u9qFvNMRtDy0GZj/FEzUF5D95jemxWk5qvuMEn05P1s9bQYutNqX7JWuuwpEV5HjELiTEoji2dpumz7PnG3lgCjTRyzuWlLQHnXA7gHQB3vPc/7px7E8AvALgA4MsA/pT3fv60c/S6XfzQ97+NW5eNufYC8fryW+esH6sQR7BBdFpGwLvLXZ2+57FtaqTKm7WgPPiQUfJbbYsvXMxNA7iABFMmQlqVkfi0WDuLl/KAJERdNSgUnU9YhlX7kMYC1Na704kth1lgOSL3Aa+pevgsaeMWSJC435OjEXbZcENF+/3SfNV2J4uOdcHPtK9VU6HGo55OfwVZJ3zEqh1IGorUCLs4Y7KsPMggjiestHvwKK4aTDVVQDeWYvsB78Ge8ZI08SVjSd7JUmD8gi+WWJF0bwe7ByiWjDdNLB5x/Y2rPHkgr48k6OBTQkM+2S+Q4vtYu6/RF69cYO3UermzGP8pZq66uCBmJDpNPg1L4D8B8Fsrn/8qgL/mvX8bwB6An/oUrtFII428InkpS8A5dwvAvw3gLwP4c85U1h8B8Ce5y88D+EsA/vrTztPvFPjCW7fQYZQ3C34To+dLW6tmlXD7Qr3ZXnlYy2r/SfXwqslfkKM+YMOJv7521awOVXONjgxJOCG6TC27xXaknPPGtmUPskwaLo7oVyt8AktqJFWGjcgGTMMAw23j5B9Rw+fiDaAPK0TdUvcmrD358HLGJ3rUaAURdQvWGGie5su6TdhEQEGes4QamxCtyXnSfHWoCdsdVlmyQk14/2WlHtuM38xj5GHdfFXzxPr52RIHfDYfPzSU4YgmQKq/wudg4cRWheo72nyPpmMX7dfbYM8D9TVgzGRAy+DB3bvYJL6kYANSV+3wGupDQUupkpZVRiLW12K9CjEi1fYnDUKc5iMUocQ3a2jYOC4ldigXYgNJVkjzIpxNlgQiEnlZS+C/A/BforY4LwLY996rwuE2gJsnHeic+2nn3DvOuXf2jo5fchiNNNLIi8oLWwLOuR8H8NB7/+vOuT+sr0/Y9URvyXv/RQBfBIAf/OzrvlO0Qr10LvYb/3Rf5nlEfnIeUHb2vSLsFy9Z++4ukW/yu3f3DFU2YxXc1o75wplq92mlbG+xz8EJWQJFisdsyXVM/LhqBI7YIHPAGMg1apk7d61yb0zmIUXoqwRZqOj3fF4jJoFa29TIQkXiEbIaygYoQq54i7gcMkfNV7CeQ3lz+fgCJyaNWcsyjgnofEL1qd5jxvk/nEzx8RO7z4f7Nh/C6OnFqTHy/Ozicy4UVODv/b40PpGWHFtbtRZVrCGVTVmUS2TEM3RY35/Vzjq3p9kn7sT90qPXsH4pU7CLMyJptiCW5M833denY3uus3wi+UMA/h3n3L8Fa/m3CbMMtp1zLVoDtwDcfYlrNNJII69YXngR8N7/BQB/AQBoCfwX3vt/zzn3dwD8UViG4CcB/NLznM85t5Y/X7kWt3G+PUSc5V+dcEzQI6H6yz5vbphm79HXfUIUWY+IriFrB7od1ssz+n1h077/6LZxEo5H5jMq9tBhDrsoitr3ZK29NLU00zHdoD1WtdVuoY/OWZUxM0yqXViUGLIGqZeX50LU1XXoHcY2Foj79knzVCVrAdh6Z5GrSarNg7DwipUsIRbjmPugTPLmM/IMHI5N6+8fH+HhnmUDxmIg1ri51XOtku5QRcGsyFSt3O3ahJug4LP15A0QfkCty9XTsaV5Ro7OlsU8Bo79KrhvzeXAMSWqXVaauAvUSyFo/pRdKlhzaQPS03EngctCvJA+QZhqP6g+Qdmmp/+ZvwqcwM/AgoTvwWIEP/cKrtFII418SvKpIAa9978C4Ff47/cB/P4XOU+Wui5c2mqOPaLU6ANPqPk6A/MBXZadgKvPo2PV3adgLn5OzdTJmB1gHv3w2HLIPTIL7bCr7qNHFsHeYxZBPQQuXrDfVQlZVT60Fr948QK/i2vG5dtOA7MQ23uHluRJFaG0aup+UpbU0miJKVhIQ7vX0gGVOAvIpquuRtJIuu8FzYsptak4CsRaE7ri6H6d8v4ci7IBATVp2wljLiPWfRxNxqH1eMqiv+ZtKzpOi0B1CK0wj3b/R7SwZEnJakHo/mzH98kdsLFj9SKtIkcxNEtvu7Bz7R9YbGhBayIIx9IKvjzvW9kWVbxy9KFqM4+rCVNEYfo5z9y6dRDaxufRvi7ESvhMSvE+xgjDVBrEYCONnHM5E7UDp0lwuwIzj2mXg0Nb6R+JM44c+Ne3hqHqa07/cJ8cgg8eGJ/89777IYDap1OEeIfaYHPD8v+PH9q5K3L4zxfExLMa7PveeAsAcESLIDDJKApeLtAms6yYbOSbKVtwxDr2Ga2TKS0A/T6nthQjUWh3p63mh9ulEJQnh1aQeWAxj/np2owNbDPWIS07HtvY5tSu6i/QoQWlKkGpEZVUhOxL6KokHL98YvnAqiIsQ0wjTSOdFocX8nLG+QkoT567JEahZKRd9fiZOAyJ21DMQZ2XW+0eFp43RpB/n5yVriJ+QnyOVP25/oRcbBEIvedV2RdSHPHNJLCBld9VTVjXMcoyrBLkqaSuPVGGJ86InSaNJdBII+dczoQlYD0HXORPA/WKJyKVak6kHCPXvms5+veptQ+qDzAis80BWWkKZz7tEaPR99jZVjx1C66z99gb7hK7787F4kOfeTazz6/dNP9+vDeKznN0ZNfdJLXrZDQLeXox5czIY/jorlkGVUhvC+vP6O8y9l2DdklUY5Zo3xBV545a4VURl/uV6D97EeSsrZA23N6SxmZsgPe9COxI7GZMP75UN11ey3n5o3FGR6y7bWpYdRdaLJd1/f8p2yzZ+kSjh96MYgJW9SWRpqUza2bGaxYcyy5xGBeuGvP04ZNH8FtmEXlyKRbkk2g5XePkXLzmda2iL/XnT7lbt7a/TISyRgAqpiSLIGTJdESMQkRqfZwijSXQSCPnXM6EJWDiUK9JcZxYte0zRawZA9gji65W+LsPHqKienzjzdcBAMO+5X1v37Mo7wP65+ORIvHKNFBbtFg/D/mT5IKjX9lWB2T2qDuk/75HrdLaZDbCFQGTMCeK2lObFEQnpt139VmaLriJVeIDrkSOASALjLfcJB2GVxmJavahMnwHAAV93Z464mzbD0dHdn+zaYxVUCRazm6lewzthmlp0Boqku5Soe6+qn3zlE0/VWCBH0BcBIwRrXV4crGJNGX6odNWNaE9y0Pe2yGtxvlkBlcSL3HLLIJQRqlrrIHvUqc+/t6tvdOn3d3JYn0IlCWKt7K2JKE7df3iPNeVzswikGdZSOfV4B+aoGrWyIf5kH9wjqmvt65fA2AFIt/+yEA8j1mOentpBCUffWRQ3MXCzrUxNLO+1bWHLhhrqRRYYYvHFinLlyQq/egDW0yuXbLjRXW1zRLbDboTj4+Og3kWypH1R0lKLlrk6HKhWs7jF8UlpqH+1vMkABREi0DycqwCTgKXa5KikivW4UKla7T4+3FLZcycJzX9DOAZ/inngiUnQSrBhzUngUZr3Rw9DeSquwjpR36eJ63URMkuuLX+QAcb9mw6pFFfcNW9c8dArVd2trGYibDGto4kK/rDq4E5YcWNvl8n+IjN/mq9Z9pT77qqqjownqSYT8v8hcXCa6F/tQVEjTTSyL/gciYsAecc8qwIpqMIFFWmW1Hb7o1oehPQ8/rr1lz0R3/khwGYpfDhAwPz3L5n2yMCcTxLQS9fspbkuXJcU0vxtQSsoc0pbVGQgrpFOPBkaW7DR48twNgmRvXWhUs83k7baRWhwUcns2O11vcJSd7e3ObnLofCYp6EVkxpPKVIXRIoUtqqbjOemIErBkVIVcmloAWgRigKmi0FWhFZJ62uOct+J7ScQiqwtkGxKlKMrVDOKgvAfmgXLbTWHIFY0jOnoCIdLcUYNJssJ/5jk+Sxb3/W0rtf/brRYIwZNO5euYgjBoinhCKrRf0K/tf+n0DZ16m8ZBnIgkgCq3pGiSUR0sCCFVdV3Zxlja5OV4qTxSGgqvjg0w2BxhJopJHzLmfDEkCGvNWuyQ9CGWoMvTwWvRa1z5375u//sy//hv2+LPGYRJHoqJ2YCC9NuoLLUtN3MttW1GxXN0w7X6LWeOszFm+4sGWgkW0SkSp9Kajqzatmldx9YGP6x//0n4dg2lLwXwYfCy7hNy+bVbK3bynOR4GEk6nQZez01S3IVcxymh8ai6C9RVaDfjQjWWj9RjirLIS6wsXOgRjqrIBZJkh3KS0VB+t0vdDkpFZfNqa8qAk8klSn5LTA1mned+oqiwFuSZLUz739JgDgwX2LEz14bFbj48dPMJ0y2MztkkFcl6routYX0agTMhHRr9WUaPZOZIm5okCr/Pkw9jxDeFYJTXwdfIyBahmtl6x9SuwokcYSaKSRcy5nwxLIM3SHm2HFEhmE69qqeZU+8+eUhhpYNNdTy0xD6WSBL3z/ZwEAN6+ZBr9Eyq02LQPBfkU+efuuRfuvXrQCoJssBBrSD++L3vqUMudaTdk/LnaMSOn6j/+RcB+BSirsSb+Y3zw+sBjHf/83fwEAsHdomsmRSdRVakNG/zJT840EFpq0OatZp6jtsxztIqHrTsAq2iqVpxiBflnyXkKhUBV74n6tRRq1mBqXlmrGakf1XIYBmclGrH8SK+3JUYZ1zaWXOMCnk+M0widPLGP0nXe/CwDoqtUYW7/vLvcx3BSpCN8ptXZLMNvBCkviDgHZ5gT0UoYr/j3QjCnLovZjbNai+fcur59JrtJhROeSFaH254FgNDvNhoqlsQQaaeScy5mwBPI8x8bm5prvovJLEWj+wOd/AABwxEIb6QqVc+ZZFiCkKh/NgjqUT2Z6YpdR4C22orpJuvN6DGke9+SP6fdqenJl58IJFTExnPMOIczfeM8008NHFk8IOfRMDUIW0ec6BCB4dRn9LqCT7qRgnrjfLtBtxxRkAYEjDIPuX63RdA5pdGkdTSt3WC6VoRChqDIZ3E+RaypKxR6KzKFPuvP+PB53TKhdi0t+12cdp5c6zcBP2IXl/Q/Mklws1ZqO2ZfFHNu0/LokUXG0K1q0oFxC6+0C6EqR+WzlVwSW1zxYRvaLKOFEBLPHojj2X8H2juFQNgb9QKUu8hMXGs/a2FQopIYygWwkbyyBRhpp5DnkTFgCzmXodDpBk4nwQ8U7qluVd3SRbcdqWYHN1pUYtlFuVaSTVEX7pBPb3iKB6GkO6NMD76s3EW+BU2s4j4lz+J/+3v8GAPi//sn/CwC4/eHHAGpEXGhDlgSmQ/NPFRglkWsXaNVZMMObGnbb6DHW0ZLGV5whDtqHc2WILaOQz24FTzU6rlwKGacoueqEZVkQ28Dz9DotbJDDbF6adhwTHn5MwJ/oPE7TWCkdWWoJiGas39+I9g+U9SxqKlwLm5yfx48sVjSf2DGbRBv2e6R1L2KK9ixPMjUB3ikCFN53wBWwLRsjIFM24f36t9+3e59+h2Pu4hLjVDeuWTbpMuNXm30bQ4+I04xjl3kma+tZr3BjCTTSyDmXM2IJWFygzdUVooMS5lm+jkv9dKkv/Y56BRZNN5FssiqWzN1fZV5/i3n/U+s8XlS8BrQyJm7fv2356W991zT/IZGQKbNmyPtSe4jySwgyKfNQOBKyB3YdWQAb1Bg7wy569HmD0gjjpaZPUWghRiCfl/l/xLUDLTVhUTCdz6zGD8SY+Zw7dvqtuh0bB3VMVGK5b9pxdgqgMI3+63Mni62ygtiQIaniumxSooYh47Fd5/UbV9BuM1PAjM2UGZojojkvXJBlYOfq94koJT1dUdhn+el1sxWOtVJTVfvcG9rN9bfNIr311usAgEckw7l/fxcPSWn3tXc/BAC02/b+XGAT3dduWDzr1k3bbm3aO61n7Z6h6xtLoJFGzrmcEUsgMwx+noCcT2WeTCyBmoEyNKEUiUVG2uXp3Fb2BX2wLZJHnNoI8rkHn3xeDUm7elwAcJ/ZgF/8B/8YALB3wHZkSdWXcOWBVKSKc8tqgJKOPZQWM0J/ccu01RU2SNke9tDrk3wzi+dO187TWtm0joPPJAsanaeRJaAYjEIyPI3apIfaAcZ5er0uOozO96klp4FMhhbBLmMFGhJOFl1hLgJSvk5qiTY6tjqRBYlnJswytbu244Ur2xg9Mi07Yiah7Ij5xc55PFJtASnqaWWoenRra4f3ZQjTTou+f2g224nvIRMhq93z5W17Lz97g+S0n/9swBCksaKQuSlE2GJjKdpqTHtKhiuRxhJopJFzLmfCEoCDLduBODEpdUvbNysGkGjCyXiKbo/OLi2AOUPM88q0yTabUtZwumQddOmqeYreeZYF4X24HxFj/so//yoA4Lt3TNsITTanRpqPSWxCq0WNTHPl1kUGoBiJNB6HwFvGlshQb1jcY5M02qU3zWv7qp6BbcNWGqmu3naoguMl8ySrkDMXLRrs0IgTyjro2cXxm4LtzTqdDvK2/TZQIoFxhP5VUp8507ofPjE8v+hN0liAQgch+q/258Trz+fkoVC4I1Cz23y9+8H30KdV0qNPv83Xpd1lU9hqGp17cmjPbnrEWpKR7be9TYtz03z9PmnKcloh4l9YelYrElcR8IFCFlad8EfaYrNUkZjW5Qd2Dr/kO78UQjXlLjhZGkugkUbOuZwJS6CqPMajSYhiulQ7V7FvIzYcKZcjNswYH45w5fpVAMAhm1t+dM+07huvGaZ/TIB6HWZ4XiCA9k8k5MP1UZqwpgq/89AozX7969+2sbE1+f179wDUbdC9APlkIlJRpcvi4Ihakcv16/D7IUlD37puTVbfuGWWgNqu7x2P0O2SJ0FWiKenXT29pj9IEpdxScDG15zb3NpGDEWOPnRGau9etxssGALgBLvHgEi5TqEWb4yYk6p9nzX/s6RYIDXSEnuy3q8SdbldcHf3GK1Lprk/pc1PrgAAIABJREFUQ5+8Sytl/9iuPWQ2qZ0Lr28XV4Vryav0B8QPwEyJxWIcj8HF8R54Ed9q9MzGuGVo7FLEpSg1jZjuMCkdyZ9RXSppLIFGGjnnciYsgf2DQ/zS//6PV3jcxHKj1fRk7j3lXHf3zdfrtHIUXfN/7zLPmtFhvv4t08Jt5dRTjMFpkiIJk88+UT/6XAJYEAi+yxyzsgPH+za2BZucinWnRZaioiSmoRKvt2kZNc9QhVuXGmKna/P0xnXLE7/1GbOGLjBfLD8dmQ/1/X1q14mqAxlBrxL2omdJ2vAiJwMRvLAMwszTJ2aOfklrZ9jvo+J9epoCoivscl4uMQfvYTn6VkFf/7HFCoL1RYsgrSJM70QwgroewD7nrQJzWmPfvXPH9iWKscdYwQ1vY9nq20lUk1lyvyzwG06iseotWzKy/2jXagUyWhpdztvjw3F0T71BG0cHZtW2+Hdx47pZK4OO5to2z+KVOE0aS6CRRs65nAlLYDIZ42tf/SoO9211zJkH75DVpUxyzMFrUj0AKcirIsNEbLiMIwwZld1nVqDFoxV5d4LpJbh8+brBxU3iEgGnn2DFZQksvcMBG3c8ZK3A/sRW9EPyHs7JbCvtK2SgVmZpGcH0RX++wcj6xQ2zHD5z1SyA128YtnyLLcXatBhkcVzcGGL/2K5dELKW9RgboCafc2wlI+pVEvNIIZVrra8E8gyGFusX1ApMDEZU991uBw8Z9T9mbKRHjP82cfpd4kc2aSluzm0ehh02hgnoUERbvTUxu2GdV1eNSiuvMxUuJ5rQsV058/pHNC8+eGhjvECf/+qmnaMjbkY1jxUjMt/dlrf9j8ln+LV3LR6UEcX4u24Yo9XBsR13h1Wuo9kCN3aIJrxq77DnudKYx4tKYwk00sg5l5eyBJxz2wD+BoDfDVto/wMA7wL4RQBvAPgAwB/33u899TweaFVLtOU/yq8mcLvKpI34vXjYqRmOWA9wuFygw5D5G1dMKxal+O5j1ldPP3uZIORSSyDV9C6pp1dswScac4k2ps60yOZl02xuwvy/fFc2LhnTopF1IpbiHv3Ji0T5XWJk+hY1/2vXbLu9SUy8EGOKxAdWYvt+0MlD9H5Cq0T48g7Hqnr3KfsszBahI0o0DynTbZg3ghpkvKkGvttlbwXyOVy+tM17XeLJrsV0bpP5p+iZNXLjimU5tvt8TcmF0OubRtzcMAvncKFa/EU0VEmKOmkRrSjOCeXsF/MSLltE1xowxjTYtGsKtVnynTxg1L/PZ7jVU82AsByyKG376IGhFgtiOWZTm4+7D+x5PCSq8dJVm5/icIoy2DDCTDIWkBabvKC8rCXwswD+kff+8wB+L4DfAvDnAfyy9/5tAL/Mz4000sgZlRe2BJxzmwD+FQD/PgB47+cA5s65nwDwh7nbzwP4FQA/89RzwaNABUK4UVZkDhJnHrfit6u4nfD7Xebd260Mg45pnCE13Ij++IzbdocNMYncqjvCEKPg0xgBon8EraKyAGlGRnfzjmmM9uACLnTt33Pm4Je5aayNHcYt2A2p4jVnI/tdvvwNMyDww6+bRrzOCjb5/F3VEIjBll6wkGNLNRFd1tZKn3EFtT1fkmW53zfNlKkqjnGHks6w2I3y0M1GOABiF2RpLYVAFCOyGJTt+62BzckGLYPZfIYN9mGoPNvI75o2fMImr72e3ecOLSLhKZasqlTMJGUYSiWENWozzo7LNU9zLPjqZc6uOYJ9oXiFMgvbm/YsuqzkG9MaHSq/v1LYCgALvgPjhf0wIHJzyXd9RIv18kXDJ3zfa/bMy/EE7z+w92LPNrhGHsQ8tVYTed6C2JexBN4C8AjA/+ic+w3n3N9wzg0AXPXe3wMAbq+cdLBz7qedc+84594ZTWcn7dJII438NsjLxARaAH4YwJ/13n/JOfez+ASmv/f+iwC+CAA3L1/wWeYD+kpNLYWnVoTVB2od1qqLx1+r86Af/GHViCvTcMxoKxj1lW9aJWy5tUMZZwfCuLN4fc3Zt2C4af55e2hVZMu8F6rZ1CcA1C4F8QDdwYDHWvRX/AEtXmNry/Z/45YxJ28VwrpzfqbsrqTqMloloSuQItVVfU8Zc8sbQ9Nk8kn7fRtLj3l8oQxl6YxGbMUemItb4ZxAzStQeGUD2I6dYxzyntt8WMdHquibY9ix767Rwlk6akdaME+oAg/2aW1Qp8/ZR6CURhQHo6wSjjDQC/BhLhXnUOaCVlzLtdHn89tk/4kJsyULauwZLaiPDox56DGpkl+7ZFWEM7WX97HVqsHcumkavqLVVjlaWJxX0QK2mdEpO8Dbn7GxiHNTXATPCeV4pryMJXAbwG3v/Zf4+e/CFoUHzrnrAMDtw5cbYiONNPIq5YUtAe/9fefcx8657/fevwvgxwB8k//9JIC/wu0vPdcJ3YqGU38BL00mfkDbVSu/cv50GdHKfOB+06qpGmudUxFVRcFV1y5EW4d8dNLaXltea6G6eVoY/b5ZAK0ONSsHOV2WmNLfnmlVZ/Q/9OGjf94jnj/EQFgVtuD95sRLbBIXIHTffCItRa2j7jW8V9VYtANXo0OLmr5DzVwuiQ+Y27k6HTLjMOGv3gliGS7nZfR7RgSiE3cht/t7lhC6tGOaVVj7eWgnziHNM/TpDl7kmPSMHrF1+EOO4UjzqIQN9w48kmWMs5D0xSREfL66AuXMAIDIwSVyHE1YkceKvQF5AjYukIFK4QRWpVZL8VTYfe0zozXhvIZW7WovvrTfZa0JJakO1V4lkuyB2e52AneDtnqVVU34gkDBIC8LFvqzAP6Wc64N4H0Afxr2DP62c+6nAHwE4I+95DUaaaSRVygvtQh4778C4EdO+OnHPum5rG5d/dfjrrwlseV1gx36pdQAbUVkl3O0GOXeOzSfc4PRbq3csxmPocYK/HbSmkK2UcPVSDmytxDFNlf3I3Y7zjg25dVni0WwAKa0AOYzcc3H5V5CrlVtxStYq06tOaZW6W9t8z6JT++b9l7OlRXgCZNouXgIvEOoSGsJMcjo/SPy2I1GQhQK0SfLgDUXyyQmwrHntKCOiH0YDkyDbnL+xXq0Re0b+hWUFTYGFhOZzj/g/VqW4LM3rwMArl6y+33EuM7jkf2uzsjCJKhf4FxsxYx3DNh3csLeCJ4zk3EsVa56kgxzzT15FlS512X9wgazAltE8Q2GrGtgrODwtvWQOCAOowosR0SwMsMBIhNzWZYJH6VYkTK4EFcIXZ2E0VipWAWa2oFGGmnkBeVM1A7YWuZDr/cQG6hUc60td5efxc+dXFqrHTT6gtpyOuXKSh9+ye+l6da4CzQixSGobbodW/krr/yuHT+iX95qy4rJwvWVg18wizFX/TsjzjMiCH2o3Itr8hVfuPPIrJrP32IGIlO8g6yyvJe6voH3wO1q7cWMEXWxHQnxuMF894MnZhEoS6Ba/mHfPpdJ12FpVfWGKFm7sS1GncBuzNgBq/FqTgiPNjM2h7Tebu+Zpj+ixt/qm1XxWVZJvlYq/jLjfnZPo4n52wfEFyxoCajr0pTBBFl5hWpTsjqin6tylVwGPlNMiXPJfyir0qHl2Rtafv/yG7zW4Yc2lrmd+0JPHZzY0QgnP6ualyGoe2RCHSLOfnxa0lgCjTRyzuVMWAIODplzAdcf+roxVNriCi6uvRAbqGKfKKsqZESmtaA8tdiHFa2NfTBx4ImtJ1MXWflXHdOQkw7zwKztq+j7U9ljwbhFrkizr7MZ8qMXM+WaTXNNJwRJ0VJwxBMIw6Cx3nlg2PqDI8uXq5bAe/m4ErEWC59QRltfVVgqKl3JIiLTD6P9m2QmfsIOTZcumIYrGENR9V/gJGTcZkyrpuA8C2egPnpi0Kl5Cqh9qzLALze2DE34g59/0+6bjEwPntj9z5guEaKyQ6jgkGxFHUe0I9+TEBNgVeIh4YCZ+lUStdhuCYvvkDFWEjAcG0QGMuYh5qAx0Z0ZsQYlLYI+YyiDy28BAO5PrbdEr2PPOOBVmCVwikfICuZINnndwaAbskl1TwjBEePq2jomwHe5er4YwZlYBABjR05bYemFqW/FJZ9T2ogq0FTrRZkwKKdCDqUO05exhgdzdy04fGEUy5ObMQ9gI/t+zkBSW62mSw/P1JYWgSoQX/KPXRBblTXrj5aLn0a4N7bP37tn5vGFt9g8lYATUVHXxUy6hyra0uviGMrwFQAUbRt3n3BWMLU3obvTDwSlsfEoF+7g0AJh+uNXqrZujSbquJU/ftgCJRCY2qZf2bKF5+KWAWvu7Nhi8J2PjejjgwcGPVkwnddlQFULvQA9r12xBW3J75XWzWjyK5jZ52LgihYylg4XXLyKbvxZRKt5ILu1sR8eHPJ+6NqJ3LViWfTMnl2vYDkwXZAWF7IywKz1WYvrFG2OW9vnLhhaI809WRp3oJFGzrmcCUvAOaCV5yGNpGaZwdhxsamUElysmsOBxIEQ2x4BItLUHWqoYAlk8WqZJSm0EQOAYxKAeBXjqGpFASNq1sWiLmctl8pd2SY0RAlaUaagTOu4+YiKUEb0K969Y0G7t29Z0E3gGtX0JDyT9ayohVhVIZNPFTJSiTVFzbY5HEZf78sVYYqsYHBNQK7FXPBjFlBRO4drh6vEpdfOueCS9HjNyZhQY7ak39g2jX6DJdSfYzrzA1K333tkWlZkLVcv2n5Xd8yS+M07ZoI7jlkgoXabJjy1dqvdqVlRGGyWxg/QawF0pD65/3REa64luLrd/yyzsX98aPN0s80mL5WNSRBmpXFFTjudi5q8wpApzkLPJDVbJSnG/TkDiI0l0Egj51zOhiUAK4tUCywF1OpyXbXgts8hAiANWkq7VCvpE7U5ZzqtJUiqVKCCj9RIgRCTBUgKLs3sHwKPKD1ZqORYYBmBaaT5Sl+3jZJvLotAhULUgH4myychNBElNi2he0em6d5joHDrjVs2Flo92Yp2XZXV2EDdGptEGrJcQkkwZ4nn2qLmf7hvgcI5I6Ft+s4h5doWLXg7GkNNyqLRJJZXlgVIcpar7Nh+m7EIrMv05CUSp+4QgPR9N41QVaAqQZJLxjHe+3iPvzOIR+3cYtPQQL9Ovz9vFcGsUgm0E2XZQoFNPmf+MGeAdaEYEYORGdOeanvuPYORsGtdH1jcIgtEKAx6iuiE79NiMQtW5jFL5jdpIeVBhad175ImJtBII408h5wJSwCQqyVVH6eTlD6pEoBKyB7IYpjNgtWgltqB8DJZJesiDJYl038X5HTKktfxjMAdEXWQWhokhOwwlZQXKczYrWQa4jZqThTjtCJKHRssB0bQQxqT5bdzO8HXvmfR8c+QQu3W9oauyP0TIIqXleID2EXE3GnLbB+sLlpGPOkOwUTHpMTuknZLXBAdRc+lzZPMhEtiD6uWgiwB0ZVvDO23o0MCc45I6MHcn3x7bfvcdpkyOyQ0/FAFXIH6jG3NCrMAWi3FLTi0sgxZjQAiC9YmiUy8CGrjwjTdtzJCE6YQi2AZ2fbO1LYbXYOAX8z2ef55NF+FiFudC98tCH0vS2Uqkga+LyiNJdBII+dczoglYGCh2gIwERlElkSU60aMAvgQA1AuUc1Uomm7yA9P89QhNMBS4YpAlCn3Oybcc0oasiVPNGUEWqQYHbZAu3iJJcXSLlm2ls0ITTtLgXk0VsJChWHwyWMRjHhu+3+8Z2P6yvsWHb/8QxaBHqhkmFiJUFpMyTJX+42KaUCNL/kpELowak0rosf7GuUEvQjwxFhJvyvADXhvsUWRJRZAtvI8UpCLYiWbtHBmEzv3ZGxzPp+KFj2ObyiW4Akaun9AiLQsAGZn+DFodRVgFUUHRDWHdy8E3BV3UgGVYiiMx7Ryca3bZkFg0pjvz4wxhh4tpkluWIjtDWIAnOEskOAP4Ku6eEst6k/BCaSZnufECjWWQCONnHc5I5bAyRJKYaVJU43KrbplZ96HiHqAEoc2zrQyMjWSVFGGtDHxBGzsMKJFoYhzpRU6ydHu7ZlPp/ZeW9uGtENRhJVY6C9propxB1/GsE854DmEZeDxZZwdmTCe8c2PLF/+xlXLk/+e1y9zLDGcuOLxzq+jLeWPByRqUsAS0uO0UgaEx4rcVZmPZ/mn0vYq5Eqp3ZOduaEGZx5/wKzAlIVCaqoSnmlmz+6jD63d2+MJ7zuL8RmSOY9Xc9S2SEZQK2LND1nb4Rln0DNxTlsOPVCzM4ZA/MSS71mLOz44MivkNULALxeENMvqjWImcfT/2Qo+xRE8PUvQWAKNNHLO5UxaAnXu/vmcGu/XV7oQVwg5d302qYLrxUg8ncEZi5VECCIaqADLg3Dutv+Yn/eJHZc/WwyGa4U8ATdQxqQpktDU89QyU92vHbdLlNqvfv17AIDLLMC5udOJj1tJ1aflxuHaSa4+NFZNipCE7z8aWYZi0BFpSBJzSTALtTJ7igVwypiU/pFVNxhYrMAnVt/e2LSpkJVzrzoRDYlxH8Z1ChAvEDIC9aiCZRMsSVpImo9FYsWpqY06ywQ8RlwQo8KtPVLgP9y0+dzme9Nea6fqTny/Nd6ny/P9/TSWQCONnHM5O5bASa6haKCy2LepW35JQyRaaFUSCiZpuHBufl9S64wZBVbDEDUsDdVvPJFy/NrKPz1mfribtZBTw9TU37QI5G8nNx9KRgMUTDgIASG4X6DOtvN9ROz8r/zmdwAA//rvszLWDXZzqa2ieiKkoOp5ifWB5kcIyuADa/9KCEsiMttxm2yf2C8vwnxVJ3KUNYktROFGVJX57m2zAD7etUj7UjEAVU6qFDRX2TepzlasFc2CagbCs+BWv88rkrwuxQzKZyoy2XCieKKXzPWP2Obt4ZH9Cb4+UAs54hBcbQk8r6QWgw+pjaefo7EEGmnknMuZsAScYxQ0CyHqVydJ0ZwLKzSrwYgRF/lGkWgjIcO01HeoAaesYT86Jk696ISW2qn2rIO3imrHkfUQE0nQjqqXD8UV9Hln1NLf+tj89O2Bne9HP/8aAGDIhiNmCcTaQU0vZJWkhWiBwIJDXNKa6bTFG8AIfqDhiisiJWlk/mRRHCG21oIFoL10DZ7zAfEAX3n/NgBgNJelqHtSxJ44gYDG05jqm1YcR23ii9AQhliExCqdjIjiYyxAsSIXrBS2TVfDFKhhjp3vYEIavCVp3AplsRREWJ03WbUxgvb0v5cmJtBII408h5wJSwAAWsjQEoZ8Lc8pnz+OBaRx85NiAim5StCuahMVNCE/KxtAi2CqZp/knu5vbnJktr8w88qjz8k7MJ/P0OmLrttF+7gqqZpLfDnl9ZWkdy42W6S9Q6ggadH+zrcNSShl8i+/bdWGG0UR8BJB6yaNLTQfIXYS8uW8lujOQyOVPu9RZKexBRAQgi4+b+3xOsgOUQVo+nhDdWT4nbUBU7vWl779EQDgLqssQ04+xFY0z4pz5NE9hbfNOSw0P6yhKAQQIFuDjDHXIzKUOP45CW1lZQgfQF7RECMIQEA+4/FECEwb04WW0K/S9u6EeIoGHj/LVPFnyd/HadJYAo00cs7lzFgCv/0ifLUtn3NqZzX+UHXYlBjwJfHq4pobsKqu04sJNVVvP1/OQ664FbDrjKCrSlCIwir2YcP6HqonoyHjNF9Pxx+xpv3XvmWcfHNqzB/53C1ssWdbIRyAWoeHLIjQhYmVFbgYbZ6mbKSyTNCMp6HT0myBpPJVqFx0PrYipALlT2f0j49Y5/HP37MYwFdZQ0HYRLBKXCaeQ33PewixpzTLUAXtmhd6JjHyT9F/yPcPvrttFnz+spgUvwnWoOIUgUHI7vnBsb1n/dLQkAWUnarxEKpwFD38pxU7ayyBRho553J2LAG37tOnec8UdZZKisADVlhhT5GKmlCWQClLIGC4iR8Ys8rrseWihYHXtk8eOOHRq7IMHIOKoBfct2ZCYi26GncE31e1BAzJiwVJLccDe1FCKa7W5Nx/n+jHX3/PNOXBaILf+/YNAMDNC6ZNumqDFcB5aa5ZPny81b2JmahuSqKxxAxPgXEpwXis3h98mlGIPmKPMY8vv3cPAPDOt43O+2hKbe1izELGJrOSuuyj5lcAYkRnW3UGHJMatY53rUZEjVYL1jNskPfPi1lIDWc4LzqfE3oxzUqRs3JfvBUd/UkSL1BVa759WnX5stJYAo00cs7l7FgCq4phjYVGv8gvPcW/rKp1ayKJAIdzB9SeTcFsGTMRBV5Aeeg8wf7+werhuHbdeO4G5H2bssHIeD4JnP15Xje3iEVZD7HvKOof9wTwiSVQLaW5FuG+sXKEKiSlV3d53Dc/3sMuW3Z94a1rAIC3b1rV42ZBq8TF2IZUg4f8P8csHL4Qkz4wFtEXrml5efw62jPgAnQN+eiMU4wZ4/gNWjRfef8BAODA3GeUtOKKjhqpxA1V6lZzSWYivEeBTAG5avar2BJ4/MgwGIEJmTUUi4I4ATaQUWOZUC0ZGtsmCEAfW1RTckXkZJDuC57hfZ0jy+Ks0Mu2JJc0lkAjjZxzeSlLwDn3nwH4D2GL69cA/GkA1wH8AoALAL4M4E957+dPPQ8Q1U3Xvv+n4/OEi0RbRZ5NW4g5yAduPB9thQeYsanoLn1DYcsvXzKO++HQLILRwxH22VizTYy6KG0CFkF560xINkS/q5HoQh2MpG2XqkWQlk7OJ4RhYKQhW+18ju89sXPsjQzheGfPmJG+8MYVAMC1oWm4bqiajC0nxTHY+QuLpbIEbKlGCSw4SYJcbn+IDWQ5HPH1mZieuc+9XVP13/iuZQHeu2udiHZnio3Yfm0iIpE0mQ3chdTKIlOSpaFn50Sk6DzSasBjWn4jMkgVPYsBdNmifsn7Hx0fRvffJ4NxHdiQ5UlRxkJ9G0K7PIstDbuyDt1KZoZnOgUn86LywpaAc+4mgP8YwI947383DJHxJwD8VQB/zXv/NoA9AD/1aQy0kUYaeTXysjGBFoCec24BoA/gHoA/AuBP8vefB/CXAPz1Z57JAS3CsbR4Vqqac8JvU5uovvyERGnKd5+QssArGi4sPM+xDGxFtjIrJ50LNJ8x0t8l8xDZbZ7sGouN/PhtIgqLLMc+m08esytPt2/aI/D3ZXFMoOYgJJc9mXzn9LuVo1ZMIADFpPkCxRJ9R1kEzDKU1QIL4uofLlnr8F3zs+8/MU321hWLEXzuOlttD8WXTz+dlkCXfH2z0JNRsQIXfU7z6DW4TRBEH87xhPGKb3xsGZjv3qHmJ0+AGJUUAlEthSyIoNIUY8jjZ523YtxA8NKVw89c6BE4Y2znkBbfgvezuWnPskdOxdHBKNp/Y7jBaxFBGd6/GDFZaoy8lwmf/Xyp+BFZkwIq8JkwkTWpntNSeGFLwHt/B8B/A+Aj2B//AYBfB7DvZTMCtwHcPOl459xPO+fecc69czSZnbRLI4008tsgL2wJOOd2APwEgDcB7AP4OwD+zRN2PXE58t5/EcAXAeCNqxefWg51mlZHElXPXRZW3BBRDRx6yvvbxzLgA+zznL5dYKvh9x121pky+quuu+INnLJLzqPHZhEs5sLWlyH/PBqZtsjEQZBg1xX1V3vvydgqEdVddzHT2NSyXPPAqHiRzhstpuB389s8R+bjeMKIVsZtxgqe7NtYv/2x3c+NHdNst3YsL35x2OXY7KSH1IDbF5ijV6/DNR4HzhfneUKk3O7BFB/dN7/7gydmOT2hUlgQGbhUTCS8B9Sq6vK8VNcfWgYtofn07IUUjK3EIFkdQxEP5JTPbHJk25xdkLY2bR4KdSDi/cvaEiak7pER90qoX3EhVnkeckMcMiAyads89dv5SwADX30V4b8K4Hve+0fe+wWAvw/gDwLYdmKIBG4BuPsS12ikkUZesbxMTOAjAH/AOdcHMAHwYwDeAfB/A/ijsAzBTwL4pec6m/fPwZlm8jTkYPpNzXsfWxOqKFsqas39l+r/R+SXkGN95m/H7Bbbl0VBLTQZm9Y+PDJt1spbQdtOmDvu9Kg1eC5psiktgNGRacTjkZ1DFoBQaAF/rmxCHmu2mmFJcyFsfI3iS5mVxJw0FwJwOeZ92pgfUiN+8x57AcgPZ6ZCE36fXXcHSoQkcL8J73GP8/NkRitp7jAPPR7YaSfEGXQuRf3B+4gthCzEiuz3QM+geWAWQBZYnrAxK8bkK48FOwHJwhMfwM6GMTkLg3i4Z5bSbGb3VbCrkXAmqkHRX5iaWAdWpFAlyBgB+SwOiImY82Z6n+DvYl1esSXgvf8SgL8LSwN+jef6IoCfAfDnnHPvAbgI4Ode9BqNNNLIq5eXyg547/8igL+YfP0+gN//IucLmixdwdaYaxPf7kS0IaJ9QhWbcONQV17i7YnhliWgFXrOCH2bsYFu4A+guhkwZ02LYMbYwdItg+8uNN2Cv6nz7JxWxYjdjI6pJSdTshOJzUi9C6XJeLz4CUJ9fMjJx6i8aEZ8MmeJJSSuQcVKpI7nzjTiLMlgKNvy6END8ZVk010hDOBHRcXt/As9h9IB6upTxtmgXBo+qV8IufzQzyFGe2a5PYuiLdQizyuLoBX756UyH8slQCuknBKOyKrKNlX5/btWmam4DXIyLLWVFeFhtNKKYKXxe5URBiyIslP2+ZhWn2uph6Fi7C8gKU3UKXJGYMMOWZ7XhKLPGHuAg4pkIpzG1cGgNYkXB5GILEr98dv3IXXDscwJDhKl+JCBobnSdXxBW4UIR/kSLRb1HxbTclO9OFwUZgzKTY7N5J7P7MXTS69mlm21PxfZZTBrY4LVOg2VQH3Drft6QU3mJeCyQhRR7gK3Kk6qYtp0tVRTYLU8ZREOGUGVyOoP1jm49H6YGi7DYkaKdkSnWoFV83elTplsUul2zoKiAARTuzvOl6jRstwBxLVNlJblH+3BwSMAwBEDp11SrWf6I2VCTAtKwXOr7ZiAYF5g7qRx64LHH/F9Y99X9LoeLdGhnRbTVvQBAAAgAElEQVTYPFVefWCwkUYa+f+BnBFLAADcGtnD+pqP6HuXmL9uxRKozxBKYWzXgLAlgCbpIeGphepgkk1RKCWWpuuQ4IJwUpUH19gYXxNu0MScHNJlEFSZ34uqS4HComBDD8KNM5qvea5tDA5KCTtEWLFMmnOYWXwygCSepdrUDtBjQXCTjp0qmQ3PRNZIeCZ8HtJiSC2RVQtGUO7YJRP5SvBkahPAfldjVwXl1JKc7lY2k2kd1LHdS0st4vmsXYZJ4tbpmocHBxx3XJwUSqyDaWObLslmukwZKt07X5CcplDJstK+JI0lrd2YdGVTtwxuoeZDachOoZZufPcSivUqJWk5RRpLoJFGzrmcGUugggsEHi6h9Rb8s44yxUG+4HZm+RpYyAXKZvqdCioJLixCiUAwoePsc4/pPAX1BPzZaBmIps9Ckr0nRjqhlGLm6hU2+OaKF8h/DMVKcbGN4gshABi2MdhIvm1K7pnQTwbxWEmzpdhZAYyStKLmIZOmS7GrCW13sN1OCeb69DQrH9bivElKsEosAJcGDANZLNOdpATzMC0+oTGXM+gmi6tPv72V5xgfH/DYKc+p4KOds9dXKlAWAMlonD2b7oAQccLLx7QUH7EUWeXfFy8YvDxQ2CtVS9DQgk1py2oZYh26TzWf8SGWIeISPROB6DRfT48NNJZAI42cczkzlgCwDnoJPq+rtat9tm2egD86nc5ak4vAcqXP4Xu79WWILCcUXgSzeNbMSgMsDm0lP2a2YMCCITUhmRFmWmIZVHLQghq/4hKBUtx+UOpKvqriErIE1OxUqa7a5Yu1VUqzthph8amaXdtLY+aZpX4FN07Udd0qLpn3lEY9abxSWwbrMYo01hHmy8f5Afnha1ZGUsQ0ZWHSnMAeF2jIzBKYcJu1MkwZ+1HZtk4a0rEhtSk4sGn8fp+WIclGDnet+OkeU4pLWhZtvk+Tsb0vvW5vdVoCUW1JKvdWUSJvKfYRU7dN56q5kUUc/zlXCe3cadJYAo00cs7lbFgCzlZzRb0Fjqk1XrpWxbFsEX4MBv11DRRosRT1Z3HJROW1Y+5Ha0MH0uyYEiwkbSza5/GI8FrGCLqM5C9z0zbLhQ85dPn2gZY68eFD8QkBSSEbwK2IMwvGBGoq8jimUPokGuBP0pQxyYek/hg77YFeLMlNhyh/CuAKAeo4Uv00ScFAaZZIrdAyWQQJ8EvFYWE+ZIJxmgUZV5RdRT8lNWlZ2LbV6eD/a+/KYiW5zvJ3er19e7n7vXNn89iO48QkIosBB3iISBBJhAgPIBEhxUKR8hKJgJDAEQ+ItyAhCBFRRAQhCUJhCRGxLASKTCQEgoRYCcbxNrbHs3nm7lvf3rsPD//3neqqu1nGnm649Uujmq6urjp1qu75t+///p6IW0K8hXTfKg0ORC527hI1f4Fj2bxteIKNNSuZ6Q3s3EViPpTrV+ZC2ac8b0Xt71a37T0qdJqhp2myrXteFjN4n4lmpj7xLI+S1BJIJZVTLuNhCcBZjj/wXnMT4J4aZsLXdfHyTUXygcgCCMUoyhKwIUW/Ec9FK96QYc44T18vr9JSZgcChJfX7LLsNVuM58kzcEEz+wQUWVThQo2F6D+tjEzB9ivXXCoyIi2ykbZaXDGjwehxQMLF3e8EyWUcentAErtdIAilpZQVtFfWRTzeEf0waTkkTz+MWYzDnDM4cDBPmcg0hGBH3CLCQAhDm6ecIN9lQ3uGFvHUxl0v33pwIM4QMg48dy/gB/g9o/3bq6b594n+VBOX0KBUVi1fNJUsd/UukOi2y2dZZIzhzNkF5JAck4Ybf+6CrofnfaBt3+GSWgKppHLKZUwsAcRak8tP6gVUVrLJpSBf1Eq5SKMG/1GaJqdcsVZFoa/i+6XhsqGJJVfmdrz4p9FUYQmP4lZ1ALJWev1+oHeS/xwsAw1NtNT5+LZI7MGkNFdXTU7tGn0ScvR7CQvjANIymFTgzcIhAZFMiEP8/n3C+hK2QZptELMyhv3TeMQ+SDxcYb8LUf245joaI89nHLqlqsAqnhVQGbAo1UqTbB0n7Aebx8oPz/R9VL9AVlKRZPVCtkAoTPt+f2+bd6H4j6wVPRtlE2RxqngpUTugMmpmB8TNVSmWkcvES8kR3mH9ltYuswhRTQli26MktQRSSeWUyxhZAj4izUhQRouCKZlbTqa8M5lMWGGl0aIbFCJQ59CKHvefIrw9aZ6ofUUs2iF9mGpGswdqZgM4IMrTChWWKGlVc9McK9ImCub7TzIG0G2Lboy5a5Xv9pTZiKMfg/5PRNUj8UPjxKESNWuJ5/+jXL38UHAMcbSjqjNdwHgkItQaZD/S+pEPe/iYJAcyERGzKDfUhHzqKmtu85m5DMvCAwW8PdM+/XrvBwcaq6q9WldzD2E67ByyiIQ5CPSaIbaAcJ+2tc+BVr4nQla+I3xnWt3oegctIs0DY2K0gHJF1RIkMR8pYjCVVFI5RsbEEvBx3zOBRw+7ExTaQ/2kbJvJHIjqBprGA41JXfyn3Lbok4m8skicuWczCNV2awXvizBE/AJcyTMuM9Q2Kl45F1kAzP8TRSYS0x7Vbr2pNmOykBQ70Hl5wgSJSJbqPLT1UtQ8g+BPJqGUOlXwsxPhfBe+ZwVkNx6J1u+FdgzkngnLIyI/1dhwpAWQbK92QMMlSGgygVJcMRhlhpgpYlwnx9y+6OObu01eLzOEuovPqeY6yg4xy6SxclsivbyyRxqDKMmjxjFCQ9LKTaA92yHeM5SBOVB4QWs0gZzVu6tzRfUJh0tqCaSSyimXMbEEQGdUsYA4U+TBSrS4ZkgSbPKEtk9U42E1VQSZVwiViHH8fYcR+FyILcjnVV0DV1nmoLXadphN6Pd64VwaZ4GosTwtgCy3EwWzAArcNgfxXHFAq7HVl3zYAXEC0i6qJvQJzHiExbc7ABCq38JsJTT6wbR/cG5j85S0diRJFqBkKMJHpkX40ilOk7RgEs1m5AsHRKTwFgq9sAJvMBD7D2MoqtBTLp6xlwa2eU+9CPGYaAobtaJX3j/eNr7H595kzKlcse9rU1ZbAjMQUK8bu9SA2jrJryDLoB/a0A/FaRJWbvQucwy9BOuTLMPUEkgllVSOkzGxBByAbEBZRVxqJ/wqkQgdin0PiXLowpcTsTWIr6rh6BBxp/YR4ks0111q436ceUfR3Ww+akOuFVqsOqoBCKhExhvU3FI8AXMVUxuzs6xNZ8xAZ1Yd+fq2abCbr1grsS756fqBGyDuU2cQ+bYBGRd89TjzUlDUioEwmh+aqbps7DyZmJ0RnScgMUMGI45pcD4aRJKVSBLxHMY1WqgWDCF3Hc+ai1z8vFm+XyrdyPF5FJidGXQbkYXkEs+OVgNCq3HbLytNNf7CcnR2iEHgNZcXjbK8zKpB1Z6IVDbEXvhe9gIK1B/I/kSWgCymk5CBKWIwlVRSOUbGxBIAhlergIxKrHiSZP14FPEGhuhUAESaf6CIMVf4HuO60uQZ8gs4r/bfjOp6ixxXyRswyJh23mXbca3CYpkdBDakbODj6/eFTLNtnuCzTCGO/S9Pm2VwZm7KjqfV0d2zMWSY356YsHt684WzAIACJ+KVVWOvYddx9LoJfgEXaeIMOe5A2INTIw/i7UN+RliN0J6N8xsq/VhlSW68gTgIXSILERifaEGFTiEZRHEK7kvwLIR4QsLq8IGLkJqS/rhy91nm7oWv0LMNsQee/+z58wCAV648H/nRvL9O4C4QtwMzOhnGcUql2FhLtAQrtOaKzN2rmUk/ZJ+6sc+e18vm4jGBZq8VqgXztAAdDqchP/h3As7P8VyDqSWQSiqnXMbGEvAuioImt/87kR/JFVbrnpY/aR0pOCEOuYyq9lyNQao1i9j3qrbS7+2ZT9cP7cIjX1vnkEbeZavyArVwhx0+tLKXz9i511eskcc73vl2u/auIQbFTtzlyq66hhqx8G52DgBw5aZVtPkoXB5mYLJm8Yf777kLAPDsM1dsLA3VPsTnK0AMOObJalzz9RqMGTAvXmUDU/Hy765bfb16L/jQOk1OcAb5oLnzw8NFh9aY2q15cvyzxQHmpqd5CvvdBtuI9/hwi05bu1YT4nhke7eWaePK3Bm7t/IU6uQEdCFOEW9ZpviEDJweW8zVpircmsWo9nabG2xtzp4T4gZQ/Ce0bleVIT8KmbqzW0exZnMuXIk7otIxkrg13O+nlkAqqaRyjIyPJTDUeDHiGIx/TnLnHR/0DPFt28h3VyBZpeeDeE5aFVhChIV2ZLQIGnVbwau0BOR41XdNg/gQ1e2Fc4Y8Lptdek57D+YnFlumnVdXWJHGQV5+6hkAwLXbG3Y8MQ+VCfv9+TNLAID5mVkAwPau8dqF6kpNxUARbI8LS8sAgHe/+X4AwN6Gaagr167Z2AKegj/lKYRhuHTBfp9lHGZv18a8tm/zc+/5RQDAJP3xlxoWO9lqsz+DFxYE4ToXztp9TE1ZlZ9q9NUktVK2+REPwNMvXQcAnFmcARD553PT9kxeurbGwTNfnhFeQhfVzNh1VDi6uHgGe3yOHcYRMlm7j3w2zu5UYYxol5p+46ZZb2IWEnJQqMTQ+ixkV+LxjiQaUtkB0+KK/kcowuH7iVCNcQvABeRkagmkkkoqx8jYWAKHy/FBgeOzn5Q46Cwso4Gzn1tVmjXIKaheg+pLMKBfv9O0lX+/qayBRfJn6J9uM3ffa3cPIPaUAw74AWnTBpuYrpsmn6SPe5P+tLjotWJvUnWt3Lb22ItzloMWFWOIsiv8IcRiPociJ+Rf//0JAEC+SC3CntuuLV5HNRaVBuSrwizCNvH2oFXS7bOf4r797pWrhl0QBqK3YxaHH8Is2NxkUSLq8tZ106azVfvcJttOg7+tzdhc54nvzxA3cP3GLQDAA/ffCyCqF7l8g7GRvnoSMvZAC4GJIpQZv+jnOiGqX69vc0KU7eF9CqXJ8VfmLQ6zt0vfv2kxAt80i6JSMIuhOmXvh6wZ1TEcyISEeRG2pBjeo37gwVTdwfG4AF2r3z++qemJloBz7ovOuVXn3FND+2adc990zl3mdob7nXPus865F5xzTzrn3nXS+VNJJZXRyquxBL4E4I8BfGVo3yMAHvfef9o59wg//xaADwK4j/9+DMDnuX0VEuWLo1oB+2Ygt/4IdN+w752sMEx2YQkddYKJIGQYPybQigGjlmiH3WJ/gTY5BicrpvEm2J8AA49+WzEN7VLfOTvHBBGDhQm7yiwj68rNt/dElxvP0asybadp2mTnpmlCddKZ5BgctfeAc7K0NB9QhZvEsM/kbdw1+ry7jKBrrgXSy7HCsVw1v73dUzv0eKVjnbGTiTIj2swqFPj7fUbT9SirxQlUJu27vbJp+q63WEmP/AoF8j22FBVnlLzdFrOO7X/i6ecBAPfffQ4AcGHOYgY3V00rd1l9me+T1Ye/K9IScIUGKhWyOTEF0ZISdepbaGPd3bMxTs3P236yQIUXjz79Tl0tzO1ZzC6Y5VCYtHlv7Vv/Chcs1ril0Wg2UOM8DLLxTkQhw5KosQkxBMT/Po6SEy0B7/2/ANhM7P4wgC/z/18G8PND+7/iTf4DwLRzbvmka6SSSiqjk9caE1jy3t8CAO/9LefcIvefA3B96Lgb3HcreQLn3McBfBwAZmsVwwlETiyACEUVqsagqjzEtqF6buDgFepNrJLqQpxlojaXSB74kInQcUISShvH87lwYq8hsoswvXwhqi6Lqrl0Cn7HasJCsB5Mu8yzjx2oPVvUVJmcoruMZg/s90XuDtyFPKrNasM876VYsjFXSgVsbZpWvLhoGqlOxt2FGfu80zK/fEB/XOX1Exzr8l2GrtsZmL9d5rUyjMDvNU2zLdXMB94hUm6RefTrxCOo1mKqVkGTfnSpZPv2ttXtyM4t7EEbYvCVZWBjK8oy4L28/LIhJ88v2RiWZmxsa3Wi9ko2lm7Pjm/xvKWJXBSBZxwmsBdxf4kWzj4twH2eszZpvv8+O1d1nD2rHN8fZR0GfI/OLhs2oVzgc2DXY5kEeqbb9QZmiRAtZOKYBb1PYuDSnOoeVIsjFuKj5PXODhwWyTs0euG9/4L3/kHv/YPV0sRhh6SSSip3QF6rJbDinFumFbAMYJX7bwC4MHTceQCvnHg2R19cOeqwm76OcAOJfKjqAQZy6HMT8PRtB6E8Lu4nBecrk1ivQhCA2po+74CrqQt8+/LH7DpFdZAhRrzDPvStdvMAn0CeufYc8eZ5+sn6XrxyVVoTRUbNQY0Fx88ymDjUQfD95M+L096256qmSVdXtzFd5jnEyEvNNVm2hTjHCkfP3gYZT2uF81qihfCmJdNk82Xb/9w1e8xtovBKy+Yjb9Dnnaqxtn9FlYKcz0IPt6lNF2csy9Eg5qDHKsDJro25y/vLi0uQ81Cp2LmXqhYD6NbtuFvMrly6aGPNT9pYb2/YmBSvEFtQq9vF9q591yBjlDoXi8FYyMlKjjgB9hzsMx5TZLaot2fZAoWepG1bvNeNDcN+nCPWY5JYiBazTlm+Ewtzc6hwjouK8cgSDpAD25ENzEqqOiTW4QTk7Wu1BB4F8DD//zCAbwzt/yizBA8B2JHbkEoqqYynnGgJOOe+CuC9AOadczcA/A6ATwP4G+fcxwBcA/CLPPwfAHwIwAsAGgB+5dUOxFazwxGDoWafx4pBRSueKvcyhSJyBVtRk731FEkdUNuq68/R1PbCdsd564L2yEtz2mf1s280bCXvDwZDSzDvg1aD+iqEij4fj30ow1EukhGnSZw+LZ6+j0eJIzwCz8s4hXj4p0iuu7LfhO+ZFXKLTMbLRBvus8tyhUGATeWpOSZpqBVWKnbJbpDvitWH/miX/jdjD3ki7gYgk3LZ/PRGnbx+7U7AQShkophGjlmB8iTrENjdp8QI/i4xHdW8KvtYv1CzG84y63L1uo35bW8xHIHy5mvbds871ObbW5toMn8/yA5XOUbWQofXrBHd2GLNxPa2+fRzSxYeqxFRus9ahGDlMsPT4L3ss5dljfgEVRVKO1eKRVSEb+C+YAHLIla/Do4lq3eUf97ZEyyBExcB7/1HjvjqfYcc6wF84qRzHikqFQ0BwbihkqQZC5BLkVtms6HkU8jQQHQZUoIKsh1+LrU0U9pI8M2wpYntGVns8rx6eQQu8hgq49D7FGDRajEuczSnA+ycHPz8rJmWjYYCn4mUqF6CrsZGk5Vm8zTBNavrZnrO1OYA0nDL5N7P28t7dc2OyaqJqppj8Jm0CR7a4Uvd9IQ+051wwZWz7R4XmYvztsjsMkBYnWE7cC7G+XwGTQZA97YJOOK89VlK3SHde4vPoETz3zfY7JML/x4bwOySsKPIAGBl2tyEHzz/EgDgHpZgC+rd5liajVYofGqr2YjTH5gtLC0Cu6oVG0ttxs69umILTYPNSGY594E4dF9mPu+N+3cIu67SHRB9WVgG+h6+q+I3vgdOJK/6O7HFUotoSHO/KjRdChtOJZVTL2MEGx5etuJaOkkjpcBIoKgaIhodBEilziGgRJyAIklBLi2tIKMQma2Omk/YVukpwWHVfkyU4+EOXCaQXoRW20o7ymxTOWm4FrUL7fdq0UzOhQU7QAGtLOehwCCeaM871O4CIdUISLlx07RNodBGqULikkXTkm5g4y9wbF2SWihFOGBrbe+LnBdaUjRbmyp9zfMmOE99avcK054NAptqM5o3O7w8kUWDJdX7+yzL7ovclSfv8T3guZUSVBs2le2WqmZ1FCvmclx+wbLVb7pgabjJnJnq129awPBeBgzXNu269d06Oi01eJFFR1OcLofnfW0zpTfLwN7UvFkEDboWDQaKJ8uaZ05PS81POW8sB5cbKdLZyNUtoq8IMAN9AfrTC4wwnC+RpchifJ3AQqmkksr/bxkzSyCutYf6atleZf1CKaj8UAbQMEDG9WP7AtmmqKFVjBGuRD+cK3+LTSqV3lOjyDKDUVNMAQkWWqf26tIiEOzTORdaqmdIlhFiBEx9iWFCII8urZcpkkh0uYJfOGfaZm7eUmh5wl+XaqZlXrhqxCAvvXwDALAQ/FHwd6YhG70etm5thPEBQJHBtzZjHdWq+cRZauMGU56BbEXAJE5gyLj2FIthipYxhA4thhn62mD85r5Ldk+NjXVUazaGzXUBZmiFECx0beWmXUNtvQk8GvDZ5Fiuq5hJgfMqkMzzVy19+SNvf4vd675dZ311k+cl9Xu+gLoCwongrd61StVSgztbZk3Ut+wcsywk6tPE2d22eINAZzOzZimIWEapZL2f+4RzVyqkmZ+065+7936cXZjnsXY/W1d/YGNYs3kJACcVRjFekxko8IxjJbUEUknllMv4WAJuEIGDAimlSm+TpAkm/sj83qEXSHyKxxPyeVkb1IBNW1UnWBiiUlDFI+p188+bLBBxApPI/89kIm2SSHEKuqxMRIW+bIPAGrUev+uM+axPXjHCj1KBqUaO6dpV00LzRfOz16dMG83NkqZsy7T++QX7vLK2h4kZ+21FTU95rULXfNE6I/FVxhN2aeEELlfRdTPuENqWqcRWuC1aCLfWDYZ8ada02d6ene/e+6yk5Hsra5gmUGlF1cvUvqIJO7NgWrTJNKWee55WzCRBV2vMcGRKpAXnPDdp3X3n+08CAN56t+HZ1kkI0mHcJ5ctIZshCUgmDhdX6e9klQVR9N3rOxZvmWAMYIop1/WOlXnvbNAiYAS/SkvCnjTQpkXQZVxHDVCRt+dTO3sR8yRCVVZgb4Ml0mvE4envgpmdPu930BFpbkoqkkoqqRwj42MJxCRe3ZOkTzqulXXUTitOtaRtiCsIqCMwSKgZVosqwkCpEVXQITBHsyUKaRaKQOQTUoW5KIqbaJsmuu1yzjTge95kIJZnbrwMALi9budGy3z8i4sW1d7mNXdFkFkzrS7SywvLpm0bO2YhzEyb1lndNo13c3075PELzEeXua1NWRyhzVjIwgz95B27n5LKoHn/fcYlOv04kCWQxHIaNmkxqYBoj4StL183IGnP50K9cktcrYGElHELNf7Q93xmImw9v2T3vUlLQGXMUT2aHVcnduFFEp6cZ6zlMmMpQC6U/IJxApF79qhlRRE3UbX3orNlls32tmID9kwXCBraJHR5c3OLY7FBzalgiziBXj9OZhMyY8iEG/EDkdPILOMm0XBVx2eUsRrc2QKiVFJJ5f+YjIkl4I9poRRJaIAR2i8l9w+C1k1aAqH9VQJqq992u/Gy3/KkadkCLQJRQHeI/Aq+MX1DRYF1fHvQQ4MlsvkQaQa3duwEswYi2ijmiHzLmMa/QUhpk79cmDdtXWQsoc5ilD1G4u8+Z77z1X07f42IucvXTfM5lxE7GJpd0arZtVsiPgntxki4oVbcIT6j+mVmU7o2Bh8ou0gLTkRljsQfdc5biW24nmPE/t6Ld2GPfnUvxE7iz6zNFu2KBexzXmWFra6YBSC/WpotEMzoLe/bf9aIKMxu2HXPnbfYy+bGFvb27Bqdjv1WeBDFDeTDCykocpBukzDgPeIHZi2Ts7hs1sYOUZs7pJ8TXds0LbBtWm9dvmdC/Q3QDyQyTohBL9g832U+qybp74VALeYFv8exkloCqaRyymVMLIHjJUndHZBiovL2tgrni5PI5uNIr2RMQEQdyimH2gL6ahMTbFLJMl5pny5zzg1Gy+U7Z6jZhCNQw87e3vZQrUAiM6FcdE4oPZJiDNiwlEOukLz0HH39G7foX9KP7LDt1tykjXV/y7TU2TPmj16hr6sxZYs5+FY8zuICdZlds0ofuMm4Q7VELDuns9knwSgtp6mK3X9vWhaVadktlSLzFdvZM5/3rrOkUKPWqpXyuH2LWAQhLOUDOzULEULS7rOnAipuV0jy2RGBx74aqXDe5ebTEtDNbrBseKJs1tvC4jS2d+xcTVLDFQuKL3R5braf66t9vMVduiQ0EXJ0v27HlVkTMMVn2aB1t8kajCXGNaq0PDuMOej8vcEgxLFkjQUQbNTQ3IaomJNwFqHhTpodSCWVVI6RsbAEvI+3Sop8fH1m/lOro2jAiSkXLr3Z6iEjUkYI4SatwQgqI9GdrpZTEniwJgABaUgLgBqvTb+9TYtgQD9/mhH4PMvDNjcsP+z7PRSycUtAK3qe016eNK1Y75hWbBOnXyE5xg+dt3Nn6AP+6H3mu/7X06yWcxaJrrG12OXblpPv3ORY2fpKwZNBZhDowjL9OO3agOZHrab8t2n0BSIkN+mreuLXN1kq3CeuIM/26UtEzmXXTMveZkS+17UxZUiYssjmHXBAg1HtCNmm+g1iNqhlc0JaKjqunl5sK5an9dbuiBqMloUAqHmRc6gWwbZbW3YPbq6KEmMAu4xhBEY4lo7vkixEBLPlmj2rvjR50+a8xfvtN5U9YX0DW8YV+LnD81TV3o2T4BW8GUT6XnGavnAzoYaGlZ9ZZbziVaYnhdtSSyCVVE65jIUlcJIk6+h9gpZZFNuFUgXIyZdnrX6CXISBZuy16zwnV1Fi2hX87qr9GKmnO4w8y4+frpiGnGSEf3vD/PVuU1h7F8UEQmWiVmxW/3EFb1GbLiybL38XtcKtl4xCe0Dsf2nData3qOHPXjRL4r3vfgAA8G9P2Xm/86RhywfM6QuXkHER0WXkJlIjKd7Cr++5dNF+Sz2xv8WoNvPn2yU7wQYj9V1aY6HhKLMEntZalzwGz774IgBgjk1Ar99eQZPVcIEDQpquH8/kTJBcZK8ez0g0OX8A6x6yio4zvqGSRY1NlYE5VUra1+sbu+i01L48XrkqqrccTakeQQ2ZmmJJzBIIP8KTFkPTFsYrGAuoVe39KRHtKKITNSodbiwijR8alwTi0Hjca0CLSMQujojKiKj3cEktgVRSOeXiXk1+/g0fhHNrAPYBrI96LEfIPNKxvRYZ17GN67iAN3Zsd8DN2+AAAAPmSURBVHnvF5I7x2IRAADn3He99w+OehyHSTq21ybjOrZxHRcwmrGl7kAqqZxySReBVFI55TJOi8AXRj2AYyQd22uTcR3buI4LGMHYxiYmkEoqqYxGxskSSCWVVEYg6SKQSiqnXMZiEXDOfcA595xz7gXn3CMjHMcF59y3nHPPOOd+4Jz7JPfPOue+6Zy7zO3MCMeYdc59zzn3GD/f7Zz7Nsf2107taO78uKadc19zzj3L+XvPuMybc+7X+Tyfcs591Tk3Map5c8590Tm36px7amjfofPEnp6f5d/Fk865d70RYxr5IuAM0/g5AB8E8ACAjzjnHhjRcHoAfsN7/1YADwH4BMfyCIDHvff3AXicn0clnwTwzNDn3wPwhxzbFoCPjWRUwB8B+Efv/VsA/DBsjCOfN+fcOQC/CuBB7/3bYBViv4TRzduXAHwgse+oefoggPv47+MAPv+GjMh7P9J/AN4D4J+GPn8KwKdGPS6O5RsAfhrAcwCWuW8ZwHMjGs95viQ/BeAxGPB/HUDusLm8g+OqAbgCBpqH9o983gCcA3AdwCysVuYxAD8zynkDcAnAUyfNE4A/AfCRw457Pf+N3BJA9JAkN7hvpOKcuwTgnQC+DWDJs8U6t4sjGtZnAPwmoi4tcwC2feCbGtnc3QNgDcCf01X5U+dcGWMwb977mwB+H9Y9+xaAHQBPYDzmTXLUPN2Rv41xWAQOax4w0rylc64C4O8A/Jr3fvek4++EOOd+FsCq9/6J4d2HHDqKucsBeBeAz3vv3wmrAxmlyxSE/vWHAdwN4CyAMszMTso45srvyPMdh0XgBoALQ5/PA3hlRGOBcy4PWwD+0nv/de5ecc4t8/tlAKsjGNpPAPg559zLAP4K5hJ8BsC0c6IKGdnc3QBww3v/bX7+GmxRGId5ez+AK977Ne99F8DXAfw4xmPeJEfN0x352xiHReA/AdzHaG0BFrR5dBQDcUb2/mcAnvHe/8HQV48CeJj/fxgWK7ij4r3/lPf+vPf+EmyO/tl7/8sAvgXgF0Y8ttsArjvn7ueu9wF4GmMwbzA34CHn3CSfr8Y28nkbkqPm6VEAH2WW4CEAO3IbXle504GaIwIlHwLwPIAXAfz2CMfxkzBz60kA3+e/D8F878cBXOZ2dsTz9V4Aj/H/9wD4DoAXAPwtgOKIxvQOAN/l3P09gJlxmTcAvwvgWQBPAfgLAMVRzRuAr8JiE12Ypv/YUfMEcwc+x7+L/4ZlOF73MaWw4VRSOeUyDu5AKqmkMkJJF4FUUjnlki4CqaRyyiVdBFJJ5ZRLugikksopl3QRSCWVUy7pIpBKKqdc/gdg9CvnnNwJjAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import cv2\n",
    "import random\n",
    "image,_ = train_data[1000]\n",
    "im_plot(image[100,:,:,:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch\n",
    "import numpy as np\n",
    "from torch.autograd import Variable\n",
    "from torchvision.models import resnet152\n",
    "from efficientnet_pytorch import EfficientNet\n",
    "##############################\n",
    "#         Encoder\n",
    "##############################\n",
    "\n",
    "class Encoder(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(Encoder, self).__init__()\n",
    "        resnet = EfficientNet.from_pretrained('efficientnet-b0')\n",
    "        resnet._fc = Identity()\n",
    "        for param in resnet.parameters():\n",
    "            param.requires_grad = False\n",
    "        self.resnet = resnet\n",
    "        self.bn1 = nn.BatchNorm1d(1280, momentum=0.01)\n",
    "        #self.final = nn.Sequential(nn.Linear(1280, latent_dim), nn.BatchNorm1d(latent_dim, momentum=0.01))        \n",
    "    def forward(self, x):\n",
    "        x = self.resnet(x)\n",
    "        x = x.view(x.size(0), -1)\n",
    "        return x\n",
    "\n",
    "##############################\n",
    "#           LSTM\n",
    "##############################\n",
    "\n",
    "class LSTM(nn.Module):\n",
    "    def __init__(self, latent_dim, num_layers, hidden_dim, bidirectional):\n",
    "        super(LSTM, self).__init__()\n",
    "        self.lstm = nn.LSTM(latent_dim, hidden_dim, num_layers, batch_first=True, bidirectional=bidirectional)\n",
    "        self.hidden_state = None\n",
    "\n",
    "    def reset_hidden_state(self):\n",
    "        self.hidden_state = torch.randn(150, 2, 1024)\n",
    "    def forward(self, x):\n",
    "        self.lstm.flatten_parameters()\n",
    "        x, self.hidden_state = self.lstm(x, self.hidden_state)\n",
    "        return x\n",
    "\n",
    "\n",
    "##############################\n",
    "#      Attention Module\n",
    "##############################\n",
    "\n",
    "\n",
    "class Attention(nn.Module):\n",
    "    def __init__(self, latent_dim, hidden_dim, attention_dim):\n",
    "        super(Attention, self).__init__()\n",
    "        self.latent_attention = nn.Linear(latent_dim, attention_dim)\n",
    "        self.hidden_attention = nn.Linear(2*hidden_dim, attention_dim)\n",
    "        self.joint_attention = nn.Linear(2*attention_dim, 1)\n",
    "        self.relu = Swish()\n",
    "    def forward(self, latent_repr, hidden_repr):\n",
    "        if hidden_repr is None:\n",
    "            hidden_repr = [\n",
    "                Variable(\n",
    "                    torch.zeros(latent_repr.size(0), 1, self.hidden_attention.in_features), requires_grad=False\n",
    "                ).float()\n",
    "            ]\n",
    "\n",
    "        latent_att = self.latent_attention(latent_repr)\n",
    "        hidden_att = self.hidden_attention(hidden_repr)\n",
    "        att = self.relu(torch.cat([latent_att,hidden_att],dim = 1))\n",
    "        joint_att = self.joint_attention(att)\n",
    "        attention_w = F.softmax(joint_att, dim=-1)\n",
    "        return attention_w\n",
    "\n",
    "\n",
    "##############################\n",
    "#         ConvLSTM\n",
    "##############################\n",
    "\n",
    "\n",
    "class ConvLSTM(nn.Module):\n",
    "    def __init__(\n",
    "        self, num_classes, latent_dim=512, lstm_layers=1, hidden_dim=1024, bidirectional=True, attention=True\n",
    "    ):\n",
    "        super(ConvLSTM, self).__init__()\n",
    "        self.encoder = Encoder(latent_dim)\n",
    "        self.lstm = LSTM(latent_dim, lstm_layers, hidden_dim, bidirectional)\n",
    "        self.output_layers = nn.Sequential(\n",
    "            nn.Linear(2 * hidden_dim if bidirectional else hidden_dim, hidden_dim),\n",
    "            Swish(),\n",
    "            nn.Linear(hidden_dim, num_classes),\n",
    "        )\n",
    "        self.attention = attention\n",
    "        self.attention_layer = Attention(512,1024,128)\n",
    "        \n",
    "    def forward(self, x):\n",
    "        batch_size, seq_length, c, h, w = x.shape\n",
    "        x = x.view(batch_size * seq_length, c, h, w)\n",
    "        x1 = self.encoder(x)\n",
    "        x = x1.view(batch_size, seq_length, -1)\n",
    "        x = self.lstm(x)\n",
    "        if self.attention:\n",
    "            attention_w = self.attention_layer(x1,x.reshape(-1,2048))\n",
    "            attention_w = attention_w.reshape(-1,150)\n",
    "            x = torch.sum(attention_w.unsqueeze(-1)*x, dim = 1)\n",
    "        else:\n",
    "            x = x[:, -1]\n",
    "        return self.output_layers(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "class FAN(nn.Module):\n",
    "    def __init__(\n",
    "        self, num_classes,latent_dim=1792, lstm_layers=1, hidden_dim=1024, bidirectional=True):\n",
    "        super(FAN, self).__init__()\n",
    "        self.encoder = Encoder(latent_dim)\n",
    "        self.lstm = LSTM(latent_dim, lstm_layers, hidden_dim, bidirectional)\n",
    "        self.relu = Swish()\n",
    "        self.attention1 = nn.Linear(1792,1)\n",
    "        self.sig = nn.Sigmoid()\n",
    "        self.attention2 = nn.Linear(1792*2,1)\n",
    "        self.dp = nn.Dropout(0.2)\n",
    "        self.output_layers = nn.Sequential(\n",
    "            nn.Linear(1792*2, 1024),\n",
    "            Swish(),\n",
    "            nn.Dropout(0.2),\n",
    "            nn.Linear(1024, num_classes))\n",
    "    def forward(self, x , p = None):\n",
    "        batch_size, seq_length, c, h, w = x.shape\n",
    "        x = x.view(batch_size * seq_length, c, h, w).detach()\n",
    "        x = self.encoder(x)\n",
    "        v1 = []\n",
    "        v2 = []\n",
    "        x = x.view(batch_size,150,1792).detach()\n",
    "        for i in range(x.shape[0]):\n",
    "            alpha = self.sig(self.attention1(self.dp(x[i])))\n",
    "            x1 = torch.sum(x[i]*alpha,dim = 0)\n",
    "            with torch.no_grad():\n",
    "                x1 = x1.unsqueeze(dim = 0).repeat(150,1)\n",
    "                x2 = torch.cat([x[i],x1],dim = 1)\n",
    "            betas = self.sig(self.attention2(self.dp(x2)))\n",
    "            a1 = torch.sum(alpha , dim = 0)\n",
    "            a2 = torch.sum(alpha*betas, dim = 0)\n",
    "            x2 = torch.sum(x2*alpha*betas,dim = 0)\n",
    "            v1.append(torch.div(x1,a1))\n",
    "            v2.append(torch.div(x2,a2))\n",
    "        v1 = torch.stack(v1)\n",
    "        v2 = torch.stack(v2)\n",
    "        #print(v2.shape)\n",
    "        x = torch.cat([v1,v2],dim = 1)\n",
    "        return self.output_layers(v2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "class FAN(nn.Module):\n",
    "    def __init__(\n",
    "        self, num_classes,latent_dim=1792, lstm_layers=1, hidden_dim=1024, bidirectional=True):\n",
    "        super(FAN, self).__init__()\n",
    "        self.encoder = Encoder()\n",
    "        self.lstm = LSTM(latent_dim, lstm_layers, hidden_dim, bidirectional)\n",
    "        self.relu = Swish()\n",
    "        self.attention1 = nn.Linear(1792,1)\n",
    "        self.sig = nn.Sigmoid()\n",
    "        self.attention2 = nn.Linear(2048,1)\n",
    "        self.dp = nn.Dropout(0.5)\n",
    "        self.output_layers = nn.Sequential(\n",
    "            nn.Linear(1792 + 2048, 1024),\n",
    "            Swish(),\n",
    "            nn.Dropout(0.6),\n",
    "            nn.Linear(1024, num_classes),\n",
    "            nn.Dropout(0.6))\n",
    "    def forward(self, x):\n",
    "        batch_size, seq_length, c, h, w = x.shape\n",
    "        x = x.view(batch_size * seq_length, c, h, w).detach()\n",
    "        x = self.encoder(x)\n",
    "        v1 = []\n",
    "        v2 = []\n",
    "        x = x.view(batch_size,150,1792).detach()\n",
    "        x_lstm = self.lstm(x)\n",
    "        for i in range(x.shape[0]):\n",
    "            alpha = self.sig(self.attention1(self.dp(x[i])))\n",
    "            alpha1 = self.sig(self.attention2(self.dp(x_lstm[i])))\n",
    "            x1 = torch.sum(x[i]*alpha,dim = 0)\n",
    "            x2 = torch.sum(x_lstm[i]*alpha1,dim = 0)\n",
    "            a1 = torch.sum(alpha , dim = 0)\n",
    "            a2 = torch.sum(alpha1, dim = 0)\n",
    "            v1.append(torch.div(x1,a1))\n",
    "            v2.append(torch.div(x2,a2))\n",
    "        v1 = torch.stack(v1)\n",
    "        v2 = torch.stack(v2)\n",
    "        x = torch.cat([v1,v2],dim = 1)\n",
    "        return self.output_layers(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "class FAN(nn.Module):\n",
    "    def __init__(\n",
    "        self, num_classes,latent_dim= 1280, lstm_layers=1, hidden_dim=1024, bidirectional = False):\n",
    "        super(FAN, self).__init__()\n",
    "        self.encoder = Encoder()\n",
    "        self.lstm = LSTM(latent_dim, lstm_layers, hidden_dim, bidirectional)\n",
    "        self.lstm1 = LSTM(1024, lstm_layers, 2048, bidirectional)\n",
    "        \n",
    "        self.relu = Swish()\n",
    "        self.attention1 = nn.Linear(latent_dim,1)\n",
    "        self.sig = nn.Sigmoid()\n",
    "        self.attention2 = nn.Linear(1024*2,1)\n",
    "#        self.attention4 = nn.Linear(1024*2,1)\n",
    "        \n",
    "        self.attention3 = nn.Linear(latent_dim*2,1)\n",
    "        self.bn1 = nn.BatchNorm1d(latent_dim*2, momentum=0.01)\n",
    "        self.bn2 = nn.BatchNorm1d(1024*2, momentum=0.01)\n",
    "        \n",
    "        self.dp = nn.Dropout(0.4)\n",
    "        self.output_layers1 = nn.Sequential(\n",
    "            nn.Linear(latent_dim*2, 1024),\n",
    "            Swish(),\n",
    "            nn.Dropout(0.5),\n",
    "            nn.Linear(1024, num_classes),nn.Dropout(0.2))\n",
    "        self.output_layers2 = nn.Sequential(\n",
    "            nn.Linear(1024*2, num_classes),nn.Dropout(0.2))\n",
    "    def forward(self, x):\n",
    "        batch_size, seq_length, c, h, w = x.shape\n",
    "        x = x.view(batch_size * seq_length, c, h, w).detach()\n",
    "        x = self.encoder(x)\n",
    "        v1 = []\n",
    "        v2 = []\n",
    "        v3 = []\n",
    "        x = x.view(batch_size,seq_length,1280).detach()\n",
    "        x_lstm = self.lstm(x)\n",
    "        x_lstm = self.lstm1(x_lstm)\n",
    "        for i in range(x.shape[0]):\n",
    "            alpha = self.sig(self.attention1(self.dp(x[i])))\n",
    "            alpha1 = self.sig(self.attention2(self.dp(x_lstm[i])))\n",
    "            x1 = torch.sum(x[i]*alpha,dim = 0)\n",
    "            with torch.no_grad():\n",
    "                x11 = x1.unsqueeze(dim = 0).repeat(seq_length,1)\n",
    "                x3 = torch.cat([x[i],x11],dim = 1)\n",
    "            betas = self.sig(self.attention3(self.dp(x3)))\n",
    "            x2 = torch.sum(x_lstm[i]*alpha1,dim = 0)\n",
    "            x3 = torch.sum(x3*alpha*betas,dim = 0)\n",
    "            a1 = torch.sum(alpha , dim = 0)\n",
    "            a2 = torch.sum(alpha1, dim = 0)\n",
    "            a3 = torch.sum(alpha*betas, dim = 0)\n",
    "           # v1.append(torch.div(x1,a1))\n",
    "            v2.append(torch.div(x2,a2))\n",
    "            v3.append(torch.div(x3,a3))\n",
    "\n",
    "#        v1 = torch.stack(v1)\n",
    "        v2 = self.bn2(torch.stack(v2))\n",
    "        v3 = self.bn1(torch.stack(v3))\n",
    "        x1 = self.output_layers1(v3)\n",
    "        x2 = self.output_layers2(v2)\n",
    "        out = (x1+x2)/2\n",
    "#        x = torch.cat([v2,v3],dim = 1)\n",
    "        return x1,x2\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loaded pretrained weights for efficientnet-b0\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "torch.Size([8, 4])"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = FAN(4).cuda()\n",
    "model = nn.DataParallel(model)\n",
    "'''model_dict = model.state_dict()\n",
    "pretrained_dict = torch.load('./results/FAN/save.pth')['state_dict']\n",
    "pretrained_dict = {k: v for k, v in pretrained_dict.items() if k in model_dict}\n",
    "model_dict.update(pretrained_dict)\n",
    "model.load_state_dict(model_dict)'''\n",
    "im_size = 112\n",
    "#model.load_state_dict(torch.load('./results/FAN/save.pth')['state_dict'])\n",
    "model(torch.from_numpy(np.empty((8,150,3,im_size,im_size))).type(torch.cuda.FloatTensor))[0].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ConvLSTM(nn.Module):\n",
    "    def __init__(\n",
    "        self, num_classes, latent_dim=512, lstm_layers=1, hidden_dim=1024, bidirectional=True, attention=True):\n",
    "        super(ConvLSTM, self).__init__()\n",
    "        self.encoder = Encoder(latent_dim)\n",
    "        self.lstm = LSTM(latent_dim, lstm_layers, hidden_dim, bidirectional)\n",
    "        \n",
    "        self.relu = Swish()\n",
    "        self.output_layers = nn.Sequential(\n",
    "            nn.Linear(2 * hidden_dim if bidirectional else hidden_dim, 512),\n",
    "            Swish(),\n",
    "            nn.Dropout(0.5),\n",
    "            nn.Linear(512, num_classes),\n",
    "            nn.Dropout(0.5))\n",
    "        self.attention = attention\n",
    "        self.attention_layer1 = nn.Linear(1792, 256)\n",
    "        self.attention_layer2 = nn.Linear(2048,256)\n",
    "        self.joint_att = nn.Linear(512,1)\n",
    "        self.soft = nn.Sigmoid()\n",
    "    def forward(self, x , p = None):\n",
    "        batch_size, seq_length, c, h, w = x.shape\n",
    "        x = x.view(batch_size * seq_length, c, h, w)\n",
    "        x = self.encoder(x)\n",
    "        x = x.view(batch_size, seq_length, -1)\n",
    "        attention_w = self.relu(self.attention_layer1(x).squeeze(-1))\n",
    "#        a = torch.argsort(attention_w,dim =1)\n",
    "#        a = a[:,:50]\n",
    "#        if(p):\n",
    "#            print(attention_w[0])\n",
    "#        with torch.no_grad():\n",
    "#            x1 = []\n",
    "#            for i in range(len(x)):\n",
    "#                x1.append(torch.index_select(x[i],0,a[i]))\n",
    "#            x1 = torch.stack(x1)\n",
    "        x = self.lstm(x)\n",
    "        attention_w1 = self.relu(self.attention_layer2(x).squeeze(-1))\n",
    "        a = torch.cat([attention_w1,attention_w],dim = 2)\n",
    "        att = self.soft(self.joint_att(a).squeeze(-1))\n",
    "        x = torch.mean(x*att.unsqueeze(-1),dim = 1)\n",
    "        return self.output_layers(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loaded pretrained weights for efficientnet-b4\n"
     ]
    }
   ],
   "source": [
    "model = ConvLSTM(\n",
    "        num_classes=4,\n",
    "        latent_dim=1792,\n",
    "        lstm_layers=1,\n",
    "        hidden_dim=1024,\n",
    "        bidirectional=True,\n",
    "        attention=True,\n",
    "    ).cuda()\n",
    "model = nn.DataParallel(model)\n",
    "#model.load_state_dict(torch.load('./results/cnnlstm-effb0-224-attention-sig/save.pth')['state_dict'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([8, 4])"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "im_size = 112\n",
    "model(torch.from_numpy(np.empty((8,150,3,im_size,im_size))).type(torch.cuda.FloatTensor)).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Identity(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(Identity, self).__init__()\n",
    "    def forward(self, x):\n",
    "        return x\n",
    "from torchvision import models\n",
    "\n",
    "from efficientnet_pytorch import EfficientNet\n",
    "class SwishImplementation(torch.autograd.Function):\n",
    "    @staticmethod\n",
    "    def forward(ctx, i):\n",
    "        result = i * torch.sigmoid(i)\n",
    "        ctx.save_for_backward(i)\n",
    "        return result\n",
    "\n",
    "    @staticmethod\n",
    "    def backward(ctx, grad_output):\n",
    "        i = ctx.saved_variables[0]\n",
    "        sigmoid_i = torch.sigmoid(i)\n",
    "        return grad_output * (sigmoid_i * (1 + i * (1 - sigmoid_i)))\n",
    "\n",
    "\n",
    "class Swish(nn.Module):\n",
    "    @staticmethod\n",
    "    def forward(x):\n",
    "        return SwishImplementation.apply(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loaded pretrained weights for efficientnet-b1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/sanjaymoto75/.conda/envs/env/lib/python3.7/site-packages/torch/nn/modules/rnn.py:51: UserWarning: dropout option adds dropout after all but last recurrent layer, so non-zero dropout expects num_layers greater than 1, but got dropout=0.5 and num_layers=1\n",
      "  \"num_layers={}\".format(dropout, num_layers))\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<All keys matched successfully>"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from torchvision import models\n",
    "\n",
    "from efficientnet_pytorch import EfficientNet\n",
    "class SwishImplementation(torch.autograd.Function):\n",
    "    @staticmethod\n",
    "    def forward(ctx, i):\n",
    "        result = i * torch.sigmoid(i)\n",
    "        ctx.save_for_backward(i)\n",
    "        return result\n",
    "\n",
    "    @staticmethod\n",
    "    def backward(ctx, grad_output):\n",
    "        i = ctx.saved_variables[0]\n",
    "        sigmoid_i = torch.sigmoid(i)\n",
    "        return grad_output * (sigmoid_i * (1 + i * (1 - sigmoid_i)))\n",
    "\n",
    "\n",
    "class Swish(nn.Module):\n",
    "    @staticmethod\n",
    "    def forward(x):\n",
    "        return SwishImplementation.apply(x)\n",
    "class Model(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(Model, self).__init__()\n",
    "        model = EfficientNet.from_pretrained('efficientnet-b1')\n",
    "        model._fc = Identity()\n",
    "        for param in model.parameters():\n",
    "            param.requires_grad = False\n",
    "        self.model1 = model\n",
    "        self.lstm = nn.LSTM(1280,1024,1,batch_first=True ,dropout = 0.5 , bidirectional = True)\n",
    "        self.l1 = nn.Linear(2048,512)\n",
    "        self.dp = nn.Dropout(0.4)\n",
    "        self.relu = Swish()\n",
    "        self.l2 = nn.Linear(512,4)\n",
    "    def forward(self,x):\n",
    "        self.lstm.flatten_parameters()\n",
    "        out = self.model1(x)\n",
    "#        print(out.shape)\n",
    "        out1 = out.view(-1,150,1280).detach()\n",
    "        out,_ = self.lstm(out1,None)\n",
    "        out = torch.mean(out,dim=1)\n",
    "        out = self.dp(self.relu(self.l1(out)))\n",
    "        out = self.l2(out)\n",
    "        return out\n",
    "model = Model().cuda()\n",
    "model = nn.DataParallel(model)\n",
    "model.load_state_dict(torch.load('./results/lrcn-effb4-224/save.pth')['state_dict'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 0.0312,  0.0350, -0.0293,  0.0418],\n",
       "        [ 0.0340,  0.0345, -0.0242,  0.0434],\n",
       "        [ 0.0245,  0.0262, -0.0234,  0.0406],\n",
       "        [ 0.0267,  0.0314, -0.0220,  0.0394]], device='cuda:0',\n",
       "       grad_fn=<GatherBackward>)"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "im_size = 112\n",
    "model(torch.from_numpy(np.empty((600,3,im_size,im_size))).type(torch.cuda.FloatTensor))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "from torch.autograd import Variable\n",
    "class FocalLoss(nn.Module):\n",
    "    def __init__(self, gamma=0, alpha=None, size_average=True):\n",
    "        super(FocalLoss, self).__init__()\n",
    "        self.gamma = gamma\n",
    "        self.alpha = alpha\n",
    "        if isinstance(alpha,(float,int)): self.alpha = torch.Tensor([alpha,1-alpha])\n",
    "        if isinstance(alpha,list): self.alpha = torch.Tensor(alpha)\n",
    "        self.size_average = size_average\n",
    "\n",
    "    def forward(self, input, target):\n",
    "        if input.dim()>2:\n",
    "            input = input.view(input.size(0),input.size(1),-1)  # N,C,H,W => N,C,H*W\n",
    "            input = input.transpose(1,2)    # N,C,H*W => N,H*W,C\n",
    "            input = input.contiguous().view(-1,input.size(2))   # N,H*W,C => N*H*W,C\n",
    "        target = target.view(-1,1)\n",
    "        logpt = F.log_softmax(input)\n",
    "        logpt = logpt.gather(1,target)\n",
    "        logpt = logpt.view(-1)\n",
    "        pt = Variable(logpt.data.exp())\n",
    "        if self.alpha is not None:\n",
    "            if self.alpha.type()!=input.data.type():\n",
    "                self.alpha = self.alpha.type_as(input.data)\n",
    "            at = self.alpha.gather(0,target.data.view(-1))\n",
    "            logpt = logpt * Variable(at)\n",
    "        loss = -1 * (1-pt)**self.gamma * logpt\n",
    "        if self.size_average: return loss.mean()\n",
    "        else: return loss.sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from torch.autograd import Variable\n",
    "import time\n",
    "import os\n",
    "import sys\n",
    "import os\n",
    "def train_epoch(epoch, num_epochs, data_loader, model, criterion, optimizer, epoch_logger, batch_logger, batch_size , onecyc , writer , result_path):\n",
    "    print('Training Epoch {}'.format(epoch))\n",
    "    model.train()\n",
    "    batch_time = AverageMeter()\n",
    "    data_time = AverageMeter()\n",
    "    losses = AverageMeter()\n",
    "    accuracies = AverageMeter()\n",
    "    accuracies_att = AverageMeter()\n",
    "    accuracies_lstm = AverageMeter()\n",
    "    \n",
    "    start_time = time.time()\n",
    "    end_time = time.time()\n",
    "    flag = 0\n",
    "    for i, (inputs, targets) in enumerate(data_loader):\n",
    "        data_time.update(time.time() - end_time)\n",
    "        if(inputs.size(0)<batch_size):\n",
    "            continue\n",
    "        if torch.cuda.is_available():\n",
    "            targets = targets.cuda()\n",
    "            inputs = inputs.cuda()\n",
    "        #with torch.no_grad():\n",
    "        #    inputs = inputs.reshape(-1,3,im_size,im_size)\n",
    "        x1,x2 = model(inputs)\n",
    "        outputs = (x1+x2)/2\n",
    "        #outputs = torch.mul(outputs,class_weights)\n",
    "        #loss1 = CB_loss(targets, x1, samples_per_cls,no_of_classes,loss_type, beta, gamma)\n",
    "        #loss2 = CB_loss(targets, x2, samples_per_cls,no_of_classes,loss_type, beta, gamma)\n",
    "        loss = criterion(outputs, targets)\n",
    "        acc = calculate_accuracy(outputs, targets)\n",
    "        acc_att = calculate_accuracy(x1, targets)\n",
    "        acc_lstm = calculate_accuracy(x2, targets)\n",
    "        #loss2.backward()\n",
    "        \n",
    "        accuracies_att.update(acc_att, inputs.size(0))\n",
    "        accuracies_lstm.update(acc_lstm, inputs.size(0))\n",
    "        losses.update(loss.item(), inputs.size(0))\n",
    "        accuracies.update(acc, inputs.size(0))\n",
    "        lr,_ = onecyc.calc()\n",
    "        update_lr(optimizer, lr)\n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        batch_time.update(time.time() - end_time)\n",
    "        end_time = time.time()\n",
    "        del inputs,targets,outputs\n",
    "        torch.cuda.empty_cache() \n",
    "    \n",
    "        batch_logger.log({\n",
    "            'epoch': epoch,\n",
    "            'batch': i + 1,\n",
    "            'iter': (epoch - 1) * len(data_loader) + (i + 1),\n",
    "            'loss': losses.val,\n",
    "            'acc': accuracies.val,\n",
    "            'lr': optimizer.param_groups[0]['lr']\n",
    "        })\n",
    "        writer.add_scalar('data/acc', accuracies.val, (epoch - 1) * len(data_loader) + (i + 1))\n",
    "        writer.add_scalar('data/loss', losses.val, (epoch - 1) * len(data_loader) + (i + 1))        \n",
    "        sys.stdout.write(\n",
    "                \"\\r[Epoch %d/%d] [Batch %d / %d] [Time %.2f %.2f] [Data %.2f %.2f] [Loss: %f, Acc: %.2f%% , att: %.2f%% , lstm: %.2f%%]\"\n",
    "                % (\n",
    "                    epoch,\n",
    "                    num_epochs,\n",
    "                    i,\n",
    "                    len(data_loader),\n",
    "                    batch_time.val,\n",
    "                    batch_time.avg,\n",
    "                    data_time.val,\n",
    "                    data_time.avg,\n",
    "                    losses.avg,\n",
    "                    accuracies.avg,\n",
    "                    accuracies_att.avg,\n",
    "                    accuracies_lstm.avg\n",
    "                    \n",
    "                    )\n",
    "                )\n",
    "    print('\\nEpoch time {} mins'.format((end_time-start_time)/60))\n",
    "    epoch_logger.log({\n",
    "        'epoch': epoch,\n",
    "        'loss': losses.avg,\n",
    "        'acc': accuracies.avg,\n",
    "        'lr': optimizer.param_groups[0]['lr']\n",
    "    })\n",
    "    writer.add_scalar('data/acc_epoch', accuracies.avg, epoch)\n",
    "    writer.add_scalar('data/loss_epoch', losses.avg, epoch)        \n",
    "    torch.cuda.empty_cache() \n",
    "    save_file_path = os.path.join(result_path,'save.pth')\n",
    "    states = {\n",
    "        'model':model.module,\n",
    "        'state_dict': model.state_dict()\n",
    "    }\n",
    "    torch.save(states, save_file_path)\n",
    "\n",
    "def test(epoch,model, data_loader ,criterion, batch_size, result_path,best_acc = 0 , logger = None ):\n",
    "    print('Testing')\n",
    "    if(epoch == 1):\n",
    "        best_acc = 0\n",
    "    model.eval()\n",
    "    losses = AverageMeter()\n",
    "    accuracies = AverageMeter()\n",
    "    accuracies_att = AverageMeter()\n",
    "    accuracies_lstm = AverageMeter()\n",
    "    pred = []\n",
    "    pred_att = []\n",
    "    pred_lstm = []\n",
    "    true = []\n",
    "    for i, (inputs, targets) in enumerate(data_loader):\n",
    "        if(inputs.size(0)<batch_size):\n",
    "            continue\n",
    "        if torch.cuda.is_available():\n",
    "            targets = targets.cuda()\n",
    "            inputs = inputs.cuda()\n",
    "        inputs = Variable(inputs)\n",
    "        #with torch.no_grad():\n",
    "        #    inputs = inputs.reshape(-1,3,im_size,im_size)\n",
    "        targets = Variable(targets)\n",
    "        x1,x2 = model(inputs)\n",
    "        outputs = (x1+x2)/2\n",
    "        #outputs = torch.mul(outputs,class_weights)\n",
    "        _,p = torch.max(outputs,1)\n",
    "        _,p_att = torch.max(x1,1)\n",
    "        _,p_lstm = torch.max(x2,1)\n",
    "        #loss = CB_loss(targets, outputs, samples_per_cls, no_of_classes,loss_type, beta, gamma)\n",
    "        loss = criterion(outputs, targets)\n",
    "        acc = calculate_accuracy(outputs, targets)\n",
    "        acc_att = calculate_accuracy(x1, targets)\n",
    "        acc_lstm = calculate_accuracy(x2, targets)\n",
    "        true += targets.detach().cpu().numpy().reshape(len(targets)).tolist()\n",
    "        pred += p.detach().cpu().numpy().reshape(len(p)).tolist()\n",
    "        pred_att += p_att.detach().cpu().numpy().reshape(len(p_att)).tolist()\n",
    "        pred_lstm += p_lstm.detach().cpu().numpy().reshape(len(p_lstm)).tolist()\n",
    "        \n",
    "        losses.update(loss.item(), inputs.size(0))\n",
    "        accuracies.update(acc, inputs.size(0))\n",
    "        accuracies_att.update(acc_att, inputs.size(0))\n",
    "        accuracies_lstm.update(acc_lstm, inputs.size(0))\n",
    "        del inputs,targets,outputs\n",
    "        torch.cuda.empty_cache() \n",
    "        \n",
    "        sys.stdout.write(\n",
    "                \"\\r[Batch %d / %d]  [Loss: %f, Acc: %.2f%% , att: %.2f%% , lstm: %.2f%%]\"\n",
    "                % (\n",
    "                    i,\n",
    "                    len(data_loader),\n",
    "                    losses.avg,\n",
    "                    accuracies.avg,\n",
    "                    accuracies_att.avg,\n",
    "                    accuracies_lstm.avg\n",
    "                    )\n",
    "                )\n",
    "    if(logger):\n",
    "        logger.log({'epoch': epoch, 'loss': losses.avg, 'acc': accuracies.avg})\n",
    "    print('\\nAccuracy {}'.format(accuracies.avg))\n",
    "    if(accuracies.avg>best_acc):\n",
    "        best_acc = accuracies.avg\n",
    "        result_path = os.path.join(result_path,'best.pth')\n",
    "        state = {\n",
    "        'acc':best_acc,\n",
    "         'state':model.state_dict()\n",
    "        }\n",
    "        torch.save(state,result_path)\n",
    "    return true,pred,pred_att,pred_lstm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "arch = 'FAN'\n",
    "result_path = os.path.join('./results',arch)\n",
    "from tensorboardX import SummaryWriter\n",
    "writer = SummaryWriter()\n",
    "os.makedirs(result_path,exist_ok = True)\n",
    "train_logger = Logger(\n",
    "            os.path.join(result_path, 'train{}.log'.format(arch)),\n",
    "            ['epoch', 'loss', 'acc', 'lr'])\n",
    "train_batch_logger = Logger(\n",
    "            os.path.join(result_path, 'train_batch{}.log'.format(arch)),\n",
    "            ['epoch', 'batch', 'iter', 'loss', 'acc', 'lr'])    \n",
    "val_logger = Logger(\n",
    "            os.path.join(result_path, 'val{}.log'.format(arch)), ['epoch', 'loss', 'acc'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 8\n",
    "train_loader = DataLoader(train_data,batch_size = batch_size , num_workers = 8,shuffle = True)\n",
    "val_loader = DataLoader(val_data,batch_size = batch_size , num_workers = 8)\n",
    "test_loader = DataLoader(test_data,batch_size = batch_size , num_workers = 8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def set_lr(optimizer, lr):\n",
    "    for group in optimizer.param_groups:\n",
    "        group['lr'] = lr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import torch\n",
    "import torch.nn.functional as F\n",
    "\n",
    "def focal_loss(labels, logits, alpha, gamma):\n",
    "    \"\"\"Compute the focal loss between `logits` and the ground truth `labels`.\n",
    "    Focal loss = -alpha_t * (1-pt)^gamma * log(pt)\n",
    "    where pt is the probability of being classified to the true class.\n",
    "    pt = p (if true class), otherwise pt = 1 - p. p = sigmoid(logit).\n",
    "    Args:\n",
    "      labels: A float tensor of size [batch, num_classes].\n",
    "      logits: A float tensor of size [batch, num_classes].\n",
    "      alpha: A float tensor of size [batch_size]\n",
    "        specifying per-example weight for balanced cross entropy.\n",
    "      gamma: A float scalar modulating loss from hard and easy examples.\n",
    "    Returns:\n",
    "      focal_loss: A float32 scalar representing normalized total loss.\n",
    "    \"\"\"    \n",
    "    BCLoss = F.binary_cross_entropy_with_logits(input = logits, target = labels,reduction = \"none\")\n",
    "\n",
    "    if gamma == 0.0:\n",
    "        modulator = 1.0\n",
    "    else:\n",
    "        modulator = torch.exp(-gamma * labels * logits - gamma * torch.log(1 + \n",
    "            torch.exp(-1.0 * logits)))\n",
    "\n",
    "    loss = modulator * BCLoss\n",
    "    weighted_loss = alpha * loss\n",
    "    focal_loss = torch.sum(weighted_loss)\n",
    "\n",
    "    focal_loss /= torch.sum(labels)\n",
    "    return focal_loss\n",
    "\n",
    "\n",
    "\n",
    "def CB_loss(labels, logits, samples_per_cls, no_of_classes, loss_type, beta, gamma,p = 0):\n",
    "    \"\"\"Compute the Class Balanced Loss between `logits` and the ground truth `labels`.\n",
    "    Class Balanced Loss: ((1-beta)/(1-beta^n))*Loss(labels, logits)\n",
    "    where Loss is one of the standard losses used for Neural Networks.\n",
    "    Args:\n",
    "      labels: A int tensor of size [batch].\n",
    "      logits: A float tensor of size [batch, no_of_classes].\n",
    "      samples_per_cls: A python list of size [no_of_classes].\n",
    "      no_of_classes: total number of classes. int\n",
    "      loss_type: string. One of \"sigmoid\", \"focal\", \"softmax\".\n",
    "      beta: float. Hyperparameter for Class balanced loss.\n",
    "      gamma: float. Hyperparameter for Focal loss.\n",
    "    Returns:\n",
    "      cb_loss: A float tensor representing class balanced loss\n",
    "    \"\"\"\n",
    "    effective_num = 1.0 - np.power(beta, samples_per_cls)\n",
    "    weights = (1.0 - beta) / np.array(effective_num)\n",
    "    weights = weights / np.sum(weights)*4\n",
    "    labels_one_hot = F.one_hot(labels, no_of_classes).float()\n",
    "    weights = torch.tensor(weights).float().cuda()\n",
    "    weights1 = weights\n",
    "    weights = weights.unsqueeze(0)\n",
    "    weights = weights.repeat(labels_one_hot.shape[0],1) * labels_one_hot\n",
    "    weights = weights.sum(1)\n",
    "    weights = weights.unsqueeze(1)\n",
    "    weights = weights.repeat(1,no_of_classes)    \n",
    "    if loss_type == \"focal\":\n",
    "        cb_loss = focal_loss(labels_one_hot, logits, weights, gamma)\n",
    "    elif loss_type == \"sigmoid\":\n",
    "        cb_loss = F.binary_cross_entropy_with_logits(input = logits,target = labels_one_hot, weights = weights)\n",
    "    elif loss_type == \"softmax\":\n",
    "        pred = logits.softmax(dim = 1)\n",
    "        if(p == 1):\n",
    "            pass\n",
    "            #print(pred,labels_one_hot)\n",
    "        cb_loss = F.binary_cross_entropy(input = pred, target = labels_one_hot, weight = weights1)\n",
    "    return cb_loss\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import confusion_matrix\n",
    "lr = 1e-3\n",
    "batch_size = 32\n",
    "train_loader = DataLoader(train_data,batch_size = batch_size , num_workers = 16,shuffle = True)\n",
    "val_loader = DataLoader(val_data,batch_size = batch_size , num_workers = 16)\n",
    "test_loader = DataLoader(test_data,batch_size = batch_size , num_workers = 16)\n",
    "class_weights = torch.from_numpy(np.asarray([2, 1, 0.21102427, 0.22006205])).type(torch.FloatTensor).cuda()\n",
    "#optimizer = torch.optim.SGD(model.parameters() , lr = lr , momentum = 0.9 , weight_decay = 1e-3)\n",
    "optimizer = RAdam(model.parameters(), lr= lr,weight_decay = 1e-4)\n",
    "onecyc = OneCycle(len(train_loader)*num_epochs, lr)\n",
    "#criterion = nn.CrossEntropyLoss(weight = class_weights).cuda()\n",
    "#criterion = nn.CrossEntropyLoss().cuda()\n",
    "\n",
    "criterion = FocalLoss(gamma = 3).cuda()\n",
    "best_acc = 0\n",
    "beta = 0.9999\n",
    "gamma = 0.5\n",
    "no_of_classes = 4\n",
    "samples_per_cls = [204,426,2615,2493]\n",
    "loss_type = \"focal\"\n",
    "num_epochs = 10\n",
    "for epoch in range(1,num_epochs+1):\n",
    "    train_epoch(epoch,num_epochs,train_loader,model,criterion,optimizer,train_logger,train_batch_logger,batch_size,onecyc,writer,result_path)\n",
    "    true,pred,pred_att,pred_lstm = test(epoch,model,val_loader,criterion,batch_size,result_path,best_acc)\n",
    "    cm1 = confusion_matrix(true,pred)\n",
    "    cm2 = confusion_matrix(true,pred_att)\n",
    "    cm3 = confusion_matrix(true,pred_lstm)\n",
    "    print('Comb\\n',cm1 ,'Att\\n\\n' ,cm2,'Lstm\\n\\n' , cm3)\n",
    "    if(epoch%5 == 0):\n",
    "        true,pred,pred_att,pred_lstm = test(epoch,model,test_loader,criterion,batch_size,result_path,best_acc)\n",
    "        cm1 = confusion_matrix(true,pred)\n",
    "        cm2 = confusion_matrix(true,pred_att)\n",
    "        cm3 = confusion_matrix(true,pred_lstm)\n",
    "        print('Comb\\n',cm1 ,'Att\\n\\n' ,cm2,'Lstm\\n\\n' , cm3)\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}