9_gradient_accumulation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from gpt2 import GPT, GPTConfig # our GPT class\n",
    "\n",
    "import math\n",
    "import time\n",
    "import tiktoken\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "enc = tiktoken.get_encoding('gpt2')\n",
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') # dynamic device\n",
    "\n",
    "torch.manual_seed(13) # for reproducibility\n",
    "if torch.cuda.is_available():\n",
    "    torch.cuda.manual_seed_all(13)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Loader Lite"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class DataLoaderLite:\n",
    "\n",
    "    def __init__(self, B, T):\n",
    "\n",
    "        self.B, self.T = B, T\n",
    "\n",
    "        with open('data/input.txt', 'r') as file:\n",
    "            text = file.read().replace('\\n', '')\n",
    "        \n",
    "        enc = tiktoken.get_encoding('gpt2')\n",
    "        tokens = enc.encode(text)\n",
    "        self.tokens = torch.tensor(tokens, dtype=torch.long, device=device)\n",
    "\n",
    "        self.current_batch = 0\n",
    "        self.number_of_batches = len(self.tokens) // (B * T)\n",
    "\n",
    "        print(f'Loaded {len(self.tokens)} tokens, {self.number_of_batches} batches of size {B}x{T}')\n",
    "\n",
    "    \n",
    "    def next_batch(self):\n",
    "\n",
    "        B, T = self.B, self.T\n",
    "\n",
    "        buf = self.tokens[self.current_batch * B * T : (self.current_batch + 1) * B * T + 1]\n",
    "        x = buf[:-1].view(B, T)\n",
    "        y = buf[1:].view(B, T)\n",
    "\n",
    "        self.current_batch += 1\n",
    "        if self.current_batch >= self.number_of_batches:\n",
    "            self.current_batch = 0\n",
    "        \n",
    "        return x, y"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Learning Rate Scheduler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "max_lr = 6e-4\n",
    "min_lr = max_lr / 10\n",
    "warmup_steps = 10\n",
    "num_steps = 50\n",
    "def get_lr(step):\n",
    "    if step < warmup_steps:\n",
    "        return min_lr + (max_lr - min_lr) * step / warmup_steps\n",
    "    elif step < num_steps:\n",
    "        return min_lr + (max_lr - min_lr) * (1 + math.cos((step - warmup_steps) / (num_steps - warmup_steps) * math.pi)) / 2\n",
    "    else:\n",
    "        return min_lr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlEAAAHHCAYAAACfqw0dAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/TGe4hAAAACXBIWXMAAA9hAAAPYQGoP6dpAABx1UlEQVR4nO3deVxU5f4H8M8szAzbsMoqIiruCoqKKF4zKVzKcMklSzPTFi1Nq5uVS127pmU/syyzRe1eza3ymiVluCuCIO6ioChu7LLLNvP8/kCmJlGBgMPMfN6v17yUc54ZPnMi58s5z/k+MiGEABERERHVilzqAERERESmiEUUERERUR2wiCIiIiKqAxZRRERERHXAIoqIiIioDlhEEREREdUBiygiIiKiOmARRURERFQHLKKIiIiI6oBFFBFJpmXLlnj66aeljmGW9uzZA5lMhi1btjT491qwYAFkMlmdnrtmzRrIZDJcunSpfkMRNQIWUUQmrupDKC4uTuooFqW4uBgLFizAnj17pI5CRBJRSh2AiCzXuXPnIJeb5u9yxcXFeOeddwAADzzwgLRhiEgSLKKIqF5UVFRAr9dDpVLV+DlqtboBE9VOXfITkWUzzV8BiajWrl27hmeeeQbu7u5Qq9Xo1KkTvvnmG6MxZWVlmDdvHoKCguDg4ABbW1v069cPu3fvNhp36dIlyGQyfPjhh1i2bBlat24NtVqNM2fOGObHJCcn4+mnn4ajoyMcHBwwadIkFBcXG73OX+dEVV2aPHjwIGbNmoVmzZrB1tYWw4cPR2ZmptFz9Xo9FixYAC8vL9jY2GDAgAE4c+ZMjeZZ3St/TY7BpUuX0KxZMwDAO++8A5lMBplMhgULFhjGJCYmYtSoUXB2doZGo0GPHj2wbdu2+/1nAgBs2LABQUFBsLe3h1arRZcuXfDxxx8bjcnNzcUrr7yCli1bQq1Wo3nz5pgwYQKysrLuOE7vvfcemjdvDo1Gg4EDByI5OfmO7xkTE4NBgwbBwcEBNjY26N+/Pw4ePHjHuAMHDqBnz57QaDRo3bo1vvjii7se3zVr1tyx76/H6W527NiBfv36wdbWFvb29hg6dChOnz593+cRNSaeiSKyAOnp6ejduzdkMhmmT5+OZs2aYceOHZg8eTLy8/Mxc+ZMAEB+fj6++uorjBs3DlOmTEFBQQG+/vprhIeHIzY2FoGBgUavu3r1apSUlGDq1KlQq9VwdnY27Bs9ejT8/PywaNEiHD16FF999RXc3NywePHi++Z96aWX4OTkhPnz5+PSpUtYtmwZpk+fjo0bNxrGzJkzB0uWLMGjjz6K8PBwHD9+HOHh4SgpKanxcakuf02OQbNmzfD555/jhRdewPDhwzFixAgAQNeuXQEAp0+fRt++feHt7Y033ngDtra22LRpEyIiIvD9999j+PDhd820c+dOjBs3DgMHDjQcq7Nnz+LgwYOYMWMGAKCwsBD9+vXD2bNn8cwzz6B79+7IysrCtm3bcPXqVbi6uhpe7/3334dcLserr76KvLw8LFmyBOPHj0dMTIxhzK5duzB48GAEBQVh/vz5kMvlWL16NR588EHs378fvXr1AgCcPHkSDz/8MJo1a4YFCxagoqIC8+fPh7u7e42PeU385z//wcSJExEeHo7FixejuLgYn3/+OUJDQ5GQkICWLVvW6/cjqjNBRCZt9erVAoA4cuTIXcdMnjxZeHp6iqysLKPtY8eOFQ4ODqK4uFgIIURFRYUoLS01GnPz5k3h7u4unnnmGcO2lJQUAUBotVqRkZFhNH7+/PkCgNF4IYQYPny4cHFxMdrm6+srJk6ceMd7CQsLE3q93rD9lVdeEQqFQuTm5gohhEhLSxNKpVJEREQYvd6CBQsEAKPXrM698tf0GGRmZgoAYv78+Xe8/sCBA0WXLl1ESUmJYZterxd9+vQR/v7+98w2Y8YModVqRUVFxV3HzJs3TwAQP/zwwx37qo7b7t27BQDRoUMHo/fz8ccfCwDi5MmThvH+/v4iPDzc6JgXFxcLPz8/8dBDDxm2RURECI1GIy5fvmzYdubMGaFQKMSfP06qju/q1avvyPfXY1b13zwlJUUIIURBQYFwdHQUU6ZMMXpeWlqacHBwuGM7kZR4OY/IzAkh8P333+PRRx+FEAJZWVmGR3h4OPLy8nD06FEAgEKhMMwJ0uv1yMnJQUVFBXr06GEY82cjR440XNb6q+eff97o6379+iE7Oxv5+fn3zTx16lSjW+b79esHnU6Hy5cvAwCioqJQUVGBF1980eh5L7300n1f+375a3sM/ionJwe7du3C6NGjUVBQYDjW2dnZCA8PR1JSEq5du3bX5zs6OqKoqAg7d+6865jvv/8eAQEB1Z7R+murgUmTJhnN8+rXrx8A4OLFiwCAY8eOISkpCU888QSys7MNeYuKijBw4EDs27cPer0eOp0Ov/76KyIiItCiRQvD63Xo0AHh4eH3PS41tXPnTuTm5mLcuHFGP6sKhQLBwcF3XFomkhIv5xGZuczMTOTm5mLVqlVYtWpVtWMyMjIMf1+7di2WLl2KxMRElJeXG7b7+fnd8bzqtlX58wctADg5OQEAbt68Ca1We8/M93ouAEMx1aZNG6Nxzs7OhrE1cbf8tTkGf5WcnAwhBObOnYu5c+dWOyYjIwPe3t7V7nvxxRexadMmDB48GN7e3nj44YcxevRoDBo0yDDmwoULGDly5H2zAPc/lklJSQCAiRMn3vU18vLyUFpailu3bsHf3/+O/e3atcMvv/xSozz3U5XnwQcfrHb//X52iBoTiygiM6fX6wEATz755F0/KKvm8vz3v//F008/jYiICLz22mtwc3ODQqHAokWLcOHChTueZ21tfdfvq1Aoqt0uhLhv5r/z3NqoLn9tj8FfVR3vV1999a5naP5a/P2Zm5sbjh07hl9//RU7duzAjh07sHr1akyYMAFr166t4Tv7w/2OZVXeDz744I45b1Xs7OxQWlpa4+95t8abOp3uvs+tyvOf//wHHh4ed+xXKvmxRU0HfxqJzFyzZs1gb28PnU6HsLCwe47dsmULWrVqhR9++MHog3D+/PkNHbNWfH19AVSe9fnz2aHs7GzDGZa6qukxuFuh0KpVKwCAlZXVfY/33ahUKjz66KN49NFHodfr8eKLL+KLL77A3Llz0aZNG7Ru3RqnTp2q02v/VevWrQFUnuG5V95mzZrB2tracKboz86dO2f0ddXZrtzcXKPtVWcQa5LHzc2tzsePqLFwThSRmVMoFBg5ciS+//77aj94/9w6oOqsxZ/P+MTExCA6Orrhg9bCwIEDoVQq8fnnnxtt//TTT//2a9f0GNjY2AC4s1Bwc3PDAw88gC+++AI3bty44/X/2qrhr7Kzs42+lsvlhjOFVWeDRo4ciePHj+PHH3+84/m1PVsXFBSE1q1b48MPP0RhYeFd8yoUCoSHh2Pr1q1ITU017D979ix+/fVXo+dotVq4urpi3759Rts/++yz++YJDw+HVqvFv//9b6NLqX/NQ9QU8EwUkZn45ptvEBkZecf2GTNm4P3338fu3bsRHByMKVOmoGPHjsjJycHRo0fx+++/IycnBwDwyCOP4IcffsDw4cMxdOhQpKSkYOXKlejYsWO1H7BScXd3x4wZM7B06VIMGzYMgwYNwvHjx7Fjxw64urrWeR03oObHwNraGh07dsTGjRvRtm1bODs7o3PnzujcuTNWrFiB0NBQdOnSBVOmTEGrVq2Qnp6O6OhoXL16FcePH7/r93/22WeRk5ODBx98EM2bN8fly5fxySefIDAwEB06dAAAvPbaa9iyZQsef/xxPPPMMwgKCkJOTg62bduGlStXIiAgoMbvVy6X46uvvsLgwYPRqVMnTJo0Cd7e3rh27Rp2794NrVaLn376CUBlT6zIyEj069cPL774IioqKvDJJ5+gU6dOOHHixB3v4/3338ezzz6LHj16YN++fTh//vx982i1Wnz++ed46qmn0L17d4wdOxbNmjVDamoqfv75Z/Tt27deimWieiHZfYFEVC+qbhG/2+PKlStCCCHS09PFtGnThI+Pj7CyshIeHh5i4MCBYtWqVYbX0uv14t///rfw9fUVarVadOvWTWzfvl1MnDhR+Pr6GsZV3cL+wQcf3JGnqsVBZmZmtTmrbmUX4u4tDv7arqHqdv3du3cbtlVUVIi5c+cKDw8PYW1tLR588EFx9uxZ4eLiIp5//vl7HrN75a/pMRBCiEOHDomgoCChUqnuuHX/woULYsKECcLDw0NYWVkJb29v8cgjj4gtW7bcM9uWLVvEww8/LNzc3IRKpRItWrQQzz33nLhx44bRuOzsbDF9+nTh7e0tVCqVaN68uZg4caKhjUXVMdu8eXO17/2v7QcSEhLEiBEjhIuLi1Cr1cLX11eMHj1aREVFGY3bu3ev4T23atVKrFy50vDf/M+Ki4vF5MmThYODg7C3txejR48WGRkZ921xUGX37t0iPDxcODg4CI1GI1q3bi2efvppERcXd8/jR9SYZELU80xNIiKJ5ObmwsnJCQsXLsRbb70ldRwiMnOcE0VEJunWrVt3bFu2bBkALghMRI2Dc6KIyCRt3LgRa9aswZAhQ2BnZ4cDBw7gu+++w8MPP4y+fftKHY+ILACLKCIySV27doVSqcSSJUuQn59vmGy+cOFCqaMRkYXgnCgiIiKiOuCcKCIiIqI6YBFFREREVAecE9WA9Ho9rl+/Dnt7+7/V/I+IiIgajxACBQUF8PLyglx+9/NNLKIa0PXr1+Hj4yN1DCIiIqqDK1euoHnz5nfdzyKqAdnb2wOo/I+g1WolTkNEREQ1kZ+fDx8fH8Pn+N2wiGpAVZfwtFotiygiIiITc7+pOJxYTkRERFQHLKKIiIiI6oBFFBEREVEdsIgiIiIiqgMWUURERER1wCKKiIiIqA5YRBERERHVAYsoIiIiojpgEUVERERUByyiiIiIiOpA8iJqxYoVaNmyJTQaDYKDgxEbG3vP8Zs3b0b79u2h0WjQpUsX/PLLL0b7hRCYN28ePD09YW1tjbCwMCQlJRmNycnJwfjx46HVauHo6IjJkyejsLDwjtf58MMP0bZtW6jVanh7e+O9996rnzdNREREJk/SImrjxo2YNWsW5s+fj6NHjyIgIADh4eHIyMiodvyhQ4cwbtw4TJ48GQkJCYiIiEBERAROnTplGLNkyRIsX74cK1euRExMDGxtbREeHo6SkhLDmPHjx+P06dPYuXMntm/fjn379mHq1KlG32vGjBn46quv8OGHHyIxMRHbtm1Dr169GuZAEBERkekREurVq5eYNm2a4WudTie8vLzEokWLqh0/evRoMXToUKNtwcHB4rnnnhNCCKHX64WHh4f44IMPDPtzc3OFWq0W3333nRBCiDNnzggA4siRI4YxO3bsEDKZTFy7ds0wRqlUisTExL/1/vLy8gQAkZeX97deh6Sh1+tFRn6JKCgpFzqdXuo4RETUSGr6+a2UqngrKytDfHw85syZY9gml8sRFhaG6Ojoap8THR2NWbNmGW0LDw/H1q1bAQApKSlIS0tDWFiYYb+DgwOCg4MRHR2NsWPHIjo6Go6OjujRo4dhTFhYGORyOWJiYjB8+HD89NNPaNWqFbZv345BgwZBCIGwsDAsWbIEzs7Od31PpaWlKC0tNXydn59fq2NCTcs/vz+BTXFXAQAyGWCnVsJerYS9xgr2GiXsNJV/d7FVoY2bHdp52KOtmz0cbKwkTk5ERI1BsiIqKysLOp0O7u7uRtvd3d2RmJhY7XPS0tKqHZ+WlmbYX7XtXmPc3NyM9iuVSjg7OxvGXLx4EZcvX8bmzZvx7bffQqfT4ZVXXsGoUaOwa9euu76nRYsW4Z133rnfWycTUFqhw0/Hbxi+FgIoKKlAQUkFkFdyj2cCHloN2nrYo527Hdq626Odhz383exhrVI0dGwiImpEkhVRTZler0dpaSm+/fZbtG3bFgDw9ddfIygoCOfOnUO7du2qfd6cOXOMzpTl5+fDx8enUTJT/Tp6ORe3ynVwtVNh3+sDUFhaYSiiCksqUFBSXvl1aQUy8ktwPr0A59MLcS33FtLyS5CWX4J95zMNr6eUyxDk64QH2rnhgXbN0N7DHjKZTMJ3SEREf5dkRZSrqysUCgXS09ONtqenp8PDw6Pa53h4eNxzfNWf6enp8PT0NBoTGBhoGPPXiesVFRXIyckxPN/T0xNKpdJQQAFAhw4dAACpqal3LaLUajXUavU93zeZhgPJlQVQ3zausFEpYaNSws3+/s/LLylHUnohzqcX4Fxawe3iqgBZhWWISclBTEoOFkcmwkOrQf+2zfBAu2bo6+8KrYaXAImITI1kRZRKpUJQUBCioqIQEREBoPIMUFRUFKZPn17tc0JCQhAVFYWZM2catu3cuRMhISEAAD8/P3h4eCAqKspQNOXn5yMmJgYvvPCC4TVyc3MRHx+PoKAgAMCuXbug1+sRHBwMAOjbty8qKipw4cIFtG7dGgBw/vx5AICvr2+9Hgdqmg4kZwMAQtu41up5Wo0VgnydEOTrZLQ9NbsYe85nYM+5TBy6kIW0/BJsjLuCjXFXoJTL0N3XCQ+2d8Pwbt5w12rq7X0QEVHDkQkhhFTffOPGjZg4cSK++OIL9OrVC8uWLcOmTZuQmJgId3d3TJgwAd7e3li0aBGAyhYH/fv3x/vvv4+hQ4diw4YN+Pe//42jR4+ic+fOAIDFixfj/fffx9q1a+Hn54e5c+fixIkTOHPmDDSayg+nwYMHIz09HStXrkR5eTkmTZqEHj16YP369QAqi7mePXvCzs4Oy5Ytg16vx7Rp06DVavHbb7/V+P3l5+fDwcEBeXl50Gq19Xz0qKHkFZej279+g14Ah+cMhIdD/RY1JeU6xKbkYPe5DOw9l4mLWUWGfQq5DA+2d8O4Xj7o39YNCjkv+RERNbaafn5LOidqzJgxyMzMxLx585CWlobAwEBERkYaJoanpqZCLv+jlVWfPn2wfv16vP3223jzzTfh7++PrVu3GgooAHj99ddRVFSEqVOnIjc3F6GhoYiMjDQUUACwbt06TJ8+HQMHDoRcLsfIkSOxfPlyw365XI6ffvoJL730Ev7xj3/A1tYWgwcPxtKlSxvhqJDUDl3Igl4Abdzs6r2AAgCNlQL/aNsM/2jbDHj0j7NU24/fQOylHOw8k46dZ9Lh5aDB6J4+GN3DB16O1vWeg4iI/h5Jz0SZO56JMk1v/XgS62JS8XSfllgwrFOjfu/kjAJ8F3sF3x+9itzicgCAXAY80M4N43q1wIB2zaBUSL7QABGRWTOJM1FETdGB5CwAtZ8PVR/auNlj7iMd8Vp4O/x6Og3fxabi8MUc7ErMwK7EDHhoNZj6j1Z4IrgFNFZsmUBEJCWeiWpAPBNleq7kFKPfkt1QymU4Nv9h2Kml/z3jYmYhNh65gi3xV5FdVAYAcNeqMW1AG4zp6QO1ksUUEVF9qunnN68LEP3J/qTKs1DdWjg2iQIKAFo1s8OcIR0QPWcg/j28C7wcNEjPL8W8/53GAx/swX8PX0ZZhV7qmEREFodFFNGfHLx9Ka+vBJfy7kellOOJ4BbY/doD+FdEZ3hoNbiRV4K3t57CgA/34LvYVJTrWEwRETUWFlFEt+n0AgcvVBZR/fybXhFVRa1U4Knevtjz2gNY8GhHuNmrcS33Fub8cBIPLt2DTXFXoNPzKj0RUUNjEUV02+nrecgtLoedWomA5o5Sx7kvjZUCT/f1w77XB2DuIx3haqfGlZxbeH3LCUSsOIgTV3OljkhEZNZYRBHdVnVXXu9WLibVRkBjpcDkUD/sf30A3hzSHvYaJU5ey8NjKw5i7tZTyLtVLnVEIiKzZDqfFEQN7EBS07+Udy/WKgWm/qM1omb3R0SgF4QA/nP4MgYu3YMfE66CN+ISEdUvFlFEAG6V6RB36SYAINREi6gqbvYaLBvbDeufDUarZrbIKizDKxuP44kvY5CcUSh1PCIis8EiighA7KUclOn08HLQoJWrrdRx6kWfNq7YMaMfXgtvB7VSjuiL2Rj88T588GsibpXppI5HRGTyWEQRwbi1gUxmPov+qpUKTBvQBr/P6o8H27uhXCewYvcFPPR/exF9IVvqeEREJo1FFBH+aLJp6pfy7sbH2QZfT+yBL54KgpeDBldv3sITXx3GkshE9pYiIqojFlFk8TILSnH2Rj6Aptlks77IZDKEd/LAzln9MaaHD4QAPttzAaM+P4SUrCKp4xERmRwWUWTxDt1usNnRUwtXO7XEaRqerVqJxaO64rPx3aHVKHH8ah6GLt+PTUeu8A4+IqJaYBFFFu+AmV/Ku5shXTwROfMfCPZzRnGZDq9/fwLT1ycgr5h9pYiIaoJFFFk0IYShyWaoGV/KuxsvR2usn9Ibr4W3g1Iuw88nb2Dwx/tw+CInnRMR3Q+LKLJoFzKLcCOvBCqlHL38nKWOIwmFXIZpA9rg+xf6oKWLDa7nlWDcl4fxwa+cdE5EdC8sosiiVbU26OHrBI2VQuI00grwccTPL/fD6B7NIQSwYvcFTPwmFjeLyqSORkTUJLGIIotm7q0NastWrcSSUQH49IlusFUpcOhCNoatOIDEtHypoxERNTksoshilev0hrk//do0kzhN0/JIVy/88GJftHC2wZWcWxjx2SHsOHlD6lhERE0KiyiyWCeu5qKwtAKONlbo5KWVOk6T087DHtum90VoG1cUl+nwwrqj+Oi3c9Dr2QaBiAhgEUUWrOpSXt/WrpDLzWepl/rkaKPCmkk9MTnUDwCwfFcypv4nHgUlbINARMQiiiyWpfaHqi2lQo65j3TEh48HQKWU4/ez6Rjx2SFcYpdzIrJwLKLIIhWUlCPhSi4Ay+wPVRejgppj49TecLNXIymjEMM+PYB95zOljkVEJBkWUWSRYi7mQKcXaOliAx9nG6njmIxuLZzw00uhCPRxRH5JBZ5eHYv/Hr4sdSwiIkmwiCKLVNWl3JwXHG4o7loNNkztjVFBzaEXwNtbT2HZ7+e57h4RWRwWUWSR9idVXobqx/lQdaKxUuCDUV0xY6A/AGDZ70l4a+sp6HjnHhFZEBZRZHFu5N3ChcwiyGVASCsWUXUlk8nwykNt8a+IzpDJgPUxqZi27ihKynVSRyMiahQsosjiVN2V16W5IxxsrCROY/qe6u2LFU90h0ohR+TpNEz4JhZ5t9gCgYjMH4sosjhV86H6cT5UvRnSxRNrn+kFe7USsSk5GPNFNNLzS6SORUTUoFhEkUXR64Vh0WH2h6pfIa1dsPG5EDSzVyMxrQAjPjuEi5mFUsciImowLKLIopxLL0BWYRmsrRTo3sJJ6jhmp6OXFj+80ActXWxwLfcWRq2MxvHb/biIiMwNiyiyKFXzoYJbOUOl5I9/Q/BxtsGWF/qga3MH5BSVYdyXhw1n/4iIzAk/Rcii7K+6lMf5UA3K1U6N9VN6o59/5eLFz6w5YihgiYjMBYsoshgl5TrEpmQDAPr5N5M4jfmzUyvx1cQeGNjeDaUVekxee8TQn4uIyBywiCKLcTT1JkrK9Whmr0Zbdzup41gEtVKBz57sjrAOVYVUHPZyvT0iMhMsoshiVF1OCm3jCplMJnEay6FWKvDZ+CA81NEdZRV6TPk2DnvOZUgdi4job2MRRRbjAOdDSUallGPFE93x8O1Cauq38djNQoqITByLKLIIucVlOHktDwAXHZaKSinHivHdMaiTB8p0ejz3bTx2J7KQIiLTxSKKLMKhC9kQAvB3s4OHg0bqOBbLSiHHJ090w+DOtwup/8Qj6my61LGIiOqERRRZhP1J7FLeVFgp5Fg+rhuGdKkspJ7/bzx+P8NCiohMD4sosggHkivvCON8qKbBSiHHx2O7YWhXT5TrBF5YF49diSykiMi0sIgis5eaXYwrObeglMsQ3MpF6jh0m5VCjo/HBOKRqkLqv0cRczFb6lhERDXGIorM3v7bZ6G6t3CCnVopcRr6M6VCjv8bE2joI/Xs2jicun0DABFRU8ciiszeAc6HatKsFHJ8+kR3BPs5o6C0AhO+iUVyRqHUsYiI7otFFJk1nV7g0IXKS0RsbdB0aawU+GpiD3Txrly0eMLXMbiWe0vqWERE98QiiszaqWt5yLtVDnuNEgHNHaSOQ/dgr7HCmkk90bqZLa7nleCpr2KQVVgqdSwiortiEUVmrapLeUgrFygV/HFv6lzs1PjP5GB4O1rjYlYRJn4Ti/yScqljERFVi58qZNb2J91ubcD5UCbDy9Ea/5ncC652Kpy+no9n18ThVplO6lhERHdgEUVmq7isAkcv5wJgfyhT06qZHdY+0wv2aiViL+XgxXXxKNfppY5FRGSERRSZrdiUHJTp9PB2tIafq63UcaiWOnk54JtJPaGxkmP3uUzM3nQcOr2QOhYRkUGTKKJWrFiBli1bQqPRIDg4GLGxsfccv3nzZrRv3x4ajQZdunTBL7/8YrRfCIF58+bB09MT1tbWCAsLQ1JSktGYnJwcjB8/HlqtFo6Ojpg8eTIKC/+4rfrSpUuQyWR3PA4fPlx/b5waVFVrg75tXCCTySROQ3XRs6UzPh8fBKVchm3Hr2Phz2ekjkREZCB5EbVx40bMmjUL8+fPx9GjRxEQEIDw8HBkZFS/uvuhQ4cwbtw4TJ48GQkJCYiIiEBERAROnTplGLNkyRIsX74cK1euRExMDGxtbREeHo6SkhLDmPHjx+P06dPYuXMntm/fjn379mHq1Kl3fL/ff/8dN27cMDyCgoLq/yBQg6iaVB7q30ziJPR3DGjvhqWjAwAAqw9ewjcHUiRORER0m5BYr169xLRp0wxf63Q64eXlJRYtWlTt+NGjR4uhQ4cabQsODhbPPfecEEIIvV4vPDw8xAcffGDYn5ubK9Rqtfjuu++EEEKcOXNGABBHjhwxjNmxY4eQyWTi2rVrQgghUlJSBACRkJBQ5/eWl5cnAIi8vLw6vwbVTXr+LeH7z+3C95/bRVZBidRxqB58tjtZ+P5zu2j5xnax4+QNqeMQkRmr6ee3pGeiysrKEB8fj7CwMMM2uVyOsLAwREdHV/uc6Ohoo/EAEB4ebhifkpKCtLQ0ozEODg4IDg42jImOjoajoyN69OhhGBMWFga5XI6YmBij1x42bBjc3NwQGhqKbdu23fP9lJaWIj8/3+hB0jiUXNlgs6OnFi52aonTUH14vn8rPBHcAkIAMzYkICH1ptSRiMjCSVpEZWVlQafTwd3d3Wi7u7s70tLSqn1OWlraPcdX/Xm/MW5ubkb7lUolnJ2dDWPs7OywdOlSbN68GT///DNCQ0MRERFxz0Jq0aJFcHBwMDx8fHzudwiogey/PR+qH1sbmA2ZTIZ3h3XCgHbNDOvsXc4ukjoWEVkwyedENVWurq6YNWsWgoOD0bNnT7z//vt48skn8cEHH9z1OXPmzEFeXp7hceXKlUZMTFWEEDiQzP5Q5kh5e529Tl5aZBeVYdLqI7hZVCZ1LCKyUJIWUa6urlAoFEhPTzfanp6eDg8Pj2qf4+Hhcc/xVX/eb8xfJ65XVFQgJyfnrt8XAIKDg5GcnHzX/Wq1Glqt1uhBje9CZiHS80uhUsrRs6Wz1HGontmqlfjm6Z7wctDgYlYRpv4nDiXlbMZJRI1P0iJKpVIhKCgIUVFRhm16vR5RUVEICQmp9jkhISFG4wFg586dhvF+fn7w8PAwGpOfn4+YmBjDmJCQEOTm5iI+Pt4wZteuXdDr9QgODr5r3mPHjsHT07P2b5QaVdWlvJ4tnaCxUkichhqCu1aD1ZMqm3EeuXQTr205AT17SBFRI1NKHWDWrFmYOHEievTogV69emHZsmUoKirCpEmTAAATJkyAt7c3Fi1aBACYMWMG+vfvj6VLl2Lo0KHYsGED4uLisGrVKgCV8yZmzpyJhQsXwt/fH35+fpg7dy68vLwQEREBAOjQoQMGDRqEKVOmYOXKlSgvL8f06dMxduxYeHl5AQDWrl0LlUqFbt26AQB++OEHfPPNN/jqq68a+QhRbVX1hwptw9YG5qydhz1WPhWEid/E4qfj19HcyRr/HNRe6lhEZEEkL6LGjBmDzMxMzJs3D2lpaQgMDERkZKRhYnhqairk8j9OmPXp0wfr16/H22+/jTfffBP+/v7YunUrOnfubBjz+uuvo6ioCFOnTkVubi5CQ0MRGRkJjUZjGLNu3TpMnz4dAwcOhFwux8iRI7F8+XKjbP/6179w+fJlKJVKtG/fHhs3bsSoUaMa+IjQ31Gu0+Pwxco787jUi/nr28YV74/silc3H8fney7Ax8kGTwS3kDoWEVkImRCC58AbSH5+PhwcHJCXl8f5UY3kyKUcPL4yGk42Voh/+yHI5exUbgmW/X4ey35PgkIuw9cTe+CBdm73fxIR0V3U9PObd+eRWamaD9WnjSsLKAsyY6A/RnZvDp1e4KX1CUjOKLz/k4iI/iYWUWRWDt5e6qUfL+VZFJlMhkUjuqBnSycUlFZgyrdxyCsulzoWEZk5FlFkNvJLynHsSi4A9oeyRCqlHJ8/GQRvR2ukZBVh+ndHUaHTSx2LiMwYiygyG4cvZEOnF2jpYoPmTjZSxyEJuNqpsWpCEKytFNiflIVFOxKljkREZoxFFJmNA7cv5fEslGXr5OWAj0YHAAC+PpCCTUe4cgARNQwWUWQ2DEUU+0NZvMFdPDEzzB8A8NbWk4i7lCNxIiIyRyyiyCxcz72Fi5lFkMuAkNYuUsehJuDlB/0xuLMHynUCz/83Htdyb0kdiYjMDIsoMgtVXcq7NneEg7WVxGmoKZDLZVg6OgAdPLXIKizDlLVxKC6rkDoWEZkRFlFkFqou5fXjfCj6ExuVEl9OCIKLrQpnbuTjtc0nwP7CRFRfWESRydPrhaE/FJd6ob9q7mSDlU8FwUohw88nb+CTXclSRyIiM8Eiikze2bR8ZBeVwUalQLcWTlLHoSaoZ0tnLIyoXF/zo53n8evpNIkTEZE5YBFFJq/qLFSwnzNUSv5IU/XG9GyBSX1bAgBmbzqOC5lcGoaI/h5+4pDJq1ovL9SfrQ3o3t4c0gG9/JxRWFqB5/8Tj6JSTjQnorpjEUUmraRch9iUyh5AnA9F92OlkGPFE93hrlUjKaMQr3/PieZEVHcsosikHb18E6UVerjZq9HW3U7qOGQCmtmr8dn47pUTzU/cwNcHUqSOREQmikUUmbT9f7orTyaTSZyGTEWQrzPmPtIRALBoRyKiL2RLnIiITBGLKDJpB5K4Xh7VzVO9fTG8mzd0eoGXvjuKtLwSqSMRkYlhEUUm62ZRGU5dzwMA9OV8KKolmUyGfw/vgvYe9sgqLMML6+JRVqGXOhYRmRAWUWSyDl3IhhBAW3c7uGs1UschE2StUuCLp4Kg1SiRkJqLhT+fkToSEZkQFlFksg4kZwIAQtuwtQHVna+LLZaNDQQAfBt9GT8cvSptICIyGSyiyCQJIf7UH8pF4jRk6h5s744ZA/0BAHN+OInTty8TExHdC4soMkmpOcW4evMWrBQyBPuxiKK/b8ZAfzzQrhlKK/R4/r/xyCsulzoSETVxLKLIJFWdherWwgm2aqXEacgcyOUyLBsTCB9na1zJuYVZm46xEScR3ROLKDJJVa0N+vGuPKpHjjYqfD4+CCqlHFGJGfhy/0WpIxFRE8YiikyOTi9w6EJlEdWX/aGonnX2dsC82404F0eeQ/zlHIkTEVFTxSKKTM7Ja3nIL6mAvUaJrt4OUschMzQ+uAUeDfCqbMS5PgE3i8qkjkRETRCLKDI5B5IqWxv0ae0CpYI/wlT/ZDIZFo3oAj9XW1zPK8GsTceg13N+FBEZ4ycQmZwDf1ovj6ih2KmVWPFEd6iVcuw+l4lVnB9FRH/BIopMSnFZBeIv3wQAhPqzySY1rI5eWiwY1gkA8MGv53DkEudHEdEfWESRSYlJyUG5TsDb0RotXWykjkMWYGxPH0QE/jE/KruwVOpIRNREsIgik3KwqrWBvytkMpnEacgSyGQyvDe8C1o1s0VafglmbTrO+VFEBIBFFJmYqvlQfTkfihqRrVqJz8Z3h8ZKjr3nM/H53gtSRyKiJoBFFJmMjIISJKYVQCZjEUWNr72HFu8O6wwAWPrbOcRczJY4ERFJjUUUmYyDt89CdfLSwtlWJXEaskSP92iOEd29oRfAyxsSkMX5UUQWjUUUmYwDSZW/+fMsFElFJpNhYURntHGzQ3p+KWZzfhSRRWMRRSZBCIEDyZVNNvu1YWsDko6NqnJ+lFpZOT/qm4MpUkciIomwiCKTkJxRiPT8UqiVcvRo6SR1HLJwbd3tMe/RqvX1EnHyap7EiYhICiyiyCRU3ZXXy88ZGiuFxGmIgCd6tcDgzh4o1wm89N1RFJZWSB2JiBoZiygyCQeS2NqAmhaZTIb3R3SFt6M1LmUXY97WU1JHIqJGxiKKmrxynR6Hb99OzvXyqClxsLHCx2MDIZcBPyRcww9Hr0odiYgaEYsoavISUnNRVKaDs60KHT21UschMtKjpTNmhrUFAMzdegopWUUSJyKixsIiipq8qvlQfVq7QC7nUi/U9Ewb0AbBfs4oKtPh5e8SUFahlzoSETUCFlHU5B1Iut3awJ+X8qhpUshlWDY2EI42Vjh5LQ8f/JoodSQiagQsoqhJyy8px/Hbt4+H+rM/FDVdng7W+GBUAADgy/0p2H0uQ+JERNTQWERRkxZ9IRs6vUArV1t4O1pLHYfonh7q6I6JIb4AgFc3HUdGfonEiYioIbGIoiatar08tjYgUzFnSAe097BHdlEZZnFZGCKzxiKKmrSq/lChnA9FJkJjpcCnT3SDxkqOA8lZWLX/otSRiKiBsIiiJuta7i1czCqCQi5DSGsXqeMQ1VgbN3sseLQTAGDpb+e4LAyRmWIRRU3WwdtnoQKaO0CrsZI4DVHtjOnpg0GdKpeFmbExAcVlXBaGyNywiKIma//t+VDsUk6mSCaTYdGILnDXqnExswgLfz4rdSQiqmcsoqhJ0uuFYVI5WxuQqXKyVeGj0YEAgPUxqfjtdJq0gYioXjWJImrFihVo2bIlNBoNgoODERsbe8/xmzdvRvv27aHRaNClSxf88ssvRvuFEJg3bx48PT1hbW2NsLAwJCUlGY3JycnB+PHjodVq4ejoiMmTJ6OwsLDa75ecnAx7e3s4Ojr+rfdJNXfmRj5yispgq1KgWwtHqeMQ1VnfNq6Y0s8PAPDGDyfZ9oDIjEheRG3cuBGzZs3C/PnzcfToUQQEBCA8PBwZGdU3qjt06BDGjRuHyZMnIyEhAREREYiIiMCpU3+soL5kyRIsX74cK1euRExMDGxtbREeHo6Skj/+8Ro/fjxOnz6NnTt3Yvv27di3bx+mTp16x/crLy/HuHHj0K9fv/p/83RXVWehglu5wEoh+Y8p0d/yang7dPDUIqeoDLM3s+0BkdkQEuvVq5eYNm2a4WudTie8vLzEokWLqh0/evRoMXToUKNtwcHB4rnnnhNCCKHX64WHh4f44IMPDPtzc3OFWq0W3333nRBCiDNnzggA4siRI4YxO3bsEDKZTFy7ds3otV9//XXx5JNPitWrVwsHB4davbe8vDwBQOTl5dXqeSTEk18dFr7/3C6+3n9R6ihE9eJ8Wr5o+9Yv/LkmMgE1/fyu06/4FRUV+P333/HFF1+goKAAAHD9+vW7Xg67m7KyMsTHxyMsLMywTS6XIywsDNHR0dU+Jzo62mg8AISHhxvGp6SkIC0tzWiMg4MDgoODDWOio6Ph6OiIHj16GMaEhYVBLpcjJibGsG3Xrl3YvHkzVqxYUaP3U1paivz8fKMH1V5JuQ6xKTkAuF4emQ9/d3u8PbQDAOD9HYk4e4P/PhCZuloXUZcvX0aXLl3w2GOPYdq0acjMrFwcdvHixXj11Vdr9VpZWVnQ6XRwd3c32u7u7o60tOonYKalpd1zfNWf9xvj5uZmtF+pVMLZ2dkwJjs7G08//TTWrFkDrVZbo/ezaNEiODg4GB4+Pj41eh4Zi798E6UVerhr1WjjZid1HKJ682RvXzzY3g1lOj1mbjiGknKd1JGI6G+odRE1Y8YM9OjRAzdv3oS19R9rmQ0fPhxRUVH1Gk5KU6ZMwRNPPIF//OMfNX7OnDlzkJeXZ3hcuXKlAROar/1Jfyz1IpPJJE5DVH9kMhmWjOoKVzsVzqUX4P0diVJHIqK/odZF1P79+/H2229DpVIZbW/ZsiWuXbtWq9dydXWFQqFAenq60fb09HR4eHhU+xwPD497jq/6835j/jpxvaKiAjk5OYYxu3btwocffgilUgmlUonJkycjLy8PSqUS33zzTbXZ1Go1tFqt0YNq70By5dlNXsojc+Rqp8YHowIAAGsOXcKec9XfRENETV+tiyi9Xg+d7s5T0FevXoW9vX2tXkulUiEoKMjoDJZer0dUVBRCQkKqfU5ISMgdZ7x27txpGO/n5wcPDw+jMfn5+YiJiTGMCQkJQW5uLuLj4w1jdu3aBb1ej+DgYACV86aOHTtmeLz77ruwt7fHsWPHMHz48Fq9T6q5nKIynL5eOVeEiw6TuRrQ3g0TQ3wBAK9uPoHswlKJExFRXdS6iHr44YexbNkyw9cymQyFhYWYP38+hgwZUusAs2bNwpdffom1a9fi7NmzeOGFF1BUVIRJkyYBACZMmIA5c+YYxs+YMQORkZFYunQpEhMTsWDBAsTFxWH69OmGPDNnzsTChQuxbds2nDx5EhMmTICXlxciIiIAAB06dMCgQYMwZcoUxMbG4uDBg5g+fTrGjh0LLy8vw5jOnTsbHt7e3pDL5ejcuTOcnJxq/T6pZg5dyIIQQDt3e7jZa6SOQ9Rg5gzpAH83O2QVluKf35+EEGx7QGRqlLV9wtKlSxEeHo6OHTuipKQETzzxBJKSkuDq6orvvvuu1gHGjBmDzMxMzJs3D2lpaQgMDERkZKRhYnhqairk8j9qvT59+mD9+vV4++238eabb8Lf3x9bt25F586dDWNef/11FBUVYerUqcjNzUVoaCgiIyOh0fzxobxu3TpMnz4dAwcOhFwux8iRI7F8+fJa56f6dSCpqks5z0KRedNYKfDx2G6IWHEQv59Nx8YjVzC2VwupYxFRLchEHX79qaiowMaNG3H8+HEUFhaie/fuGD9+vNFEc6q8jOjg4IC8vDzOj6oBIQRCF+/GtdxbWD2pJwa0c7v/k4hM3Bd7L2DRjkTYqBTYMaMffF1spY5EZPFq+vld6yJq37596NOnD5RK45NYFRUVOHToUK3uZjN3LKJqJyWrCAM+3AMrhQzH5z8MG1WtT5QSmRydXmDcl4cRm5KD7i0csem5ECjZpZ9IUjX9/K71/6kDBgxATk7OHdvz8vIwYMCA2r4ckcGB20u9dG/hxAKKLIZCLsNHowNgp1biaGouVu69IHUkIqqhWhdRQohqe/dkZ2fD1panoanuDiSxtQFZpuZONnhnWCcAwLLfk3Dyap7EiYioJmr86/6IESMAVN799vTTT0OtVhv26XQ6nDhxAn369Kn/hGQRKnR6HLqQDYCtDcgyjejujajEdPxyMg0zNyZg+0v9YK1SSB2LiO6hxkWUg4MDgMozUfb29kaTyFUqFXr37o0pU6bUf0KyCCev5aGgpAJajRJdmztKHYeo0clkMrwX0QVxl27iQmYRFkcmYsHts1NE1DTVuIhavXo1gMrO5K+++iov3VG9qmpt0Ke1KxRyLvVClsnJVoUPHg/AxG9isebQJTzY3g3/aNtM6lhEdBe1nhM1f/58FlBU7/Ynsz8UEQD0b9sME253M39ty3HkFpdJnIiI7qZOt0Bt2bIFmzZtQmpqKsrKjP8HP3r0aL0EI8tRVFqBhNSbAIBQzociwpzBHXAgOQsXM4vw1o+n8OkT3bgYN1ETVOszUcuXL8ekSZPg7u6OhIQE9OrVCy4uLrh48SIGDx7cEBnJzMWm5KBcJ9DcyRq+LjZSxyGSnLVKgWVjAqGUy/DzyRvYeqx2i7sTUeOodRH12WefYdWqVfjkk0+gUqnw+uuvY+fOnXj55ZeRl8fbcqn29t+eD9XP35W/bRPd1rW5I2YM9AcAzNt6Gtdyb0mciIj+qtZFVGpqqqGVgbW1NQoKCgAATz31VJ3WziM6kFzZHyq0DSfQEv3ZCw+0RrcWjigorcDsTceg13ORYqKmpNZFlIeHh6FjeYsWLXD48GEAQEpKClchp1rLyC/B+fRCyGRAn9YuUschalKUCjn+b3QgrK0UOHwxB6sPXZI6EhH9Sa2LqAcffBDbtm0DAEyaNAmvvPIKHnroIYwZMwbDhw+v94Bk3qqWeuns5QAnW5XEaYianpautnhraAcAwOLIRCSlF0iciIiq1PruvFWrVkGv1wMApk2bBhcXFxw6dAjDhg3Dc889V+8BybxV9YdiawOiuxsf3AI7z6Rj7/lMzNp0HD+82AdWXKSYSHK1+r+woqICCxcuRFpammHb2LFjsXz5crz00ktQqXgmgWpOCGE4E8XWBkR3J5PJsGRUVzhYW+HktTx8sitZ6khEhFoWUUqlEkuWLEFFRUVD5SELkpRRiIyCUqiVcgT5Okkdh6hJc9dqsDCiMwBgxe5kHLuSK20gIqr9nKiBAwdi7969DZGFLExVa4Nefs7QWHGhVaL7eTTAC48GeEGnF5i16RhulemkjkRk0Wo9J2rw4MF44403cPLkSQQFBd2xBMywYcPqLRyZtwNJVa0NeCmPqKb+9VgnxKZk4yIXKSaSnEzUsi+BXH73k1cymQw6HX8zqpKfnw8HBwfk5eVBq9VKHadJKavQI/Dd31BcpsPPL4eik5eD1JGITMbe85mY+E0sAOC/k4N5YwZRPavp53etL+fp9fq7PlhAUU0lpN5EcZkOLrYqdPBggUlUG/3bNsOTvVsAqFykOO9WucSJiCwT75ElSVTdlde3jSvkci71QlRbbw7pAF8XG9zIK8E7205LHYfIIrGIIklUTSrnfCiiurFRKfHR6EDIZcAPCdew4+QNqSMRWRwWUdTo8m6V48TVXABsskn0dwT5OuGFB1oDAN788SQyCkokTkRkWVhEUaOLvpANvQBaNbOFl6O11HGITNqMgW3RwVOLm8XlmPP9Sa5hStSIWERRozuQzNYGRPVFpZTj/8YEQKWQIyoxA5vjrkodichi1LqIys/Pr/ZRUFCAsrKyhshIZuYA50MR1av2HlrMergtAODd7WdwJadY4kRElqHWRZSjoyOcnJzueDg6OsLa2hq+vr6YP3++YZFioj+7klOMS9nFUMhl6N3aReo4RGZjSr9W6OHrhMLSCry25Tj0el7WI2potS6i1qxZAy8vL7z55pvYunUrtm7dijfffBPe3t74/PPPMXXqVCxfvhzvv/9+Q+QlE3fwdmuDgOYO0GqsJE5DZD4Uchk+fDwA1lYKHL6YgzWHLkkdicjs1XrZl7Vr12Lp0qUYPXq0Ydujjz6KLl264IsvvkBUVBRatGiB9957D2+++Wa9hiXTt/92ERXq30ziJETmp6WrLd4c2gFzt57C4shE/KNtM7Rxs5M6FpHZqvWZqEOHDqFbt253bO/WrRuio6MBAKGhoUhNTf376cis6PUCh24XUf3Y2oCoQTwZ3AL9/F1RWqHH7M3HUaHj1AqihlLrIsrHxwdff/31Hdu//vpr+Pj4AACys7Ph5OT099ORWTlzIx83i8thq1Ig0MdR6jhEZkkmk2HJqK6w1yhx/EouPt9zQepIRGar1pfzPvzwQzz++OPYsWMHevbsCQCIi4tDYmIitmzZAgA4cuQIxowZU79JyeRVdSnv3coFVgp21yBqKJ4O1nj3sU54ZeNxfByVhAHt3dDZm4t8E9W3Wn+SDRs2DImJiRg8eDBycnKQk5ODwYMHIzExEY888ggA4IUXXsBHH31U72HJtBn6Q/FSHlGDiwj0xqBOHqjQC8zedBylFVwgnqi+1fpMFAD4+fnx7juqlZJyHY5cugmA86GIGoNMJsN7wzsj7nIOzqUX4KOd5zFncAepYxGZlToVUbm5uYiNjUVGRsYd/aAmTJhQL8HIvBy5lIOyCj3ctWq0bsa7hYgag4udGotGdMWUb+Owat9FhHVwR8+WzlLHIjIbtS6ifvrpJ4wfPx6FhYXQarWQyWSGfTKZjEUUVeuPLuXNjH5miKhhPdTRHaOCmmNL/FXM3nQcO2b0g626Tr8/E9Ff1HpO1OzZs/HMM8+gsLAQubm5uHnzpuGRk5PTEBnJDBxgawMiycx7tCO8Ha2RmlOMf/9yVuo4RGaj1kXUtWvX8PLLL8PGxqYh8pAZyi4sxenr+QCAPm241AtRY9NqrPDBqK4AgHUxqdh7PlPiRETmodZFVHh4OOLi4hoiC5mpgxeyAQDtPezhZq+ROA2RZerTxhVP92kJAHh9y3HkFZdLG4jIDNT6wvjQoUPx2muv4cyZM+jSpQusrIzXPxs2bFi9hSPzcNAwH4qX8oik9M9B7bHvfCYuZhVh/rZTWDb2ztUniKjmZEKIWi31LZff/eSVTCaDTsdeJFXy8/Ph4OCAvLw8aLVaqeNIQgiB0MW7cS33FlZP6okB7dykjkRk0RJSb2Lk54egF8Bn47tjSBdPqSMRNTk1/fyu9eU8vV5/1wcLKPqrlKwiXMu9BZVCjmA/3lpNJLVuLZzw4gNtAABv/XgSGQUlEiciMl1ce4MaVNVded19HWGj4m3VRE3BywP90dFTi5vF5Xjzh5Oo5QUJIrqtRp9qy5cvx9SpU6HRaLB8+fJ7jn355ZfrJRiZh6r+UP38m0mchIiqqJRyfDQmAMM+OYjfz2Zgc/xVjO7hI3UsIpNTozlRfn5+iIuLg4uLC/z8/O7+YjIZLl68WK8BTZmlz4mq0OnR7d2dKCitwNZpfRHo4yh1JCL6k8/3XMDiyETYqZWInNkPzZ3YuoYIqPnnd43ORKWkpFT7d6J7OX41DwWlFXCwtkIXriBP1ORM/Ucr/H42HfGXb+K1zSew7tlgyOVcUYCopjgnihrMwdvzofq0doGC/zATNTkKuQxLHw+AtZUC0RezsebQJakjEZmUWs/01el0WLNmDaKioqpdgHjXrl31Fo5Mm2G9PC71QtRktXS1xZtDO2Du1lNYHJmIf7RthjZuXCScqCZqXUTNmDEDa9aswdChQ9G5c2cuJkvVKiytwNHUmwDYZJOoqXsyuAV+O52G/UlZmL35OL5/PgRKBS9UEN1PrYuoDRs2YNOmTRgyZEhD5CEzEZuSjQq9gI+zNXxdbKWOQ0T3IJPJsGRUVzz8f/tw/EouPttzAS8P9Jc6FlGTV+tfNVQqFdq0adMQWciM7Dcs9cLWBkSmwNPBGv96rDMAYHlUEk5dy5M4EVHTV+siavbs2fj444/rtTnbihUr0LJlS2g0GgQHByM2Nvae4zdv3oz27dtDo9GgS5cu+OWXX4z2CyEwb948eHp6wtraGmFhYUhKSjIak5OTg/Hjx0Or1cLR0RGTJ09GYWGhYf+5c+cwYMAAuLu7Q6PRoFWrVnj77bdRXs5FO2viANfLIzI5jwV6YUgXD1ToBV7ZeAwl5VyFguheal1EHThwAOvWrUPr1q3x6KOPYsSIEUaP2tq4cSNmzZqF+fPn4+jRowgICEB4eDgyMjKqHX/o0CGMGzcOkydPRkJCAiIiIhAREYFTp04ZxixZsgTLly/HypUrERMTA1tbW4SHh6Ok5I/lDcaPH4/Tp09j586d2L59O/bt24epU6ca9ltZWWHChAn47bffcO7cOSxbtgxffvkl5s+fX+v3aGnS80uQlFEImazyzjwiMg0ymQwLI7rA1U6NpIxCfPjrOakjETVptV6AeNKkSffcv3r16loFCA4ORs+ePfHpp58CqFybz8fHBy+99BLeeOONO8aPGTMGRUVF2L59u2Fb7969ERgYiJUrV0IIAS8vL8yePRuvvvoqACAvLw/u7u5Ys2YNxo4di7Nnz6Jjx444cuQIevToAQCIjIzEkCFDcPXqVXh5eVWbddasWThy5Aj2799fo/dmqc02v4+/itmbj6Nrcwdsmx4qdRwiqqWos+mYvDYOMhmw/tneCOEvQ2Rh6rXZZpWKigoMGDAADz/8MDw8PP52yLKyMsTHx2POnDmGbXK5HGFhYYiOjq72OdHR0Zg1a5bRtvDwcGzduhVAZTPQtLQ0hIWFGfY7ODggODgY0dHRGDt2LKKjo+Ho6GgooAAgLCwMcrkcMTExGD58+B3fNzk5GZGRkXU622ZpqtbL46U8ItM0sIM7xvb0wYYjV/Dq5uOInNkP9horqWMRNTm1upynVCrx/PPPo7S0tF6+eVZWFnQ6Hdzd3Y22u7u7Iy0trdrnpKWl3XN81Z/3G+Pm5ma0X6lUwtnZ+Y7v26dPH2g0Gvj7+6Nfv35499137/p+SktLkZ+fb/SwNEIIFlFEZuDtRzrCx9ka13Jv4V/bz0gdh6hJqvWcqF69eiEhIaEhsjRJGzduxNGjR7F+/Xr8/PPP+PDDD+86dtGiRXBwcDA8fHwsb0HP8+mFyCwohcZKjqCWTlLHIaI6slMrsfTxQMhkwKa4q/jtdPW/2BJZslr3iXrxxRcxe/ZsXL16FUFBQbC1Ne4B1LVr1xq/lqurKxQKBdLT0422p6en3/VyoYeHxz3HV/2Znp4OT09PozGBgYGGMX+duF5RUYGcnJw7vm9VIdSxY0fodDpMnToVs2fPhkKhuCPbnDlzjC415ufnW1whtT8pEwDQy88FauWdx4iITEcvP2dM7dcKX+y7iDk/nER3Xye42qmljkXUZNT6TNTYsWORkpKCl19+GX379kVgYCC6detm+LM2VCoVgoKCEBUVZdim1+sRFRWFkJCQap8TEhJiNB4Adu7caRjv5+cHDw8PozH5+fmIiYkxjAkJCUFubi7i4+MNY3bt2gW9Xo/g4OC75tXr9SgvL79jqZsqarUaWq3W6GFpDhou5XEiKpE5eOWhtmjnbo/sojK89ePJem1vQ2Tqan0mKiUlpV4DzJo1CxMnTkSPHj3Qq1cvLFu2DEVFRYa7ACdMmABvb28sWrQIQOWyM/3798fSpUsxdOhQbNiwAXFxcVi1ahWAylt0Z86ciYULF8Lf3x9+fn6YO3cuvLy8EBERAQDo0KEDBg0ahClTpmDlypUoLy/H9OnTMXbsWMOdeevWrYOVlRW6dOkCtVqNuLg4zJkzB2PGjIGVFSdYVqesQo+YlBwAbLJJZC40Vgp8NCYAESsO4tfT6fj+6DWMCmoudSyiJqHWRZSvr2+9BhgzZgwyMzMxb948pKWlITAwEJGRkYaJ4ampqZDL/zhh1qdPH6xfvx5vv/023nzzTfj7+2Pr1q3o3LmzYczrr7+OoqIiTJ06Fbm5uQgNDUVkZCQ0Go1hzLp16zB9+nQMHDgQcrkcI0eOxPLlyw37lUolFi9ejPPnz0MIAV9fX0yfPh2vvPJKvb5/c3I09SaKy3RwtVOhvYe91HGIqJ508nLAzLC2+ODXc3hn22n0buWM5k42Usciklyt+0RVOXPmDFJTU1FWVma0fdiwYfUSzBxYWp+oD389h093J+OxQC98PLZ2l3aJqGmr0Okx+otoHE3NRUgrF6x7NhhyORegJ/PUIH2iAODixYsYPnw4Tp48CZlMZrg+LpNV/s+k03GZAEtV1dqgL1sbEJkdpUKOpaMDMeTj/Yi+mI3Vhy5hcqif1LGIJFXrieUzZsyAn58fMjIyYGNjg9OnT2Pfvn3o0aMH9uzZ0wARyRTkFZfjxNVcAEA/fxZRRObIz9UWbw7tAABYHJmI8+kFEiciklati6jo6Gi8++67cHV1hVwuh1wuR2hoKBYtWoSXX365ITKSCYi+mAW9AFo3s4Wng7XUcYiogTwZ3AL92zZDWYUeMzccQ1lF9XcrE1mCWhdROp0O9vaVk4ZdXV1x/fp1AJUTzs+d42KVlopdyoksg0wmwwejusLJxgpnbuRj2e/npY5EJJlaF1GdO3fG8ePHAVQuHrxkyRIcPHgQ7777Llq1alXvAck0HEi6XUT5s7UBkblz02qwaEQXAMDKvRdw5FKOxImIpFHrIurtt982NJt89913kZKSgn79+uGXX34xahFAluNKTjEuZRdDIZehdytnqeMQUSMY1NkTI7s3h14Ar2w8hoKScqkjETW6Wt+dFx4ebvh7mzZtkJiYiJycHDg5ORnu0CPLUtWlvJuPI1d6J7IgC4Z1xOGL2bh6s3KR4iWjAqSORNSoan0mqkpycjJ+/fVX3Lp1C87OPPtgyfaztQGRRbLXWOGj0QGGRYp/5SLFZGFqXURlZ2dj4MCBaNu2LYYMGYIbN24AACZPnozZs2fXe0Bq2vR6gUO3iyi2NiCyPMGtXDD1H5XzYef8cBIZBSUSJyJqPLUuol555RVYWVkhNTUVNjZ/tP0fM2YMIiMj6zUcNX2nr+fjZnE57NRKBPg4Sh2HiCQw66G26OCpRU5RGf655QQXKSaLUesi6rfffsPixYvRvLnxApT+/v64fPlyvQUj01DV2qB3K2dYKep8dZiITJhaqcCyMYFQKeXYfS4T62NTpY5E1Chq/alXVFRkdAaqSk5ODtRqdb2EItNxIDkTAPtDEVm6dh72eD28HQBg4fazSMkqkjgRUcOrdRHVr18/fPvtt4avZTIZ9Ho9lixZggEDBtRrOGraSsp1OHLpJgD2hyIi4Jm+fujT2gW3ynV4ZeMxVOjYzZzMW61bHCxZsgQDBw5EXFwcysrK8Prrr+P06dPIycnBwYMHGyIjNVFHLuWgrEIPTwcNWjezlToOEUlMLpfhw8cDEL5sH45dycWnu5MxM6yt1LGIGkydOpafP38eoaGheOyxx1BUVIQRI0YgISEBrVu3boiM1ERVdSnv28aVPcKICADg5WiNhRGdAQCf7ErG0dSbEiciaji1PhMFAA4ODnjrrbeMtl29ehVTp07FqlWr6iUYNX37k9jagIju9FigN3YlZuB/x65j5oZj+GVGP9ip6/RxQ9Sk1dvtVNnZ2fj666/r6+WoicsqLMWZG/kA2GSTiO707mOd4e1ojdScYizYdlrqOEQNgvekU50cupANAOjgqYWrHe/KJCJjDtZW+L8xgZDLgC3xV7H9xHWpIxHVOxZRVCcHkqpaG7hInISImqpefs6YNqANAODNH07ieu4tiRMR1S8WUVRrQgjDpHK2NiCie3l5oD8CfByRX1KBVzYeg07PbuZkPmo802/EiBH33J+bm/t3s5CJSMkqwvW8EqgUcvRqycWniejurBRyfDwmEEOW70dMSg5W7buIFx7gndxkHmp8JsrBweGeD19fX0yYMKEhs1ITUbXUS5CvE6xVConTEFFT19LVFguGdQIAfLTzHE5ezZM4EVH9qPGZqNWrVzdkDjIh+w2X8nhXHhHVzONBzbHnXAZ+OZmGGRsTsP2lUNio2PaATBvnRFGtVOj0OHz7zjz2hyKimpLJZPj38C7w0GpwMbMIC38+K3Ukor+NRRTVyvGreSgorYCjjRU6eTlIHYeITIijjQofjQmATAasj0nFb6fTpI5E9LewiKJaqborr09rFyjkXOqFiGqnT2tXTP1HKwDAP78/gYz8EokTEdUdiyiqlQPJVf2h2NqAiOpm9kPt0MlLi5vF5Zi9+Tj0bHtAJopFFNVYYWkFElJzAXA+FBHVnUopx8dju8HaSoH9SVlYtf+i1JGI6oRFFNVYzMVsVOgFfF1s4ONsI3UcIjJhbdzssGBYRwDAh7+ew7ErudIGIqoDFlFUY1WtDbjgMBHVh9E9fDC0qycq9AIvf5eAgpJyqSMR1QqLKKqxqiab/VhEEVE9qGp70NzJGqk5xXjrx1MQgvOjyHSwiKIaScsrQXJGIWSyyrtriIjqg4O1FT4e2w0KuQzbjl/HlvirUkciqjEWUVQjVWehuno7wMHGSuI0RGROgnydMOuhtgCAef87jQuZhRInIqoZFlFUIweSbrc24F15RNQAnu/fGn1au+BWuQ4vrU9AaYVO6khE98Uiiu5LCIEDyZVLvbA/FBE1BIVchv8bEwhnWxXO3MjH+zsSpY5EdF8soui+zqUXIKuwFNZWCnT3dZQ6DhGZKXetBh8+3hUAsPrgJUSdTZc4EdG9sYii+6pa6qWXnzPUSoXEaYjInD3Y3h3P9PUDALy6+TjSuSwMNWEsoui+qvpDsUs5ETWGfw7+Y1mYmRuOQcdlYaiJYhFF91RaoUNMyu35UCyiiKgRqJUKfDKuG2xUCkRfzMbne5KljkRULRZRdE9HL+eipFwPVzs12rnbSx2HiCxEq2Z2eGdYJwDARzvP4/DFbIkTEd2JRRTd04Hk260N2rhAJpNJnIaILMmooOYY0d0begG8/F0CsgpLpY5EZIRFFN1T1aTyUH+2NiCixiWTybAwojPauNkho6AUr2zk/ChqWlhE0V3lFZfjxLU8AEAo18sjIgnYqJT4bHx3WFspsD8pCyt2c34UNR0souiuDl3IghBAGzc7eDhopI5DRBaqrbs9/hXRGQCw7PfzOHQhS+JERJVYRNFd7b+9Xh7PQhGR1EYFNcfjQc2hF8CMDceQWcD5USQ9FlF0VweT2R+KiJqOdx/rjLbudsgsKMWMDQmcH0WSYxFF1bqSU4zL2cVQymUIbuUidRwiIlirFPhsfHfYqBQ4dCEby6OSpI5EFo5FFFWrqkt5txaOsFMrJU5DRFSpjZs93hteOT9q+a4kwxlzIimwiKJq/dEfiq0NiKhpGd6tOcb29IEQwIwNCcjg+nokERZRdAedXuDQhaqlXngpj4iangXDOqG9hz2yCsvw8oYEVOj0UkciC8Qiiu5w+noecovLYa9WIqC5o9RxiIjuoLFSYMX47rBVKXD4Yg6W/c75UdT4WETRHarmQ/Vu7QKlgj8iRNQ0tW5mh3+P6AIA+HR3Mn4/ky5xIrI0TeITcsWKFWjZsiU0Gg2Cg4MRGxt7z/GbN29G+/btodFo0KVLF/zyyy9G+4UQmDdvHjw9PWFtbY2wsDAkJRn/lpKTk4Px48dDq9XC0dERkydPRmFhoWH/nj178Nhjj8HT0xO2trYIDAzEunXr6u9NN2FsbUBEpuKxQG9MDPEFALyy6RguZRVJnIgsieRF1MaNGzFr1izMnz8fR48eRUBAAMLDw5GRkVHt+EOHDmHcuHGYPHkyEhISEBERgYiICJw6dcowZsmSJVi+fDlWrlyJmJgY2NraIjw8HCUlf0w+HD9+PE6fPo2dO3di+/bt2LdvH6ZOnWr0fbp27Yrvv/8eJ06cwKRJkzBhwgRs37694Q5GE3CrTIe4SzcBAH3ZZJOITMBbQzsiyNcJBSUVeP6/8Sguq5A6ElkImRBC0m5lwcHB6NmzJz799FMAgF6vh4+PD1566SW88cYbd4wfM2YMioqKjIqZ3r17IzAwECtXroQQAl5eXpg9ezZeffVVAEBeXh7c3d2xZs0ajB07FmfPnkXHjh1x5MgR9OjRAwAQGRmJIUOG4OrVq/Dy8qo269ChQ+Hu7o5vvvmmRu8tPz8fDg4OyMvLg1arrdVxkcre85mY+E0svBw0OPjGg5DJZFJHIiK6r/T8EgxdfgBZhaV4LNALy8YE8t8vqrOafn5LeiaqrKwM8fHxCAsLM2yTy+UICwtDdHR0tc+Jjo42Gg8A4eHhhvEpKSlIS0szGuPg4IDg4GDDmOjoaDg6OhoKKAAICwuDXC5HTEzMXfPm5eXB2dn5rvtLS0uRn59v9DA1VZfyQv1d+Q8QEZkMd60GK57oBoVchv8du461hy5JHYksgKRFVFZWFnQ6Hdzd3Y22u7u7Iy0trdrnpKWl3XN81Z/3G+Pm5ma0X6lUwtnZ+a7fd9OmTThy5AgmTZp01/ezaNEiODg4GB4+Pj53HdtUVU0q56U8IjI1wa1cMGdwewDAwp/PIu5SjsSJyNxJPifKFOzevRuTJk3Cl19+iU6dOt113Jw5c5CXl2d4XLlypRFT/n2ZBaU4e6Py7BmLKCIyRZND/fBIV09U6AVeXHcUGQVsxEkNR9IiytXVFQqFAunpxrelpqenw8PDo9rneHh43HN81Z/3G/PXiesVFRXIycm54/vu3bsXjz76KP7v//4PEyZMuOf7UavV0Gq1Rg9TcuhC5Vmojp5auNqpJU5DRFR7MpkMi0d2RVt3O2QUlGL6ugSUsxEnNRBJiyiVSoWgoCBERUUZtun1ekRFRSEkJKTa54SEhBiNB4CdO3caxvv5+cHDw8NoTH5+PmJiYgxjQkJCkJubi/j4eMOYXbt2Qa/XIzg42LBtz549GDp0KBYvXmx05565OpD0x3woIiJTZatWYuWTQbBXKxF7KQeLfkmUOhKZKckv582aNQtffvkl1q5di7Nnz+KFF15AUVGRYe7RhAkTMGfOHMP4GTNmIDIyEkuXLkViYiIWLFiAuLg4TJ8+HUDlbyEzZ87EwoULsW3bNpw8eRITJkyAl5cXIiIiAAAdOnTAoEGDMGXKFMTGxuLgwYOYPn06xo4da7gzb/fu3Rg6dChefvlljBw5EmlpaUhLS0NOjnleYxdC4EDVpHJeyiMiE9eqmR2Wjg4AAHxzMAX/O3ZN4kRklkQT8Mknn4gWLVoIlUolevXqJQ4fPmzY179/fzFx4kSj8Zs2bRJt27YVKpVKdOrUSfz8889G+/V6vZg7d65wd3cXarVaDBw4UJw7d85oTHZ2thg3bpyws7MTWq1WTJo0SRQUFBj2T5w4UQC449G/f/8av6+8vDwBQOTl5dX8YEgkOaNA+P5zu/B/6xdxq6xC6jhERPVi8Y6zwvef20X7t3eIxBv5UschE1HTz2/J+0SZM1PqE7X20CXM33YafVq7YP2U3lLHISKqFzq9wMRvYnEgOQu+Ljb437S+cLRRSR2LmjiT6BNFTcd+zociIjOkkMuwfFw3NHeyxuXsYkxbf5QTzanesIgilOv0OHwxGwDQr00zidMQEdUvZ1sVvpzQAzYqBQ4mZ+O9n89KHYnMBIsowomruSgsrYCjjRU6ejXty45ERHXRwVOL/xsTCABYc+gSvotNlTYQmQUWUfRHl/LWrlDIudQLEZmn8E4emP1QWwDA3K2nEHP7DDxRXbGIIvaHIiKLMf3BNoaO5i+sO4orOcVSRyITxiLKwhWUlCPhSi4A9ociIvMnk8nwwagAdPbWIqeoDFO+jUNRaYXUschEsYiycDEXc6DTC/i62MDH2UbqOEREDc5apcCqp3rA1U6NxLQCzNp0DHo9u/1Q7bGIsnDsUk5ElsjL0RpfPBUElUKOX0+nY1lUktSRyASxiLJwVUVUP86HIiILE+TrhH+P6AIAWB6VhJ9P3JA4EZkaFlEW7EbeLSRnFEIuA0JasYgiIsszKqg5ng31AwDM3nwMp67lSZyITAmLKAtWdVdel+aOcLCxkjgNEZE05gzpgP5tm6GkXI9n18bhRt4tqSORiWARZcEOVl3K43woIrJgVUvDtHGzQ1p+CSatPoKCknKpY5EJYBFloYQQOJBc2WiuL4soIrJwDtZWWP10T8Mdey+u4xp7dH8soixUYloBsgpLYW2lQHdfR6njEBFJzsfZBt883QPWVgrsT8rC2z+eghBsfUB3xyLKQlXNhwpu5Qy1UiFxGiKipqFrc0d8Mq4b5DJgY9wVfLbngtSRqAljEWWh2B+KiKh6YR3dsWBYJwDAB7+ew/+OXZM4ETVVLKIsUGmFDjEplfOhuF4eEdGdJoS0xJR+la0PXtt8Aoe5WDFVg0WUBYq/fBMl5Xo0s1ejnbu91HGIiJqkOYM7YEgXD5Tp9Jj6bRySMwqkjkRNDIsoC3TwT5fyZDKZxGmIiJomuVyGj0YHonsLR+SXVODp1UeQWVAqdSxqQlhEWaCqSeVsbUBEdG8aKwW+mtgTLV1scPXmLUxeewTFZRVSx6ImgkWUhcktLsOJ28sacFI5EdH9OduqsGZSLzjZWOHE1Ty8tD6BPaQIAIsoi3PoQjaEAPzd7ODhoJE6DhGRSWjpaouvJvaAWilHVGIGXt9yAno9e0hZOhZRFqaqtQEv5RER1U6QrzM+f7I7lHIZfky4hnd+Os1mnBaORZSFqZoP1Y+tDYiIau3B9u5YOjoAMhmwNvoy/u/3JKkjkYRYRFmQ1OxipOYUQymXIbiVi9RxiIhM0mOB3nj3djPO5VFJ+PpAisSJSCosoizI/uRMAED3Fk6wUyslTkNEZLqeCmmJVx9uCwD41/Yz2Bx3ReJEJAUWURbkIOdDERHVm2kD2uDZ0Mqu5v/8/gR+PZ0mcSJqbCyiLIROL3AwmUu9EBHVF5lMhreGdsDjQc2hF8BL6xMMv6ySZWARZSFOXctD3q1y2GuUCGjuIHUcIiKzIJPJsGhEFwzqVLk8zJRv45CQelPqWNRIWERZiKrWBiGtXKBU8D87EVF9USrk+HhcIPq2cUFxmQ6T1hzB+XSus2cJ+GlqIapaG/BSHhFR/VMrFVj1VA8E+jgit7gcT3wZwwWLLQCLKAtwq0yH+MuVp5e51AsRUcOwVSuxZlJPtPewR1ZhKcauikESz0iZNRZRFiAmJRtlOj28Ha3h52ordRwiIrPlaKPCd1N6o4OnFlmFpRj35WFe2jNjLKIswB+tDVwgk8kkTkNEZN6cbFVY/2wwOnpqkVVYhnGrDuNcGgspc8QiygLsN8yHaiZxEiIiy+Bkq8L6KcHo5KVFdlEZxn15GIlp+VLHonrGIsrMZRaUIvH2b0B9W3OpFyKixuJoo8K6Z4PRxdsBOUVleOLLGJy9wULKnLCIMnOHLlSeherkpYWLnVriNERElsXRRoX/Tg5G1+ZVhdRhnLnOQspcsIgyc4ZLebwrj4hIEg42VvjP5GAENHfAzeJyPPHVYZy+nid1LKoHLKLMmBCC/aGIiJoAB2sr/OfZYKM+UqeusZAydSyizNiFzEKk5ZdApZSjZ0tnqeMQEVk0rcYK307uhW4tHJF3qxzjVh3G4YvZUseiv4FFlBmrOgvVs6UTNFYKidMQEZFWY4Vvn+mFXi2dUVBagQnfxCLy1A2pY1EdsYgyY1Xr5YW2YWsDIqKmwv72GamHO7qjrEKPF9cdxX8PX5Y6FtUBiygzVa7T4/DFHABAP86HIiJqUjRWCnz+ZBDG9WoBvQDe3noKy34/DyGE1NGoFlhEmaljV3JRWFoBJxsrdPTUSh2HiIj+QiGX4d/DO+Plgf4AgGW/J+Htraeg07OQMhUsosxU1XyoPm1cIZdzqRcioqZIJpNh1kNt8a+IzpDJgHUxqZi27ihKynVSR6MaYBFlpqrmQ/Vjfygioibvqd6+WPFEd6gUckSeTsPEb2KRd6tc6lh0HyyizFBBSTmOXckFAPRlEUVEZBKGdPHEmmd6wl6tRExKDsZ8EY2M/BKpY9E9sIgyQ4cv5kCnF2jpYgMfZxup4xARUQ31ae2KDc/1RjN7NRLTChCx4iCbcjZhLKLM0IGkTADsUk5EZIo6eTnghxf6oJWrLa7nlWDUykPYdvy61LGoGiyizNB+9ociIjJpPs42+HFaXzzQrhlKyvV4+bsELI5M5J17TQyLKDNzPfcWLmYWQS4DQlq7SB2HiIjqyMHaCl9P7Inn+7cGAHy+5wKeXXsE+SWccN5USF5ErVixAi1btoRGo0FwcDBiY2PvOX7z5s1o3749NBoNunTpgl9++cVovxAC8+bNg6enJ6ytrREWFoakpCSjMTk5ORg/fjy0Wi0cHR0xefJkFBYWGvaXlJTg6aefRpcuXaBUKhEREVFv77ehVd2V17W5IxysrSROQ0REf4dCLsMbg9vj47GBUCvl2H0uExErDuJCZuH9n0wNTtIiauPGjZg1axbmz5+Po0ePIiAgAOHh4cjIyKh2/KFDhzBu3DhMnjwZCQkJiIiIQEREBE6dOmUYs2TJEixfvhwrV65ETEwMbG1tER4ejpKSP+5wGD9+PE6fPo2dO3di+/bt2LdvH6ZOnWrYr9PpYG1tjZdffhlhYWENdwAaQFV/KHYpJyIyH48FemPL833g6aDBxcwiRHx6ELsTq/+spMYjExL2mA8ODkbPnj3x6aefAgD0ej18fHzw0ksv4Y033rhj/JgxY1BUVITt27cbtvXu3RuBgYFYuXIlhBDw8vLC7Nmz8eqrrwIA8vLy4O7ujjVr1mDs2LE4e/YsOnbsiCNHjqBHjx4AgMjISAwZMgRXr16Fl5eX0fd8+umnkZubi61bt9b6/eXn58PBwQF5eXnQahu+a7heL9Dzvd+RXVSGjVN7I7gVL+cREZmTzIJSvLguHkcu3YRMBrwW3g4v9G8NmYxNletTTT+/JTsTVVZWhvj4eKMzPXK5HGFhYYiOjq72OdHR0XecGQoPDzeMT0lJQVpamtEYBwcHBAcHG8ZER0fD0dHRUEABQFhYGORyOWJiYv7WeyotLUV+fr7RozElphUgu6gMNioFurVwatTvTUREDa+ZvRrrnu2Ncb1aQAhgSeQ5TFt/lI05JSJZEZWVlQWdTgd3d3ej7e7u7khLS6v2OWlpafccX/Xn/ca4ubkZ7VcqlXB2dr7r962pRYsWwcHBwfDw8fH5W69XWweSK1sbBPs5Q6WUfLobERE1AJVSjkUjumBhRGco5TL8cjINQz7ej9iUHKmjWRx+0tajOXPmIC8vz/C4cuVKo37//bfnQ4X6s7UBEZG5e7K3LzY/H4IWzja4lnsLY1dFY+lv51Cu00sdzWJIVkS5urpCoVAgPT3daHt6ejo8PDyqfY6Hh8c9x1f9eb8xf524XlFRgZycnLt+35pSq9XQarVGj8ZSUq4z/BbCSeVERJahWwsn/DKjH0Z2bw69AD7ZlYzHV0bjcnaR1NEsgmRFlEqlQlBQEKKiogzb9Ho9oqKiEBISUu1zQkJCjMYDwM6dOw3j/fz84OHhYTQmPz8fMTExhjEhISHIzc1FfHy8YcyuXbug1+sRHBxcb++vsR29fBOlFXq42avh72YndRwiImokdmollo4OwCfjusFeo8SxK7kY8vF+bIm/CgnvHbMISim/+axZszBx4kT06NEDvXr1wrJly1BUVIRJkyYBACZMmABvb28sWrQIADBjxgz0798fS5cuxdChQ7FhwwbExcVh1apVAACZTIaZM2di4cKF8Pf3h5+fH+bOnQsvLy9Dr6cOHTpg0KBBmDJlClauXIny8nJMnz4dY8eONboz78yZMygrK0NOTg4KCgpw7NgxAEBgYGCjHZ/a+KNLuSvv0iAiskCPBnihWwtHzNp4HLGXcvDq5uPYcy4D7w3vwr6BDUTSImrMmDHIzMzEvHnzkJaWhsDAQERGRhomhqempkIu/+NkWZ8+fbB+/Xq8/fbbePPNN+Hv74+tW7eic+fOhjGvv/46ioqKMHXqVOTm5iI0NBSRkZHQaDSGMevWrcP06dMxcOBAyOVyjBw5EsuXLzfKNmTIEFy+fNnwdbdu3QCgyVb1BwzzoXgpj4jIUjV3ssF3U3vjs93JWBaVhO0nbiAhNRdLRwegN9ve1DtJ+0SZu8bqE3WzqAzdF+6EEEDMmwPhrtXc/0lERGTWjqbexMwNx5CaUwwAeDyoOeYM6QBnW5XEyZq+Jt8niurPoQvZEAJo627HAoqIiAAA3W9POh/Xq7Ldzub4q3hw6R5siE2FngsZ1wsWUWagqj9UaBu2NiAioj/YqZVYNKIrvn8hBO097JFbXI43fjiJUSsP4cz1xm0IbY5YRJmBqkWH2dqAiIiqE+TrjO0vheLtoR1gq1LgaGouHv30AP61/QwKSyukjmeyWESZuMvZRbiScwtWChl6+TlLHYeIiJoopUKOZ/u1wu+z+2NIFw/o9AJfH0hB2NK9+OXkjSZ741RTxiLKxFV1Ke/Wwgm2aklvtiQiIhPg6WCNz8YHYc2knmjhbIO0/BK8uO4oxn8Vg/jLN6WOZ1JYRJm4qtYG/drwUh4REdXcA+3c8Nsr/8DLA/2hUshx6EI2Rn5+CBO/icWxK7lSxzMJLKJMmE4vcOhCZRHVl/OhiIioljRWCsx6qC2iZvfHmB4+UMhl2Hs+ExErDuKZNUdw8mqe1BGbNBZRJuzktTzkl1TAXqNEV28HqeMQEZGJ8nG2weJRXbFrdn+MCmoOuQzYlZiBRz89gGfXxuH0dRZT1WERZcIOJFW2NujT2gVKBf9TEhHR3+PrYosPHw9A1OwHMKKbN+Qy4Pez6Ri6/ACe/088Tl1jMfVn/OQ1YVWtDUL92R+KiIjqj5+rLT4aE4jfXumPYQFekMmAyNNpeOSTAxj26QF8F5vK1gjgsi8NqiGXfSkuq0DAO7+hXCew+9UH4OdqW6+vT0REVCUpvQCf7ErGjlM3UK6rLBtsVQoMC/TCuF4t0MXbATKZTOKU9aemn9+8J95ExaTkoFwn4O1ojZYuNlLHISIiM+bvbo/l47ohu7Ajvj96FRtir+BiVhG+i72C72KvoKOnFuOCW+CxQC9oNVZSx200PBPVgBryTNS/tp/B1wdSMLanD94f2bVeX5uIiOhehBCIScnBd7Gp2HEqDWUVegCAtZUCD3V0x4D2zfAP/2ZwsVNLnLRueCbKzB00zIdiawMiImpcMpkMvVu5oHcrFywoKsMPCdfwXWwqkjMKse34dWw7fh0yGdDV2wH927lhQLtm6NrcEQq5+VzyA3gmqkE11JmojIIS9HovCjIZEP/2Q3C2VdXbaxMREdWFEAJHU2/i97MZ2HMuE2dvGC9w7GRjhX7+zfBAu2bo3coFng6aJjuPimeizFjVWahOXloWUERE1CTIZDIE+TojyNcZ/xzUHun5Jdh7LhN7zmdgf1IWbhaXG85SAYC9Rol27vZo62Ff+ae7Pdp52JvU5xqLKBN0ICkbABDahq0NiIioaXLXajC6pw9G9/RBuU6PhNRc7DmXgb3nM5GYVoCCkgrEXb6JuL+s1+dqp0Y7Dzt4O1rDXmMFe40SdmoltFV/1ygN2+01SrjaqiGX6DIhiygTI4TAgeTKJpuhXC+PiIhMgJVCjl5+zujl54zXB7VHaYUOKVlFOJdWgPPpBTiXVojz6QVIzSlGVmEpspJLa/zap94Jh51amnKGRZSJySkqg61KCbWyHD1aOkkdh4iIqNbUSgXae2jR3sN4vlFRaQWSMwpxLr0AGfklKCitQEFJ5aOwpNzw94KSchSUVuBWmQ62KoVE74ITyxtUQ7Y4yC4sNdlbR4mIiOqDXi8a5FJeTT+/ueyLiWIBRURElk6quVCG7y/pdyciIiIyUSyiiIiIiOqARRQRERFRHbCIIiIiIqoDFlFEREREdcAiioiIiKgOWEQRERER1QGLKCIiIqI6YBFFREREVAcsooiIiIjqgEUUERERUR2wiCIiIiKqAxZRRERERHWglDqAORNCAADy8/MlTkJEREQ1VfW5XfU5fjcsohpQQUEBAMDHx0fiJERERFRbBQUFcHBwuOt+mbhfmUV1ptfrcf36ddjb20Mmk9Xb6+bn58PHxwdXrlyBVqutt9clYzzOjYPHufHwWDcOHufG0ZDHWQiBgoICeHl5QS6/+8wnnolqQHK5HM2bN2+w19dqtfwftBHwODcOHufGw2PdOHicG0dDHed7nYGqwonlRERERHXAIoqIiIioDlhEmSC1Wo358+dDrVZLHcWs8Tg3Dh7nxsNj3Th4nBtHUzjOnFhOREREVAc8E0VERERUByyiiIiIiOqARRQRERFRHbCIIiIiIqoDFlEmaMWKFWjZsiU0Gg2Cg4MRGxsrdSSTtm/fPjz66KPw8vKCTCbD1q1bjfYLITBv3jx4enrC2toaYWFhSEpKkiasCVu0aBF69uwJe3t7uLm5ISIiAufOnTMaU1JSgmnTpsHFxQV2dnYYOXIk0tPTJUpsmj7//HN07drV0IAwJCQEO3bsMOznMW4Y77//PmQyGWbOnGnYxmP99y1YsAAymczo0b59e8N+qY8xiygTs3HjRsyaNQvz58/H0aNHERAQgPDwcGRkZEgdzWQVFRUhICAAK1asqHb/kiVLsHz5cqxcuRIxMTGwtbVFeHg4SkpKGjmpadu7dy+mTZuGw4cPY+fOnSgvL8fDDz+MoqIiw5hXXnkFP/30EzZv3oy9e/fi+vXrGDFihISpTU/z5s3x/vvvIz4+HnFxcXjwwQfx2GOP4fTp0wB4jBvCkSNH8MUXX6Br165G23ms60enTp1w48YNw+PAgQOGfZIfY0EmpVevXmLatGmGr3U6nfDy8hKLFi2SMJX5ACB+/PFHw9d6vV54eHiIDz74wLAtNzdXqNVq8d1330mQ0HxkZGQIAGLv3r1CiMrjamVlJTZv3mwYc/bsWQFAREdHSxXTLDg5OYmvvvqKx7gBFBQUCH9/f7Fz507Rv39/MWPGDCEEf57ry/z580VAQEC1+5rCMeaZKBNSVlaG+Ph4hIWFGbbJ5XKEhYUhOjpawmTmKyUlBWlpaUbH3MHBAcHBwTzmf1NeXh4AwNnZGQAQHx+P8vJyo2Pdvn17tGjRgse6jnQ6HTZs2ICioiKEhITwGDeAadOmYejQoUbHFODPc31KSkqCl5cXWrVqhfHjxyM1NRVA0zjGXIDYhGRlZUGn08Hd3d1ou7u7OxITEyVKZd7S0tIAoNpjXrWPak+v12PmzJno27cvOnfuDKDyWKtUKjg6OhqN5bGuvZMnTyIkJAQlJSWws7PDjz/+iI4dO+LYsWM8xvVow4YNOHr0KI4cOXLHPv4814/g4GCsWbMG7dq1w40bN/DOO++gX79+OHXqVJM4xiyiiKjRTZs2DadOnTKa20D1p127djh27Bjy8vKwZcsWTJw4EXv37pU6llm5cuUKZsyYgZ07d0Kj0Ugdx2wNHjzY8PeuXbsiODgYvr6+2LRpE6ytrSVMVomX80yIq6srFArFHXcepKenw8PDQ6JU5q3quPKY15/p06dj+/bt2L17N5o3b27Y7uHhgbKyMuTm5hqN57GuPZVKhTZt2iAoKAiLFi1CQEAAPv74Yx7jehQfH4+MjAx0794dSqUSSqUSe/fuxfLly6FUKuHu7s5j3QAcHR3Rtm1bJCcnN4mfZxZRJkSlUiEoKAhRUVGGbXq9HlFRUQgJCZEwmfny8/ODh4eH0THPz89HTEwMj3ktCSEwffp0/Pjjj9i1axf8/PyM9gcFBcHKysroWJ87dw6pqak81n+TXq9HaWkpj3E9GjhwIE6ePIljx44ZHj169MD48eMNf+exrn+FhYW4cOECPD09m8bPc6NMX6d6s2HDBqFWq8WaNWvEmTNnxNSpU4Wjo6NIS0uTOprJKigoEAkJCSIhIUEAEB999JFISEgQly9fFkII8f777wtHR0fxv//9T5w4cUI89thjws/PT9y6dUvi5KblhRdeEA4ODmLPnj3ixo0bhkdxcbFhzPPPPy9atGghdu3aJeLi4kRISIgICQmRMLXpeeONN8TevXtFSkqKOHHihHjjjTeETCYTv/32mxCCx7gh/fnuPCF4rOvD7NmzxZ49e0RKSoo4ePCgCAsLE66uriIjI0MIIf0xZhFlgj755BPRokULoVKpRK9evcThw4eljmTSdu/eLQDc8Zg4caIQorLNwdy5c4W7u7tQq9Vi4MCB4ty5c9KGNkHVHWMAYvXq1YYxt27dEi+++KJwcnISNjY2Yvjw4eLGjRvShTZBzzzzjPD19RUqlUo0a9ZMDBw40FBACcFj3JD+WkTxWP99Y8aMEZ6enkKlUglvb28xZswYkZycbNgv9TGWCSFE45zzIiIiIjIfnBNFREREVAcsooiIiIjqgEUUERERUR2wiCIiIiKqAxZRRERERHXAIoqIiIioDlhEEREREdUBiygiIiKiOmARRUQWLzMzEy+88AJatGgBtVoNDw8PhIeH4+DBgwAAmUyGrVu3ShuSiJocpdQBiIikNnLkSJSVlWHt2rVo1aoV0tPTERUVhezsbKmjEVETxmVfiMii5ebmwsnJCXv27EH//v3v2N+yZUtcvnzZ8LWvry8uXboEAPjf//6Hd955B2fOnIGXlxcmTpyIt956C0pl5e+nMpkMn332GbZt24Y9e/bA09MTS5YswahRoxrlvRFRw+LlPCKyaHZ2drCzs8PWrVtRWlp6x/4jR44AAFavXo0bN24Yvt6/fz8mTJiAGTNm4MyZM/jiiy+wZs0avPfee0bPnzt3LkaOHInjx49j/PjxGDt2LM6ePdvwb4yIGhzPRBGRxfv+++8xZcoU3Lp1C927d0f//v0xduxYdO3aFUDlGaUff/wRERERhueEhYVh4MCBmDNnjmHbf//7X7z++uu4fv264XnPP/88Pv/8c8OY3r17o3v37vjss88a580RUYPhmSgisngjR47E9evXsW3bNgwaNAh79uxB9+7dsWbNmrs+5/jx43j33XcNZ7Ls7OwwZcoU3LhxA8XFxYZxISEhRs8LCQnhmSgiM8GJ5UREADQaDR566CE89NBDmDt3Lp599lnMnz8fTz/9dLXjCwsL8c4772DEiBHVvhYRmT+eiSIiqkbHjh1RVFQEALCysoJOpzPa3717d5w7dw5t2rS54yGX//FP6+HDh42ed/jwYXTo0KHh3wARNTieiSIii5adnY3HH38czzzzDLp27Qp7e3vExcVhyZIleOyxxwBU3qEXFRWFvn37Qq1Ww8nJCfPmzcMjjzyCFi1aYNSoUZDL5Th+/DhOnTqFhQsXGl5/8+bN6NGjB0JDQ7Fu3TrExsbi66+/lurtElE94sRyIrJopaWlWLBgAX777TdcuHAB5eXl8PHxweOPP44333wT1tbW+OmnnzBr1ixcunQJ3t7ehhYHv/76K959910kJCTAysoK7du3x7PPPospU6YAqJxYvmLFCmzduhX79u2Dp6cnFi9ejNGjR0v4jomovrCIIiJqINXd1UdE5oNzooiIiIjqgEUUERERUR1wYjkRUQPhbAki88YzUURERER1wCKKiIiIqA5YRBERERHVAYsoIiIiojpgEUVERERUByyiiIiIiOqARRQRERFRHbCIIiIiIqoDFlFEREREdfD/yn816f3bn9YAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot([get_lr(step) for step in range(0, num_steps + 1)]);\n",
    "plt.title('Learning rate schedule');\n",
    "plt.xlabel('Step');\n",
    "plt.ylabel('Learning rate');"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loaded 297884 tokens, 72 batches of size 4x1024\n",
      "step    0 | loss 10.978139 | lr: 6.0000e-05 | norm: 16.5712 | dt: 52202.80 ms | throughput: 10043.29 tok/s\n",
      "step    1 | loss 10.102369 | lr: 1.1400e-04 | norm: 6.4550 | dt: 25268.63 ms | throughput: 20748.57 tok/s\n",
      "step    2 | loss 9.759232 | lr: 1.6800e-04 | norm: 2.5760 | dt: 25297.78 ms | throughput: 20724.66 tok/s\n",
      "step    3 | loss 9.622756 | lr: 2.2200e-04 | norm: 3.0623 | dt: 25862.73 ms | throughput: 20271.95 tok/s\n",
      "step    4 | loss 9.364845 | lr: 2.7600e-04 | norm: 1.8434 | dt: 26559.22 ms | throughput: 19740.34 tok/s\n",
      "step    5 | loss 9.120868 | lr: 3.3000e-04 | norm: 1.8474 | dt: 27252.84 ms | throughput: 19237.92 tok/s\n",
      "step    6 | loss 8.846601 | lr: 3.8400e-04 | norm: 1.8089 | dt: 27364.27 ms | throughput: 19159.58 tok/s\n",
      "step    7 | loss 8.519905 | lr: 4.3800e-04 | norm: 1.5765 | dt: 27340.18 ms | throughput: 19176.47 tok/s\n",
      "step    8 | loss 8.187926 | lr: 4.9200e-04 | norm: 1.4141 | dt: 27321.78 ms | throughput: 19189.38 tok/s\n",
      "step    9 | loss 7.844541 | lr: 5.4600e-04 | norm: 1.3066 | dt: 27275.72 ms | throughput: 19221.78 tok/s\n",
      "step   10 | loss 7.521743 | lr: 6.0000e-04 | norm: 1.2493 | dt: 27267.65 ms | throughput: 19227.48 tok/s\n",
      "step   11 | loss 7.251814 | lr: 5.9917e-04 | norm: 0.9833 | dt: 27473.72 ms | throughput: 19083.25 tok/s\n",
      "step   12 | loss 7.051537 | lr: 5.9668e-04 | norm: 0.7831 | dt: 27248.34 ms | throughput: 19241.10 tok/s\n",
      "step   13 | loss 6.914010 | lr: 5.9254e-04 | norm: 0.5955 | dt: 27401.79 ms | throughput: 19133.35 tok/s\n",
      "step   14 | loss 6.874609 | lr: 5.8679e-04 | norm: 1.5581 | dt: 27638.85 ms | throughput: 18969.24 tok/s\n",
      "step   15 | loss 6.827901 | lr: 5.7945e-04 | norm: 1.9500 | dt: 27720.94 ms | throughput: 18913.07 tok/s\n",
      "step   16 | loss 6.783661 | lr: 5.7057e-04 | norm: 0.7548 | dt: 27709.95 ms | throughput: 18920.56 tok/s\n",
      "step   17 | loss 6.793988 | lr: 5.6021e-04 | norm: 1.1699 | dt: 27758.29 ms | throughput: 18887.62 tok/s\n",
      "step   18 | loss 6.758572 | lr: 5.4843e-04 | norm: 0.9165 | dt: 27818.44 ms | throughput: 18846.78 tok/s\n",
      "step   19 | loss 6.766694 | lr: 5.3531e-04 | norm: 2.7475 | dt: 27904.36 ms | throughput: 18788.75 tok/s\n",
      "step   20 | loss 6.697817 | lr: 5.2092e-04 | norm: 1.9794 | dt: 27832.40 ms | throughput: 18837.33 tok/s\n",
      "step   21 | loss 6.667795 | lr: 5.0535e-04 | norm: 1.1136 | dt: 27826.11 ms | throughput: 18841.58 tok/s\n",
      "step   22 | loss 6.632704 | lr: 4.8870e-04 | norm: 1.0731 | dt: 27843.76 ms | throughput: 18829.64 tok/s\n",
      "step   23 | loss 6.578654 | lr: 4.7107e-04 | norm: 0.7037 | dt: 27857.07 ms | throughput: 18820.64 tok/s\n",
      "step   24 | loss 6.548522 | lr: 4.5258e-04 | norm: 1.3790 | dt: 27866.81 ms | throughput: 18814.07 tok/s\n",
      "step   25 | loss 6.496417 | lr: 4.3332e-04 | norm: 0.6279 | dt: 27894.70 ms | throughput: 18795.25 tok/s\n",
      "step   26 | loss 6.485850 | lr: 4.1343e-04 | norm: 0.4647 | dt: 27849.37 ms | throughput: 18825.85 tok/s\n",
      "step   27 | loss 6.465448 | lr: 3.9303e-04 | norm: 0.8199 | dt: 27864.15 ms | throughput: 18815.86 tok/s\n",
      "step   28 | loss 6.407889 | lr: 3.7224e-04 | norm: 0.6891 | dt: 27904.00 ms | throughput: 18788.99 tok/s\n",
      "step   29 | loss 6.386956 | lr: 3.5118e-04 | norm: 0.3912 | dt: 27927.82 ms | throughput: 18772.97 tok/s\n",
      "step   30 | loss 6.368323 | lr: 3.3000e-04 | norm: 1.0433 | dt: 27917.25 ms | throughput: 18780.07 tok/s\n",
      "step   31 | loss 6.348032 | lr: 3.0882e-04 | norm: 0.7479 | dt: 27869.37 ms | throughput: 18812.34 tok/s\n",
      "step   32 | loss 6.313011 | lr: 2.8776e-04 | norm: 0.4351 | dt: 27882.00 ms | throughput: 18803.82 tok/s\n",
      "step   33 | loss 6.300664 | lr: 2.6697e-04 | norm: 0.6875 | dt: 27886.06 ms | throughput: 18801.08 tok/s\n",
      "step   34 | loss 6.268357 | lr: 2.4657e-04 | norm: 0.6187 | dt: 27903.25 ms | throughput: 18789.50 tok/s\n",
      "step   35 | loss 6.267078 | lr: 2.2668e-04 | norm: 0.5530 | dt: 27888.96 ms | throughput: 18799.13 tok/s\n",
      "step   36 | loss 6.243991 | lr: 2.0742e-04 | norm: 0.3257 | dt: 27904.12 ms | throughput: 18788.91 tok/s\n",
      "step   37 | loss 6.209406 | lr: 1.8893e-04 | norm: 0.4796 | dt: 27919.70 ms | throughput: 18778.43 tok/s\n",
      "step   38 | loss 6.198496 | lr: 1.7130e-04 | norm: 0.5119 | dt: 28025.41 ms | throughput: 18707.60 tok/s\n",
      "step   39 | loss 6.184638 | lr: 1.5465e-04 | norm: 0.4016 | dt: 28168.50 ms | throughput: 18612.56 tok/s\n",
      "step   40 | loss 6.173847 | lr: 1.3908e-04 | norm: 0.3243 | dt: 28172.49 ms | throughput: 18609.93 tok/s\n",
      "step   41 | loss 6.149192 | lr: 1.2469e-04 | norm: 0.3221 | dt: 28123.55 ms | throughput: 18642.31 tok/s\n",
      "step   42 | loss 6.138320 | lr: 1.1157e-04 | norm: 0.4532 | dt: 28086.42 ms | throughput: 18666.96 tok/s\n",
      "step   43 | loss 6.118985 | lr: 9.9787e-05 | norm: 0.4207 | dt: 28092.84 ms | throughput: 18662.69 tok/s\n",
      "step   44 | loss 6.122375 | lr: 8.9428e-05 | norm: 0.3157 | dt: 28097.57 ms | throughput: 18659.55 tok/s\n",
      "step   45 | loss 6.113974 | lr: 8.0553e-05 | norm: 0.2504 | dt: 28106.67 ms | throughput: 18653.51 tok/s\n",
      "step   46 | loss 6.082560 | lr: 7.3215e-05 | norm: 0.3116 | dt: 28101.73 ms | throughput: 18656.79 tok/s\n",
      "step   47 | loss 6.079542 | lr: 6.7460e-05 | norm: 0.2769 | dt: 28070.77 ms | throughput: 18677.37 tok/s\n",
      "step   48 | loss 6.076788 | lr: 6.3324e-05 | norm: 0.3431 | dt: 27932.13 ms | throughput: 18770.07 tok/s\n",
      "step   49 | loss 6.071138 | lr: 6.0832e-05 | norm: 0.2716 | dt: 28080.20 ms | throughput: 18671.09 tok/s\n"
     ]
    }
   ],
   "source": [
    "model = GPT(GPTConfig(vocab_size=50304)).to(device).train()\n",
    "model = torch.compile(model)\n",
    "\n",
    "total_batch_size = 524288\n",
    "B, T = 4, 1024\n",
    "assert total_batch_size % (B * T) == 0, 'Total batch size must be divisible by B * T'\n",
    "grad_accumulation_steps = total_batch_size // (B * T)\n",
    "\n",
    "data_loader = DataLoaderLite(B, T)\n",
    "scaler = torch.cuda.amp.GradScaler() # create a gradient scaler for mixed precision training\n",
    "\n",
    "# Create optimizer with weight decay\n",
    "param_dict = {pn: p for pn, p in model.named_parameters() if p.requires_grad} # all the params require grad anyways\n",
    "weight_decay_params = [p for pn, p in param_dict.items() if p.dim() >= 2]\n",
    "non_weight_decay_params = [p for pn, p in param_dict.items() if p.dim() < 2]\n",
    "optim_groups = [\n",
    "    {'params': weight_decay_params, 'weight_decay': 0.01},\n",
    "    {'params': non_weight_decay_params, 'weight_decay': 0.0}\n",
    "]\n",
    "optimizer = torch.optim.AdamW(optim_groups, lr=1, betas=(0.9, 0.95), eps=1e-8, fused=True)\n",
    "\n",
    "for step in range(num_steps):\n",
    "    t0 = time.time()\n",
    "\n",
    "    loss_accumulator = torch.tensor(0.0, device=device, requires_grad=False)\n",
    "    optimizer.zero_grad()\n",
    "    for micro_step in range(grad_accumulation_steps):\n",
    "\n",
    "        # forward pass\n",
    "        x, y = data_loader.next_batch()\n",
    "        with torch.autocast(device_type=device.type, dtype=torch.float16):\n",
    "            logits, loss = model(x, y)\n",
    "\n",
    "        # backward pass\n",
    "        loss = loss / grad_accumulation_steps # loss is averaged over each micro-batch in the batch\n",
    "        loss_accumulator += loss.detach() # this is preferred over `.item()` since it does not force GPU-CPU sync\n",
    "        scaler.scale(loss).backward()\n",
    "    \n",
    "    # learning rate schedule\n",
    "    lr = get_lr(step)\n",
    "    for param_group in optimizer.param_groups:\n",
    "        param_group['lr'] = lr\n",
    "\n",
    "    # optimizer step\n",
    "    scaler.unscale_(optimizer)\n",
    "    norm = torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n",
    "    scaler.step(optimizer)\n",
    "    scaler.update()\n",
    "    \n",
    "    # track throughput\n",
    "    torch.cuda.synchronize() # wait for GPU to finish work\n",
    "    t1 = time.time()\n",
    "    dt = (t1 - t0) * 1000 # time difference in milliseconds\n",
    "    throughput = (B * T * grad_accumulation_steps) / (t1 - t0) # tokens per second\n",
    "\n",
    "    # print stats\n",
    "    print(f\"step {step:4d} | loss {loss_accumulator:.6f} | lr: {lr:.4e} | norm: {norm:.4f} | dt: {dt:.2f} ms | throughput: {throughput:.2f} tok/s\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}