micrograd.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import math\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Value:\n",
    "    \n",
    "    def __init__(self, data, _children=(), _op='', label='', grad=''): #children is a tuple\n",
    "        self.data = data\n",
    "        self.grad = 0\n",
    "        self._backward = lambda: None\n",
    "        self._prev = set(_children) #tuple stored as a set\n",
    "        self._op = _op\n",
    "        self.label = label\n",
    "        \n",
    "    def __repr__(self):\n",
    "        return f\"Value(data={self.data})\"\n",
    "    \n",
    "    def __add__(self, other):\n",
    "        other = other if isinstance(other, Value) else Value(other)\n",
    "        out = Value(self.data + other.data, (self, other), '+')\n",
    "        def _backward():\n",
    "            self.grad += out.grad\n",
    "            other.grad += out.grad #dL/dother = dL/dout * dout/dother = dL/dout * 1 = out.grad\n",
    "        out._backward = _backward\n",
    "        \n",
    "        return out\n",
    "   \n",
    "    def __radd__(self, other):\n",
    "        return self + other\n",
    "        \n",
    "    def __mul__(self, other):\n",
    "        other = other if isinstance(other, Value) else Value(other)\n",
    "        out = Value(self.data * other.data, (self, other), '*')\n",
    "        def _backward():\n",
    "            self.grad += out.grad * other.data\n",
    "            other.grad += out.grad * self.data #dL/dother = dL/dout * dout/dother = dL/dout * d(self * other)/dother = out.grad * self = out.grad\n",
    "        out._backward = _backward\n",
    "        \n",
    "        return out\n",
    "    \n",
    "    def __rmul__(self, other):\n",
    "        return self * other\n",
    "    \n",
    "    def __truediv__(self, other): #self/other = self * (other**-1)\n",
    "        return self * other**-1\n",
    "    \n",
    "    def __pow__(self, other): #self**other\n",
    "        assert isinstance(other, (int, float)), \"only supporting int/float powers for now\"\n",
    "        out = Value(self.data**other, (self, ), f'**o{other}')\n",
    "\n",
    "        def _backward():\n",
    "            self.grad += out.grad * other * (self.data ** (other - 1))# x^2 -> 2x^1\n",
    "        out._backward = _backward\n",
    "        \n",
    "        return out\n",
    "    \n",
    "    def __neg__(self):\n",
    "        return self * -1\n",
    "    \n",
    "    def __sub__(self, other):\n",
    "        return self + (-other)\n",
    "    \n",
    "    def tanh(self):\n",
    "        x = self.data\n",
    "        t = (math.exp(2*x) - 1)/(math.exp(2*x) + 1)\n",
    "        out = Value(t, (self,), 'tanh')\n",
    "        def _backward():\n",
    "            self.grad += out.grad * (1 - t**2) #dL/dself = dL/dout * dout/dself = dL/dout * d(tanh(self))/dself = out.grad * (1 - tanh**2(self))\n",
    "        out._backward = _backward\n",
    "        \n",
    "        return out\n",
    "    \n",
    "    def exp(self):\n",
    "        x = self.data\n",
    "        out = Value(math.exp(x), (self, ), 'exp')\n",
    "        def _backward():\n",
    "            self.grad += out.grad * out.data #dL/dself = dL/dout * dout/dself\n",
    "        out._backward = _backward\n",
    "        \n",
    "        return out\n",
    "    \n",
    "    \n",
    "    def backward(self):        \n",
    "        topo = []\n",
    "        visited = set()\n",
    "        def build_topo(v):\n",
    "            if v not in visited:\n",
    "                visited.add(v)\n",
    "                for child in v._prev:\n",
    "                    build_topo(child)\n",
    "                topo.append(v)\n",
    "        build_topo(self)\n",
    "        \n",
    "        self.grad = 1.0\n",
    "        for node in reversed(topo):\n",
    "            node._backward()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Value(data=-1.0)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a = Value(1.0)\n",
    "b = Value(2.0)\n",
    "a-b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjgAAAGdCAYAAAAfTAk2AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAABHWklEQVR4nO3de1yUdd4//tfMMAwHYRSQARQBjRREUyERTM1VUMvOnu6K1lZt/dnJqF/FWpu623J32KQsK3fdJQ+pd7dr1h0Zo+WhQPMAmqampoLIWWBAZI7X9w9kcuQgo8xczDWv5+MxD5hrPnPxvt4N+Oo6fC6ZIAgCiIiIiCRELnYBRERERF2NAYeIiIgkhwGHiIiIJIcBh4iIiCSHAYeIiIgkhwGHiIiIJIcBh4iIiCSHAYeIiIgkx0PsAsRgsVhw4cIF+Pn5QSaTiV0OERERdYIgCKivr0dYWBjk8o730bhlwLlw4QLCw8PFLoOIiIhuQHFxMfr27dvhGLcMOH5+fgCaG+Tv7y9yNd2D0WhEbm4uUlNToVQqxS5H8thv52K/nYv9di536rdOp0N4eLj13/GOuGXAaTks5e/vz4BzhdFohI+PD/z9/SX/C9IdsN/OxX47F/vtXO7Y786cXsKTjImIiEhyGHCIiIhIchhwiIiISHIYcIiIiEhyGHCIiIhIchhwiIiISHIYcIiIiEhyGHCIiIhIchhwiIiISHIcGnB27dqFe+65B2FhYZDJZPj888+v+56dO3ciPj4eXl5e6N+/Pz766KNWYzZt2oTY2FioVCrExsZi8+bNDqieiIiIXJVDA86lS5dw22234f333+/U+DNnzuCuu+7CmDFjUFBQgD/96U945plnsGnTJuuY/Px8zJw5E2lpaTh06BDS0tIwY8YM7N2711GbQURERC7GofeimjJlCqZMmdLp8R999BH69euHrKwsAEBMTAz279+Pt99+Gw899BAAICsrCykpKcjIyAAAZGRkYOfOncjKysL69eu7fBuIiIjI9XSrm23m5+cjNTXVZtmkSZOwatUqGI1GKJVK5Ofn47nnnms1piUUtUWv10Ov11uf63Q6AM03KDMajV23AS6spQ/sh3Ow387FfjuXlPtttggwmCwwWSwwmgWYLAJMZguMFgEm85XvzQJMFgvMFgFmQWj+agHMFgvMAmA2Ny+3XHndIgDClXEt31sEwCwI1u9bvgKARRAgCL99NZnNOHlehtPbT0Iml0MAAAEQ0Py6AFz5euUJflvW/L1w1fdXvrYsuMq1Y64eJ7QxLtDXEwvu7H9T/b6WPZ+pbhVwysrKoNFobJZpNBqYTCZUVVUhNDS03TFlZWXtrjczMxNLlixptTw3Nxc+Pj5dU7xEaLVasUtwK+y3c7HfziVWvy0CcNkENJqAy2ag0STDZRPQZAYMlisPswz6K8+v/mq0yGASAJMFMFrQ6nuLcP27WItDARSfEbsIG8FeAiIbj3fpOhsbGzs9tlsFHKD1LdBb0uHVy9sa09Gt0zMyMpCenm59rtPpEB4ejtTUVPj7+3dF2S7PaDRCq9UiJSUFSqVS7HIkj/12Lvbbubq634IgoEFvQkW9AZX1elTU61HZoLd+X9VgQG2jEbomI+oum9CgN3XBVnSOXAZ4KORQymXwUMjgIZfDQyGDUi6DQi6HQi6DQg4oZDIoFLLmr/Lmh1z221e5DM1f5Ve+ymSQtSyTNf+7JwOuWg7gymuCRUDphRL07dsHcrkCMhlw5WXIILM+x5V1tPxz2fy97KrvbbfN5t9d67KW57JrxqLV2F6+nrgrOeImO2yr5QhMZ3SrgBMSEtJqT0xFRQU8PDwQGBjY4Zhr9+pcTaVSQaVStVquVCr5x+4a7Ilzsd/OxX47lz39NpgsKK5pxNmqSzhTdQlnqy/hbFUjimsaUa5rQpPRYvfP9/FUQO2thNpbCX8vJXp4ecDbUwEfpQK+qt++91F5wMdTAR9PBVQeCngp5fD0kEPloYDKQ37lobiyTA6lhxxKhQxKuRxyufh7dIxGI3JyinHXXUMk//m2Z/u6VcBJSkrCl19+abMsNzcXCQkJ1o1KSkqCVqu1OQ8nNzcXycnJTq2ViIjsZzJbcLKiAYXFtTheqsOZ6uZQc76m0XqOSXv8vDwQ7KdCsJ8Xgv1V1u97+6nQ00dpDTNqbyX8vJTw9OBUb+7MoQGnoaEBp06dsj4/c+YMCgsLERAQgH79+iEjIwMlJSVYvXo1AGD+/Pl4//33kZ6ejnnz5iE/Px+rVq2yuTrq2WefxdixY/HGG2/gvvvuw5YtW7Bt2zZ8//33jtwUIiKykyAIuFB7GYeKa1FYXIuC4locKalDo8Hc5ngfTwUiA30RFeSLyCAfRAb6IiLQFyH+zSHG21Ph5C0gV+bQgLN//36MHz/e+rzlPJjf//73yM7ORmlpKYqKiqyvR0VFIScnB8899xw++OADhIWF4b333rNeIg4AycnJ2LBhA1555RW8+uqrGDBgADZu3IjExERHbgoREXXCuepL2PrTBfzfcTleP7ILFfX6VmN6qDwwtK8aQ/qo0b+3rzXU9PZTdXg+JZE9HBpw7rzzzjYvNWuRnZ3datm4ceNw8ODBDtc7bdo0TJs27WbLIyKimyQIAo5e0CH3aBlyfy7H8bL6K6/IAeihkMswUOOHYf16Ylh4TwwP74kBvXt0i3NXSNq61Tk4RETU/ZnMFuw/V4NvjpYh92g5SmovW19TyGUYGdkLvU2VmJUyCsMjAnloiUTBgENERJ1yrFSHT/LO4pujZahp/G3CNS+lHGOje2PS4BD8blAwenjKkJOTg9sje0GpZLghcTDgEBFRuwRBQP7pany061fs+qXSurynjxITBmkwabAGY6J72+ylkeIMxuR6GHCIiKgVk9mCr4+U4eNdp3GkpHlyNbkMmDIkFI+M7IeRUQHwUPAybOq+GHCIiMiq0WDCZ/vP45/f/4rii83n1ngp5ZiREI65d/RHv0De3oZcAwMOERGhQW/Cyl2/YnX+WdReOb8mwNcTjyVF4LGkSAT4eopcIZF9GHCIiNzc7pOVeHnTT9arofoF+GDemChMiw/nFVDkshhwiIjclK7JiMycY1j/YzEAIDzAGy9PjsHkuBAoOE8NuTgGHCIiN7TjRAUy/vMTSuuaAACzkyPx4uSB8PHkPwskDfwkExG5kbrLRrz+1c/4n/3nAQARgT5486GhSOwfKHJlRF2LAYeIyE18d7x5r02ZrgkyGfB4chT+/0kDeZ4NSRIDDhGRxNVdNmLplz9j08HmvTZRQb54a9pQJEQGiFwZkeMw4BARSViFrgmPrtqLX8obIJMBc++IQnoK99qQ9DHgEBFJVPHFRjy6ai/OVTdC46/CikdGID6Ce23IPTDgEBFJ0KmKBjz6z70o0zWhX4AP1s1NRHgAZyEm98GAQ0QkMUdK6vDYv37ExUsGRAf3wNq5idD4e4ldFpFTMeAQEUnI/rMX8Xj2PtQ3mTCkjxqf/GEkb7NAbokBh4hIInafrMQTqw/gstGMkZEB+OfsBPh7KcUui0gUDDhERBLwzdEyPP1pAQxmC8bd2hsfPRrPK6XIrTHgEBG5uM0F5/HCZ4dhtgiYEheCd2cNh6eHXOyyiETFgENE5MLW/1iEjP/8BACYFt8X//3gEHgoGG6IGHCIiFzU3l+r8crnRwA03yzzz1NjIeddwIkAMOAQEbmkyno9nl5fALNFwAPD++C1e2IhkzHcELXgfkwiIhdjtghYuLEAFfV6RAf3wOsPxDHcEF2DAYeIyMW8u/0kfjhVDW+lAh8+OgI+ntwZT3QtBhwiIhey65dKLP/2JAAg88EhuCXYT+SKiLonBhwiIhdRWncZCzcWQhCAhxP74f7hfcQuiajbYsAhInIBRrMFT39agIuXDBgc5o8/T40VuySibo0Bh4jIBbz1zQnsP1cDP5UHVjwyAl5KzlJM1BEGHCKibi73aBlW7voVAPDW9KGICPQVuSKi7o8Bh4ioGyuqbsTznx0CAMy5IwqT40JFrojINTgl4KxYsQJRUVHw8vJCfHw8du/e3e7Y2bNnQyaTtXoMHjzYOiY7O7vNMU1NTc7YHCIip2gymvHkpwdR32TC8H498dLkQWKXROQyHB5wNm7ciIULF2LRokUoKCjAmDFjMGXKFBQVFbU5/t1330Vpaan1UVxcjICAAEyfPt1mnL+/v8240tJSeHl5OXpziIic5vWvjuGnkjr08lHig4dH8AaaRHZw+G/LO++8gzlz5mDu3LmIiYlBVlYWwsPD8eGHH7Y5Xq1WIyQkxPrYv38/ampq8Pjjj9uMk8lkNuNCQkIcvSlERE6z7+xFrNlzDgDwzsxhCOvpLXJFRK7FodNfGgwGHDhwAC+//LLN8tTUVOTl5XVqHatWrcLEiRMRERFhs7yhoQEREREwm80YNmwY/vKXv2D48OFtrkOv10Ov11uf63Q6AIDRaITRaLRnkySrpQ/sh3Ow387lav22WAT85cujAICZCX1wR/9eLlM74Hr9dnXu1G97ttGhAaeqqgpmsxkajcZmuUajQVlZ2XXfX1paiq+//hqffvqpzfJBgwYhOzsbQ4YMgU6nw7vvvovRo0fj0KFDiI6ObrWezMxMLFmypNXy3Nxc+Pj42LlV0qbVasUuwa2w387lKv3eXynD4RIFVHIBccI55OScE7ukG+Iq/ZYKd+h3Y2Njp8c65QYm194EThCETt0YLjs7Gz179sT9999vs3zUqFEYNWqU9fno0aMxYsQILF++HO+9916r9WRkZCA9Pd36XKfTITw8HKmpqfD397dza6TJaDRCq9UiJSUFSqVS7HIkj/12Llfqd5PRjP9+9wcATXjyd9GYNa6/2CXZzZX6LQXu1O+WIzCd4dCAExQUBIVC0WpvTUVFRau9OtcSBAH/+te/kJaWBk9Pzw7HyuVy3H777Th58mSbr6tUKqhUqlbLlUql5D8M9mJPnIv9di5X6PfK78+htK4JYWovPDHuFihdeEI/V+i3lLhDv+3ZPoeeZOzp6Yn4+PhWu820Wi2Sk5M7fO/OnTtx6tQpzJkz57o/RxAEFBYWIjSU80MQkeuqqG/Ciu9OAQBemjKIsxUT3QSHH6JKT09HWloaEhISkJSUhJUrV6KoqAjz588H0Hz4qKSkBKtXr7Z536pVq5CYmIi4uLhW61yyZAlGjRqF6Oho6HQ6vPfeeygsLMQHH3zg6M0hInKYZdpfcMlgxm191bhnaJjY5RC5NIcHnJkzZ6K6uhpLly5FaWkp4uLikJOTY70qqrS0tNWcOHV1ddi0aRPefffdNtdZW1uLJ554AmVlZVCr1Rg+fDh27dqFkSNHOnpziIgc4niZDhv3FQMAXpkaC7n8+ucpElH7nHKS8YIFC7BgwYI2X8vOzm61TK1Wd3im9LJly7Bs2bKuKo+ISFSCIOD1r47BIgB3DQnB7ZEBYpdE5PI4LSYRkch2/FKJ3Ser4KmQ83YMRF2EAYeISEQmswWvf3UMAPD75AjeKZyoizDgEBGJaMO+YpyqaEAvHyWe+l3riUqJ6MYw4BARiUTXZMQy7S8AgIUTb4XaW9pzmBA5EwMOEZFIVnx3GtWXDOjf2xcPJ/YTuxwiSWHAISISQfHFRvzrhzMAgD9NiYFSwT/HRF2Jv1FERCJ485sTMJgsSB4QiAkxwWKXQyQ5DDhERE720/k6fHnoAmQyYNHdMZ26+TAR2YcBh4jIyVoOTd17WxgGh6lFroZImhhwiIicqKK+Cf93+AIAYM4dUSJXQyRdDDhERE60bk8RjGYB8RG9MLRvT7HLIZIsBhwiIifRm8xYt7f55sKzkyPFLYZI4hhwiIic5KvDpahq0CPE3wuT40LELodI0hhwiIicQBAE/PuHswCAtKQIzntD5GD8DSMicoKDRTX4qaQOnh5yzLo9XOxyiCSPAYeIyAla9t7cPywMgT1U4hZD5AYYcIiIHKy07jK+PlIGAJidzEvDiZyBAYeIyMHW7jkHs0VAYlQAYsP8xS6HyC0w4BAROVCT0YxPr1wa/vjoSHGLIXIjDDhERA70ReEF1DQa0aenNybGaMQuh8htMOAQETmIIAj4d95ZAMBjSRHw4KXhRE7D3zYiIgfZe+YijpXq4KWUYyYvDSdyKgYcIiIHyb5yafiDI/qip4+nuMUQuRkGHCIiByi+2Ijcn1suDY8UtxgiN8SAQ0TkAGv3nINFAEbfEohbNX5il0PkdhhwiIi6WKPBhPU/Xrk0nBP7EYmCAYeIqIttLiiBrsmEfgE+GD8oWOxyiNwSAw4RURcSBMF6cvHvkyOhkMvELYjITTHgEBF1oR9OVeNkRQN8PBWYntBX7HKI3BYDDhFRF/rsQDEA4MERfeDvpRS5GiL3xYBDRNRFLhvM0P5cDqB57hsiEo9TAs6KFSsQFRUFLy8vxMfHY/fu3e2O3bFjB2QyWavH8ePHbcZt2rQJsbGxUKlUiI2NxebNmx29GUREHdp2rByNBjPCA7wxPLyn2OUQuTWHB5yNGzdi4cKFWLRoEQoKCjBmzBhMmTIFRUVFHb7vxIkTKC0ttT6io6Otr+Xn52PmzJlIS0vDoUOHkJaWhhkzZmDv3r2O3hwionZtKbwAALhnaBhkMp5cTCQmhwecd955B3PmzMHcuXMRExODrKwshIeH48MPP+zwfcHBwQgJCbE+FAqF9bWsrCykpKQgIyMDgwYNQkZGBiZMmICsrCwHbw0RUdvqGo3Y+UsFAOC+YX1EroaIPBy5coPBgAMHDuDll1+2WZ6amoq8vLwO3zt8+HA0NTUhNjYWr7zyCsaPH299LT8/H88995zN+EmTJrUbcPR6PfR6vfW5TqcDABiNRhiNRns2SbJa+sB+OAf77VzO6PdXh8/DaBZwa3AP9A/0cuv/tvx8O5c79duebXRowKmqqoLZbIZGo7FZrtFoUFZW1uZ7QkNDsXLlSsTHx0Ov12PNmjWYMGECduzYgbFjxwIAysrK7FpnZmYmlixZ0mp5bm4ufHx8bmTTJEur1Ypdglthv53Lkf3O/lkOQI5oVR1ycnIc9nNcCT/fzuUO/W5sbOz0WIcGnBbXHosWBKHd49MDBw7EwIEDrc+TkpJQXFyMt99+2xpw7F1nRkYG0tPTrc91Oh3Cw8ORmpoKf39/u7dHioxGI7RaLVJSUqBU8tJWR2O/ncvR/a6o1+Pknp0AgPRp49AvwL3/x4mfb+dyp363HIHpDIcGnKCgICgUilZ7VioqKlrtgenIqFGjsHbtWuvzkJAQu9apUqmgUqlaLVcqlZL/MNiLPXEu9tu5HNXvb34+D0EAhvfriQEadZev31Xx8+1c7tBve7bPoScZe3p6Ij4+vtVuM61Wi+Tk5E6vp6CgAKGhodbnSUlJrdaZm5tr1zqJiLrKF4ear56697YwkSshohYOP0SVnp6OtLQ0JCQkICkpCStXrkRRURHmz58PoPnwUUlJCVavXg2g+QqpyMhIDB48GAaDAWvXrsWmTZuwadMm6zqfffZZjB07Fm+88Qbuu+8+bNmyBdu2bcP333/v6M0hIrJRVN2IwuJayGXA3UNDr/8GInIKhwecmTNnorq6GkuXLkVpaSni4uKQk5ODiIgIAEBpaanNnDgGgwEvvPACSkpK4O3tjcGDB+Orr77CXXfdZR2TnJyMDRs24JVXXsGrr76KAQMGYOPGjUhMTHT05hAR2fjycPPem6QBgQj28xK5GiJq4ZSTjBcsWIAFCxa0+Vp2drbN8xdffBEvvvjiddc5bdo0TJs2rSvKIyK6YVsKSwAA993GuW+IuhPei4qI6AYdL9Phl/IGeCrkmBQXInY5RHQVBhwiohv0xZVbM4wb2Btqb2lfvULkahhwiIhugCAI1vNv7hvGq6eIuhsGHCKiG1BQXIvii5fh66nAhEGdn9eLiJyDAYeI6Aa0HJ5KidXA21NxndFE5GwMOEREdjKZLfi/w6UAgHt5eIqoW2LAISKy055fL6KqQY9ePkqMie4tdjlE1AYGHCIiO31xqHnumylDQqFU8M8oUXfE30wiIjvoTWZ8faT5Zr+89xRR98WAQ0Rkhx0nKlHfZEKIvxdGRgaIXQ4RtYMBh4jIDi13Dr/ntlDI5TKRqyGi9jDgEBF10iW9CduPlQMA7uW9p4i6NQYcIqJO0v5cjiajBVFBvojr4y92OUTUAQYcIqJOapn75p7bwiCT8fAUUXfGgENE1AlNRjO+P1UJAJjCO4cTdXsMOEREnfDDqSo0GS3o09Mbg0L8xC6HiK6DAYeIqBO2HasAAPxuUDAPTxG5AAYcIqLrEAQB3x5vvnpqQkywyNUQUWcw4BARXcfRCzqU6/Tw8VRgVP9Ascshok5gwCEiuo5tV+a+GRMdBC+lQuRqiKgzGHCIiK5j+5XzbybEaESuhIg6iwGHiKgD5bom/FRSB5kMGD+Q598QuQoGHCKiDrTsvbmtb0/09lOJXA0RdRYDDhFRB1qunprIq6eIXAoDDhFRO5pnL64CwPNviFwNAw4RUTtaZi8OU3tx9mIiF8OAQ0TUjm1XXT3F2YuJXAsDDhFRGzh7MZFrY8AhImrDkRLOXkzkyhhwiIjasP04Zy8mcmUMOEREbbDOXjyIV08RuSKnBJwVK1YgKioKXl5eiI+Px+7du9sd+5///AcpKSno3bs3/P39kZSUhG+++cZmTHZ2NmQyWatHU1OTozeFiNyAzezFg3j+DZErcnjA2bhxIxYuXIhFixahoKAAY8aMwZQpU1BUVNTm+F27diElJQU5OTk4cOAAxo8fj3vuuQcFBQU24/z9/VFaWmrz8PLycvTmEJEb4OzFRK7Pw9E/4J133sGcOXMwd+5cAEBWVha++eYbfPjhh8jMzGw1Pisry+b53/72N2zZsgVffvklhg8fbl0uk8kQEhLi0NqJyD1tP8bZi4lcnUMDjsFgwIEDB/Dyyy/bLE9NTUVeXl6n1mGxWFBfX4+AgACb5Q0NDYiIiIDZbMawYcPwl7/8xSYAXU2v10Ov11uf63Q6AIDRaITRaLRnkySrpQ/sh3Ow385lT78vG36bvXjsLYH8b3QD+Pl2Lnfqtz3b6NCAU1VVBbPZDI3G9iQ9jUaDsrKyTq3j73//Oy5duoQZM2ZYlw0aNAjZ2dkYMmQIdDod3n33XYwePRqHDh1CdHR0q3VkZmZiyZIlrZbn5ubCx8fHzq2SNq1WK3YJboX9dq7O9PtIjQx6kwI9PQX8enA3znB+vxvGz7dzuUO/GxsbOz3W4YeoALSaAVQQhE7NCrp+/XosXrwYW7ZsQXDwb7uKR40ahVGjRlmfjx49GiNGjMDy5cvx3nvvtVpPRkYG0tPTrc91Oh3Cw8ORmpoKf3//G9kkyTEajdBqtUhJSYFSqRS7HMljv53Lnn7nbfkZwHncPawf7r47xjkFSgw/387lTv1uOQLTGQ4NOEFBQVAoFK321lRUVLTaq3OtjRs3Ys6cOfjss88wceLEDsfK5XLcfvvtOHnyZJuvq1QqqFStTxRUKpWS/zDYiz1xLvbbua7Xb0EQsOOXSgBAyuAQ/re5Sfx8O5c79Nue7XPoVVSenp6Ij49vtdtMq9UiOTm53fetX78es2fPxqeffoq77777uj9HEAQUFhYiNDT0pmsmIvfF2YuJpMPhh6jS09ORlpaGhIQEJCUlYeXKlSgqKsL8+fMBNB8+KikpwerVqwE0h5vHHnsM7777LkaNGmXd++Pt7Q21Wg0AWLJkCUaNGoXo6GjodDq89957KCwsxAcffODozSEiCWuZvfiOWzh7MZGrc3jAmTlzJqqrq7F06VKUlpYiLi4OOTk5iIiIAACUlpbazInz8ccfw2Qy4cknn8STTz5pXf773/8e2dnZAIDa2lo88cQTKCsrg1qtxvDhw7Fr1y6MHDnS0ZtDRBLWMv/NxBjOXkzk6pxykvGCBQuwYMGCNl9rCS0tduzYcd31LVu2DMuWLeuCyoiImnH2YiJp4b2oiIjA2YuJpIYBh4gInL2YSGoYcIjI7TUZzfjhdPPsxTw8RSQNDDhE5Pb2n61Bk9GCYD8VYkM5+SeRFDDgEJHb23WyeXK/MdG9OzXLOhF1fww4ROT2dl2ZvXjsrUEiV0JEXYUBh4jcWrmuCcfL6iGTNe/BISJpYMAhIrfWsvdmSB81Anw9Ra6GiLoKAw4RubVdJ5uvnhrLvTdEksKAQ0Ruy2wR8P3JlvNvGHCIpIQBh4jc1pGSOtQ0GtFD5YHh/XqKXQ4RdSEGHCJyWy3n3yQPCIRSwT+HRFLC32giclu7eHiKSLIYcIjILemajDhYVAsAGMeAQyQ5DDhE5JbyTlXDbBEQFeSL8AAfscshoi7GgENEbsl6eCqasxcTSREDDhG5HUEQrro9Aw9PEUkRAw4RuZ0zVZdwvuYyPBVyjOofKHY5ROQADDhE5HZa9t4kRPaCr8pD5GqIyBEYcIjI7Vhvz8DDU0SSxYBDRG5FbzIj/3Q1AN5/ikjKGHCIyK0cOFuDy0YzevupEBPqJ3Y5ROQgDDhE5FZ2Xrk8fEx0EGQymcjVEJGjMOAQkVvZ9Uvz+TecvZhI2hhwiMhtVNbrcaxUB5kMuOMWTvBHJGUMOETkNr4/1XxycVyYGoE9VCJXQ0SOxIBDRG5j96mWy8O594ZI6hhwiMgtWITf9uDw8nAi6WPAISK3UHIJqGk0oofKAyMieoldDhE5GAMOEbmFY7XNl4QnDQiEUsE/fURSx99yInILx2ub/9zx9gxE7sEpAWfFihWIioqCl5cX4uPjsXv37g7H79y5E/Hx8fDy8kL//v3x0UcftRqzadMmxMbGQqVSITY2Fps3b3ZU+UTk4uqbTDjT0Pz9OJ5/Q+QWHB5wNm7ciIULF2LRokUoKCjAmDFjMGXKFBQVFbU5/syZM7jrrrswZswYFBQU4E9/+hOeeeYZbNq0yTomPz8fM2fORFpaGg4dOoS0tDTMmDEDe/fudfTmEJEL2nvmIiyCDBEBPugX6CN2OUTkBA4POO+88w7mzJmDuXPnIiYmBllZWQgPD8eHH37Y5viPPvoI/fr1Q1ZWFmJiYjB37lz84Q9/wNtvv20dk5WVhZSUFGRkZGDQoEHIyMjAhAkTkJWV5ejNISIX1HJ5+JjoQJErISJn8XDkyg0GAw4cOICXX37ZZnlqairy8vLafE9+fj5SU1Ntlk2aNAmrVq2C0WiEUqlEfn4+nnvuuVZj2gs4er0eer3e+lyn0wEAjEYjjEajvZslSS19YD+cg/12rt0nmwNOUmRP9twJ+Pl2Lnfqtz3b6NCAU1VVBbPZDI1GY7Nco9GgrKyszfeUlZW1Od5kMqGqqgqhoaHtjmlvnZmZmViyZEmr5bm5ufDx4e7qq2m1WrFLcCvst+NVXgaKazygkAm4dKYAOUUFYpfkNvj5di536HdjY2Onxzo04LS49o69giB0eBfftsZfu9yedWZkZCA9Pd36XKfTITw8HKmpqfD39+/cRkic0WiEVqtFSkoKlEql2OVIHvvtPGv3FgGFxxHlJ2DqZPbbGfj5di536nfLEZjOcGjACQoKgkKhaLVnpaKiotUemBYhISFtjvfw8EBgYGCHY9pbp0qlgkrV+r4zSqVS8h8Ge7EnzsV+O94Ppy8CAAb1FNhvJ2O/ncsd+m3P9jn0JGNPT0/Ex8e32m2m1WqRnJzc5nuSkpJajc/NzUVCQoJ1w9ob0946icg9GUwW5J1uvj1DTE9B5GqIyJkcfogqPT0daWlpSEhIQFJSElauXImioiLMnz8fQPPho5KSEqxevRoAMH/+fLz//vtIT0/HvHnzkJ+fj1WrVmH9+vXWdT777LMYO3Ys3njjDdx3333YsmULtm3bhu+//97Rm0NELmT/uYtoNJgR1MMTYT4mscshIidyeMCZOXMmqqursXTpUpSWliIuLg45OTmIiIgAAJSWltrMiRMVFYWcnBw899xz+OCDDxAWFob33nsPDz30kHVMcnIyNmzYgFdeeQWvvvoqBgwYgI0bNyIxMdHRm0NELmTXL81XT90xIBByWedPTiQi1+eUk4wXLFiABQsWtPladnZ2q2Xjxo3DwYMHO1zntGnTMG3atK4oj4gkatcvlQCAO6KDgJJikashImfivaiISJIq6/X4ubT5ios7BgSIXA0RORsDDhFJ0u6TzXtv4vr4I7BH66soiUjaGHCISJJaDk+N5c01idwSAw4RSY7FIlhvzzD2VgYcInfEgENEkvNzqQ7Vlwzw9VRgRL9eYpdDRCJgwCEiydl55fBU0oAgeHrwzxyRO+JvPhFJTsv5N+NuDRK5EiISCwMOEUlKg96EA+dqAPD8GyJ3xoBDRJKSf7oaJouAiEAfRAT6il0OEYmEAYeIJIWXhxMRwIBDRBKz68oEfzw8ReTeGHCISDLOVV/CuepGeMhlSBoQKHY5RCQiBhwikoyWw1PxEb3QQ+WUewkTUTfFgENEkrHzF85eTETNGHCISBIMJgvyTzcHnHEMOERujwGHiCThYFENLhnMCPT1RGyov9jlEJHIGHCISBJazr8ZEx0EuVwmcjVEJDYGHCKSBF4eTkRXY8AhIpdX1aDHkRIdAGAMJ/gjIjDgEJEEfH+y+eTi2FB/9PZTiVwNEXUHDDhE5PJ2ttw9fCD33hBRMwYcInJpFouA3Sd5/ykissWAQ0Qu7edSHaoaDPD1VCA+opfY5RBRN8GAQ0QureXqqaQBgfD04J80ImrGvwZE5NJa5r/h5eFEdDUGHCJyWZf0Jhw4VwOA598QkS0GHCJyWXmnq2E0C+gX4IPIIF+xyyGiboQBh4hc1rfHywEA43l5OBFdgwGHiFySxSJg+7EKAMCEGI3I1RBRd8OAQ0Qu6ciFOlTU6+HrqUBi/wCxyyGiboYBh4hc0rYre2/G3tobKg+FyNUQUXfj0IBTU1ODtLQ0qNVqqNVqpKWloba2tt3xRqMRL730EoYMGQJfX1+EhYXhsccew4ULF2zG3XnnnZDJZDaPWbNmOXJTiKib2X6s+fyb3w0KFrkSIuqOHBpwHn74YRQWFmLr1q3YunUrCgsLkZaW1u74xsZGHDx4EK+++ioOHjyI//znP/jll19w7733tho7b948lJaWWh8ff/yxIzeFiLqR0rrLOHpBB5kMGM+AQ0Rt8HDUio8dO4atW7diz549SExMBAD84x//QFJSEk6cOIGBAwe2eo9arYZWq7VZtnz5cowcORJFRUXo16+fdbmPjw9CQkIcVT4RdWPfHm8+PDU8vCeCevDu4UTUmsMCTn5+PtRqtTXcAMCoUaOgVquRl5fXZsBpS11dHWQyGXr27GmzfN26dVi7di00Gg2mTJmC1157DX5+fm2uQ6/XQ6/XW5/rdDoAzYfEjEajnVsmTS19YD+cg/2+OdqjZQCA8bcGdaqH7Ldzsd/O5U79tmcbHRZwysrKEBzcetdxcHAwysrKOrWOpqYmvPzyy3j44Yfh7+9vXf7II48gKioKISEhOHLkCDIyMnDo0KFWe39aZGZmYsmSJa2W5+bmwsfHp5Nb5B7a6yE5BvttP4MZ+OGkAoAMHpXHkZNzvNPvZb+di/12Lnfod2NjY6fH2h1wFi9e3GZYuNq+ffsAADKZrNVrgiC0ufxaRqMRs2bNgsViwYoVK2xemzdvnvX7uLg4REdHIyEhAQcPHsSIESNarSsjIwPp6enW5zqdDuHh4UhNTbUJTu7MaDRCq9UiJSUFSqVS7HIkj/2+cduPVcD4YyH69PTCnIfGdPrvCfvtPOy3c7lTv1uOwHSG3QHnqaeeuu4VS5GRkTh8+DDKy8tbvVZZWQmNpuNJuYxGI2bMmIEzZ87g22+/vW4IGTFiBJRKJU6ePNlmwFGpVFCpWh+nVyqVkv8w2Is9cS722347TlYDACbGaODp6WnXe9lv52K/ncsd+m3P9tkdcIKCghAUFHTdcUlJSairq8OPP/6IkSNHAgD27t2Luro6JCcnt/u+lnBz8uRJfPfddwgMDLzuzzp69CiMRiNCQ0M7vyFE5HIsFsF6gjFnLyaijjjsMvGYmBhMnjwZ8+bNw549e7Bnzx7MmzcPU6dOtTnBeNCgQdi8eTMAwGQyYdq0adi/fz/WrVsHs9mMsrIylJWVwWAwAABOnz6NpUuXYv/+/Th79ixycnIwffp0DB8+HKNHj3bU5hBRN8DZi4mosxw6D866deswZMgQpKamIjU1FUOHDsWaNWtsxpw4cQJ1dXUAgPPnz+OLL77A+fPnMWzYMISGhlofeXl5AABPT09s374dkyZNwsCBA/HMM88gNTUV27Ztg0LB2UyJpKxl9uIx0Zy9mIg65rCrqAAgICAAa9eu7XCMIAjW7yMjI22etyU8PBw7d+7skvqIyLW0zF48IYaT+xFRx3gvKiJyCZy9mIjswYBDRC6BsxcTkT0YcIjIJWw/xquniKjzGHCIqNu7bDDjh1NVAHj+DRF1DgMOEXV735+qgt5kQZ+e3hioafuec0REV2PAIaJur+XqqYkxwZ26NQMREQMOEXVrFouA7Zy9mIjsxIBDRN3akQt1qOTsxURkJwYcIurWOHsxEd0IBhwi6tY4ezER3QgGHCLqtjh7MRHdKAYcIuq2Wib34+zFRGQvBhwi6ra+5dVTRHSDGHCIqFvi7MVEdDMYcIioW+LsxUR0MxhwiKhb+vpIKQDOXkxEN4YBh4i6nSajGblHmy8Pv+e2MJGrISJXxIBDRN3Ot8cr0KA3oU9Pb4zo10vscojIBTHgEFG380XhBQDA1NtCIZfz8BQR2Y8Bh4i6FV2TEd+eaL48/F4eniKiG8SAQ0TdSu7RchhMFtwS3AOxof5il0NELooBh4i6lS2FJQCa997w6ikiulEMOETUbVQ16JF3uhoAD08R0c1hwCGibiPnp1KYLQKG9lUjMshX7HKIyIUx4BBRt9Fy9RT33hDRzWLAIaJu4XxNI/afq4FMxsn9iOjmMeAQUbfw5aHmWzMkRgVA4+8lcjVE5OoYcIioW/jiUMvhqT4iV0JEUsCAQ0SiO1VRj2OlOigVMkyJCxG7HCKSAAYcIhJdy8nFY6N7o5evp8jVEJEUMOAQkagEQfjt8NQwnlxMRF3DoQGnpqYGaWlpUKvVUKvVSEtLQ21tbYfvmT17NmQymc1j1KhRNmP0ej2efvppBAUFwdfXF/feey/Onz/vwC0hIkc5fL4OZ6sb4aWUY2KMRuxyiEgiHBpwHn74YRQWFmLr1q3YunUrCgsLkZaWdt33TZ48GaWlpdZHTk6OzesLFy7E5s2bsWHDBnz//fdoaGjA1KlTYTabHbUpROQgLXtvUmJD4KvyELkaIpIKh/01OXbsGLZu3Yo9e/YgMTERAPCPf/wDSUlJOHHiBAYOHNjue1UqFUJC2j7RsK6uDqtWrcKaNWswceJEAMDatWsRHh6Obdu2YdKkSV2/MUTkEGaLgP87zMn9iKjrOSzg5OfnQ61WW8MNAIwaNQpqtRp5eXkdBpwdO3YgODgYPXv2xLhx4/D6668jODgYAHDgwAEYjUakpqZax4eFhSEuLg55eXltBhy9Xg+9Xm99rtPpAABGoxFGo/Gmt1UKWvrAfjgH+91s75mLKNfp4e/lgaSong7rB/vtXOy3c7lTv+3ZRocFnLKyMmsouVpwcDDKysrafd+UKVMwffp0RERE4MyZM3j11Vfxu9/9DgcOHIBKpUJZWRk8PT3Rq1cvm/dpNJp215uZmYklS5a0Wp6bmwsfHx87t0zatFqt2CW4FXfv94bTcgByxPobsD13q8N/nrv329nYb+dyh343NjZ2eqzdAWfx4sVthoWr7du3DwAgk8lavSYIQpvLW8ycOdP6fVxcHBISEhAREYGvvvoKDz74YLvv62i9GRkZSE9Ptz7X6XQIDw9Hamoq/P39O9wWd2E0GqHVapGSkgKlUil2OZLHfgMGkwV/fnMHABMW3H07kvoHOuxnsd/OxX47lzv1u+UITGfYHXCeeuopzJo1q8MxkZGROHz4MMrLy1u9VllZCY2m81dKhIaGIiIiAidPngQAhISEwGAwoKamxmYvTkVFBZKTk9tch0qlgkqlarVcqVRK/sNgL/bEudy537tOlaPusgnBfiqMjtZAIW//f3y6ijv3Wwzst3O5Q7/t2T67A05QUBCCgoKuOy4pKQl1dXX48ccfMXLkSADA3r17UVdX124QaUt1dTWKi4sRGhoKAIiPj4dSqYRWq8WMGTMAAKWlpThy5AjefPNNezeHiETScvXU3UNDnRJuiMi9OOwy8ZiYGEyePBnz5s3Dnj17sGfPHsybNw9Tp061OcF40KBB2Lx5MwCgoaEBL7zwAvLz83H27Fns2LED99xzD4KCgvDAAw8AANRqNebMmYPnn38e27dvR0FBAR599FEMGTLEelUVEXVvjQYTtD837+Hl1VNE5AgOnXRi3bp1eOaZZ6xXPN177714//33bcacOHECdXV1AACFQoGffvoJq1evRm1tLUJDQzF+/Hhs3LgRfn5+1vcsW7YMHh4emDFjBi5fvowJEyYgOzsbCoXCkZtDRF1k27EKNBrM6Bfgg2HhPcUuh4gkyKEBJyAgAGvXru1wjCAI1u+9vb3xzTffXHe9Xl5eWL58OZYvX37TNRKR820+2Dzz+L23hXV40QER0Y3ivaiIyKnOVF3CdycqIZMBD8X3FbscIpIoBhwicqpP8s4CAMYPDEZUkK+4xRCRZDHgEJHT1DcZ8b8Hmg9PPT46UtxiiEjSGHCIyGn+98B5NOhNuCW4B+645frTTRAR3SgGHCJyCotFsB6emp0cyZOLicihGHCIyCl2/FKBs9WN8PPywIMj+ohdDhFJHAMOETnFv384CwCYdXs4fDwdOkMFEREDDhE53qmKeuw+WQW5DHgsKVLscojIDTDgEJHDZV8592ZijAbhAT7iFkNEboEBh4gcqq7RiE0HSgAAs3lpOBE5CQMOETnU/+wvxmWjGYNC/JDUP1DscojITTDgEJHDmC0CPsk/C4CXhhORczHgEJHDbDtWjvM1l9HTR4n7h/PScCJyHgYcInKYf/9wBgDwXyP7wUupELkaInInDDhE5BDHSnXY8+tFKOQypI2KELscInIzDDhE5BAtt2WYPDgEYT29xS2GiNwOAw4RdbmLlwzYXMBLw4lIPAw4RNTlNuwrgt5kQVwffyRE9BK7HCJyQww4RNSljGYL1uSfAwDMTo7ipeFEJAoGHCLqUrlHy1Fa14RAX09MHRoqdjlE5KYYcIioywiCgFXf/woAeCSRl4YTkXgYcIioy3xztBwHi2qh8pDjEV4aTkQiYsAhoi5hMFnw318fAwDMG9MfGn8vkSsiInfGgENEXWLNnnM4W92IoB4qzL9zgNjlEJGbY8AhoptW22jAe9tPAgBeSL0VPVQeIldERO6OAYeIbtp720+h7rIRg0L8MD0hXOxyiIgYcIjo5vxa2YDV+WcBAIvujoFCznlviEh8DDhEdFP+++vjMFkEjB/YG2Oie4tdDhERAAYcIroJe36tRu7P5VDIZfjTXTFil0NEZMWAQ0Q3xGIR8NevfgYA/NfIcERr/ESuiIjoNww4RHRDNheU4EiJDn4qDyyceKvY5RAR2XBowKmpqUFaWhrUajXUajXS0tJQW1vb4XtkMlmbj7feess65s4772z1+qxZsxy5KUR0lUaDCW99cwIA8OTvbkFQD5XIFRER2XLoZBUPP/wwzp8/j61btwIAnnjiCaSlpeHLL79s9z2lpaU2z7/++mvMmTMHDz30kM3yefPmYenSpdbn3t7eXVg5EXXkH7vOoEzXhL69vDE7OVLscoiIWnFYwDl27Bi2bt2KPXv2IDExEQDwj3/8A0lJSThx4gQGDhzY5vtCQkJsnm/ZsgXjx49H//79bZb7+Pi0GktEjleua8JHO08DAF6aPIg31CSibslhASc/Px9qtdoabgBg1KhRUKvVyMvLazfgXK28vBxfffUVPvnkk1avrVu3DmvXroVGo8GUKVPw2muvwc+v7ZMc9Xo99Hq99blOpwMAGI1GGI1GezdNklr6wH44hyv3+62tx3HZaMawcDUmxQS5xDa4cr9dEfvtXO7Ub3u20WEBp6ysDMHBwa2WBwcHo6ysrFPr+OSTT+Dn54cHH3zQZvkjjzyCqKgohISE4MiRI8jIyMChQ4eg1WrbXE9mZiaWLFnSanlubi58fHw6VYu7aK+H5Biu1u/zl4BNhxUAZBivrsbXX38tdkl2cbV+uzr227ncod+NjY2dHmt3wFm8eHGbYeFq+/btA9B8wvC1BEFoc3lb/vWvf+GRRx6Bl5ftXYnnzZtn/T4uLg7R0dFISEjAwYMHMWLEiFbrycjIQHp6uvW5TqdDeHg4UlNT4e/v36lapM5oNEKr1SIlJQVKpVLsciTPFfstCAJ+n30AAi7i7iEhWDBjqNgldZor9tuVsd/O5U79bjkC0xl2B5ynnnrqulcsRUZG4vDhwygvL2/1WmVlJTQazXV/zu7du3HixAls3LjxumNHjBgBpVKJkydPthlwVCoVVKrWV3kolUrJfxjsxZ44lyv1O/uHM8j/9SI8PeR4eUqMy9R9NVfqtxSw387lDv22Z/vsDjhBQUEICgq67rikpCTU1dXhxx9/xMiRIwEAe/fuRV1dHZKTk6/7/lWrViE+Ph633XbbdccePXoURqMRoaGh198AIrJbYXEtXs85BqD5xOLwAB7aJaLuzWHz4MTExGDy5MmYN28e9uzZgz179mDevHmYOnWqzQnGgwYNwubNm23eq9Pp8Nlnn2Hu3Lmt1nv69GksXboU+/fvx9mzZ5GTk4Pp06dj+PDhGD16tKM2h8ht1TYa8OS6gzCaBUyJC8EfRkeKXRIR0XU5dKK/devWYciQIUhNTUVqaiqGDh2KNWvW2Iw5ceIE6urqbJZt2LABgiDgv/7rv1qt09PTE9u3b8ekSZMwcOBAPPPMM0hNTcW2bdugUPByVaKuZLEIeP5/DqGk9jIiAn3wxrShnT6HjohITA6d6C8gIABr167tcIwgCK2WPfHEE3jiiSfaHB8eHo6dO3d2SX1E1LGPd/2K7ccr4Okhx4pHRsDfS9rH94lIOngvKiJq095fq/F2bvPtGJbcOxiDw9QiV0RE1HkMOETUSmW9Hk+vL4DZIuCB4X0w6/ZwsUsiIrILAw4R2TBbBDy7oQAV9XpEB/fA6w/E8bwbInI5DDhEZOPd7SeRd7oa3koFVjwyAj6eDj1Vj4jIIRhwiMhq1y+VWP7tSQBA5oNDEK1p+/5uRETdHQMOEQEASusuY+HGQggC8HBiP9w/vI/YJRER3TAGHCKCwWTB058W4OIlAwaH+ePPU2PFLomI6KYw4BC5uSajGU+s2Y/952rgp/LAikdGwEvJSTOJyLXx7EEiN1bfZMTcT/Zj75mL8FLKseLREYgI9BW7LCKim8aAQ+Smai4ZMPvfP+LQ+Tr4qTywavbtGBkVIHZZRERdggGHyA1V6JqQtupHnCivRy8fJVb/IRFD+nKmYiKSDgYcIjdTfLERj67ai3PVjdD4q7B2TiIvByciyWHAIXIjpysb8Og/96K0rgnhAd5YN2cU+gX6iF0WEVGXY8AhchNHL9ThsVU/ovqSAbcE98DaOYkIUXuJXRYRkUMw4BC5gQPnajD73z+ivsmEwWH+WP2HkQjsoRK7LCIih2HAIZIwQRDwf4dL8dKmw2g0mJEQ0Qv/evx2+HspxS6NiMihGHCIJKqyXo9XPz+CrUfLAABjooPwcVo8b55JRG6Bf+mIJEYQBHxx6AJe++IoahuN8JDL8OT4W/Dk+Fvg6cHJy4nIPTDgEElIha4Jiz4/Au3P5QCA2FB/vDV9KAaHcY4bInIvDDhEEiAIAj4vLMHiL35G3WUjlAoZnv5dNP6/OwdAqeBeGyJyPww4RC6uXNeERZt/wrZjFQCAuD7+eGvabYgJ9Re5MiIi8TDgELmoS3oT1v9YhPe2n4SuyQSlQoaFE2/FE2P7c68NEbk9BhwiF1NR34RP8s5i7Z4i1F02AgCG9lXjrWm3YWAIb7lARAQw4BC5jNOVDfjn7l+x6WAJDCYLACAqyBdPjO2P6fF94cG9NkREVgw4RN3cgXM1+HjnaWiPlUMQmpcN79cTfxw7ACmxGijkMnELJCLqhhhwiLqh+iYjvjtRiTX5Z7HvbI11+cSYYPxx3AAkRPSCTMZgQ0TUHgYcom6ior4J236uwDdHy5B3ugpGc/PuGk+FHPcPD8MTY/vjlmCeY0NE1BkMOEQiOlfdiG9/qcI3R8txsKjGeggKAPoH+eKuIaFIS4qAxp93/SYisgcDDpETleuaUFhciwNnq/FloQKl+d/bvH5beE+kxmowabCGe2uIiG4CAw6Rg1zSm/BTSR0Ki2txqLgWhcW1KK1rumqEDB5yGUb1D8SkwRpMjNUgVO0tWr1ERFLCgEN0k5qMZpyrbsSZqks4W30Jv1Y24PD5OvxSXg+LYDtWLgNu1fhhaB9/eNYVYeGMiQjy9xGncCIiCXNowHn99dfx1VdfobCwEJ6enqitrb3uewRBwJIlS7By5UrU1NQgMTERH3zwAQYPHmwdo9fr8cILL2D9+vW4fPkyJkyYgBUrVqBv374O3BpyVyazBVUNBlTUN6Gsrqk5zFRfwtmq5scFm70ytkLVXhgW3hO3hffEsPCeGNJHDV+VB4xGI3JyzkHtrXTilhARuQ+HBhyDwYDp06cjKSkJq1at6tR73nzzTbzzzjvIzs7Grbfeir/+9a9ISUnBiRMn4OfXfE7CwoUL8eWXX2LDhg0IDAzE888/j6lTp+LAgQNQKBSO3CSSAKPZAt1lI+quedQ2GlFR34QKnR4V9c2PyvomVF8y2Jz82xZ/Lw9EBfkiMsgXkYG+iA3zx7Dwnjw5mIhIJA4NOEuWLAEAZGdnd2q8IAjIysrCokWL8OCDDwIAPvnkE2g0Gnz66af44x//iLq6OqxatQpr1qzBxIkTAQBr165FeHg4tm3bhkmTJjlkW8ixLBYBRosFJrMAk/m37w0mCwxmM5qMFuhNFhhMFuhN5qu+t+Cy0YzLBhMu6c24bDSj0WBCo96MRoMZjUYzGvUmNOhN1lBzyWC2uz65DAjqoYLG3wv9An0QFdgcZqKCfBAZ6IsAX0/OS0NE1I10q3Nwzpw5g7KyMqSmplqXqVQqjBs3Dnl5efjjH/+IAwcOwGg02owJCwtDXFwc8vLy2gw4er0eer3e+lyn0wEAjEYjjEZjl9Vf3aDHip1nbJa1+z/+V+0SENpeDOHKKy3L2h4n2LwuCLbvE6580/J9y+u/vdb83GyxoLxMjq9qCyCg+R9qy5X3WQThynsFWITflpstAixXLbMIAiyW35abLVceggCLRYDJ0jy2Zbk10Fx57my+KgXUXkr4eyuh9vaA2luJ3j1U6O2nQrCfJ3r7qdC7hwrBfioE+Hp2OGOwyWSy62e3fO668vNH7WO/nYv9di536rc929itAk5ZWRkAQKPR2CzXaDQ4d+6cdYynpyd69erVakzL+6+VmZlp3Zt0tdzcXPj4dN0JnhWXgdWF3aqldpIDFyvFLsJKLhPgIQOUcsBDBnjImx+/PW9+3VMBqOTNXz3lLc8Fm+UqBeCjEODtAfh4AN4egEJmAqBv/YMbmx+N5cA5ND8cRavVOnDtdC3227nYb+dyh343NjZ2eqzd/xovXry4zbBwtX379iEhIcHeVVtdu6tfEITr7v7vaExGRgbS09Otz3U6HcLDw5Gamgp/f/8brvNaFy8ZcFHd+p9DGdqu6+pyZe0ul9kMsB0ns1kmk139veyqcS2vyaxjrO+VAXKZDBazGSdOHEdsTAwUCgXkMtmV15p/glz229jmB6xjFHLZVeNl1ucechnkcsBDLodcduWrHM3LZTIoFXJ4KJrHeSjkUMptl0n5kI/RaIRWq0VKSgqUSp5o7Gjst3Ox387lTv1uOQLTGXYHnKeeegqzZs3qcExkZKS9qwUAhISEAGjeSxMaGmpdXlFRYd2rExISAoPBgJqaGpu9OBUVFUhOTm5zvSqVCiqVqtVypVLZpR8GTU8lXpoS22Xrcyaj0YicumO4KylS8r8g3UlXfwapY+y3c7HfzuUO/bZn++wOOEFBQQgKCrL3bZ0SFRWFkJAQaLVaDB8+HEDzlVg7d+7EG2+8AQCIj4+HUqmEVqvFjBkzAAClpaU4cuQI3nzzTYfURURERK7FoSeMFBUV4eLFiygqKoLZbEZhYSEA4JZbbkGPHj0AAIMGDUJmZiYeeOAByGQyLFy4EH/7298QHR2N6Oho/O1vf4OPjw8efvhhAIBarcacOXPw/PPPIzAwEAEBAXjhhRcwZMgQ61VVRERE5N4cGnD+/Oc/45NPPrE+b9kr89133+HOO+8EAJw4cQJ1dXXWMS+++CIuX76MBQsWWCf6y83Ntc6BAwDLli2Dh4cHZsyYYZ3oLzs7m3PgEBEREQAHB5zs7OzrzoEjXDODmkwmw+LFi7F48eJ23+Pl5YXly5dj+fLlXVAlERERSY1c7AKIiIiIuhoDDhEREUkOAw4RERFJDgMOERERSQ4DDhEREUkOAw4RERFJDgMOERERSQ4DDhEREUkOAw4RERFJjkNnMu6uWmZPtue261JnNBrR2NgInU4n+bvRdgfst3Ox387FfjuXO/W75d/ta++C0Ba3DDj19fUAgPDwcJErISIiInvV19dDrVZ3OEYmdCYGSYzFYsGFCxfg5+cHmUwmdjndgk6nQ3h4OIqLi+Hv7y92OZLHfjsX++1c7LdzuVO/BUFAfX09wsLCIJd3fJaNW+7Bkcvl6Nu3r9hldEv+/v6S/wXpTthv52K/nYv9di536ff19ty04EnGREREJDkMOERERCQ5DDgEAFCpVHjttdegUqnELsUtsN/OxX47F/vtXOx329zyJGMiIiKSNu7BISIiIslhwCEiIiLJYcAhIiIiyWHAISIiIslhwKF26fV6DBs2DDKZDIWFhWKXI0lnz57FnDlzEBUVBW9vbwwYMACvvfYaDAaD2KVJxooVKxAVFQUvLy/Ex8dj9+7dYpckWZmZmbj99tvh5+eH4OBg3H///Thx4oTYZbmFzMxMyGQyLFy4UOxSug0GHGrXiy++iLCwMLHLkLTjx4/DYrHg448/xtGjR7Fs2TJ89NFH+NOf/iR2aZKwceNGLFy4EIsWLUJBQQHGjBmDKVOmoKioSOzSJGnnzp148sknsWfPHmi1WphMJqSmpuLSpUtilyZp+/btw8qVKzF06FCxS+lWeJk4tenrr79Geno6Nm3ahMGDB6OgoADDhg0Tuyy38NZbb+HDDz/Er7/+KnYpLi8xMREjRozAhx9+aF0WExOD+++/H5mZmSJW5h4qKysRHByMnTt3YuzYsWKXI0kNDQ0YMWIEVqxYgb/+9a8YNmwYsrKyxC6rW+AeHGqlvLwc8+bNw5o1a+Dj4yN2OW6nrq4OAQEBYpfh8gwGAw4cOIDU1FSb5ampqcjLyxOpKvdSV1cHAPw8O9CTTz6Ju+++GxMnThS7lG7HLW+2Se0TBAGzZ8/G/PnzkZCQgLNnz4pdkls5ffo0li9fjr///e9il+LyqqqqYDabodFobJZrNBqUlZWJVJX7EAQB6enpuOOOOxAXFyd2OZK0YcMGHDx4EPv27RO7lG6Je3DcxOLFiyGTyTp87N+/H8uXL4dOp0NGRobYJbu0zvb7ahcuXMDkyZMxffp0zJ07V6TKpUcmk9k8FwSh1TLqek899RQOHz6M9evXi12KJBUXF+PZZ5/F2rVr4eXlJXY53RLPwXETVVVVqKqq6nBMZGQkZs2ahS+//NLmHwCz2QyFQoFHHnkEn3zyiaNLlYTO9rvlD9OFCxcwfvx4JCYmIjs7G3I5/9/jZhkMBvj4+OCzzz7DAw88YF3+7LPPorCwEDt37hSxOml7+umn8fnnn2PXrl2IiooSuxxJ+vzzz/HAAw9AoVBYl5nNZshkMsjlcuj1epvX3BEDDtkoKiqCTqezPr9w4QImTZqE//3f/0ViYiL69u0rYnXSVFJSgvHjxyM+Ph5r1651+z9KXSkxMRHx8fFYsWKFdVlsbCzuu+8+nmTsAIIg4Omnn8bmzZuxY8cOREdHi12SZNXX1+PcuXM2yx5//HEMGjQIL730Eg8Lgufg0DX69etn87xHjx4AgAEDBjDcOMCFCxdw5513ol+/fnj77bdRWVlpfS0kJETEyqQhPT0daWlpSEhIQFJSElauXImioiLMnz9f7NIk6cknn8Snn36KLVu2wM/Pz3quk1qthre3t8jVSYufn1+rEOPr64vAwECGmysYcIhElJubi1OnTuHUqVOtAiR3rt68mTNnorq6GkuXLkVpaSni4uKQk5ODiIgIsUuTpJbL8e+8806b5f/+978xe/Zs5xdEbo2HqIiIiEhyeCYjERERSQ4DDhEREUkOAw4RERFJDgMOERERSQ4DDhEREUkOAw4RERFJDgMOERERSQ4DDhEREUkOAw4RERFJDgMOERERSQ4DDhEREUkOAw4RERFJzv8DED+ufSCKLU8AAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(np.arange(-5,5,0.2), np.tanh(np.arange(-5,5,0.2)));\n",
    "plt.grid();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#inputs x1,x2\n",
    "x1 = Value(2.0, label='x1')\n",
    "x2 = Value(0.0, label='x2')\n",
    "\n",
    "#weights w1,w2\n",
    "w1 = Value(-3.0, label='w1')\n",
    "w2 = Value(1.0, label='w2')\n",
    "\n",
    "b = Value(6.8813735870195432, label='b') #bias\n",
    "x1w1 = x1*w1; x1w1.label='x1*w1'\n",
    "x2w2 = x2*w2; x2w2.label='x2*w2'\n",
    "x1w1x2w2 = x1w1 + x2w2; x1w1x2w2.label = 'x1*w1 + x2*w2'\n",
    "n = x1w1x2w2 + b; n.label='n' #all weights added, without activation function\n",
    "\n",
    "e = (2*n).exp()\n",
    "o = (e - 1)/(e + 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-1.5 1.0 0.5 0.0\n"
     ]
    }
   ],
   "source": [
    "o.backward()\n",
    "print(x1.grad, w1.grad, x2.grad, w2.grad)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import random"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.7071066904050358\n",
      "---\n",
      "x2 0.5000001283844369\n",
      "w2 0.0\n",
      "x1 -1.5000003851533106\n",
      "w1 1.0000002567688737\n"
     ]
    }
   ],
   "source": [
    "x1 = torch.Tensor([2.0]).double();                  x1.requires_grad = True\n",
    "x2 = torch.Tensor([0.0]).double();                  x2.requires_grad = True\n",
    "w1 = torch.Tensor([-3.0]).double();                 w1.requires_grad = True\n",
    "w2 = torch.Tensor([1.0]).double();                  w2.requires_grad = True\n",
    "b = torch.Tensor([6.8813735870195432]).double();    b.requires_grad = True\n",
    "n = x1*w1 + x2*w2 + b\n",
    "o = torch.tanh(n)\n",
    "\n",
    "print(o.data.item())\n",
    "o.backward() \n",
    "print('---')\n",
    "\n",
    "\n",
    "print('x2', x2.grad.item())\n",
    "print('w2', w2.grad.item())\n",
    "print('x1', x1.grad.item())\n",
    "print('w1', w1.grad.item())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Neuron:\n",
    "    \n",
    "    def __init__(self, nin): #number of inputs to a nueron\n",
    "        self.w = [Value(random.uniform(-1, 1)) for _ in range(nin)]\n",
    "        self.b = Value(random.uniform(-1,1)) \n",
    "        \n",
    "    def __call__(self, x):\n",
    "        #w * x + b\n",
    "        act = sum((wi*xi for wi, xi in zip(self.w, x)), self.b)\n",
    "        out = act.tanh()\n",
    "        return out\n",
    "    \n",
    "    def parameters(self):\n",
    "        return self.w + [self.b] #list of weights for each number in the input\n",
    "\n",
    "class Layer:\n",
    "    \n",
    "    #creates a list of neurons, where each Neuron has nin number of elems\n",
    "    def __init__(self, nin, nout):\n",
    "        self.neurons = [Neuron(nin) for _ in range(nout)] \n",
    "        \n",
    "    #applies activation function to each Neuron, outputs a layer of \"activated neurons\"\n",
    "    def __call__(self, x):\n",
    "        outs = [n(x) for n in self.neurons]\n",
    "        return outs[0] if len(outs) == 1 else outs\n",
    "    \n",
    "    def parameters(self):\n",
    "        #matrix - list of params for each neuron\n",
    "        return [p for neuron in self.neurons for p in neuron.parameters()] \n",
    "\n",
    "class MLP:\n",
    "    \n",
    "    def __init__(self, nin, nouts):\n",
    "        sz = [nin] + nouts\n",
    "        self.layers = [Layer(sz[i], sz[i+1]) for i in range(len(nouts))]\n",
    "    \n",
    "    def __call__(self, x):\n",
    "        for layer in self.layers:\n",
    "            x = layer(x)\n",
    "        return x\n",
    "    \n",
    "    def parameters(self):\n",
    "        return [p for layer in self.layers for p in layer.parameters()]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Value(data=-0.0775776874442727)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = [2.0, 3.0, -1.0]\n",
    "n = MLP(3, [4, 4, 1])\n",
    "n(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Value(data=0.9553033462839272),\n",
       " Value(data=-0.9686813993641871),\n",
       " Value(data=-0.9126222386309466),\n",
       " Value(data=0.9138509156778617)]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xs = [\n",
    "    [2.0, 3.0, -1.0],\n",
    "    [3.0, -1.0, 0.5],\n",
    "    [0.5, 1.0, 1.0],\n",
    "    [1.0, 1.0, -1.0]\n",
    "]\n",
    "\n",
    "ys = [1.0, -1.0, -1.0, 1.0] #desired targets\n",
    "ypred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 7.567669324937413\n",
      "1 5.35163122059849\n",
      "2 1.3717664116530643\n",
      "3 0.5232697272223047\n",
      "4 0.1328672122094652\n",
      "5 0.09171099056625065\n",
      "6 0.0717613817238342\n",
      "7 0.05899417774358762\n",
      "8 0.050012185251934904\n",
      "9 0.043336448346051264\n",
      "10 0.03818140701879926\n",
      "11 0.03408397559746218\n",
      "12 0.030751925489735717\n",
      "13 0.027991398404649342\n",
      "14 0.025668669154057626\n",
      "15 0.02368851695798824\n",
      "16 0.02198131903310138\n",
      "17 0.020495010481716418\n",
      "18 0.019189893579436797\n",
      "19 0.018035183510610203\n"
     ]
    }
   ],
   "source": [
    "for k in range(20):\n",
    "    #forward pass\n",
    "    ypred = [n(x) for x in xs]\n",
    "    loss = sum((yout - ygt)**2 for ygt, yout in zip(ys, ypred))\n",
    "    loss\n",
    "    \n",
    "    #backward pass\n",
    "    for p in n.parameters():\n",
    "        p.grad = 0\n",
    "    loss.backward()\n",
    "    \n",
    "    #update\n",
    "    for p in n.parameters():\n",
    "        p.data += -0.1 * p.grad\n",
    "    \n",
    "    print(k, loss.data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Value(data=0.9553033462839272),\n",
       " Value(data=-0.9686813993641871),\n",
       " Value(data=-0.9126222386309466),\n",
       " Value(data=0.9138509156778617)]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ypred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}