AW-SGD.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Implementing Black Box AW-SGD"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python2.7/site-packages/matplotlib/font_manager.py:273: UserWarning: Matplotlib is building the font cache using fc-list. This may take a moment.\n",
      "  warnings.warn('Matplotlib is building the font cache using fc-list. This may take a moment.')\n"
     ]
    }
   ],
   "source": [
    "# Imports of all packages used trough out this tutorial\n",
    "%matplotlib inline\n",
    "from __future__ import print_function\n",
    "import os\n",
    "import numpy as np\n",
    "from matplotlib import pyplot as plt\n",
    "import matplotlib.animation as animation\n",
    "from scipy.stats import norm\n",
    "import math"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Linear Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "N = 1000\n",
    "D = 1\n",
    "sigma = 0.2\n",
    "alpha = 0.01\n",
    "iters = 500\n",
    "batch_size = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAUEAAAE4CAYAAADFI0E4AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xd4FMTWwOHfbDq7QRGpAVSKYldQRFgkSBOwBKQoqFx7\nwQLqFVSuoGKvKOJVY4EraADpgoBowChFET4FkWJAShJ6yYb0ne+PsyEBooBks5vseZ+Hx2Szyc5q\nPMzMOXPGWGtRSqlQ5Qj0AJRSKpA0CCqlQpoGQaVUSNMgqJQKaRoElVIhTYOgUiqklUkQNMZ8aIzZ\nZoz5pcRjw4wxW4wxP/v+XFUWr6WUUmWprGaCHwOdS3n8dWttM9+fr8rotZRSqsyUSRC01qYAe0r5\nkimLn6+UUv7i7z3B+40xK4wxicaYk/z8Wkopddz8GQRHAw2ttRcBGcDrfnwtpZT6R8L99YOttTtK\nfPoBMKO05xlj9PCyUsovrLVH3ZIry5mgocQeoDGmdomv9QBW/tU3WmuD4s+wYcMCPgYdi46lIo8l\nWMZh7bHPrcpkJmiMGQ/EA9WNMZuAYUA7Y8xFgBfYCNxdFq+llFJlqUyCoLW2bykPf1wWP1sppfxJ\nT4yUEB8fH+ghHKRjKZ2OpXTBMpZgGcfxMMezdvbLAIyxgR6DUqryMcZgyzkxopRSFY4GQaVUSNMg\nqJQKaRoElVIhTYOgUiqkaRBUSoU0DYJKqZCmQVApFdI0CCqlQpoGQaVUSNMgqJQKaRoElVIhTYOg\nUiqkaRBUSoU0DYJKqZCmQVApFdI0CCqlQpoGQaVUSNMgqJQKaRoElVIhTYOgUiqkaRBUSoU0DYJK\nqZCmQVApFdI0CCqlQpoGQaVUSNMgqJQKaRoElVIhTYOgUiqkaRBUSoU0DYJKqZCmQVApFdI0CCql\nQpoGQaVUSNMgqJQKaRoElVIhTYOgUiqkaRBUSoU0DYJKqZCmQVApFdI0CCqlQpoGQaVUSNMgqJQK\naRoElVIhTYOgUiqklUkQNMZ8aIzZZoz5pcRj1Ywxc40xa4wxc4wxJ5XFaymlVFkqq5ngx0Dnwx4b\nAnxtrT0L+AZ4vIxeSymlykyZBEFrbQqw57CHrwPG+D4eAySUxWsppVRZ8ueeYE1r7TYAa20GUNOP\nr6WUUv9IeSZGbDm+llJKHZNwP/7sbcaYWtbabcaY2sD2v3ri8OHDD34cHx9PfHy8H4ellKqMkpOT\nSU5OPu7vM9aWzQTNGHM6MMNae77v85eA3dbal4wxg4Fq1tohpXyfLasxKKVUEWMM1lpz1OeVRQAy\nxowH4oHqwDZgGDAVmAjUB/4Eeltr95byvRoElVJlrlyD4InQIKiU8odjDYJ6YkQpFdL8mRjxi8zM\nTJKSkli3LpUmTRrSp08fYmNjAz0spVQFVaGWwykpKXTt2gOv101WVnOczmU4HCnMmjUZt9vt55Eq\npSqSSrcnmJmZSVxcIzIzxwEdS3xlHrGx/UhLS8XlcvltnEqpiqXS7QkmJSXh9bo5NAACdMTrdZOU\nlBSIYSmlKrgKEwTXrUslK6s5YPmcPrRk0cGvZWU1Y/361MANTilVYVWYINikSUOczmWA4X/czDSu\n4w4+AMDp/JnGjRsGdoBKqQqpwu4JnskappLAQhrypGspG9M36J6gUuqgSrcnGBsby6xZk4mN7YfT\n2YO1TOTKKo2pG/41qafXxpWZGeghKqUqoAozEyzi8XhISkpi/fpUGjduSJ9evXCNHAnvvQcTJkCr\nVn4crVKqoqh0JTJHNXMm3HYbjBgBd9114j9PKVWhhV4QBFi7FhISwO2Gt9+GqKiy+blKqQqn0u0J\nHpMzz4QlS2DnToiPh7S0QI9IKRXkKlcQBIiNhUmT4Oqr4dJL4YcfAj0ipVQQq1zL4cN9+SXceis8\n+6zsE5qjzoyVUpVEaO4JlmbdOtknbNUKRo3SfUKlQkRo7gmWpkkTWLwYdu+Gtm1h69ZAj0gpFUQq\nfxCE4n3Ca6+FFi3g++8DPSKlVJCo/Mvhw82eDf37w9NPwz336D6hUpWU7gn+nfXrZZ+wZUvZJ4yO\nLt/XV0r5ne4J/p3GjWHRIti7V/cJlQpxoRkEQfYJJ06UGWGLFpCSEugRKaUCIDSXw4f76ivZJxw2\nDO69V/cJlaoEdE/weK1fD927k3/RRXzaqhW/b9yst9kpVYFpEPwHfpg7l23drqG+10mC9w72Otfr\nbXZKVVAaBI9TcefqT3mM5TzESPqQRAo5epudUhWQZoePU/Ftdp14mcHcysdMoif3sRZvYWu9zU6p\nSio80AMIFsW32Ym5dKYVPzCF7jQ/EMGG39cGcHRKKX/RmaBP8W12xVJpxOUs4qSwbTw4aQJs3hyg\n0Sml/EX3BH0Ov82u2DxiXX3Z8diDRL37Lgc++ojxW7awbl2qZo+VCmLHuieoy2Gfotvsunbtgdfr\nJiurGU7nz77s8BSi3G5WOp3U7NqV1RHn83retURGjua++wYyaNAAhg4dqsFQqQpIZ4KHOeI2uz59\ncLlcB2eKp2a+ylSe5me2cg9XkksU8AtRUduZPv0LOnXqFOi3oJRCS2TKXGJiIgMHziIrawxVaMhH\n1KMhK+lBe7bQBlgKzGPKlPHs3LlTl8tKBZgGwTI2ePATvPyyE6gFTAMW8ygJPMxMbuBzFtIWeAN4\nEqezM1lZl+B0LtNia6UCRPcEy5hkj2eRldUU+dfWhlf5gBXMYwK9eY5HeJtXgGlkZUliJSsLYBod\nOnThvvvu5pxzmurMUKkgozPBY1ScPb4JmAXcDDwJwBmkMoU2rMDBPawlhwIgCVgAzAQuB1rrzFCp\ncqQnRspYUfbY5RoLbET2AMUG0mjFXiKpxndcTH1OR5bMM4BPgK7A92Rl5ZKZ2ZsuXRLweDzl/h6U\nUkfSIHgc3G436ekbGTjwXmCe708m0IMD3E1fGvA5W1gCtKUpcAHwL+Bx5F91K2AdHk8OI0aMCMRb\nUEodRpfD/9DcuXO57ro+5OfXp7CwAfAZUBe4kvY8wKdcxwsU8hZVkKXxoQXYERE92b17qzZlUMpP\ndDnsZ506dWLHjs107Fgf2fOLBboDLZhPBy6nObdiGIOLaA7f/+uItdqUQalgoEHwBLhcLq6/vnuJ\nM8dXAD8BmWxkGa24hHCqkUIrGvAS8ASQCGRSUNCKVatWk5iYyODBT5CYmEhmZmag3opSIUuXwyfo\n0DPHLYFGwE1AMuAFYhnEcv5NAX3pRzJ7gBTCwmoREbGFsLB2ZGU118yxUmVMi6XLUUpKSokzx9WB\n/wGPAe8A2cA0rsTBOPrxIkMYyenAjcDnwA4gFWgI1CA29k5t4KpUGdAgWM5KnjnetGkjkybtJy/v\nHGAFMAeA09jIFLqzinDuwpLNRqABUBXYD2wiOvocRo3qz+233x6ot6JUpaBBMICKl8jtgfMoKqoG\niOFrPqAzZwPdiWUTbYB84Adk1ujliiva0LJlKz17rNQJ0CAYYCkpKXTo0IXc3DbICROQmsLTgT0M\nJILBRNGXPL7FAPHIueQJvufUJDJyH5GRm5k9e6ruEyp1nLREJsDcbjcbNqwlOnoRUlQNUi/oBKrz\nJu3oSyHjyWUQtwADkVrDaN8fyMvz4vFkEx/fiVGjRmn2WCk/8PtM0BizEdiHpErzrbUtDvt6pZwJ\nFjk0abIbOXIXDTQG5tAAN1PYw2pWcicRZFMFaAtURxInrYAriIxcTFTUEs0eK3WMgmY5bIxJBZpb\na/f8xdcrdRCE4qTJF19MZs6c/8PrrYpkhS0wiBiW8T5fci4FdOcT/qQ9cA4wESm7SUIyyDm4XGNJ\nT9+o2WOljiKYlsOmnF4naLlcLm6//XaSkj4nJiYb2IBkg+sD88lmBjdzMmNoymIGciWnAy2AGKTu\ncBayjN6Ix5Or546VKkPlNRPcCxQC71trPzjs65V+JlhSSkoK7dt3IS8vG1nq/gCcgvxdcTfxvMl4\nMnmVDrzOCmA8h587jozsya5deu5Yqb8TTE1VW1tr040xNYB5xpjV1tqUkk8YPnz4wY/j4+OJj48v\nh2EFhtvtZteudN566y2GDn0Oa2sBFyPJky9IxkVLcpjM9zSnOnfQmmwyKV4SNwRakZSUpLWESpWQ\nnJxMcnLycX9fuZbIGGOGAZnW2tdLPBZSM8GSUlJSaNeuAwUFbYAfgQNAJOAmmoW8z5mcTxbd2c1G\nWgAZwBZgH927X8vkyZMCOHqlgltQ7AkaY6oYY1y+j51AJ2ClP1+zInG73fz226/A98AwZGJeALQg\nB7iFU/iIdBaRQ3u+AWoAbqABU6ZM4/nnnw/Y2JWqLPw6EzTGnAFMQdKg4cA4a+2Lhz0nZGeCRUaP\nHs2AAY8imeBkoClwGjAfaEdbkvmMKF7Dy2vEAxHI3yWpXHPNNVx77dV6skSpwwRNicxRB6BBEICM\njAyGDBnC9Okz2bMnC7gK+Bk4GahJfX5mMlVYx3buoD4HSAfOAE4lPHwPUVFbGDDgDiBMj9sphQbB\nCisxMZF77nmRwkKLtOefD7QHCojmO/5LNheRTwJV2ciVyNnkz4FNSNKkJpGR+4mM3KTH7VRIC4o9\nQXX8+vTpQ0zMXiQJkockS1YClhwK+Rdd+ZAIFmPoQBtgNJIscSI1hdXIyzuAx5NNx45X64VOSh2F\nBsEgExsby+zZU4mOjgC+AbogR+1+AloDs3ibjvRmPGN5lEepjdQYDgWWICWZBwDIydnP/fffH4B3\noVTFoUEwCLndbnbs2MKgQfficHyLBLmdyAmSU4DLWchWWtCO3vzGZ1SlCs8AbYDFwLlIK//2jBkz\njmbNLtH2/Ur9Bd0TDHIej4cRI0bw8suvYW09ZA8wD2gOfEc0v/MuBTQjjwQK2MBM5ITJVOAW5FRK\nG6KjlxIRsUgbMKiQoYmRSiY9PZ24uAZY2w5YhrTnT0Rmf98wgKr8hwPczAzmsRJp5Dqdw4/cxcb2\n0/b9KiRoYqSSqVOnDj179gSWIvt/HyPZ4N+A6rxDHr24mU+4hX/zbySjXDIAZgJr8HgctGzZWvsT\nKuWjQbAC6dSpPdHRFwAvIPcc90RacjUHDvAdW7mMs+hJOJ/zJ1XIAEYhRdg1gX9jbRirVu3jgQee\no0aNBqSkpPzVyykVEnQ5XIEU312SiMwGGwCrgShgLVImc4AozuFd0mjObroTRSoWOY4XgdyN3Ab4\nDlhIeDjs2bNdl8eq0tHlcCUUGxvLrFmTiY29g4gIAyxAlr3rkdMjEUAhueziNjL5L9H8AHQiDzm1\n+BnQASmu/gNoSUFBIW+//XZA3o9SwUBnghWQx+NhzJgxDBz4GAUFUUiD1jhgM3LmeCNSJnMmbpaT\nxEbeIo6X2IvcbHcqEjCzgR04HOHcdNONvPDCC9StWzcg70mpsqYzwUrM5XIxYMAAvv12Di5XGGFh\npyOnRiKBi5Dl8alAE1LYRAugB+kkcQAnDiRAnoQEz7Z4vecyduyXxMU10M40KuRoEKzA3G436ekb\neO+9xznzzDORwOdBmnjvRJa/Lrbi4Aqq4sHBIk6iEYuQPcQ7kFMmsUAVIIonnxzKww8/HJg3pFQA\n6HK4kkhMTOS++8aTn78Iub7zDaSDWS2kPGYf0Ix7+YVhQH/OZg7rkGN5s5B7j91ACpDMXXfdwnvv\nvReAd6JU2dBi6RBTnDkuuuy9BrJErgVkATlAGHA+rVlBEjCKurxIBjAD2Uu8GViHJFF2cvXV3di+\nfTdNmzbW/UJV4eieYIgpzhx/R3R0G9+jEUgdYS6yT1gFWMv3XEgLIrmOLUzkZJxMA84HViF7ihFA\nFDNnFrB0aVfGjt1BXFxjRo8eHYB3ppR/6Uywkim643j9+lQaN27IuHGf8+23OUg9YS5SOP0jsI9I\nHLzD2bTkNxJw8gf/Av6LzASLjtxlAmOAscBynnvuaR544AFt2KqCni6HFSB7hQ8+OJXs7GSkYLoj\ncmvdJiSBEsHdVOFpdvEv4CsMUnuYBDwFvIcso9sCzYAvCA/fyMMP38/QoUM1GKqgpUFQASX3Cgch\nTRVqAGcjS9/6wC9AQ1phmcBG3iGCF+gLTEL2EWOACb5/9gAuQ2aKvxAVtZ3p07+gU6dO5f/GlDoK\n3RNUQMm9wjdwOBoAu4DdyB7hNt+zMviBPbTgYq4BJvEJLvYje4iXIkvoHsDjSEmNAW4jN/dKOndO\nYO7cueX9tpQqMzoTDBFFp0zuv/8hJABGIjWFDmSZ7ABOJZIYRrGDVnhIIJz1DEeKqn8CfgU+QDLI\nY5CZYn0iI5exa9dWPX+sgoouh1Wp5s6dS+fO1yCB0AOcg9xnkokEwzDAcBd5PIvhX5zPbNYBnZFM\n8//5ntcGadjwK/AnvXv3ICkpqfzfkFJ/QZfDqlSdOnVizpwZREU5kCDWw/eVSGSZWwMo4H3CSaAu\n7/MLTwCGn5H9w0LgeeQ60D+BesC5TJgwjWnTppX321HqhOlMMER5PB5atmzFqlX1kb2+q5FlrxMp\nrjZAdeqwky+wpAP9OQUPzZClsUFqCnci2eYcwDJgwAAuuugCvfdYBZzOBNXfcrlc3HPPXcBCpJvM\nCKRT9QEkIeIFdpGOg3jC2EkES9hNE/KRO076Ad8DG5D7kV3AFbzzznfceecQTjmlBlOnTg3AO1Pq\n+OhMMIRlZmZSq1YDsrMdFJfOfAU0QYJbFrJHGA/M507CGEEht3IOs/gDKZv5GGnEcAPwoe+5rZEA\n+S2vvfacNmRQAaGJEXVMUlJS6NKlO9nZMRQWnoU0ao1D2vEvRWoCHwZeAQwt8TKRcN7DyXO0wFID\n6WP4EzCNwy92gutIT0+ldu3a5fm2lNLlsDo2Re24Ro4cTGTkUmRvMA3YigTAAuTIXBUAFhNGC8Lp\nwn6+YCOxTEQyy/EUH7NLRO49XgOcTseOnfXeYxW0dCaoDpJZYQIeTxaSBY5FCqulhlDKajYDEEFN\n3iKTthSQAKxlGFJY3QsJmFlIAG0JnER4+GrCwrbqCRNVbnQmqI6bzAo3MnjwQxgDkgBpgyRJiv6i\nigXCySeCe8njdbx8Rz7d+AjohmSJ9yFB9D/AcsBDQUFjcnNj6Ny5GwMGDNBZoQoaOhNUpUpPT6du\n3dOAYchy+E9khucCtiNBrgCAlsQxka28TxgjiMGSjxRh/44EwqKW/W2By4EFxMQsY+7cabjd7vJ8\nWyqEaGJEnbCrrrqKOXNA9glfQfYJayEnR6oBe5Fg2IDabGYS4ewgj1uIQOZ55yKXw1dBbrgr2jNM\nAuYTFTWTDRvWUqdOnfJ9YyokaBBUJywtLY24uMbIXSV9kYxxGjIDbIS04ypATpvkEUF93mQH7cih\nOy7WkA00Bs5CLoHvhzRgaIPcf5xCVNQiHnrobiCMJk0aapG1KjO6J6hOWN26dXnnnVeBG5EZ4H6g\nE7I/uMf3z6rIPqCXfDIYQC6v4mIh+7mGQqT+MB+5KH4xUkYzyffzTiU31/Lyyyt4+WUnAwfOIi6u\nESkpKeX9VlUI05mgOqqMjAweeeQRxo9PQoLgW8ienxf5ezS/xLPDAC+X4WAihSQSy7PkYLFAV+Df\nyHnl5silTpM5vLYwNrYfaWmp2pVGnRCdCaoyU7t2bcaNG8eUKZOAb5HTJDN9X60BNPV9bJAgaFmC\nl0uJoCOZTMFSlXrAhUACclb5O+RkSckACNARr9etHWlUudEgqI5ZQkICr732HHAd8CbQG8kUn4EE\nQJBECYBhG5YrMWylkCVkcBYjkXPGw3zPa4MkSkYhM8wLgM5kZeWxatXqcntfKrTpclgdt4yMDIYM\nGcKaNeuJjo4gOXkJci/JTCQYFv33LAqM4dxGAS9guYNmzOB3ZAl9IXKqJBe54S4eKaGZT1jYIvr0\n6UW7dldoskT9I5odVuVm7ty5dO2aQGFhNWSGtxs5cmeQgmsnkM2l1OQLtvIhp/AMe32hMhwpoUlC\nTpeMQG68cwOtcDp/wuH4nlmzJmtNoTouGgRVuRo1ahQPPDAc6USzgeLjdjlIgXU2cDK12MFEwthD\nITfjYD+nI+U2jyDL6wJgCposUSdKEyOqXPXv35/wcA9SQH0WEvTyKS6tKQS2sw1Le2ATDpZiaMpm\noDaSMGkGtKK0ZEl+/uWaLFF+oUFQlYnY2FgmTvwcuYRpBVIkXYh0nq5NccKkGvkU8gCWF6nFAgq5\njk+BS5CSmbYlfmoaUqTdkJycZIYPf5ZRo0bpuWNVpjQIqjIj2ePnkQucNgPnI0mSHUgXGgdQ1/eY\n5RNyuJoGvIWXp1mDwYUEwkxkaXwGMNX3/W3YsuVOHnhgBrVrn64F1arM6J6gKnMZGRk8+OCDTJr0\nJdbejFzTGYHs952CBDWQxEkkNXEykb3sw3IzMezD+J5bxfe8oqTJGGAWsIvIyFVs3LhOzx2rv6R7\ngipgateuzYQJE1i4cA6xsZMJC2uI9BpsgCRMQH71LJDHdizt8bIRy1KyOfvgmeMYZJlsgTrAY0jS\nZS95eVWoX/8MvfhdnTANgspv3G43aWmp9OvXivBwJ5Igsciv3ZW+Z8k55ALgQWJ4nlNZgJcEspEZ\nYx2ki00+Uk6zCQmmrSksjKNz56v1Qid1QjQIKr9yuVyMGjWKmJifgSeRpqzW96e4mLqonGYM2XQl\ngpGk8gyRGMYjZ5SLlsgjkAvgDdKV5hy6d+/DoEGDNGGi/hG/7wkaY65Czlg5gA+ttS8d9nXdEwwB\nKSkpdO3ag8LClhw4kA6sRPb89iFBMBcJbAaIoQZZTCQKDwX0w7KPCORUyo/AOCSI9kHOH18OLMTl\nWs7s2VpUrURQFEsbYxzAWuS3Nw35Db7BWvt7iedoEAwRHo+HpKQk1q9PJTV1PRMmTPB9JRIJal6k\nlEYueAonitfI5SoMCdRnNQ2QfcJHkbrCotvttFGrOtKxBkGstX77g6T0Zpf4fAgw+LDnWBWa+ve/\n1YLDgrHgshBdtE4u8XikvQXsdiJsdxwW/mMhykInC9bCHAsnW+hsoZuFGtaYSDtw4EC7f//+QL9F\nFUC+2HLUOOXvPcE4iq4nE1t8jymF292KyMjzfJ9FIUviOIp6EhaV0IzF0IUqvIGXZxmFg0KkM/Vc\nZEb4FNKxegHQAmuf4s03V1GrltYTqqMLD/QAAIYPH37w4/j4eOLj4wM2FlV++vTpw8MPP0FeXn9g\nPHLErgZSbJ2F1BTuBSzL2MelwAT2MANDX+azj+eQEybPIlvORQ1aM4FaZGdXp0OHLro8DhHJyckk\nJycf9/f5e0+wJTDcWnuV7/MhyBT1pRLPsf4cgwpuh951XICUv9QAliLniL9HApxs7YRTyCvUphsZ\nJODiN85HAuUZwJfI7LC37+ecAuwkKmoLX389UxMmISZYiqV/BBobY04zxkQCNwDT/fyaqgIpedex\nw1FUB5iNJEv2IouVouVxGAU4GMQpPAMk46EHi30/qRUSAK9Bfq3jkBvx8sjNzadjx6vxeDzl+dZU\nBVFeJTIjKS6RefGwr+tMUAGSPR4yZAjvvJNI0X6gLItBgmEhxcXW+TSjDpNJZxzwH9rj5XskYD4L\nvID0JGyOzCrn0KFDG5o1u1RvtQsRQVEicyw0CKrDSZPWayksPB+pJ8yhuGN1yaRJOKdSlQnsJgfp\nN7OXeOBX5LzyZmSfcAUyu2wDtCUycjGRkUu0prCS0yCoKrT09HTOOOMccnPfBW5FMsdFvydyz7EE\nxIaEsZ1XyOIaCkjgSlaRD/yCBL6ie5Gnc3ij1ipVbmDbtj+1UWslFSx7gkr9I3Xq1OHrr2cQG/sg\n0dGXIl1oDIde8VkI7KEQw8MUMBz4lmSuZykS/Ap8z78CCYBpQH/klMnHHDhQne7de5KYmKhH7kKY\nzgRVUCs6ZfLbb7+zYMG3LFu2Azlq50ECY9EM0QFU4WI8TCaczyhgKAYvNYAHkSTJI8DpSAnNLoqW\nx1Wq/ERYmN5jUtnoclhVOpmZmdSte4avnKYKEvgcSLeZSN9jezmV+iSxmTzC6Usce2gALEYKsotm\nkUcuj/Uek8pFl8Oq0omNjWX27KlER0chM8HdSKfqot6EewAHO9lBJ8L5jaosZRPn8R2yf+ig+L7j\nlhx63/GrZGfXYsyYMeX/xlRAaRBUFYrb7WbHji0MGjQACXx7gSbIDK+a71m5FJLPI1iGYfkGS08c\nyK13AKchxdSPAYuQQFqFgoIDPPjgI9qoNcToclhVWL169WLSpKnIzG4BMqNbg2SFi1TnInYxGekz\n8yS18JKJLI0N0pHmJaSDdSxSXrOJKVMmkJCQUH5vRpU53RNUlV5aWhpxcQ2QGZ4DuBBYiAQ3r+9Z\nZwG/U50wkvBSQBQ30oA9OIFoYDnwPEcWV89lzpxpdOrUqXzflCozuieoKr26devSt28fZDmciyQ/\nbqG4qNqBNC4y7KIBnbGspIAf2cT5ZAI/Iw1ZX0CKq69EzioXALdzzTW99KhdCNCZoKrQJGPcEI/n\nbeA236NFCRAHxSU05yGnT+BGajCSbQwgmolc4nvO70gBtgu5C8ULWNzuVsyaNUuP2FVAuhxWIaOo\ndX9+flNyclYg3ae3AzWR8hkLVEeyyU7Aw4WcyhR2MQHDE4Tj9QU9CZwRSPa4KvALDsdWZs+erkvj\nCkaDoAopJYuqd+3awZw5X5ORsZ/io3NhSOLjRuBdIJrqFPI5hXjxcgPh7MH4nle0R3ghsBM5g7yX\n8eP/x4033hiIt6f+AQ2CKqQlJiZy770fUVDwK1JTGAdkALWBrcCZwDrCgBcwXI+lO4ZfaIvcZtcX\n+BCIRzLHU4A19OqVwIcffqjL4wpAg6AKaZmZmcTFNSIzMxF4G8kE/wKkI/t9pyJLZpn93UAhb+Hl\nAS4miVikFeY0jrzVbj5RUT9y/fXdadfuCm3JFcQ0CKqQV7RXmJtbj7w8D9LT90Uk8OUjSZBMitpz\nXUAYU/Ayiao8waUUHrzV7iNgNpJN/hMppYkiPHw1YWFbmT79C90vDEJaIqNCntvtJi0tlddfv52w\nsM3IMncCskcYgcwGizLJhl+I5lK8XMw+ZrOeU7gWaIRkndORrtdPAz8BhoKCm8nNvZLOnRP0lEkF\npjNBFRJrqtr9AAAUrElEQVTmzp1L585FdxW/DKxGZoO7kCN0G5BscAFhFPA8VejJfroTwS8kIcmU\nPKTM5i1kZrgGCaI1CQ+fy6ZNf+iFTkFEZ4JKldCpUyfmzJlKdHRvwsK2AY2RhgvNkf3CWOAAAIU4\nGcx+nsDyNYXcwE3AfOAkoA4yM/wFOWIXg8wK69KgQSOdEVZAGgRVyOjUqRM7dmxm5Mi7iYxcjhRQ\nO5FscTaS+MhB7jWxJFGVDkTzHLm8QgFhLAfWAjcjxdVDkX3CfOBWCgo66NK4AtLlsApJxVd95iAX\nMw0GuiHdp5ciyZOTgPM4hRQ+IxoHOfShJbtZBjRDZoPTkMLqJCAVyCEy8kNee+05Nm9O00udAkiz\nw0odhcfjYcSIEbzxxrvk50dh7cnIZe4fIoukKCQQbseBm+dZSG/C6M7J/B9ZSFPWAUAvZDntQFp7\n5RMWFk9hoRuncxkOR4p2rQ4ADYJKHSOPx8Ndd93FZ599gXSoPhU5bld0y10MklG29CaPUXh5iJp8\nxlXAeCTTfD6yRC5ECqu1a3WgaWJEqWPkcrl47733CA8PQ/YFByD/a1RFCqtbIwExlwlE056aPIuH\nV/kfYYQhdYRrkFlkaw4NgAAd8XrdJCUllddbUsdBg6BSSOv+3r17IoHsbqQcpugGOkmUyNI4nF/J\n51LqcB6GOURSnf5Is4avkJvtjpSV1Yz161P9/TbUP6BBUCmfdu2uwOn8CdnfewhZFp+HtOCPRDLJ\nOUAOe9hIVyxLyeVH8riIDUi94QLfT8sEEoEngFEYM54VK1bo9Z5BSPcElfIpPm88Dsn4noEsh08H\n/gDCkdpCkJmhJE56sp3ROBiIYTwRSKb5dWRpXA/4HLgUqILD8ROwlwcfvJdnnnlGs8Z+pIkRpf6B\novPGXq+brKzqwDgk2dEAKYGpi9QVhiMBbjvg5TyqMYU0phPFY0AhM4BzfX+eAp5BkittkX3HBcTE\nLGPu3GmaNfYTDYJK/UNFvQnXr0+lXr267Nu3j//853m83mwkE3wy0n36ZKRRawywn2o4GY+HSE6h\nD6PZyc3IbHAFkmX+nMNrCp3OsWRkbNSssR9oEFSqDKWkpNCuXUcKCgqQ4JePnDIpRLLIeUAuDuJ4\nllj6spIehLOcS5HawabAw0AP4DJkJrka2MjAgffyxhtvBOBdVW5aIqNUGXK73WzalEp4eBQS1PKQ\nAHgeMitsCHjxUpUnWcujhDMHSz+WIJnj85AA+DiwBEmwnA3U5803RzF16tQAvCsFOhNU6rikpKTQ\nsWM3cnJyKL7VrjmSQFmEZJQBqnIu9ZjCd8zkFB6jHgXEITfiPYp0srkMWUr/BqRy33134XKdpEft\nyoguh5XyE4/Hw1NPPcXIke/j9VZD7iHxIrPDMGRWWAgUcDIexrOHaKA3zdlJIXKy5DnkHpPLgH3A\nMmT/sK0etSsjGgSV8rOSlztNmTKFDRu2I+24aiDH7rxADA4KeIY8bgJ6EMXPtEfa9z+OBMNcYDJ6\n1K5saRBUqhzJxU6vUVCQhXSiORmZ4VngLGANPYjgv+TxMI35lHpIP8LeSMv+GUf8TKezByNHduP2\n228vt/dRmWhiRKly1KdPH6KjdyAlMw7kdruayOmTPOBkJlOfeC7nKdbxJqsJpwHwCVI2cyQ9alc+\nNAgqVQZiY2OZPXsqMTERyOxvP3ATct3nNmSPMJzfWM+lNKQxGcxjBTW4CelfWPKYXSKQSXT0jzRu\n3DAg7yeUaBBUqoy43W62b9/Mq6++iDFpSNb3JaQBQy5y0mQP+2jMtZxMCqfxIzNpzizkaN4spH7w\nTaAaOTlfMnPmTNLS0gLzhkKE7gkq5QeHXuw0EZndtUDa8RfdZzKU7nzJf1nEozzE/zDAaGRu0g7J\nFicDC+jY8Up69+6ppTPHQRMjSgXY3Llzue66PhQUXEZBQTIcDHK3IWeK44D5nM25TGUlXwGPEEkB\n05FMcQpSYH0xUAVjfiUsbAs9enSnY8f2GhCPQoOgUkGgqIzm228X8NlnSXi9RfccD0EKpl1APieR\nz6fkEUtVevE7O9iLFGE/g9QTNkXuNGkFtCY6egkREYu1lvBvaBBUKsh4PB7efvttnnpqGAUF7ZE9\nwt1IOc0BDH8ynCvoz//Rk/38xBXIPccfAHcgrfy1lvBYaYmMUkHG5XLx+OOPs2rVSmSvrxbShOFU\nYDcWB8OAgWQxC+jPXsANTECWxNq23x80CCpVzs4880wGDboXWIg0af0dKaVpDSxgKq1pCzzOSt7i\nT8KZiPQhPJLWEp44DYJKBcDrr7/O+vW/cdJJVZGuNBaoBkQDTlbjpAVwOquYTwQ1+R6pJRwFdEXu\nSB5FWNhCrSU8QRoElQqQRo0aUa9efaQcphWS+KiKnCuOYj9wHfl8S3N+5CsupTaSUNmM1B6OprBw\nIfXr1w/QO6gcwgM9AKVCWVZWDrLvdytwGnAmcn2nBU7D8ifDiWI5VZhJFoNx8glNkMzxYuBPunS5\njpdeepbo6Gg2b07TVlzHyW/ZYWPMMOBO5BIGgCestV+V8jzNDquQ1bdvXz77bA8wG7mc6QnkLpI6\nSMJkL9KQ4TSasoWpVGceXXiYaPL5CNlHrIMUZEsrripVfiIs7PuQL58JeImMLwhmWmtfP8rzNAiq\nkJWWlkZcXGPkZElH3z97IcXU+5AONN8iZTSXUJX7+R/XUI1CevIx21kOvF/i+zORO0zmExU1kw0b\n1lKnTp3yf2NBIFhKZI46AKVCWd26dXnnnVeB65CEx69AY2QGmIHsEzZE9gCbsZ+bSCCM+dThJ26j\nBaOR4Fd0wqQRcga5Cbm5p1O/fiMGDx6sdx3/DX8HwfuNMSuMMYnGmJP8/FpKVUj33Xcf6emp9O9f\nk5Ytv6Jv34uJitoJXIDsJvVE2nFNAuKwOHmaLAZwNTPwcithyAywB3JF6MPAf4EmFBY+ycsvr6Bu\n3UakpKQE5g0GuRNaDhtj5iEVnwcfQnZ0n0R2bXdaa60xZgRQx1p7RHdIXQ4rdaTiBgxNkf/F7kYa\nsDZE9ghbA+dyFiuYyhK+4RIGchL5/A+ZDY4j1E+XHOty+ISyw9baw0vY/8oHlNY612f48OEHP46P\njyc+Pv5EhqVUhdepUyfmzJnKNdf0Ii9vNTK7S0D2B71IRnkta3BwGVGMZR3fEEZP3mcbbg7dH0wF\nGlJYeBlJSUmVtlN1cnIyycnJx/19/kyM1LbWZvg+HgRcaq3tW8rzdCao1F/weDzcc889jBs3GQl8\nC5HscQckefIhMAzDfxhKHe5kGz3py1LfjSbyPc2Ri5y+oV+/a/n007GBeTPlLBiyw2OBi5C/tjYC\nd1trt5XyPA2CSh1FRkYGQ4YMYfHiH1mzZi2yiLsX+Ak5dncJ8B1XcyUf8hWPE8lHfIpc+LQauRFv\nHw7HLO699x4uuuiCSl9LGPAgeKw0CCp1fObOnUvXrtdRWGiBCGT/bwewAJjOmRxgKuEkY3mI5uSz\nEtlb/B24HHATHb2UiIhFlbqWUIOgUpWYx+Ph/fffZ9Sod9i4MQOHoy2Fha2oUmUpBw58RSwxjKUp\nNfiZnrxPBoMJtWSJBkGlQkRR49b161Np3Lgh8+fP57PP9mJIYChvchfp9ORClpB8xPdW5ms9NQgq\nFaIGDXqUN9+shhRYO+nGCj5iNk/yAonEUJQtluLsOznjjDWceeZZXH11F/r3719p9gmD5cSIUqqc\nnXtuU6KjlwJ1gYV8SWfcNGUQg3iX54gkChgLnAEsZMOGpsyZ04oHHphB7dqnh1xRtc4ElapkMjMz\niYtrRGZmb2AMUqZ7G7GMZwxjqEUa17OODAzwOYfvE7pc/UhPr/j7hDoTVCpExcbGMmvWZCIiPgFu\nQJo5tSWTBK7nC2ZRgx/x0JLzKK1lf37+ZSHVsl+DoFKVkNvt5vXXXyQqKgM5cidlMJYfeI5k7uYs\nprGMO3n/iO/Nzb0spFr2axBUqpLq378/kZFLkPMKyyhusnAXs4jAzeU8xEje4y4iyT34fVFRS0Kq\nZb/uCSpViaWkpNClSwIeTy6yLN6I7BOeAVhcfMQYxlCHLVxPb9JZdrAPocvlIikpiXXrUitkt2ot\nkVFKAVJHOGLECF55ZSRe71CkyVMKcA1QgOE8Hmc59wG96M/y6AyMWYgxDoxpS1ZWc5zOZTgcKRXq\nhIkGQaXUIUaNGsWjj84hN7eooZMHeA8YDkymC/l8zK08xRO8zzMUZ44rZrdqDYJKqUMUl86UPD6X\niHS5mwZAY5YzlXZ8T1UeYB15/Mih3Wi+Jzp6EfPmzQj6GaEGQaXUEVJSUujatQder5usrGaEh39K\nQcHNFC+Re+DiZD4mknrEcD2ppJVSS1gRzhxrnaBS6ghut5u0tFRGjuzGkCHZ9O17GU7nT5Rsz+/h\nMXrRhOnUYykHaEWVw35KR7xed6WpJdSZoFIhrHiJfBOwAZiCBMRGQHuuIpJP+IphPM173E3x3Wkj\nGDIkmxdeeC5AIz86nQkqpY6q6HRJWNh/kcasALHAZGAmX7Gd1nzP/YziA+48WE8YHf1jpakl1CCo\nVIi78MILCQ8PA5aWeNQN/Aws4A820JLFnMQ+FtCWunxGTs48unXrFpgBlzENgkqFuKSkJMLC4oFF\nwLwSX1kANAP6kcUt9OYCppLJUm7iysgmfPnll4EYbpnTIKhUiFu3LpUDB1oiS+B+SIJkBPAq0AXp\nP9gNyOElHuZ2JjE+7w9OnTARKsF+vgZBpUJckyYNcTqXIUvgooCXDVwGfA+4gNuB54DbmUN3Osa0\nos2qlXDHHZCTE6ihlwkNgkqFuD59+uBwpCBL4ZIBrweQzKFLZIB5bAxfQeRPP0FmJrRtC1u2lOuY\ny5IGQaVCXFGGODa2H06nLIWdzh7Ext7JO++8Wsrj/Zg1azKu2rUhKQl69IAWLeC77wL9Vv4RrRNU\nSgFHXtjUp08fXC7XXz5+iDlz4JZb4Kmn4L77wBy1PM/v9NicUqp8/fEHdO8Ol1wCo0dDdDQgBdmB\naMmlQVApVf6ysuC222DDBvjiC1L+/LPEWeXybcmlQVApFRjWwiuv4H3jDbruy2ZO9kQC0YBBj80p\npQLDGHjsMebccANjs3MYwBqg5EQnuBowaBBUSvlFcmQMLRnAnXzAx9xKFMX1hFlZzYLmMicNgkop\nv2jSpCHbnRtoxQ9Ek8N3tKEemwFwOn8OmgYMuieolPKLQztZd+BRXmUQb3Ajj7A89iXdE1RKVW6H\nFmFfz6vkcnd0QyaYf7P8tr64nM5ADxHQmaBSys8OL7a+4bLLiL7xRv6IjWVMy9accc5Zfqkd1BIZ\npVRQSklJoWeX7jyWW41h+TdgnSv9UjuoQVApFXRKv/EO/FE7qHuCSqmgk5SUhNfr5tAACIGsHdQg\nqJQqN+vWpZKV1bzUrwWqdlCDoFKq3BQ3cD1SoGoHdU9QKVVugnFPMLxMXk0ppY5BUe1gcWeZZjid\nPx/MDvuzePqv6ExQKVXujqlR6wnSEhmlVEjTEhmllDoGGgSVUiFNg6BSKqRpEFRKhTQNgkqpkKZB\nUCkV0k4oCBpjehpjVhpjCo0xzQ772uPGmHXGmNXGmE4nNkyllPKPE50J/gp0BxaUfNAYczbQGzgb\n6AKMNiYIrqQ/iuTk5EAP4SAdS+l0LKULlrEEyziOxwkFQWvtGmvtOuDwAHcd8Lm1tsBauxFYB7Q4\nkdcqD8H0H1DHUjodS+mCZSzBMo7j4a89wTjwXSsltvoeU0qpoHLUBgrGmHlArZIPITcpP2mtneGv\ngSmlVHkok7PDxphvgUestT/7Ph8CWGvtS77PvwKGWWuXlPK9enBYKeUX5d1Kq+SLTQfGGWPeQJbB\njYGlpX3TsQxSKaX85URLZBKMMZuBlsBMY8xsAGvtb8AE4DdgFnCftopRSgWjgLfSUkqpQAqqEyPG\nmEeMMV5jzCkBHMMzxpj/M8asMMZ8bYypF8CxvOwrNl9hjPnCGFM1gGP5y8L4cnr9q4wxvxtj1hpj\nBpf36x82lg+NMduMMb8EeBz1jDHfGGNWGWN+NcY8GMCxRBljlhhjlvvG83ygxlJiTA5jzM/GmOl/\n97ygCYK+YNMR+DPAQ3nZWnuhtfYiYBowPIBjmQuc6xvLOuDxAI6l1ML48mCMcQCjgM7AucCNxpim\n5T2OEj72jSXQCoCHrbXnApcDAwL178Vamwu0s9ZeDFwAXGmMaR2IsZTwELIl97eCJggCbwD/DvQg\nrLWeEp86gZ0BHMvX1lqv79PFQMBmpX9TGF8eWgDrrLV/Wmvzgc+RgvyAsNamAHsC9folxpFhrV3h\n+9gDrCaA9bjW2gO+D6OQ2BKwf0e+SVVXIPFozw2KIGiMuRbYbK39NdBjATDGjDDGbAL+BbwQ4OEU\nuQ2YHehBBMjhxfdb0OL7QxhjTgcuAo4oQyvHMTiMMcuBDCDZlyANlKJJ1VGTHuV229zfFF0PBZ7g\n0Pv3/DrbOFoBuLV2KDDUt/f0JnBroMbie86TQL61dry/xnGsY1HBxxjjAiYBDx22kilXvlXLxb69\n67nGmLbW2kBsn3QDtllrVxhj4jlKPCm3IGit7Vja48aY84DTgf/zNVmoBywzxrSw1m4vz7GUYjxS\n4uM3RxuLMeZfyLT+Sn+O41jGEkBbgQYlPq/neyzkGWPCkQD4P2vttECPB8Bau98Y8yVwCQHYQwZa\nA9caY7oCMUCsMWastfaW0p4c8OWwtXaltba2tbahtfYMZKlzsb8C4NEYYxqX+DQBWBGIcfjGchUy\npb/Wt/EcLMp7X/BHoLEx5jRjTCRwA1KQH0iGwOyPHu4j4Ddr7chADsIYc6ox5iTfxzHIyi4g/+9Y\na5+w1jaw1jZEfle++asACEEQBEthCewv14vGmF98exvxwCMBHMvbgAuY50v1jw7UQP6qML48WGsL\ngfuRbPkqpEPR6vJ6/cMZY8YDPwBnGmM2GWP8tl1ylHG0Bvohmdjlvt+RqwIxFqAO8K3v/5vFwHRr\n7fwAjeW4aLG0UiqkBeNMUCmlyo0GQaVUSNMgqJQKaRoElVIhTYOgUiqkaRBUSoU0DYJKqZCmQVAp\nFdL+H20rTICh04c5AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x10e011390>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def GenData(N,D,sigma):\n",
    "    \"\"\"Generate Linear Data with Gaussian Noise\n",
    "        N - Number of Data\n",
    "        D - Dimension\n",
    "        simga - noise level \"\"\"\n",
    "    X = np.random.randn(N,D)\n",
    "    w_true = np.random.randn(D+1,1)\n",
    "    # Sort the X's\n",
    "    X = np.sort(X,axis = 0)\n",
    "    # Add Biases\n",
    "    X = np.concatenate((X, np.ones((N, 1))), axis=1)\n",
    "    # Get Y\n",
    "    y = np.dot(X,w_true) + sigma*np.random.randn(N,1)\n",
    "    return w_true, X, y\n",
    "    \n",
    "w_true, X, y = GenData(N,D,sigma)\n",
    "\n",
    "plt.figure(figsize=(5, 5))\n",
    "# Plot the data\n",
    "plt.scatter(X[:, 0], y, s=50)\n",
    "# Plot real line\n",
    "plt.plot(X[:, 0], np.dot(X, w_true), color='r')\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Define all the functions that we'll need\n",
    "\n",
    "def lin_reg_grad(X,y,w):\n",
    "    \"\"\"Returns Gradient of Square Loss\n",
    "    \"\"\"\n",
    "\n",
    "    if np.shape(X)[1] != D+1:\n",
    "        raise Exception('Input Matrix wrong Dimension')\n",
    "        \n",
    "    N = np.shape(X)[0]\n",
    "    gram = np.dot(X.transpose(),X)\n",
    "    grad = (np.dot(gram,w) - np.dot(X.transpose(),y))/N\n",
    "    return grad\n",
    "\n",
    "def cosine_similarity(vec1,vec2):\n",
    "    \"\"\"Calculate the cosin similarity between 2 vectors\"\"\"\n",
    "    return np.dot(vec1.T,vec2)/(np.linalg.norm(vec1)*np.linalg.norm(vec2))\n",
    "    \n",
    "def error(X,y,w):\n",
    "    \"\"\"Returns the Square Error\"\"\"\n",
    "    N = np.shape(X)[0]\n",
    "    res = np.dot(X,w) - y\n",
    "    cost = np.dot(res.transpose(),res)/N\n",
    "    return cost\n",
    "\n",
    "def get_index(N,batch_size):\n",
    "    return np.random.randint(N - batch_size + 1)\n",
    "\n",
    "def get_weighted_index(N,mean = 50):\n",
    "    \"\"\"Returns an index by rounding a sample from a gaussian\n",
    "    along with it's weight and the gaussian random variable\"\"\"\n",
    "    sigma = N/10\n",
    "    s = np.random.randn()\n",
    "    z = s*sigma + mean\n",
    "    index = max(0,min(N-1,int(z)))\n",
    "    if z < -0.5:\n",
    "        weight = 0.0\n",
    "    elif z > N + 0.5:\n",
    "        weight =0.0\n",
    "    else:\n",
    "        weight = 1.0/norm.pdf(z,mean,sigma)\n",
    "    return index,weight,z\n",
    "    \n",
    "def gradient_descent(X,y,w,mu):\n",
    "    \"Performs one step of Gradient descent on square loss objective\"\n",
    "    w = w - mu*lin_reg_grad(X,y,w)\n",
    "    return w\n",
    "\n",
    "def stochastic_gradient_descent(X,y,w,mu,batch_size, index):\n",
    "    \"Performs one step of batch SGD\"\n",
    "    x_batch = X[index:index+batch_size, :]\n",
    "    y_batch = y[index:index+batch_size]\n",
    "    return gradient_descent(x_batch,y_batch,w,mu)\n",
    "\n",
    "def weighted_stochastic_gradient_descent(X, y, w, mu, batch_size, index, weight):\n",
    "    \"Performs one step of weighted SGD with guassian weighting\"\n",
    "    mu = mu*weight\n",
    "    return stochastic_gradient_descent(X,y,w,mu,batch_size,index)\n",
    "\n",
    "def adaptive_weighted_stochastic_gradient_descent(X, y, w, mu, batch_size, tau, index, weight,z,step, base_line=0.0,\n",
    "                                                  rate = 0.000000001):\n",
    "    \"Performs AWSGD and returns both tau,w and the base line\"\n",
    "    w_new = weighted_stochastic_gradient_descent(X, y, w, mu, batch_size, index, weight)\n",
    "    \n",
    "    d = (w - w_new)/mu\n",
    "    v = np.linalg.norm(d)**2\n",
    "    tau = tau - rate*(v - base_line)*(z-tau)\n",
    "    base_line = base_line - (1.0/(step + 1))*(base_line - v)\n",
    "    w = w_new\n",
    "    return w,tau,base_line\n",
    "\n",
    "\n",
    "def learning_rate_generator(initial_rate,time,cst):\n",
    "    return initial_rate/(1 + initial_rate*time*cst)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Perform gd,sgd,wsgd and awsgd\n",
    "\n",
    "num_iter = 100\n",
    "\n",
    "errors_gd = np.zeros([num_iter,1])\n",
    "errors_sgd = np.zeros([num_iter,1])\n",
    "errors_wsgd = np.zeros([num_iter,1])\n",
    "errors_awsgd = np.zeros([num_iter,1])\n",
    "\n",
    "sim_gd = np.zeros([num_iter,1])\n",
    "sim_sgd = np.zeros([num_iter,1])\n",
    "sim_wsgd = np.zeros([num_iter,1])\n",
    "sim_awsgd = np.zeros([num_iter,1])\n",
    "taus = np.zeros([num_iter,1])\n",
    "zeds = np.zeros([num_iter,1])\n",
    "\n",
    "w_init = np.random.randn(D+1,1)\n",
    "w_gd = w_init.copy()\n",
    "w_sgd = w_init.copy()\n",
    "w_wsgd = w_init.copy()\n",
    "w_awsgd = w_init.copy()\n",
    "tau = N/2\n",
    "batch_size = 1\n",
    "base_line = 0.0\n",
    "\n",
    "\n",
    "\n",
    "for i in range(num_iter):\n",
    "    w_gd = gradient_descent(X,y,w_gd,0.1)\n",
    "    \n",
    "    sgd_rate = learning_rate_generator(1.0,i,1)\n",
    "    w_sgd = stochastic_gradient_descent(X,y,w_sgd,sgd_rate,batch_size,get_index(N,batch_size))\n",
    "    \n",
    "    wsgd_rate = learning_rate_generator(0.0001,i,1)\n",
    "    index,weight,z = get_weighted_index(N)\n",
    "    w_wsgd = weighted_stochastic_gradient_descent(X,y,w_wsgd,wsgd_rate,batch_size,index,weight)\n",
    "   \n",
    "    awsgd_rate = learning_rate_generator(1.0,i,10)\n",
    "    #awsgd_rate_mean = learning_rate_generator(0.0000001,i,1)\n",
    "    ad_index,ad_weight,z = get_weighted_index(N,tau)\n",
    "    w_awsgd,tau,base_line = adaptive_weighted_stochastic_gradient_descent(X, y, w_awsgd,\n",
    "                                                            awsgd_rate ,\n",
    "                                                            batch_size, tau, ad_index, ad_weight,z,i,base_line)\n",
    "    \n",
    "    taus[i] = tau\n",
    "    zeds[i] = z\n",
    "    \n",
    "    errors_gd[i] = error(X,y,w_gd)\n",
    "    errors_sgd[i] = error(X,y,w_sgd)\n",
    "    errors_wsgd[i] = error(X,y,w_wsgd)\n",
    "    errors_awsgd[i] = error(X,y,w_awsgd)\n",
    "    \n",
    "    sim_gd[i] = cosine_similarity(w_true,w_gd)\n",
    "    sim_sgd[i] = cosine_similarity(w_true,w_sgd)\n",
    "    sim_wsgd[i] = cosine_similarity(w_true,w_wsgd)\n",
    "    sim_awsgd[i] = cosine_similarity(w_true,w_awsgd)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABIcAAAFCCAYAAABrbL07AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzs3XmcXGWV+P/PyQokJJCENeyIsg07yE4hKosjOG4M8BsV\n5+eu464jMw6dUcdx3EFUcABXxHHHfRylZZdFkC0oihLWQNhDAoTkfP+41aRputNbVd1bfT/v16tf\nqbr3yb2nQnKoOnWe54nMRJIkSZIkSfU0qewAJEmSJEmSVB6LQ5IkSZIkSTVmcUiSJEmSJKnGLA5J\nkiRJkiTVmMUhSZIkSZKkGrM4JEmSJEmSVGMWhyRJkiRJkmrM4lCNRMRfI2JZRDwcEY80fz2lhDhm\nRsSnIuIvzTj+GhH/ExH79Buzql+M90bELyPilZ2OVVJrVCj/nBQRtzTvvygivjng/Asi4tf9cs/v\nIuK9ETGtef7kiHgiIh5q/twUEadGxMadfi2ShlaFnNPMD6/o93z/5vub/scOaMY2KSKmRsQnI+K2\n5rFbIuJTA6759xFxWUQsjYi7I+LSiHhTv/NnR8Tj/XLUtRHxHxExqzOvWqq3KuSefrFsFRErI+K0\nAce/GBGf7/d8SjOnnDbIsX2az/8xIhY288pdEfHjiJjRb/xeEfGjiLi/+XN9RHwoImY3z786Ip5s\n/nk8HBF/joizImK79v9JaKQsDtVLAi/KzFmZuW7z138abGBETB7JsTUZ4hrTgPOBnYCjgFnADsC5\nwBEDYt0lM2cBzwG+AnwuIj44mhgkVUYV8s+rgROA5zVzy17Ar/qdfwXwbeDrwBaZuQFwLLAZsHm/\nS52bmbOBOcDfARsDV0XERqOJUVJblZ5zgAuAg/s9PxhYOODYQcAlmbkKOAnYA9irmaMawO/63ePd\nwKeBjwEbZebGwBuB/SNiar9rfqyZozYATgT2BS6OiLVH85okjUkVck+fVwHXA8cOyBEXUOSePnsB\niwYc25vitVwVEYcAHwGObeaWHYBv9Ythf4rPdxcCz8nMORSf654Edu13zUuauW028HxgefP6O478\nFaudLA7VTwx6sKjmXhRFR88S4OQhjkVE/GuzKn53RHy579uoiNiy+Y3YayPiVvp96OrnVcCmwDGZ\nuTALyzPze5n57wPiDIDMvD8zvw68CTgpItZv3R+HpA4qO//sBfwiM/8KkJn3ZOZ/9zv/SaAnM8/K\nzAebY27OzLdn5p8HXiwzV2bmQooC0r3Au8f8JyOpHcrOOQOLQwdRFHYGHrug+Xgv4PuZuRggMxc1\n3//QvO8C4E2Z+f3MfLQ55veZ+Q+ZuWLgzTPzicy8CjgamEtRKJLUfmXnnj6vAnqA+4AX9zt+AbBD\nRMxpPj+I4ov6Gf2OHQhcmpkrKXLTJZl5LUBmPpiZX+vLQxR57czM/K/MvLc55vbMXJCZffntKc3P\nf3/JzLcAv2nGqAqwOKT+ngv8CdiQojo82LETKRLNIcA2wLrA5wZc52Bge+DwQe5xGMWHs8fGEN8P\ngSnAPsMNlNR1OpF/LgNeFRHviYg9I+Kp/wdGxHOA+cD3Rht48xv/H/L0b9wkVVsncs4FwE4RsV5E\nBLAnxbft6/c7dgDFhyMoctS7I+JNEbHzgGvtB0wDzhvtC83MpcAvMUdpGBFxZkQsjohrRzB224i4\nICKujohrIuLITsQ4AXQi9xARBzWv91PgO8Cr+85l5u08vVPoYIp8dckgxwB+CxweET1RTI+d1u8+\n61Dkp1G/f2r6HuamyrA4VD8/iGIe6APNX/+x37k7MvPzmbkqMx8f4tjxwKcy89bMXAZ8APj7fh+y\nEji52Q30OM80D7i770lE7NqM5aGIWLimwDPzSWAJxVQOSd2n1PyTmd8A3ga8EOgFFkfE+5qn5zV/\n7Z+fvtmM9dGIOGGY13Yn5iapasrOOYtY/QFsV+Dm5riL+x2bClze/C3/Afxn875XRMTtEfGq5rl5\nwJJmMRqAiLi4+dqWRcSBw/xZmKM0EmczRLFhEP8KfCMzdweOAz4/zPg6KfvzFhTFpR81z38bOCIi\n5vU7/xvg4GaReh+K4vRF/Y49VbjOzIuAlwK7Az8GlkSxPloA61PUFPq/f/pY87UvjYiThvmzMjdV\nyJSyA1DHHZOZ5w9x7rYRHNsUuLXf81sp/h71X2vj9jXc/z5gk74nmfl7im/QDgO+tIbfR0RMoZg/\nf/+axkmqrLLzD5n5TeCbUczRfwlwTkRc3bxXUOSnW5tjjwOIiAuB4dYAmI+5Saqa0nMOxRocBzev\nfWHz2EUUHQGLgMv7poRlZgJfAL4QEdOBfwTOiojfUrx/mhcRk/oKRJl5AEBE3MbwX/iaozSszLwo\nIrbsfywitgFOoyhQLgNel5l/BO6iWDsUYD3gjk7GWnGl5p6IWAt4BfAagMy8pjkF7Xigb3HsC4A3\nA38D/DkzH4uIi4DXNY+tRdExRPMavwB+0bz+oRTdSDcB3wBWUbx/+mNz7PuB90fE1xi+3mBuqhA7\nh+pn0DmwTTmCY3cC/f+nsSWwAlg8zHX6/Ap4YYxtUcSXNO91+XADJVVS2fln9aBivaDvAtcCOwN/\noHhj+9KR/P7+mt+cvZjV7deSqqEKOecCikLQgawuDvUVjPqvN/T0QDIfz8zPAw8AOwKXAo8Dxwxz\nv2eIiJkUi7+aozQWZwBvzcy9gfdSFDABPgq8ulmc/DFFZ64KZeeev6Mo3J0exc5id1FsrvHqfmMu\nAHYDXkQzN2XmDRQbcLwIuCIznxj0BRSFr18DOzc7m37LGN4/9Yv1wmFHqSMsDmm0vgm8M4qtEWdS\nzIs9t1+b85qSIcBXKb5p+H5E7BTF1q3TKVbEH1RErN+c0vE54D8z84HxvwxJXWhc+ae56ONRETGz\nudjjkRQfui5rfmP/boqFIP8xItZr/p7tePo3dU/dJyImR0TfbosbUewiJGniGO97Hig+gO1OUQy6\nuHnsOmBrit3InirYRMTbI+KQiFirmV9eDcwErs7Mh4B/Bz4fES/rl8d2A9YZ7MYRMS0i9gS+T9F5\n9OXRvHgpiq3K9we+3eyyPZ3V/0/8NPDfmdlXTPh6OVFOSOPNPa8GzqToANq1+XMgsGtE7ATQ3Ghj\nMfB2nl6c+W3zWP/cdHREHNvvvdE+FEXvS5tD3ge8NiLeFxEbNMdsRpHn+ut7/zSp+dpObV6nZ/g/\nEnWC08rq50cRsbLf819m5stG8fvPomgbvACYDvwc6L894xq/QcvMx5utiAuAn1DsnrEEuBJ45YDr\n/D4iEngC+D3w9sz8FpK6Van5B3iYYqvor1FME7sVeGNmXgqQmf8TEQ82x3w6Ih6nmPZxOsV8/T6v\njIhjKN7k3Emx0OuemXk3kqqk7JxDZt4cEfcA92Tmw81jGRGXU2zScUm/4csodk3ctnntPwIvzdU7\nLH48Im6n+CD2FeBR4Jbm8/7XeV9EvJ0iR90K/Ijiy7Xlo3jtEhSNBA9k5h6DnNsfOBkgMy9rFjXn\nZeaSjkZYTaXlnojYFHgesFtm3tPv1D0R8XOKwlHfeosXUEw/u7jfuAspin39Ow0faN7/1OaX+ncB\nH8vMcwEy8+KIeB5Fkeefi4ZqbqfYrOPUftfZNyIepshNSyjWf9w7M/+whj8LdVAUX5auYUBR9fsq\nRZV4FfClzDxlwJhDKP7j39I89L3M/HDrw5VUV+YiSVUwklzUHHcKcCTFB/jXZOY1HQ1Ukpoi4gPA\n/wespOhcO3GoKUMRsRXFQsZ/03x+EfCZzPxO8/kumXltRHwXOC8zv9Lsov1lZm7W/lcjqV1GUhza\nGNi4uZDVTOAqikW2buo35hDg3Zl5dFujlVRb5iJJVTDCXHQkxRodL4qI5wKfzcx9SwpZUo01F5g+\nH9g+M5+IiG8BP8nMrw4y9hyK6Y5zKaYcnUyxtswXKTpZplBMb/pwRGxLMXVpPYpC+Xsz81cdeEmS\n2mTYaWXNNvm7m4+XRrHd+HyK1cn7G8m8a0kaE3ORpCoYYS46hqK7iMz8bUTMjoiNMnPxMy4oSe31\nMMUSDTMiYhXFGlV3DjYwM48f4hpHDjL2zxSFJEkTxKgWpG62Ge5Gv23t+tkvIq6JiJ9ExI4tiE2S\nBmUuklQFa8hF83n61sR3NI9JUkc1N3L5JMUaencAD2bm/5UblaQqGnFxqNk6/R2KRYGXDjh9FbBF\nZu5GsaPUD1oXoiStZi6SVAXD5CJJqoSI2AZ4J8V26JsCMyNiqA4hSTU2ot3KImIKxRugr2XmDwee\n7/+mKDN/FhGfj4g5mXn/gOsMu6uDpO6QmR2fvmUukjRQFXMRxbfzm/d7vlnz2MDrmIukCaKMXDRC\newEX970XiojvUew0dk7/QeYjaWIYTy4aaefQWcCNmfnZwU5GxEb9Hu9DsdD1/YONzcyu/jn55JNL\nj6HO8U+E19Dt8WeW+t7BXDRB/h51e/wT4TV0e/yZ1c1FwHnAqwAiYl+KaRyDrjdU9p+hf4+6/zV0\ne/wT4TVU3B8othBfK4o9xg8DFg42sOw/xzr/HZoIr6Hb458Ir2G8hu0ciogDgBOA6yLiaiCBkyha\nEzMzzwBeHhFvAlYAy4Fjxx2ZJPVjLpJUBSPJRZn504g4KiL+RLGV/YnlRSypzjLz9xHxVYqp9yuB\nq4Ezyo1KUhWNZLeyi4HJw4w5DTitVUFJ0kDmIklVMJJc1Bz31g6EI0nDysyPAx8vOw5J1Taq3coE\njUaj7BDGpdvjh+5/Dd0ev6qh2/8edXv80P2vodvjVzVMhL9H3f4auj1+mBivQeWaCH+Huv01dHv8\nMDFew3hEK+amjfhmEdnJ+0lqj4ggq7vw4rDMRdLEYC6SVAXdnovAfCRNBOPNRXYOSZIkSZIk1ZjF\nIUmSJEmSpBqzOCRJkiRJklRjFockSZIkSZJqzOKQJEmSJElSjVkckiRJkiRJqjGLQ5IkSZIkSTVm\ncUiSJEmSJKnGLA5JkiRJkiTVmMUhSbW0alXZEUiSJElSNVgcklRLK1aUHYEkSZIkVYPFIUm19MQT\nZUcgSZIkSdVgcUhSLVkckiRJkqSCxSFJtWRxSJIkSZIKFock1ZLFIUmSJEkqWBySVEsWhyRJkiSp\nYHFIUi25W5mkKsgsOwJJkiSLQ5Jqys4hSVWwfHnZEUiSJFkcklRTFockVcGDD5YdgSRJksUhSTVl\ncUhSFVgckiRJVWBxSFItWRySNFYRcWZELI6Ia4c4PzcifhYR10TEdRHxmqGuZXFIkiRVgcUhSbVk\ncUjSOJwNHL6G828FrsnM3YBDgU9GxJTBBlockiRJVTDoGxVJmugsDkkaq8y8KCK2XMOQu4G/aT5e\nF7gvM58cbKDFIUlVsWQJXH552VFIGq2994YNNhj/dSwOSaoli0OS2uhLwK8i4k5gJnDsUAMtDkmq\nitNPh7PPhmc/u+xIJI3Ghz9scUiSxszikKQ2+gDw+8w8NCK2BX4ZEbtk5tKBAy0OSWqniHg28C0g\ngQC2AT6YmacMHLtiBZxwAixY0OEgJVWCxSFJtWRxSFIbHQB8BCAz/xwRfwG2B64cOPDHP+55Kh81\nGg0ajUbnopQ0Jr29vfT29pYdxohk5h+B3QEiYhJwO/D9wcauWgVT/HQo1Zb//CXV0ooVZUcgqctF\n82cwC4HnAxdHxEbAs4FbBhu400499PS0JT5JbTKwkLuge1ptng/8OTNvG+zkypUwbVqHI5JUGRaH\nJNWSnUOSxioizgEawNyIWAScDEwDMjPPAD4KnB0Rv6coIL0vM+8f7FpOK5PUQccC3xzq5KpVMHly\nB6ORVCkWhyTVksUhSWOVmccPc34J8OKRXMvikKROiIipwNHAPw81ZuVKmDSpczFJqhaLQ5JqyeKQ\npCqwOCSpQ44ErsrMe4cacNFFPcycCcuXuwaa1A1avf5ZZGbLLjbszSKyk/eT1B4RQWYOtdZG5UVE\n/vu/Jx/8YNmRSBqPiZCLnvWs5Oaby45E0nh0Qy6KiG8CP8/MrwxxPt/xjmTzzeFd7+pwcJJaYry5\nyMZBSbVk55CkKrBzSFK7RcQ6FItRf29N41xzSKo3p5VJqiWLQ5Kq4MEHIROi0j0HkrpZZi4DNhhu\nnGsOSfXmP39JtWRxSFIVTJ0Ky5aVHYUk2Tkk1Z3FIUm1ZHFIUhWst55TyyRVg51DUr35z19SLa1Y\nUXYEkmRxSFJ12Dkk1ZvFIUm1ZOeQpCpYf32LQ5Kqwc4hqd785y+pliwOSaoCO4ckVYWdQ1K9WRyS\nVEsWhyRVgcUhSVVh55BUb/7zl1RLFockVYHFIUlVYeeQVG8WhyTVksUhSVWw3nrwwANlRyFJdg5J\ndec/f0m1ZHFIUhXYOSSpKlatsjgk1Zn//CXVksUhSVVgcUhSVaxc6bQyqc4sDkmqJYtDkqrA4pCk\nqrBzSKo3//lLqqUVK8qOQJIsDkmqDjuHpHobtjgUEZtFxK8j4oaIuC4i/mmIcadExM0RcU1E7Nb6\nUCXVWatzkZ1DkqrA4pCkqrBzSKq3KSMY8yTwrsy8JiJmAldFxP9m5k19AyLiSGDbzNwuIp4LfBHY\nd7CLZUJEK0KXVDMtzUUWhyRVgcUhSVVh55BUb8PWhjPz7sy8pvl4KbAQmD9g2DHAV5tjfgvMjoiN\nBrve8uXjildSTbU6F1kcklQFFockVYWdQ1K9jeqff0RsBewG/HbAqfnAbf2e38EzP7QB8Mgjo7mj\nJD1TK3KRxSFJVdBXHMosOxJJdWfnkFRvIy4ONadxfAd4e/Nb+zF5+OGx/k5Jal0usjgkqQqmToW1\n1oJHHy07Ekl1Z+eQVG8jWXOIiJhC8WHsa5n5w0GG3AFs3u/5Zs1jz/Dxj/ew6abF40ajQaPRGEW4\nksrQ29tLb29v2WG0NBc98kgPPT3FY3OR1B2qkotara97aObMsiORVGd2Dkn1FjmCPuaI+CqwJDPf\nNcT5o4C3ZOaLImJf4DOZ+YxFYCMizz8/8TOY1N0igszs+NLyrcxF06Yljz/e5oAltVWJuehM4G+B\nxZm5yxBjGsCnganAvZl56CBjMjPZeWc491zYeed2Ri2pXcrKRa0UEXnggclHPgIHH1x2NJLGYry5\naNjOoYg4ADgBuC4irgYSOAnYEsjMPCMzfxoRR0XEn4BHgROHup7TyiSNRatz0RNPuHuipDE7GziV\n5gL4A0XEbOA04IWZeUdEzFvTxVyUWlIV2Dkk1duwxaHMvBgYNk1k5ltHckOLQ5LGotW5aPLk4k3Q\nlBFNrpWk1TLzoojYcg1Djge+m5l3NMcvWdP1LA5JqgLXHJLqreP//C0OSaqCadNclFpS2zwbmBMR\n50fEFRHxD2savN568MADHYpMkoZg55BUbx3/ztyt7CVVQV9xaJ11yo5E0gQ0BdgDeB4wA7g0Ii7N\nzD8NNnijjeDuuzsZniQ9k51DUr11vDhk55CkKrBzSFIb3U6xeP5jwGMRcQGwK/CM4lBPTw833AAX\nXgh77+3OiVI3mKg7J9o5JNWbxSFJtWRxSNI4RfNnMD8ETo2IycB04LnApwYb2NPTw/e/D1/+Mu7m\nKnWJRuPphdwFCxaUF0wL2Tkk1ZvTyiTVksUhSWMVEecADWBuRCwCTgamsXrnxJsi4hfAtcBK4IzM\nvHGo622xBSxa1IHAJWkNVq2yc0iqMzuHJNWSxSFJY5WZx49gzCeAT4zkehaHJLVTRMwG/hvYGVgF\nvDYzfztw3MqVdg5JdWZxSFItWRySVBXz5sGyZbB0KcycWXY0kiagzwI/zcxXRMQUYNDtOOwckurN\nrewl1dLUqRaHJFVDRNE9dNttZUciaaKJiFnAQZl5NkBmPpmZg34is3NIqreO//N3zSFJVTBtGqxY\nUXYUklRwapmkNtkaWBIRZ0fE7yLijIhYe7CBdg5J9WbnkKRaclqZpCqxOCSpTaYAewCnZeYewDLg\nnwcbaOeQVG+uOSSpliwOSaoSi0OS2uR24LbMvLL5/DvA+wcb+MADPXz2szB7NjQaDRqNRqdilDQG\nvb299Pb2tux6HS8OLVtWtCxalZZUJotDkqpkiy3gN78pOwpJE01mLo6I2yLi2Zn5R+Aw4MbBxs6c\n2cN73gPz53c2RkljM7CIu2DBgnFdr+PFoRkzit04Zs3q9J0laTWLQ5KqxM4hSW30T8A3ImIqcAtw\n4mCDXHNIqreOF4dmzSqmllkcklQmi0OSqsTikKR2yczfA3sPN841h6R66/g//3XXdd0hSeWzOCSp\nSjbbDG6/vfjmXpLKYOeQVG8dLw7NmuV29pLKZ3FIUpWsvXaxCOw995QdiaS6snNIqrdSikN2Dkkq\nm8UhSVWz+eZOLZNUHjuHpHqzOCSplqZOtTgkqVpcd0hSmewckurNNYck1dK0abBiRdlRSNJqFock\nlcnOIaneXHNIUi05rUxS1VgcklQmO4ekenNamaRasjgkqWosDkkqk51DUr05rUxSLVkcklQ1W2wB\nt95adhSS6srOIanenFYmqZYsDkmqmq22gr/+tewoJNWZxSGpvpxWJqmWLA5JqpoNNoDHH/d9kqRy\nWBiS6s3ikKRasjgkqWoiiu6hv/yl7Egk1ZHrDUn15ppDkmrJ4pCkKtp6a6eWSSqHnUNSvbnmkKRa\nsjgkqYrsHJJUFotDUr05rUxSLU2dCitWlB2FJD3d1ltbHJJUDqeVSfVmcUhSLdk5JGmsIuLMiFgc\nEdcOM27viFgRES8d6bUtDkkqi51DUr2VsuaQ08oklc3ikKRxOBs4fE0DImIS8J/AL0ZzYYtDkspi\n55BUbx0vDq21Fjz5pB/KJJXL4pCkscrMi4AHhhn2NuA7wD2juXZfcShzrNFJ0tjYOSTVW8dTQISL\nUksqn8UhSe0SEZsCL8nMLwAxmt87e3aRn5YsaU9skjQUO4ekeiulPjxrFjz0UBl3lqSCxSFJbfQZ\n4P39no+qQOR29pLKYOeQVG9TyrjpeuvBgw+WcWdJKlgcktRGewHnRkQA84AjI2JFZp43cGBPT89T\njxuNBo1G46mpZXvv3bF4JY1Cb28vvb29ZYfRcnYOSfVWSnFo/fXhgeFm6ktSG1kckjROwRAdQZm5\nzVODIs4GfjRYYQieXhzqs9VWLkotVVlfIbfPggULygumhewckuqtlOLQnDkWhySVy+KQpLGKiHOA\nBjA3IhYBJwPTgMzMMwYMH/XS0ltvDdddN+4wJWlU7ByS6q20zqH77y/jzpJUmDYNVqwoOwpJ3Sgz\njx/F2NeO9vpbbw3nDdpnJEntY+eQVG+lpACnlUkq29Spdg5Jqqa+NYckqZPsHJLqzeKQpFpyWpmk\nqtpqK1i0CFatKjsSSXVi55BUbxaHJNWSxSFJVbX22sV7pbvuKjsSSXVi55BUb+5WJqmWLA5JqrJt\ntoE//xnmzy87EkndLiL+CjwErAJWZOY+g42zc0iqNzuHJNWSxSFJVfac58BNN5UdhaQJYhXQyMzd\nhyoMgZ1DUt1ZHJJUS5MnF+t5rFxZdiSS9Ew77ggLF5YdhaQJIhjB5z47h6R6KyUFzJnjVvaSyhXh\ndvaSqmuHHeDGG8uOQtIEkcAvI+KKiHjdUIPsHJLqzTWHJNVW39SytdYqOxJJejo7hyS10AGZeVdE\nbEBRJFqYmRcNHHTnnT309BSPG40GjUajo0FKGp3e3l56e3tbdr3IzJZdbNibRWRmsnIlTJ9efCiz\nfVHqPhFBZkbZcYxVXy6aNw/+8AeYO7fsiCSNxUTJRYNZtQrWXRfuvrv4VVJ1dVMuioiTgUcy81MD\njud++yWXXFJSYJLGbby5qJTSzOTJMHMmPPRQGXeXpIKLUkuqqkmTXJRa0vhFxDoRMbP5eAbwQuD6\nwcb6pb1Ub8OmgIg4MyIWR8S1Q5w/JCIejIjfNX/+dSQ3dmqZpNFoRy6aOtXikKTq2mEHp5ZJGreN\ngIsi4mrgMuBHmfm/gw10zSGp3kay5tDZwKnAV9cw5oLMPHo0N7Y4JGmUWp6L7BySVGUuSi1pvDLz\nL8BuIxlr55BUb8OmgOZiZcOVcUY9r83ikKTRaEcusjgkqcpclFpSJ9k5JNVbq+rD+0XENRHxk4jY\ncSS/we3sJbXBqHKRxSFJVWbnkKROsnNIqrdWbGV/FbBFZi6LiCOBHwDPHmpwT3N/xFtugcsua/DK\nVzZaEIKkdmr1NoltMupcdM898OlPw333Ndh22wannNKxWCWNQZfkopZ51rPg9tvhscdgrbXKjkbS\nRGfnkFRvI9rKPiK2pFi8bJcRjP0LsGdmPqMvqP+Wre9/fzG17J//efRBSypXWVu2tjoXHXQQXHcd\nbL01bLIJ/PSn7YhaUrt00/bRg1nTVvZ9dtwRzj0Xdhk260kqS7fnIijy0VFHJT/5SdmRSBqrTm1l\nHwyxlkdEbNTv8T4UBadhJ4y55pCkMWhpLjr6aDj9dPiv/3J6maRqct0hSZ1i55BUb8NOK4uIc4AG\nMDciFgEnA9OAzMwzgJdHxJuAFcBy4NiR3Hj99YupZZI0Eu3IRe99b/HrBRfA44+3J25JGg/XHZLU\nKa45JNXbsMWhzDx+mPOnAaeN9sZ2DkkajXblIoDp0y0OSaqmnXaCb3+77Cgk1YGdQ1K9lVYftjgk\nqSqmTbM4JKmanvtcuOwyGMESkZI0LnYOSfVmcUhS7dk5JKmqttoKVq2CRYvKjkTSRGfnkFRvpRWH\n5syB+4ddtlqS2m/6dBekllRNEbDffnDJJWVHImmis3NIqjc7hyTVnp1DkkYjIs6MiMURce0Q54+P\niN83fy6KiL8Zz/3239/ikKT2s3NIqrfSikOzZ8PSpbByZVkRSFLB4pCkUTobOHwN528BDs7MXYEP\nA18az8323x8uvXQ8V5Ck4dk5JNVbaSlg0iRYd1146KGyIpCkggtSSxqNzLwIGLL/OTMvy8y+dziX\nAfPHc7899oCFC+HRR8dzFUlaMzuHpHortT7s1DJJVeCaQ5La6P8HfjaeC6y1Fuy6K1xxRYsikqRB\n2Dkk1ZvFIUm1N21aURxyq2hJrRQRhwInAu8f77Vcd0hSu9k5JNXblDJvbnFIUhVMmgRTpxYFounT\ny45G0kRG3q/kAAAgAElEQVQQEbsAZwBHZOaQ73Z6enqeetxoNGg0GoOO239/OOus1sYoaWx6e3vp\n7e0tO4yWs3NIqrfIDn5VHhHZ/36veAW8/OVw7LEdC0FSC0QEmRllxzFWA3MRFGug3XEHzJpVUlCS\nRq3MXBQRWwE/ysxn7EQWEVsAvwL+ITMvW8M1npGLhnLXXbDzznDeecXzHXcsvmSTVL5uf18ERT56\n85uT004rOxJJYzXeXFRq59CcOXYOSaoGF6WWNFIRcQ7QAOZGxCLgZGAakJl5BvBBYA7w+YgIYEVm\n7jOee26yCRx+OLzvfXDPPXDEEXDqqeN7HZLUn9PKpHpzWpkk4aLUkkYuM48f5vzrgNe1+r7nnFP8\netFF8O53t/rqkurOaWVSvZW+IPX995cZgSQVpk+3c0hSd9h9d7j+enOWpNayc0iqt1KLQ/PmwZIl\nZUYgSQWLQ5K6xYwZ8KxnwbXXlh2JpInEziGp3kpNARtuCPfeW2YEklRwzSFJ3WSffeDyy8uOQtJE\nYueQVG+lFoc22MDikKRqcM0hSd1kn33giivKjkLSRGLnkFRvpXcO3XNPmRFIUsFpZZK6iZ1DklrN\nziGp3uwckiQsDknqLjvtBIsWwUMPlR2JpInCziGp3kpNATNnwpNPwrJlZUYhSRaHJHWXKVNgt93g\nqqvKjkTSRGHnkFRvpRaHIlyUWlI1uCC1pG7j1DJJIxURkyLidxFx3lBj7ByS6q30FODUMklV4ILU\nkrqNi1JLGoW3AzeuaYCdQ1K9VaI45KLUksrmtDJJ3Wbvve0ckjS8iNgMOAr47zWNs3NIqrfSU4DT\nyiRVgcUhSd1myy3hrrtg1aqyI5FUcZ8G3gvkmgbZOSTV25SyA7BzSFIVWByS1G2mTIF114UHH4Q5\nc8qORlIVRcSLgMWZeU1ENIAYauwvf9nD0qXF40ajQaPR6ESIksaot7eX3t7ell2v9OKQnUOSqsAF\nqSV1o7lz4b77LA5JGtIBwNERcRSwNrBuRHw1M181cOBRR/Xw9rd3PD5JYzSwiLtgwYJxXa/0aWUu\nSC2pClyQWlI36isOSdJgMvOkzNwiM7cB/h749WCFIXDNIanuSk8BTiuTVAVOK5PUjSwOSWoV1xyS\n6s1pZZJEURx66KGyo5Ck0bE4JGmkMvM3wG+GOm/nkFRvpacAO4ckVYFrDknqRvPmwZIlZUchaSKw\nc0iqt9KLQ3YOSaoC1xyS1I3sHJLUKnYOSfVWegqYMQNWrYJHHy07Ekl15ppDkrqRxSFJrWLnkFRv\npReHItyxTFL5LA5J6kYWhyS1ip1DUr1VIgU4tUxS2SwOSepGFocktYqdQ1K9VaI45KLUksrmgtSS\nupHFIUmtYueQVG+VSAF2DkkqmwtSSxqpiDgzIhZHxLVrGHNKRNwcEddExG7tisXikKRWsXNIqrdK\nFIfsHJJUNqeVSRqFs4HDhzoZEUcC22bmdsAbgC+2K5C5c4ut7DPbdQdJdWHnkFRvlUgBLkgtqWwW\nhySNVGZeBDywhiHHAF9tjv0tMDsiNmpHLOusU3ygW7asHVeXVCd2Dkn1VonikNPKJJXN4pCkFpoP\n3Nbv+R3NY23h1DJJrWDnkFRvlUgBTiuTVDYXpJbUrSwOSWoFO4ekeptSdgBg55Ck8rkgtaQWugPY\nvN/zzZrHnqGnp+epx41Gg0ajMeqbWRySOqu3t5fe3t6yw2g5O4ekeqtEccjOIUllc1qZpFGK5s9g\nzgPeAnwrIvYFHszMxYMN7F8cGiuLQ1JnDSzkLliwoLxgWsjOIaneKlEc2nDDojiUCTHU2yxJaiOL\nQ5JGKiLOARrA3IhYBJwMTAMyM8/IzJ9GxFER8SfgUeDEdsbTt2OZJI2HnUNSvVWiODRjBkydCg89\nBOutV3Y0kurI4pCkkcrM40cw5q2diAXsHJLUGnYOSfVWmfrwppvCHYPOxpek9ps2zTWHJHUni0OS\nWsHOIaneKpMC5s+HO+8sOwpJdWXnkKRuNW+exSFJ42fnkFRvlSoO2TkkqSxTp8LKlbBqVdmRSNLo\n2DkkqRXsHJLqrTIpwGllksoUUUwts3tIUrexOCSpFewckupt2OJQRJwZEYsj4to1jDklIm6OiGsi\nYrexBGLnkKQ16UQusjgkqRtZHJLUCnYOSfU2khRwNnD4UCcj4khg28zcDngD8MWxBOKaQ5KG0fZc\nNH26i1JL6j5uZS+pFSwOSfU2bArIzIuAB9Yw5Bjgq82xvwVmR8RGow3EziFJa9KJXOSi1JK60ezZ\n8OijsGJF2ZFI6mZOK5PqrRX14fnAbf2e39E8NiquOSRpnMadiywOSepGkybB+uvD/feXHYmkbmbn\nkFRvUzp9w56enqceNxoNGo0GABtvDPfeC08+CVM6HpWkNent7aW3t7fsMFpqsFxkcUiqtomYi1ql\nb92hjUbduy1JBTuHpHqLzBx+UMSWwI8yc5dBzn0ROD8zv9V8fhNwSGYuHmRsrul+m2wCV15ZTDGT\nVF0RQWZGCfdtay7abTc4+2zYfffWxy6p9crKRa0y3Pui0TjwQPjoR+Ggg1pyOUmj0O25CIp8dMMN\nyY47lh2JpLEaby4aafNgNH8Gcx7wqmYw+wIPDvZhbCScWiZpGG3NRXYOSepW7lgmabzsHJLqbdgJ\nXBFxDtAA5kbEIuBkYBqQmXlGZv40Io6KiD8BjwInjjUYF6WWNJRO5CKLQ5K61QYbwD33lB2FpKqJ\niOnABRTvmaYBP8zMkwYb65pDUr0NWxzKzONHMOatrQjG7ewlDaUTucjikKRutemmvoeS9EyZ+XhE\nHJqZyyJiMnBxRByQmRcPHGvnkFRvlaoP2zkkqUwWhyR1K4tDkoaSmcuaD6dTfP57YLBxdg5J9Vap\nFOCaQ5LKNG0aPPFE2VFI0ujZfS1pKBExKSKuBu4GejPzxsHG2Tkk1VulNo23c0hSmewcktSt/IJN\n0lAycxWwe0TMAv43Ig7JzN8MHPfpT/cwa1bxuNFo0Gg0OhqnpNHp7e2lt7e3Zdcb0Vb2LbvZMFu2\nXn89vPKVcOOgtWxJVdHtW7YOlYte8xo45BA4cczL6kvqpImai8birrtgt91g8Zj2i5U0Ht2UiyLi\ng8CyzPzkgON5113JxhuXFJikcevUVvYd4bdekspk55CkbrXhhvDAA7BiRdmRSKqSiJgXEbObj9cG\nXgBcM9hY1xyS6q1SKWD99Yv1PpYuLTsSSXU0bZrFIUndafLkokB0111lRyKpYjYBzm+uOXQZcF5m\n/mqwga45JNVbpdYcili97tBznlN2NJLqZvp0F6SW1L36dizbYouyI5FUFZl5HbDHSMbaOSTVW+VS\ngLttSCqL08okdTPfQ0kaDzuHpHqrXHHIdYcklcXikKRu5nsoSeNh55BUb5VLAVtsAbfeWnYUkurI\n4pCkkYiIIyLipoj4Y0S8f5DzcyPiZxFxTURcFxGv6URcfdPKJGks7ByS6q1yxaFttoG//KXsKCTV\n0bRprjkkac0iYhLwOeBwYCfguIjYfsCwtwLXZOZuwKHAJyOi7es8Oq1M0njYOSTVW+VSwNZbWxyS\nVA47hySNwD7AzZl5a2auAM4Fjhkw5m5g3ebjdYH7MvPJdgfmtDJJ42HnkFRvldqtDIri0C23lB2F\npDqyOCRpBOYDt/V7fjtFwai/LwG/iog7gZnAsZ0IzGllksbDziGp3iqXArbcsnhjs2JF2ZFIqhuL\nQ5Ja5APA7zNzU2B34LSImNnum86fb+eQpLGzOCTVW+U6h6ZNg402gttuK9YfkqROsTgkaQTuALbo\n93yz5rH+DgA+ApCZf46IvwDbA1cOvFhPT89TjxuNBo1GY8yBrbdesW7a0qUws+2lKKm+ent76e3t\nLTsMSWqpyMzO3SwiR3K/RgM++EE47LD2xyRp9CKCzIyy4xiroXLRD38IZ54J551XQlCSRq2MXBQR\nk4E/AIcBdwGXA8dl5sJ+Yz4JPJyZCyJiI4qi0K6Zef+Aa43ofdFobLst/PznsN12Lb2spDXo9vdF\n0J58JKmzxpuLKtk86KLUkspg55Ck4WTmSordyP4XuAE4NzMXRsQbIuL1zWEfBfaKiN8DvwTeN7Aw\n1C5OLZMkSWNRuWllUEwnc1FqSZ1mcUjSSGTmz4HnDDh2er/HS4AXdzoucFFqSZI0NnYOSVKTxSFJ\n3c7ikCRJGguLQ5LUNH16sZirJHUrp5VJkqSxqGRxyGllksowbZqdQ5K6m51DkiRpLCpZHNp442Ib\n1qVLy45EUp04rUxSt5s/H84/H172Mjj+eAtFkiRpZCpZHIqArbaCv/617Egk1YnFIUndbr/94Etf\nKgpDjz0GZ59ddkSSJKkbVLI4BMW6Q04tk9RJFockdbupU+GYY4rOoXe8A771rbIjkiRJ3aDSxSEX\npZbUSdOmuSC1pInjwAPhvvtg4cKyI5EkSVVX2eLQNttYHJLUWXYOSZpIJk2CV77S7iFJkjS8yhaH\nnFYmqdMsDkmaaI49tigOZZYdiSRJqrLKFoe22Qb+/Oeyo5BUJ1OmFL8++WS5cUhSqzz3ubB8OVx3\nXdmRSJKkKovs4FdJEZEjvd/y5TBnDjzyyOoPbJKqISLIzCg7jrFaUy5aZx24916YMaPDQUkatYmc\ni1rpfe+Dq66CPfcc/PymmxaLV0sam27PRdC5fCSpfcabiypbHALYdlv42c/g2c9uY1CSRq3b3wSt\nKRett16x3tn663c4KEmjNpFzUSvdey985SuwatXg5z/0IVi0yLwnjVW35yKwOCRNBOPNRZXuydlx\nR7jxRotDkjpn+nR47LGyo5Ck1tlgA3jPe4Y+//Ofw2WXwZFHdi4mSZJULZVdcwhghx2K4pAkdcq+\n+8KXv1x2FJLUOfvvD5dcUnYUkiSpTJUuDvV1DklSp5xyCnzyk3DzzWVHIkmdYXFImrgiYrOI+HVE\n3BAR10XEP5Udk6RqqnxxaOHCsqOQVCdbbgn/8i/whje49bOketh3X7j8cndqlCaoJ4F3ZeZOwH7A\nWyJi+5JjklRBlS4O7bAD3HTT0AsoSlI7vO1t8PDD8I1vlB2JJLXfnDmw+eZudy9NRJl5d2Ze03y8\nFFgIzC83KklVVOni0LrrFm9Ybr217Egk1cmUKXDSSfD1r5cdiSR1xn77waWXlh2FpHaKiK2A3YDf\nlhuJpCqqdHEInFomqRyHHVaswbF8edmRSFL7ue6QNLFFxEzgO8Dbmx1EkvQ0ld7KHlYvSn3UUWVH\nIqlOZs+GXXaBCy+EF76w7Ggkqb323x8+8pGyo5DUDhExhaIw9LXM/OFQ43p6ep563Gg0aDQabY9N\n0tj19vbS29vbsutFdnDF1YjI0d7vjDPgssvgrLPaFJSkUYsIMjPKjmOsRpqLPvQhePDBYvcySdVT\nl1zUCatWwbx5cMMNsMkmZUcjdZeq56KI+CqwJDPftYYxlclHksZmvLmoK6aVuZ29pDIccQT84hdl\nRyFJ7TdpEhx4ILz2tfClL8Ftt5UdkaRWiIgDgBOA50XE1RHxu4g4ouy4JFVP5YtDO+xQrDlkIVtS\np+2xByxeDLffXnYkkqokIo6IiJsi4o8R8f4hxjSaH8Suj4jzOx3jWJx1FrzylXDBBbD77vDe9xY7\nN0rqXpl5cWZOzszdMnP3zNwjM39edlySqqfyxaG5c2GtteDOO8uORFLdTJ4Mz3++3UOSVouIScDn\ngMOBnYDjImL7AWNmA6cBf5uZOwOv6HigYzBvHpx4Inzta8X0svvvh+23d2MQSZLqoPLFIYCddoLr\nry87Ckl1dPjhFockPc0+wM2ZeWtmrgDOBY4ZMOZ44LuZeQdAZi7pcIzjttFGcOaZ8IEPwBvfaAe3\nJEkTXVcUh/bcE666quwoJNXRC14A53fFhBBJHTIf6L8iz+3NY/09G5gTEedHxBUR8Q8di67F3vxm\nWLoUvv71siORJEnt1BXFob32giuvLDsKSXW0ySZw333FTj6SNEJTgD2AI4EjgA9GxLPKDWlsJk+G\nL3wB3vc+eOCBsqORJEntMqXsAEZir72KRRElqdMmTSrWPVu+HGbMKDsaSRVwB7BFv+ebNY/1dzvF\nttGPAY9FxAXArsCfBl6sp6fnqceNRoNGo9HicMdvn33gJS+BE04odjKbP7BPSqqZ3t5eent7yw5D\nkloqcgSTyJvbHX6GotPozMz82IDzhwA/BG5pHvpeZn54kOvkSO43UCbMmQM33VTMgZdUroggM6OE\n+5aSizbcsFj3bMMNxxy6pDYoIxdFxGTgD8BhwF3A5cBxmbmw35jtgVMpuoamA78Fjs3MGwdca0zv\ni8qwfDl8+MNw+unFOkTvehdEx/8vIFVTWe+LWqmb8pGkwY03Fw3bOdRvV47DgDuBKyLih5l504Ch\nF2Tm0WMNZM0xFN1DV10FRx3VjjtIqroyc9GMGcWaGxaHJGXmyoh4K/C/rC5UL4yINxSn84zMvCki\nfgFcC6wEzhhYGOo2a68NH/kIvOY18NKXFl/anXhi2VFJkqRWGcm0sqd25QCIiL5dOQZ+IGtrtbxv\n3SGLQ1JtlZaLZsyARx9t9VUldavM/DnwnAHHTh/w/BPAJzoZVydstx184xtw2GHFgv2bbVZ2RJIk\nqRVGsiD1SHblANgvIq6JiJ9ExI4tia4fF6WWaq+0XDRzpsUhSeqzyy7wtrfB61/vFveSJE0Urdqt\n7Cpgi8zcjWLaxw9adN2nWBySNAJtyUV908okSYUPfADuusst7iVJmihGMq1s2F05MnNpv8c/i4jP\nR8SczLx/4MXGuivHFlvAihVw552w6aYj+i2SWqQiu3KUloucViZVQ0VykYCpU+Gzn4XXvhaOOw6m\ndMX+t5IkaSjD7lY2wl05NsrMxc3H+wD/k5lbDXKtca2Cf+SR8KY3wdFtWfZa0khVeIegtuSi446D\nF78Yjj9+fK9BUmt1+w5B3b47UCYcdBC85S1FnpTqqttzEXR/PpI0/lw07LSyzFwJ9O3KcQNwbt+u\nHBHx+uawl0fE9RFxNcU208eONaA1cWqZVF9l5qKZM51WJkkDRcC//Av8x3/AqlVlRyNJksZj2M6h\nlt5snBXp886DU0+FX/6yhUFJGrVu/4ZstLnoHe+ALbeEd76zjUFJGrW65aIqyoQ994QFC4oOS6mO\nuj0XwcTIR1Ldtb1zqEoOOgguuwyeeKLsSCTViWsOSdLgIuCkk+BDH7J7SJKkbtZVxaH114fttoPL\nLy87Ekl14rQySRraS19aLFD9xS+WHYkkSRqrrioOARx6KJx/ftlRSKoTO4ckaWiTJsGZZ8LJJ8Nf\n/1p2NJIkaSy6sjjkLraSOsnikCSt2fbbw3veA69/fbEOkSRJ6i5dVxw66KBiWtnjj5cdiaS6mDnT\n4pAkDefd74YHH4SXvAQuvbQ4lum0XEmSukHXFYdmz4YddigWppakTpgxww83kjScKVOKqf8veAGc\ncAJsvnmRP+fOhV12gf/6L/jjH124WpKkKuq64hC47pCkznJamSSNzIwZ8Na3FkWgX/8aFi+G5cvh\nc5+Dm2+G5z8f5swpOsFf+MLi55vfLDtqSZI0pewAxuLQQ+GjH4WenrIjkVQHTiuTpNGZMqXYYbbP\nwQcXPwD33gs33ABPPFHk1je+ETbYoCgcSZKkcnRlcejAA+Gqq2DZMlhnnbKjkTTROa1Mklpngw2g\n0Vj9fO5cePnLi67wnXZ6+thVq4p1Jp94AmbNgoiOhipJUm10ZXFo5kzYe2/4v/+Do48uOxpJE53T\nyiSpfQ4+GD71Kdhnn6LjaNkyWLmyOJcJa60FkyfD2mvDHnvA1lsXRaJZs4ou8rXXLjV8SZImhMgO\n7jcaEdmq+51yClx9NZx9dksuJ2kUIoLM7Nrvb0ebi5Ysgec8B+67r41BSRq1uuWiie6BB2DSpKLY\nM6X59WXE6m6hO++EK6+EO+4onv/kJ0UX0lln2VGkcnV7LgLzkTQRjDcXdW1xaNGi4tuju+9e/QZC\nUmd0+5ug0eaixx4rdkp8/PE2BiVp1OqWi/R0jz4K++4Lb3lLsW6RVJZuz0VgPpImgvHmoq7crQxg\niy1gq63gwgvLjkTSRDd9Ojz5ZPEjSaqGGTPge9+Df/s3uOiisqORqikizoyIxRFxbdmxSKq2ri0O\nAfzd3xVvCiSpnSJcd0iSqmi77eDrX4eXvhQuvrjsaKRKOhs4vOwgJFVf1xeHfvCDYrFCSWqnmTPd\nsUySquiFL4SvfQ1e8pJis5JVq8qOSKqOzLwIeKDsOCRVX1ev1rPDDsVW9ldeWexeJkntYueQJFXX\n4YfDuefCa15TbCKw5ZYwb15R2F9rrWcuWH3EEfCGN5QSqiRJldTVxaGIoo34O9+xOCSpvSwOSeoT\nEUcAn6HowD4zMz82xLi9gUuAYzPTifBtdthhcNttRa7+61+L3c+WLoXly58+buVKeO97i53RXve6\nUkKVJKlyuro4BPCqV8Hzngcf/jBMnVp2NJImqpkzLQ5JgoiYBHwOOAy4E7giIn6YmTcNMu4/gV90\nPsp6mzEDdtppzWN23RUOOQTmzIGXvawzcUlV19PT89TjRqNBo9EoLRZJw+vt7aW3t7dl1+varez7\nO+ggeNe7ijWIJLVft2/ZOpZcdPjh8M53FlMRJFVDGbkoIvYFTs7MI5vP/xnIgd1DEfF24Algb+DH\ng3UOuXV0ua6+usjps2bBvvvCttvC7NnF8+nTi5/Jk5/+e1auhNtvh1tuKbqSZs4silGTJhUd7Xvu\nCX/7tzBtWjmvSeWo+vuiiNgK+FFm/s0axpiPpC433lzU9Z1DULQEf+lLFocktY/TyiQ1zQdu6/f8\ndmCf/gMiYlPgJZl5aEQ87ZyqY/fd4c47YeFCuOyyYkrarbfCww/D448XPwMXt46A+fOLQtK66xb/\nX3j00WLcypXw2c/Cm98MRx9dFI76ikZDmT696F6aPXt1IeqQQ2Drrdv3ulUvEXEO0ADmRsQiiuL2\n2eVGJamKJkRx6OUvL77Rv/XWYgFCSWo1p5VJGoXPAO/v97yyHQV1N3ky7Lxz8dMK//Zv8Mc/wi9+\nAStWFAWjNVm+vChKXX99UWBasQLe8x544xvhAx8ovphop0y49lpYtKi41zrrFMs0TJ5cnHvyyeIH\nimN77vnMbipVW2YeX3YMkrrDhCgOrbMOHH88nHUWLFhQdjSSJqIZM9zKXhIAdwBb9Hu+WfNYf3sB\n50ZEAPOAIyNiRWaeN/BirvEx8Tz72cXPWN1xB7z//bD55rDPPrDXXkV3ERSFmsceK34iis6k5cvh\nnnvg/vsH73SaMqUo+MycWXQ7rbNOcXz5cvjpT4tr7rTT6i6olSuLY5MmFb938uRi/P33w6abwjnn\nwCabjP31TQStXudDkqpgQqw5BMW3HkcdBX/5iwtTS+1W9bn1wxlLLnrPe2CjjYodbiRVQ0lrDk0G\n/kCxIPVdwOXAcZm5cIjxZ1Os9eGaQxqVu++GK66AK6+ERx4pjk2eDGuvXUxHg6IYNH06bLhhUUCa\nMuBr377pbk88URR+Hn4Yli0ruoImTy52eNtjjzVPfeuzcmWxAczpp8NJJxVT4aZPL4pImUXB6sEH\niy9SMoufZcuKXeMee6woTs2aBTvsUBS81loLLr4YrrqqiGXmzKJwtc46xbmI4hqPP17E/vjjRRyZ\nq1/XuusWhbjtt4dnPauIpQzd/r4IzEfSRDDeXDRhikMAL3gBHHccvPa1bbuFJLr/TdBYctHJJxdv\nVPt9yS+pZGXlouZW9p9l9Vb2/xkRb6BYmPqMAWPPwgWpNYH8+tfwjW8UBZ8nniiKNVAUitZff/Va\nS1B03a63XlHsWbq0KB5df31REFq2DA44APbeuxj/yCPFseXLi58+06cX15k+fXURa9KkoqD00EPF\nNL4bbyyKUHvtBRtvXMS24YbwyU8WxbR26/b3RWA+kiYCi0P9XHghvOY1cNNNdg9J7dTtb4LGkos+\n/nFYvBg+8Yk2BSVp1OqYiyQNbskSuPzyYvrb9Onw7W8XRaLvfrf9nwu6PReB+UiaCCwODXDooUWB\n6NWvbuttpFrr9jdBY8lFn/88XHcdfOELbQpK0qjVMRdJGpkVK+CYY2DuXPjoR4tupBUrinN9U94e\neaToLNp++2KK2q9/Dd/6Ftx3XzGVfN681V1Qs2YV3UgbbFBM4Zs7F7baqig8dXsuAvORNBG4lf0A\nJ58Mr389nHDCM+ddS9JYuZW9JEndY+pU+M53il2Nn/vcoghUFHKK8+usUxSEli6FP/yhKBztsgv8\n/d8XRZ/Fi4tupL71k+6+u1jj9N57i+6k++6D//s/2HrrUl+mJLXMhCufHHJIMdf4a1+DE08sOxpJ\nE8XMme5WJklSN1lnnWJHtuFkFl1Es2a1PyZJqqqS1vRvnwj41KeKXRTuv7/saCRNFHYOSZI0MUVY\nGJKkCVccgmKngpe9DD7wgbIjkTRRWBySJEmSNFFNyOIQwEc+Aj/6EVxySdmRSJoIZs60OCRJkiRp\nYpqwxaHZs4vpZa97XbEbgSSNx4wZrjkkSZIkaWKasMUhgGOPhT32gDe/uVhoTpLGymllkiRJkiaq\nCV0cioAvfhGuvBLOPLPsaCR1M6eVSZIkSZqoJtxW9gPNmAHf/S4cdBDstBPst1/ZEUnqRn3TyjKL\nwrMkSZIkTRQTunOoz3P+X3v3HmNFecZx/PvssoqCIqhg5KKoaK0RqzGCV7aiCJh4ibGBNqaatGka\nTbU1RjRp0j/8p4kNYmxtjNZUY61pbStN1Foi6wVB0a3xBoo0ImBFQYuCCOzu0z/eme5wmLN72GXP\nzHvO75NM2DM7Z+Z5z+XHmWdn5pwEDz0EV1wBnZ1FVyMiMRo2LEw7dxZdiYiIiIiIyP7VFM0hgNmz\nwylmc+fCW28VXY2IxEjXHRIRERERkUbUNM0hgCuvhIULYeZMWLq06GpEJDYjR+oby0REREREpPE0\nVbSj4IwAAAvcSURBVHMIYP58ePRRmDcP7ruv6GpEJCY6ckhERERERBpR0zWHAC68EF58Ee66K3zd\n/ebNRVckIjFQc0hERERERBpRUzaHAKZMgddeg4kT4dRT4ZFHoKen6KpEpMx0WpmIiIiIiDSipm0O\nARx0ENx5Z/iq+0WL4Mwz4emnw1dVi4hU0pFDIiIiIiLSiJq6OZQ65xx4+WW4/Xa4+WY47TS4/37t\nBIrInkaMgC++gK1bYcMGePttWL4c1q1TU1lEREREROJlXsc9GjPzem5vINxhyRK4+2544QWYMweu\nvhouuggOPbTo6kTKwcxwdyu6joEaaBbddFPIhpEjw3TYYeHfDRvC6WaTJkFbW5jGjoWjj4ZZs+Cq\nq8CifbREyqtZs0hEyiX2LALlkUgjGGwWqTnUh08/DaecPf44rFgBp58OF1wAZ58N06bBEUcUXaFI\nMWL/EDTQLErvktfo+fxz2LgRdu+GnTvhk09C0+jee2HCBLjnHjj++EEWLiJ7aNYsEpFyiT2LQHkk\n0gjUHKqTr76C55+HZcvCaSSvvgoHHxwuZn3iieEC15MnhyMHJk6E0aN1pIA0rtg/BNUzi3bvhoUL\n4Y47wnXOJk+Gs86CuXNhxowwT0QGRlkkImUQexaB8kikEag5VBB3WL8e3nwT1qwJ0wcfwIcfhvk7\ndsC4cXDkkXD44TBmDIwaFU5DOeSQcCrKiBGhwXTQQWEaPhwOPBAOOCBM6ekpbW0wbFiYWlv3nlpa\neiczNaVk6MX+IaiILOrpgU2bYO3acMrqk09CZydMnQrTp4d86OnpPfJo167wXm5tDRkyZQoce2zI\niba2kCOHHx4yZCje83kPT3d3qC+ddu0KNad1VmZWa+u+1+YettPTEyb33pxL8y3mnOvpCeMz6x0T\nVB+P+95HrOUtm10u/Tk7ZVXeP/t7997HPv0Gz3See2/dLZkrFmb//0lvZ5+zoaQsEpEyiD2LQHkk\n0gjq0hwys9nAXYQLWD/g7r/MWeZuYA6wHbjW3V/PWaZpQufrr+Hjj2HzZtiyBT77LFzEdutW+PLL\ncH2SbdtCE2n79rAzuGNH2NnauTNMXV1hB6yrq/fn7u7eKd3JSP+t3IlIp8odkLydq8qdjr5+V81A\ndwJi3clrNuPHh2YoFPchqNGyaPt2WLkSXnkl/Jw2fIcPDw3itFHy6aehAb1uXciI3bvDhbG3bAlZ\nkar23m5p2btRUNkQSBsAfWlp6W38HHhg+Dldd2XjqKurt3GULpNtJOU1RtzD+CBsI21A9JdxleNP\n15VtcFQ+Htnlqq2z2uOSt460vsrlsuvLNrsg3CdPuv6+lkmXS7efvV257coxpdJmT+V906Zedozp\nvMqxpo9RWmf6eOU9HtmGYWUd/T0Olctt25auq5xZZGbfBW5Nbn4J/Njd38xZTymySEQGp+zNoRo/\nPymPRCI35M0hM2sB3gNmAh8BK4F57r46s8wc4AZ3v9TMpgGL3H16zrqiD52Ojg7a29uLLqOqvL8Y\nZ//6+9xzHZx/fnvuB/NqO42V86ptd6D17qsXX+zgvPPaB7bBEoi1/paWcOQbFPMhSFm0pzSLurrC\n7XSHO3uEB/TuuFfu7Kf3SVUembM/ZBsH6frTGpcu7eCCC9r3yBiz0BRLmyf9rTfbtMlmXuURLJXN\nnsqGSzrmvMcv74jMdB3PPdfBjBntQG/DK6+e7HaqjS3v6J3sUVN5409/HugROkP9/1nlmLq7+f/r\nFaq/FtPlsw3FyuWGD0+fl9Jm0XRglbtvTXbKftHoWRSz2McQe/0Q/xjK3ByqJbOS5aLOo9hfQxD/\nGGKvH+Ifw2CzaFgNy5wFrHH3dckG/whcDmQD5XLgIQB3f9nMRpnZOHffNNDCyqrsL5j+duyWL+/g\nkkva61bPUOjs7OCyy9qLLmPAYq+/QMqijDSLhtWS4gXJHr2Y1doKL73UwaxZ7ft1vUN1vzytrbBs\nWQcXX9xedVv7Wlv2575q3F8NvKH+/yxvTG1tQ7a5euo3i9x9RWb5FcD4ulZYR2X/XFSL2McQe/3Q\nGGMosVo+P0WvEV5DsY8h9vqhMcYwGLV8RB4PrM/c3sDeH3Iql9mYs4yIyGAoi0SkDGrJoqwfAE8N\naUUiItXta2aJSJMq8d+cRUREROJlZt8GrgPOK7oWERERkb7Ucs2h6YRz5WcntxcAnr2QmZn9Fljq\n7o8lt1cDMypP5TCzeE9kFZE9FHCdD2WRiOyljFmUzJ8KPA7Mdve1VdalLBJpECW+5lCtmaU8EmkA\nQ33NoZXACWZ2DPAfYB4wv2KZxcD1wGNJAP037xofZQ1NEYmCskhEyqDfLDKzSYTG0DXVGkOgLBKR\nuqjl85PySET6bw65e7eZ3QA8Q+/XH64ysx+FX/t97v6kmc01s/cJXx993dCWLSLNRlkkImVQSxYB\nPwfGAL8xMwN2u/tZxVUtIs2qWmYVXJaIlFC/p5WJiIiIiIiIiEjj2g9f6FsbM5ttZqvN7D0zu7Ve\n2x0oM5tgZs+a2dtm9qaZ/SSZP9rMnjGzd83sH2Y2quha+2JmLWbWaWaLk9ux1T/KzP5kZquS52Ja\nhGO4Lan9DTN7xMwOKPMYzOwBM9tkZm9k5lWtNxnfmuQ5mlVM1bVTFhVDWVS82LIIGjuPlEXFiTmP\nlEXFUBaVh7KoPGLPI2XR3urSHDKzFuAe4BLgFGC+mX2jHtsehC7gZ+5+CnA2cH1S8wJgibufBDwL\n3FZgjbW4EXgnczu2+hcBT7r7ycBpwGoiGoOF87t/CJzu7lMJp3LOp9xjeJDwXs3KrdfMvgl8BzgZ\nmEPvKRSlpCwqlLKoQJFmETRoHimLChdzHimLiqEsKg9lUXlEm0fKoircfcgnYDrwVOb2AuDWemx7\nP47hb8BFhBf9uGTeUcDqomvro+YJwD+BdmBxMi+m+g8F1ubMj2kMo5N6RxNCZ3EMryPgGOCN/h7z\nyvcy8BQwrej6+xiXsqiYmpVFxY8hyixK6mq4PFIWFVp3tHmkLCq8dmVRCSdlUWH1R51HyqL8qV6n\nlY0H1mdub0jmRcHMjgW+BawgPPCbANz9Y2BscZX1ayFwC5C9sFRM9U8GNpvZg8khl/eZ2cFENAZ3\n/xz4FfAhsBHY6u5LiGgMibFV6q18b2+k3O9tZVExlEUFa6AsgsbII2VRcWLOI2VRuSiLCqYsKlTU\neaQsyle3aw7FysxGAn8GbnT3bez5BibndimY2aXAJnd/Hejr8LFS1p8YBpwB/NrdzyB8+9QCInkO\nAMzsOOCnhA7v0cAIM/seEY2hitjqjZ6yqFDKonKLseZoxZpF0BB5pCwqtxhrjpayqHBR55GyKF+9\nmkMbgUmZ2xOSeaVmZsMIofOwuz+RzN5kZuOS3x8FfFJUff04F7jMzP4NPApcaGYPAx9HUj+Ev16s\nd/dXk9uPE0IolucA4Exgmbt/5u7dwF+Bc4hrDFC93o3AxMxyZX9vK4vqT1lUDo2SRdAYeaQsKkbs\neaQsKhdlUUGURaUQex4pi3LUqzm0EjjBzI4xswOAeYTz+srud8A77r4oM28xcG3y8/eBJyrvVAbu\nfru7T3L34wiP97Pufg3wdyKoHyA5PG69mZ2YzJoJvE0kz0HiXWC6mQ1PLgA2k3DhubKPwdjzLxnV\n6l0MzLNwdf/JwAnAK/UqcgCURXWmLCqNWLMIGjOPlEUFiD2PlEWFUxaVh7KoYA2QR8qiPIO5GNK+\nTMBswpOwBlhQr+0Oot5zgW7gdeBfQGcyhjHAkmQszwCHFV1rDWOZQe+FzqKqn3Dl+5XJ8/AXYFSE\nY7iFEJZvAL8H2so8BuAPwEfATsJ5uNcRLtaWWy/hivjvA6uAWUXXX8P4lEXFjUVZVOwYosqipOaG\nzSNlUeHjiTKPlEWF1awsKsmkLCrPFHseKYv2niy5k4iIiIiIiIiINCFdkFpEREREREREpImpOSQi\nIiIiIiIi0sTUHBIRERERERERaWJqDomIiIiIiIiINDE1h0REREREREREmpiaQyIiIiIiIiIiTUzN\nIRERERERERGRJqbmkIiIiIiIiIhIE/sf4exDnT9uyxIAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x10e102510>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Plot the Errors\n",
    "\n",
    "plt.figure(figsize=(20,5))\n",
    "z = np.arange(num_iter)\n",
    "plt.subplot(1,4,1)\n",
    "plt.plot(z,errors_gd)\n",
    "plt.title('Error GD')\n",
    "plt.subplot(1,4,2)\n",
    "plt.plot(z,errors_sgd)\n",
    "plt.title('Error SGD')\n",
    "plt.subplot(1,4,3)\n",
    "plt.plot(z,errors_wsgd)\n",
    "plt.title('Error WSGD')\n",
    "plt.subplot(1,4,4)\n",
    "plt.plot(z,errors_awsgd)\n",
    "plt.title('Error AWSGD')\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXEAAAEKCAYAAADkYmWmAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAFWNJREFUeJzt3Xu0ZGV95vHvgw3K3QuKBBRBB6IxqCARQ5SKRlBZQTOZ\nRNREQKJrJjoQjXhhzKLXMuNcWIomjrPGpWGIw2VEUdHRBA2WDLeJchcQW1DudOQiILiApn/zx95t\nl6fPpdpTXc17zvezVq2u2vutt956u/o5b/323qdTVUiS2rTF5h6AJOlXZ4hLUsMMcUlqmCEuSQ0z\nxCWpYYa4JDXMEJekhhniS1ySHyd5MMl9Se7v//ybzTCO45Pc0L/+TUlOn7H/VUnO7ff/JMmlSY5L\nslW//4QkDye5t799P8nfJnn6FN/DDUm+N2Pb4UmumbHtG7NsOyfJe+fod8ckn0ly+8h7e++MNu9M\nckWSB5Lc1s/VG0b2D5P8vH/+T5N8J8n71s2fli5DfOkr4NCq2qGqtu//PGa2hkkeN862+czRxxHA\nm4FXVNUOwIuBfxrZ/0fAmcD/Ap5ZVU8F3gDsBjxjpKszqmpH4MnAHwBPBy5JsvPGjPFXkeTlwOOB\npyXZb2TXecDeSZ7St3scsA/whBnbXgp8e47uTwK2Bfbu399hwA9HXvtvgWOAd9G9912BDwKHjPRR\nwJ/3z98F+EvgcOBri3jbaoAhvjxk1o3JEUnOT/LRJHcCJ8yxLUk+2K/q70jyP5Ps0Pexe5K1Sd6a\n5EZGwnnEi4F/rKofA1TVv1TVp0f2fwRYWVV/V1U/7dusqqpjq+r6mZ1V1aNVdS1d0P+ELrBmvret\nktyT5Hkj23bqv5XslOQpSb7St7kryVwBu84RwOeBLwFHjozlNuBHwMv7TfsCV9MF9rpt+9H9HXx3\njr73B06rqvv6Pn9QVWf1Y/5XwL8D3lBV51bVQ9W5sKreOvNt98//eVWdR/fD4KVJXrvAe1PDJhri\n/VfC1UmuHKPts5Ocl+SyJJcnec0kx6KxvYRu1fc04D/Ose0o4C3AQcCewPbAJ2b083Lg1/nl1eE6\nFwNvSfKeJPsl+cXnLsnedCvLszZ24FW1Fvgy8LJZ9j0MfAF448jmPwaGVXUnXfDfDDylf5/Hz/U6\nSbYG/g3wObpvDIcnWTHS5DzWB/bL+8fnj2x7GXBxVT06x0tcDHw4yZFJnjNj3yuAm6rqsrnGN5eq\nupnuB8cG86OlY9Ir8ZOZ/R/xbD4InFpVL6L7h/bJCY9F630pyd39qvPuJEeP7Lu1qj5ZVWur6qE5\ntr0J+GhV3VhVDwIfoAuydZ+fAk7oV4APMUNVnQr8e+BgYAisHqn57tT/ece69klO78f6QJI3L/De\nbqMrMczmdH45xN8EnNrff4Su7LBHv7K/YJ7X+EPgvr7NufQlqpH9o6vulwH/lw1DfL6V/jvpSknv\nAK5OsirJq/t9OzEyNwBJbu7n5+dJnsH85psfLQETDfGqOh+4Z3Rbkj2TfL0/0PLtJHv1u24Hdujv\nPxG4dZJj0S95XVU9uaqe1P/5mZF9N8/Sfua2XwNuHHl8I7ACGK1F3zLfAKrq9Ko6mO7v+t8CH0ry\nKuAuujLALiNt31hVTwIuBRaqye8K3D3Hvm8BWyfZP8nuwAvoyiEA/xW4HjgnyQ+TvG+e13gL3aqe\nfjX9JbryyjrnAfskeSJwAHBRVV0H7NJv+52+DUnelPUHmP9P3+dDVfWfq2p/um8GnwM+1z/3rtG5\n6ds/gy7ct2KOUtmY86MlYBo18U8B7+w/oMcB/73f/p+AI5LcDHyVbqWmTWO+f+iz/RrLmdtuA3Yf\nebw73Up29QL9bNhxt+r9AnAl8HzgOrof4P96nOePShLg9+kDcpbXWksXiG+iW5F/taoe6Pc9UFXv\nqapn09WO353kd2d5jV3pShpH9GeP3E5Xlnltkif3ff2Ibo7eDqz7tgJwUb9tW7qSCVV12sgB5kNn\nvBxV9TPgw8B2wB50K//dkuw72xQsMD/PoKvHzzo/Who2aYgn2Rb4beDMJJcB/4P1q7eTgE/3q4pD\n6b5O6rHpdOBdSZ6VZDu6OvkZfUjCwmFyRJLXJtmuP0j6GuB5dHXioqtPn5Dk6H71ue6A3syzTtLv\ne1yS5wJn9G1OWmDsb6AL8tNGxnRokmf3D+8H1gBrN3w6b6H7QbMX3Ur+Bf39W/jlUs35wLvpSinr\nXNBv++5sZaaRsXwwyYuTbJnk8cBf0H2jva6qfkC3EDojye8leUJfxjqQOX5wJtk6yUF03xgurqqv\nz/XaWgKqaqI3ulXalf397enqq7O1uwbYdeTx9cBOkx7Pcr/RnTnxAHDfyO0L/b4jgPNmtJ9tW+iO\nYdxEt/o+Bdhx5O/7UWCLecbwB3QhdxfwU+AK4E9ntFlXL7+P7oyTS+jCfet+/wnAQ/3+++mC9RPA\nLmPMwaq+zxUj2/6in5v7+/d1/Dyf0z+fZftxwD+PPH57Pw+vG9m2f7/trxcY338Arurn5k661fdL\nZrR5J923lwfovrl8C/jDkf3fAh4E7u1vlwDvB7ba3J9Bb5v2lv4DMK8kHwD+pP9AXgUcVd3R/9na\nPgv4SlX9Zv/4fOBjVfX5/vE+VXVlki8AZ1fVKf2q6htVtduCg5Ek/cKC5ZT+gNDbgBdV1T50B7QO\nn6PtacCFwF7prso7iu4ij6P70wi/R1d/BHgvcFSSy+nOGDhitj4lSXNbsXAT7gMeBrZNshbYhu4g\nzgaq6k1z9LHBOeDVXcQxGG+YkqTZLLgSr6p76K6ou4muFvfTqvrmph6YJGlh45RT9qT7nQ27050v\nvF2SuVbckqQpGqec8mLggqq6GyDJWXSnDZ422ijJWOcJS5LWq6qFLtia1zjniV8HHNCfnxrglcC1\ncwzGWxUnnHDCZh/DY+HmPDgXzsX8t0kYpyZ+BfD3dOedXkF3zvCnJvLqkqRFGaecQlWdCJy4icci\nSdpI/j7xTWAwGGzuITwmOA/rORfrOReTNdYVm2N1lNSk+pKk5SAJNYUDm5KkxyhDXJIaZohLUsMM\ncUlqmCEuSQ0zxCWpYYa4JDXMEJekhhniktQwQ1ySGmaIS1LDDHFJapghLkkNM8QlqWGGuCQ1zBCX\npIaN9d+zbZRVq+DCCyferSQtCc9/Puy338S6WzDEk+wF/G+g6P6T5D2Bv6qqv5n1CR/7GFxyCey9\n98QGKUlLxjbbTDfEq+oHwIsAkmwB3AJ8cc4nPPwwvPWt8Pa3T2qMkqQ5bGxN/PeA66vq5jlbrFkD\nKyZfpZEkbWhjQ/wNwOnztlizBrbc8lcekCRpfGOHeJItgcOAM+dt6EpckqZmY9L2NcAlVfWTuRqs\nXLkSrrgC1qxhsMsuDAaDxY5PkpaM4XDIcDicaJ+pqvEaJqcD/1BVp8yxv6oKXv96OPLI7k9J0pyS\nUFVZTB9jlVOSbEN3UPOsBRtbTpGkqRkrbavqQeCpY/X4yCOGuCRNyeQvu3clLklTs2lC3FMMJWkq\nXIlLUsMmH+LWxCVpaiynSFLDLKdIUsMsp0hSw1yJS1LDrIlLUsMsp0hSwyynSFLDLKdIUsNciUtS\nw6yJS1LDLKdIUsMsp0hSwyYb4mvXdrctJv+zQZK0ocmm7bpVeBb1X8ZJksY0+RC3Hi5JUzPuf5S8\nY5Izk1yb5OokL5m1ofVwSZqqcRP348DXquqPkqwAtpm1lacXStJULZi4SXYAXlZVRwJU1Rrgvlkb\nW06RpKkap5yyB3BnkpOTXJrkU0m2nrWl5RRJmqpxEncFsC/wjqr6bpKPAe8HTpjZcOWJJ8LPfgYr\nVzIYDBgMBpMdrSQ1bDgcMhwOJ9pnqmr+BsnOwEVVtWf/+HeA91XV789oV7VqFRxyCFx//UQHKUlL\nURKqalHnZC9YTqmq1cDNSfbqN70SuGbWxtbEJWmqxi1gHwOcmmRL4AbgqFlbWROXpKkaK3Gr6gpg\n/wUbeoqhJE2VV2xKUsM2ze9OkSRNxWRD3HKKJE2V5RRJapjlFElqmOUUSWqYK3FJapg1cUlqmCtx\nSWqYNXFJapjlFElqmOUUSWqY5RRJapgrcUlqmDVxSWqYK3FJapg1cUlqmOUUSWqY5RRJathYiZvk\nx8C9wFrgkar6rVkbPvIIPOEJExucJGl+4y6b1wKDqrpn3laWUyRpqsYtp2SstpZTJGmqxg3xAr6R\n5DtJ3jZnK0NckqZq3MQ9sKpuT/JUujC/tqrOn9lo5UUXwQ03wF13MRgMGAwGEx2sJLVsOBwyHA4n\n2meqauOekJwA3F9VH52xveroo+GAA+DP/mySY5SkJSkJVZXF9LFgOSXJNkm26+9vCxwMfG/WxpZT\nJGmqxkncnYEvJqm+/alVdc6sLb1iU5KmasHEraofAS8cqzdPMZSkqfKKTUlqmL8AS5Ia5kpckhrm\nbzGUpIa5EpekhlkTl6SGWU6RpIZZTpGkhllOkaSGWU6RpIZZTpGkhhniktQwa+KS1DBr4pLUMMsp\nktQwyymS1DDLKZLUMMspktQwyymS1LCxQzzJFkkuTXL2nI1ciUvSVG3MSvxY4Jr5e9uiu0mSpmKs\nxE2yG/Ba4NPzNnQVLklTNe6y+STgOKDmbWWIS9JULZi6SQ4FVlfV5UkGQOZqu/LRR2HlSgAGgwGD\nwWAyo5SkJWA4HDIcDifaZ6rmX1wn+TDwJ8AaYGtge+CsqnrLjHZVT3kK3HnnRAcoSUtVEqpqzoXx\nWH0sFOIzXvAg4C+r6rBZ9lXtvDPcccdixiNJy8YkQnyyp5J4taYkTdVGHYmsqm8D3567Nw9sStI0\nTXYlbohL0lQZ4pLUMGviktQwV+KS1DBDXJIaZjlFkhrmSlySGmaIS1LDDHFJapg1cUlqmCtxSWqY\nIS5JDbOcIkkNcyUuSQ0zxCWpYYa4JDXMmrgkNcyVuCQ1zBCXpIYtmLpJHg+cB2zV375cVcfP2thy\niiRN1YIhXlUPJfndqnowyeOAC5IcWFUXbNibK3FJmqaxyilV9WB/9/H9c+6ZtaEhLklTNVaIJ9ki\nyWXAHcCwqq6ZtaHlFEmaqrGWzlW1FnhRkh2Ac5IcVFXfntlu5XAIa9YAMBgMGAwGExyqJLVtOBwy\nHA4n2meqauOekPwV8GBVfWTG9qoTT4T3vGeS45OkJSsJVZXF9LFgOSXJTkl27O9vDbwKuHzWxtbE\nJWmqxkndXYBTkoQu9D9bVf80a0tr4pI0VeOcYngVsO94vbkSl6Rp8opNSWqYvwBLkhrmSlySGmaI\nS1LDDHFJapg1cUlqmCtxSWqYIS5JDbOcIkkNcyUuSQ0zxCWpYZZTJKlhrsQlqWGGuCQ1zBCXpIZZ\nE5ekhrkSl6SGGeKS1DDLKZLUsHH+t/vdkpyb5OokVyU5Zs7GrsQlaapSVfM3SJ4OPL2qLk+yHXAJ\n8Lqq+v6MdlX33Qfbb7/pRitJS0gSqiqL6WPBlXhV3VFVl/f3fwZcC+w6a2NX4pI0VRtVE0/yLOCF\nwP+btYE1cUmaqrGXzn0p5fPAsf2KfAMrP/QhSPfNYDAYMBgMJjFGSVoShsMhw+Fwon0uWBMHSLIC\n+Crw9ar6+Bxtapy+JEmdSdTExw3xvwfurKp3z9PGEJekjTCVEE9yIHAecBVQ/e34qvqHGe0McUna\nCFNbiY85GENckjbCVE4xlCQ9dhniktQwQ1ySGmaIS1LDDHFJapghLkkNM8QlqWGGuCQ1zBCXpIYZ\n4pLUMENckhpmiEtSwwxxSWqYIS5JDTPEJalhhrgkNcwQl6SGGeKS1LAFQzzJZ5KsTnLlNAYkSRrf\nOCvxk4FDNvVAJEkbb8EQr6rzgXumMBZJ0kayJi5JDTPEJalhKybZ2cqVK39xfzAYMBgMJtm9JDVt\nOBwyHA4n2meqauFGybOAr1TVb87TpsbpS5LUSUJVZTF9jHOK4WnAhcBeSW5KctRiXlCSNDljrcTH\n6siVuCRtlKmsxCVJj12GuCQ1zBCXpIYZ4pLUMENckhpmiEtSwwxxSWqYIS5JDTPEJalhhrgkNcwQ\nl6SGGeKS1DBDXJIaZohLUsMMcUlqmCEuSQ0zxCWpYYa4JDXMEJekho0V4kleneT7SX6Q5H2belCS\npPGM87/dbwF8AjgE+A3gjUl+fVMPrGXD4XBzD+ExwXlYz7lYz7mYrHFW4r8FrKqqG6vqEeAM4HWb\ndlht80PacR7Wcy7Wcy4ma5wQ3xW4eeTxLf02SdJm5oFNSWpYqmr+BskBwMqqenX/+P1AVdV/mdFu\n/o4kSRuoqizm+eOE+OOA64BXArcD/wy8saquXcwLS5IWb8VCDarq0STvBM6hK798xgCXpMeGBVfi\nkqTHrkUf2FzOFwIl2S3JuUmuTnJVkmP67U9Kck6S65L8Y5IdN/dYpyXJFkkuTXJ2/3hZzkWSHZOc\nmeTa/vPxkmU8Fx/o5+DKJKcm2Wq5zEWSzyRZneTKkW1zvvd+rlb1n5uDx3mNRYW4FwKxBnh3Vf0G\n8FLgHf37fz/wzaraGzgX+MBmHOO0HQtcM/J4uc7Fx4GvVdVzgRcA32cZzkWS3YG3AS+qqn3oSrhv\nZPnMxcl0+Thq1vee5HnAHwPPBV4DfDLJggc9F7sSX9YXAlXVHVV1eX//Z8C1wG50c3BK3+wU4PWb\nZ4TTlWQ34LXAp0c2L7u5SLID8LKqOhmgqtZU1b0sw7kA7gMeBrZNsgLYGriVZTIXVXU+cM+MzXO9\n98OAM/rPy4+BVXQZO6/FhrgXAvWSPAt4IXAxsHNVrYYu6IGnbb6RTdVJwHHA6IGW5TgXewB3Jjm5\nLy19Ksk2LMO5qKp7gI8AN9GF971V9U2W4VyMeNoc731mnt7KGHnqxT4TkGQ74PPAsf2KfObR4iV/\n9DjJocDq/pvJfF8Bl/xc0JUM9gX+W1XtCzxA9xV6OX4u9gTeBewO/BrdivzNLMO5mMei3vtiQ/xW\n4Jkjj3frty0b/VfEzwOfraov95tXJ9m53/904F821/im6EDgsCQ3AKcDr0jyWeCOZTgXtwA3V9V3\n+8dfoAv15fi5eDFwQVXdXVWPAl8EfpvlORfrzPXebwWeMdJurDxdbIh/B3hOkt2TbAUcDpy9yD5b\n83fANVX18ZFtZwNH9vePAL4880lLTVUdX1XPrKo96T4H51bVnwJfYfnNxWrg5iR79ZteCVzNMvxc\n0F0oeECSJ/QH6V5Jd+B7Oc1F+OVvp3O997OBw/uzd/YAnkN3ceX8qmpRN+DVdH9Rq4D3L7a/lm50\nq89HgcuBy4BL+/l4MvDNfl7OAZ64ucc65Xk5CDi7v78s54LujJTv9J+Ns4Adl/FcHEf3Q+xKugN5\nWy6XuQBOA24DHqI7LnAU8KS53jvdmSo/pDtJ4uBxXsOLfSSpYR7YlKSGGeKS1DBDXJIaZohLUsMM\ncUlqmCEuSQ0zxCWpYYa4JDXs/wPT6sRcPPlsBgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x10b46d650>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(z,errors_sgd,c = 'blue')\n",
    "plt.title('Error SGD vs AW-SGD')\n",
    "plt.hold\n",
    "plt.plot(z,errors_awsgd, c = 'red')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA3cAAAFCCAYAAABSGyQhAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzs3Xm4XFWZ9/3vL4QpDEmAMCQhA4MZUEIi0sh4FBTEFhTf\nFqItCDZNKzzw6vN0g7Q2ibb9Ag4tCH1JWpoHEKUbREBFRMQDggIBQmTIAAQyMSUhkIQACcn9/rF2\nQdXhDJVzap/aVfX7XNe5UrWH2ncl587e915rr6WIwMzMzMzMzBrbgHoHYGZmZmZmZn3n4s7MzMzM\nzKwJuLgzMzMzMzNrAi7uzMzMzMzMmoCLOzMzMzMzsybg4s7MzMzMzKwJuLgrMEmPSTqsn4+5u6RV\nktSfxzXrT84ts9pxPpnVnvPKesvFXY1I+qykmZJWS1oq6deSDu7LZ0bEeyPi7lrFWCJphKQbJC2T\ntFLSXySdlB1zcURsHzWaAFHS5pL+RdLc7O9mcfZ385GybZ6VtFbSq5JelnSPpNP9n4uBc6ubY31R\n0pwsb56X9CtJ25St31/SL7Ocejm7UPiWpMHZ+pMlvZWdyFdJelrSf0nauxbxWTE5nzo9zm2S/rHs\n/XBJG7tYtnP2/jxJC7LcWSTpZx0+8yOS7szWL5P0sKR/lLRFtv58Seuy/H01O0f+UNKuff0+1v+c\nV90ebxtJayT9usPycyXd2mHZk51sN1/SZ7LXx0maJekVSS9JukPS6LJt95L0s2zdK5LmSbpY0vBs\n/eGSNpSd9xZJ+m9J+9fq+xaBi7sakPRV4PvAvwI7A6OAy4BP1DOublwDLAR2B3YEPg+8mNOxfk76\ne/hbYCgwFrgYOKZsmwA+HhGDgdHABcA5wBU5xWQNwrnVOUmHA98GTsjyZgLw32XrDwL+APwRGBcR\nOwBHA28Bk8o+6k8RsT0wGDgSeB14SNLEWsds9ed86tLdQHkLyWHAnE6WzY+IlySdDHwO+HCWP/sD\nvy9tKOlvgOuBnwCjImIYcAIwMvsuJddl+bsD8ClgV1L+7VLj72c5cl716NPAIuBDpZsjmbuBD5Zu\n5Gc3NgYCkzss2xO4S9KewFXAVyJiCOl68jJgQ7btXsD9wBJgv2ybg4GngUPKjrs0K2C3Bw4E5gJ/\nlPShXL59PUSEf/rwA2wPrAaO72abLYAfAEtJv3T/DmyerdsR+CWwElgB3FW23zOkkwfA+aSLt6uA\nVcCjwJSybXcDbgBeIv0i/69u4lkN7NvFutHARmBA9v4PwLeAe7P9bs5i/gnwKimRRnXxWUcCrwG7\n9fB3+Pb3LFv2AVLCTqz3v7F/6vPj3Oo2t/43cGM3cfwR+EEPf78nA3d3svyXwP/U+9/fP7X9cT51\nm0+HAC+Xvb8MOA14vsOyGdnrHwLf7ybuRcD/28O/x/nA1R2WDQAeAS6q9++Lf6r7cV51nVdln/l7\n4CvA7cBXy5ZvTrpGnJy9/xvgv7Jjli+bn73+NPBwN8e5Bri5h1gOBxZ1svyHwAP1/n2q1Y9b7vru\ng8CWwE3dbPN14ABgX9Jd8wOyZZAu0haTkmVn4LxuPucTwE9Jd9l/STrZkN3h+CUwi5TgRwBnq6zr\nYwd/Bv5D0gmSdu9kfcfm+BNIdymHA3tl+19BaombS/pPpzNHAPdHxPPdfKdORcRM0n+Ch27qvtY0\nnFtd59b9wFGSpkk6qNTVK4t5EOnv7sZuvm93bsR514ycT13n0wPAVpJKrdqHAb8DnuqwrNRF7j7g\nJEn/R9L7Jb19LSVpHDCCXuRfRGwkXTw7/xqH86rrvCLrMnkYqSX7etJNxXSQiPWkc1mphbyUY/d0\nsgzgYWC8pO9LalPZYwiZI0m9xXrjRmCKpK17uX+huLjrux2B5dl/yl35LDA9IlZExApgOqkZHGA9\nKRnHRsSGiLi3m8+5JyJ+G+k2wzWk/ygg/UexU0R8O/uMZ4EfAyd28Tl/Q0qWrwMLsmcBuutvfGVE\nPBsRq4HfAE9GxB+y73w9MLmL/XYCXii9kTQ069/9iqTXuzleyXOk7irWmpxbXeRWRNwDHJ+t/xWw\nXNL3spP8UNL/7eW5d2GWe2skdXfxAM67ZuV86jqf1pFdZEoaCmyfxXZP2bKJwF3Z9tcC/wv4KNAO\nvCjpn7KP2yn7szz/fpbl32uSPtdN/OD8azTOq66vAcm+58yIWEIqoCaU3TCBlFOlQu5QUq+Tezos\nK+XdM0Abqcj8b2CZpCuzG5rw7mvOM7K8Wy3p8m5ihJR3Aob0sF1DcHHXdyuAncrv3HViOKmbRsnC\nbBnAd0hN6LdLekrSOd18zgtlr9eS7jQOIPXvHqF3Bk5YCXyNdBfoXSLi1Yg4LyLeB+wCzAZ+0c1x\ny/tiv97J+2272G8F6T+t0nFXRsRQ4P2kbgo9GQG8XMV21pycW13nFtlJ/rhIz9MdB3wB+DtS956N\nVObeOVnu/YL0TEN3nHfNyfnUTT7xznN3h5K6oEG6yDw8W7YoIhaXxfaziPgo6WLwH4BvZS0lK0gX\nieX5NzXLv4eBzbqJAZx/jcZ51X1efR74n+y4K0g3Q04uW383cEh2A2WniHga+BNwULbsvbzTckdE\nPBARJ0bELqS8PAz452x1x2vOy7K8+wGpC2h3RpBaLF/pYbuG4OKu7/4MvAl8spttlpL6MZeMJt0l\nICLWRMT/iYg9gWOBr/bioc7FwIKI2CH7GRoRgyOix4d5I+Jl4LvA8CyRaun3wAeUjVLUQbcjYUr6\nAOk/v3tqHJM1DudWlSLiD8CdwHsjYi2pFeL4Xn7cp0h3T625OJ+6VyruDuOd3/97SQMylHcN6xjX\nhoj4OfAX0oXoPNLf4ybnX9by/omujmWF5LzqgqQPAnsDX1ca0fl5UjfWz5YVw38m3SA5jeymStZC\n+Fy2bGlELOwi9odIrYHvzRb9nt6f944nPc9XTa+ywnNx10cRsYrU3/gypSFat5Y0UNLHJF2QbXYd\n6Zd7J0k7Ad8gNakj6ePZCECQHlZ9i2zknyqUCqQHgNWS/knSVpI2k7RPV83ski7I1m8maTvgy8BT\nEbGyw+f2SUT8jvRg7E2SDlCaFmEgKbk7HWZX0naS/hr4GXBNRDxei1is8Ti3uglOOlbpeYkh2fsD\nSC0Mf842+Sfg1CzuYdk2I0mji73re0oaIGmMpB9mnzOtFnFacTifelS6yPwcWXEXEa8Ay0ijPb9d\ncClNI3KMpG2VfIzUbfO+rMvc/wbOV5qupJSje5NaSSq+YrZuM0kTSH//u5AG3LAG4Lzq1hdIg6hM\nID1rOAl4H7A18DGAiHgDeBD4KpU3Fe/NlpXn3cGS/q7snDaeVBCXznvTgEMlfVfvTH2wU3b8cm9/\nP6UpTs4HTiW1djYFF3c1EBHfJ/0Sfp00UtEiUrKUHrD9V9Iv719Izd8PkoYxh3RX4w5Jq0m/zJfF\nO/Oa9DTPSGTH3wj8NbAfaXSll4D/JI3i1JlBpCb4lcBTpOFwj+34uVXG0JNPkZ4J+kl2vAXAVNKz\nCuV+KelV0t/d10h3kk7t47GtwTm3urSSdFdzfpY3VwMXRsR1Wdz3Ah8mFWrzJL0M3Eq62fLDss85\nUNIq0qhnfyB1r/lARDzRh9isoJxP3QSYWrwfIo1i+FjZqj8Cw6hsTVtFGvhiYRbbBcA/RMSfs8/6\nH+AzpC5piyQtI13gX056RqnkM1n+vUL6N1gGvD8iyrvfWcE5r95N0pbA/wNcEhHLIuKl7OdZUmFb\n3jXzLlKOlffUKuXdXWXLXsnifDTLm1tJA6h8ByAingT+Kvs+s7Nz4x9JLaffKPuc3ZTmuFtNKoz3\nAQ6PiN/TJJRuMnWzgXQF6ZfmxYjYt4ttLiFV4a8BX4iIR7LlR5P6ug4AroiIC2sYu1lTca6Z1U4e\n+aTUbem/Sd2qngU+ExGv5vxVzBpST+clSceShtnfSGqt+qeIuFPSe0h5FqRWlj2Ab0TEJf0Zv1mj\nqqbl7krgqK5WZt0R9oyIvYHTgR9lywcAl2b77gNMzZpQzaxzzjWz2skjn84F7oiIcaRnHJumG49Z\nLVV5XrojIiZFxGTgFGAGQETMj4jJETGFNADba3Q/4IeZlemxuIs05PbKbjY5jtQliIi4HxgsaRfS\n0KxPRsTCSHNZXJdta2adcK6Z1U5O+XQcaRJhsj+7G0TBrJX1eF7KusKWbAss7+RzjgSejrKRSs2s\ne7V45m4EaaSekiXZsq6Wm1nvONfMaqc3+bRLRLwIkD0X1elQ42ZW3XlJ0iclzSE9P3VWJ59zAmmA\nNTOrUh4DqtRy9Coz65pzzax2epNPfR1wyqylRcRNETGBNAXENeXrJG1OGkDj+s72NbPO9TSZbTWW\nkkamKRmZLduCNLFix+WdkuSTpDWliKhVEdbnXHOeWbPqRZ71Jp9ekLRLRLwoaVfSqHTv4jyzZrUJ\nebaUTbgGjIh7sikEdow02TWkwY4eiohlXe3nXLNm1Zdrx2pb7kTXdzVvAU4CkHQg8ErWbWUmsJek\n0ZK2AE7Mtu1SRNTt58kngx/9KPjc54K99w4GDjyfSZOCz3wm+NrX0rpf/Sp46KFgyZLg9df7N77z\nzz+/rn8/jRBT0eKJ6N0IwnnnWjVxv+c9wRNPVPcdN2xorn+zosXkeHr+6cd8uoU0dxOkobxv7inP\npk4NPvKRdF656Sb/uzmexo1pE/V4Xiqb3w1JU7K8WVG2yVSq6JJZ77+XIv+bFS2eIsZUtHgi+n6/\noseWO0k/BdqAHSUtIk3WuEXKp5gREbdmk3k+RRrR6JQs2TZIOpM0gWFpGNw5fY64hp56Cv7v/4Wf\n/xxefRWOPBLa2uCcc+D66+Gb36x3hNZKipJrEbBkCYwcWd32AzxbphVQTvl0IfA/kk4lzXH2mZ7i\neOIJuOIKuPZamDevxl/SrKC6yiNJp5PlIPBpSScB60g5eEJpf0mDSIOp/H3/R2/W2Hos7iLis1Vs\nc2YXy28DxvUirlzdfTd8+9swaxZ8/vNw1VWw//6VF6k//3n94rPWVJRce+UVGDgQttuuFp9mVh95\n5FNEvEy64KzKhg0wfz6MH59+7r+/2j3NGl9neRQRl5e9vgi4qIt915ImsTazTVSLZ+4axlNPwZln\nwpNPwj//M9xyC2y5ZefbtrW19WtsPSlaPFC8mIoWT6NaurT6Vru+KuK/WdFicjyN69lnYeedYZtt\nYNy4dCOxXor27+Z4elbEmKx7Rfs3K1o8ULyYihZPLagWfTtrQVLkFUsEXHIJfOtbcO65cPbZsPnm\nuRzKrIIkonYDqvRZNXl2223w/e/D7bf3U1BmfVTUPPvVr+Cyy+A3v4EXXoD3vheWdzaTl1kDKFqe\nQcq1f/u34LXX6h2JWd+deirssUffc63pW+5eew2++MXUWjdzJowdW++IzIptU563M7OuPfEETJiQ\nXu+yC6xfDytWwI471jcus2byzW+m3lhmja5WYxg0dXG3Zg18/OOw++5w772w1Vb1jsis+PqzW6ZZ\nM5szBw4+OL2WUtfMefPgoIPqG5dZMxk0CL7+9XpHYVYcTTvO3euvwzHHwN57w9VXu7Azq9aSJTBi\nRL2jMGt85S13kAZV8YiZZrXlEZvNKjVlSkTA6aenC9QZM5z4ZpvC3TLN+i4itdyVF3fjxsHcufWL\nyawZ+RrPrFJTpsRll8Ff/pLmFnLSm20aF3dmfbd0aeoutsMO7ywrdcs0s9rxdZ5ZpaZ75m7uXJg2\nDR54IJ1YzWzTLF3qbplmfdWx1Q5c3JnlwcWdWaWmSokI+NKX4F/+JQ0lamab5rXXYO1aj+Zn1ldP\nPw177VW5bO+94Zln0qiZZlYbKtTkDGb111TF3U9+AqtWwZe/XO9IzBpTaaRMnyzN+uaNN97de2Sr\nrWD48FTgmVltuOXOrFLTpMS6dXDeeel5u4FN19nUrH94GgSz2li3DrbY4t3LJ0xIXTbNrDZc3JlV\napqUuOYamDgRDjyw3pGYNS5Pg2BWG+vXu7gz6w8u7swqNUUb14YNcMEF8OMf1zsSs8bmwVTMaqOr\nlrvx4+Gee/o/HrNm5eLOrFJTpMQNN8DOO8Nhh9U7ErPGtmxZyiUz65t162Dzzd+93C13ZrXlZ8TN\nKjVFcfejH8FXvuIEN+ur5cthp53qHYVZ4+uu5W7OnDS6s5n1nVvuzCo1fEosXJgmLP/EJ+odiVnj\nW7HC0yCY1UJXz9ztuGMaNfO55/o/JrNm5OLOrFLDp8S118Lf/A1suWW9IzFrfG65M6uNrrplQuqa\nOXdu/8Zj1qxc3JlVauiUiEijZJ50Ur0jMWsOLu7MaqOrbpng5+7MasnFnVmlhk6Jhx5KXV8++MF6\nR2LWHNwt06w2uuqWCS7uzGrJ4y2YVWro4u7mm1OXTCe2Wd+tXw+rV8OQIfWOxKzxdddyN348PPII\nPPkkrFrVv3GZNRu33JlVauiU+O1v4aij6h2FWXN4+WXYYQefKM1qobtn7iZPhldegbY2+Pzn+zUs\ns6bjc5ZZpYZNieXLYd48OOigekdi1hzcJdOsdrprudt5Z3j8cbj1VliwoH/jMms2Lu7MKjVsSvz+\n92nS8q5Onma2aTyYilntdPfMXcmoUbBoUf/EY9asXNyZVWrYlHCXTLPacnFnVjvdtdyVDBkCGzem\nLppm1jsu7swqNWRKRMDtt8NHP1rvSMyah7tlmtVOd8/clUgwerRb78z6woPqmVVqyOJu7lwYOBD2\n3rvekZg1D7fcmdVONS134K6ZZn3lljuzSg2ZEvffnwZS8d0as9pxcWdWO9U8cwcu7sz6ysWdWaWG\nTIkHH4QPfKDeUZg1F3fLNKudalvuRo+GhQvzj8esWbm4M6vUkCkxcybsv3+9ozBrLm65M6udap65\nA7fcmfWVizuzSg2XEuvWwWOPpUlgzax2XNyZ1Y67ZZr1Dz+iY1ap4Yq7xx6DsWNh223rHYlZc1m+\n3N0yzWrFA6qY9Q+33JlVariU8PN2ZvlYscItd2a1Um23zBEj4MUXU0ufWTORdLSkuZLmSzqnk/XH\nSpotaZakByV9uGzdYEnXS5oj6XFJf9XVcVzcmVVquJTw83Zmtbd+PaxenSZVNrO+q7blbuBA2HVX\nWLo0/5jM+oukAcClwFHAPsBUSeM7bHZHREyKiMnAKcCMsnUXA7dGxARgEjCnq2O5uDOr1HAp8eCD\nLu7Mau3ll2HoUJ8kzWql2mfuwF0zrSkdADwZEQsjYj1wHXBc+QYRsbbs7bbAcgBJ2wOHRsSV2XZv\nRcSqrg7k85ZZpYZKiTffTBOYT5pU70jMmou7ZJrVVrXdMiEVd54OwZrMCGBx2fsl2bIKkj4paQ5w\nK3BWtngssFzSlZIeljRD0tZdHcgDqphVaqjibsEC2H132Gqrekdi1lw8UqZZbW22WfUtCm65s1YV\nETdlXS+PBa7JFg8EpgCXRcQUYC1wblef4ZY7s0oD6x3Appg3D8aNq3cUZs3npZdg2LB6R2HWPKrt\nkgmw886weHHP25k1kKXAqLL3I7NlnYqIP0oaKGlHUivf4oh4MFt9A/CuAVlK5s+fxrRp6XVbWxtt\nbW19Ctysv7W3t9Pe3l6zz2uo4m7+fHjPe+odhVnzWboURo6sdxRmzWNTirsdd4TZs/OLxawOZgJ7\nSRoNPA+cCEwt30DSnhHxdPZ6CkBErMjeL5b0noiYDxwBPNHVgSZOfKe4M2tEHW9KTJ8+vU+fV1Vj\ndhXD2Q6RdGM2pO19kiaWrTtb0qPZz1kd990U8+a5uLPm1cc8+1o2XPRfJF0raRMuLWHJEhd3ZrW0\nKcXdDjuk517NmkVEbADOBG4HHgeui4g5kk6X9PfZZp+W9Jikh0mjY55Q9hFnAddKeoQ0Wua/dXUs\nd8s0q9Rjy13ZcLZHAM8BMyXdHBFzyzY7D5gVEcdLGgdcBhwpaR/gi8D+wFvAbyT9KiIW9CbY+fPh\n85/vzZ5mxdbHPBsNnAaMj4h1kv6bdJf06mqPv2QJTJ5cq29jZtUOpgKpuHv55fxiMauHiLgNGNdh\n2eVlry8CLupi39lAVbMau7gzq1RNSvQ4nC0wEbgTICLmAWMkDQMmAPdHxJvZXZy7geN7G6xb7qyJ\n9SXPVgHrgG0kDQQGkQrEqi1ZkiZTNrPa2NSWOxd3Zr3j0TLNKlVT3FUznO1ssqJN0gGkh2hHAo8B\nh0oaKmkQcAywe28CXbkSXn8ddtutN3ubFV6v8ywiVgLfAxaRHlh/JSLu2JSDu1umWW25uDPrH265\nM6tUqwFVLgAuzvpNPwrMAjZExFxJFwK/A9aUlnf1IdPKnojt+HBhaTAV36GxIqv1iEcddJpnkvYA\nvgKMBl4FbpD02Yj4aWcf0jHPDjusjeeec8udNY6c86wmNrW4W7kSInyOM9tULu7MKikiut9AOhCY\nFhFHZ+/PBSIiLuxmn2eA90XEmg7Lv00a3vZHnewT3cVyzTXwm9/ATzu9XDUrJklERI+Xa73MswXA\nvqQW8Y9ExGnZ8s8DfxURZ3ayz7vy7MUX4b3vhWXLNuGLmRVItXnWXyTF5MnBww9Xv8/226fpEAYP\nzi8us74oWp5ByrXPfS74yU/qHYlZ7fQ116q53/H2cLbZCHwnArd0CGKwpM2z16cBd5UKu+yZICSN\nAj4F9Ko88/N21uR6k2d3Z3k2DzhQ0laSRBqUZU61B3aXTLPa25SWO3DXTLPecsudWaUeu2VGxAZJ\npeFsBwBXlIazTatjBmnglKskbSQNefvFso/4uaQdgPXAlyNiVW8CnT8fPvWp3uxpVnx9ybOImC3p\nauAhUrfnWcCMao/t4s6s9npb3I0dm088Zs3KXZnNKlX1zF0Vw9ne13F92brD+hJgyfz5sPfetfgk\ns2LqY559B/hOb47r4s6s9jZlKgRwy51Zb7nlzqxSw6TE4sUwenS9ozBrPi7urBlJOlrSXEnzJZ3T\nyfohkm6UNFvSfZImlq07W9Kj2c/ZZcvPl7RE0sPZz9FdHX9TW+523NHFnVlvuLgzq9QQKfH66/Da\na7DTTvWOxKz5uLizZiNpAHApcBSwDzBV0vgOm50HzIqIScDJwCXZvvuQujzvD+wH/HU2Im3J9yNi\nSvZzW1cx+Jk7s/7h4s6sUkOkxNKlMHy4+1Wb5cHFnTWhA4AnI2JhRKwHrgOO67DNROBOgIiYB4zJ\nBgCbANwfEW9GxAbgLrL5JTNVnYl6U9ytWLFp+5iZizuzjhoiJXzxaZYf55c1oRHA4rL3S7Jl5WaT\nFW2SDgBGASOBx4BDJQ2VNIg01cjuZfudKekRST+W1OXEBX7mzqx/uLgzq9QQKbF0qS8+zfIQkYo7\nT2BuLegCYKikh4EzSKPMboiIucCFwO+AW0vLs33+A9gjIvYDXgC+39WHu1umWf9wry6zSlWNlllv\nvvg0y8fLL8OWW8K229Y7ErOaWkpqiSsZmS17W0SsBk4tvZf0DLAgW3clcGW2/NtkrYARsazsI/4T\n+GVXATz66DSmTUuv29raaGtr6zZgD6hiRdPe3k57e3u9w+iRW+7MKjVMcbfXXvWOwqz5rF4Ng7vs\nWGbWsGYCe0kaDTwPnAhMLd8g61K5NiLWSzoNuCsi1mTrhkXEMkmjgE8BB2bLd42IF7KPOJ7UhbNT\nH/zgO8VdNdxyZ0XT8abE9OnT6xdMN1zcmVVqiOJu6VI4/PB6R2HWfDZsgM02q3cUZrUVERsknQnc\nTnr84IqImCPp9LQ6ZpAGTrlK0kbgcdIImSU/l7QDsB74ckSsypZfJGk/YCPwLHB6VzH05pk7D6hi\ntulc3JlVaojizgM+mOVj40YXd9acsmkKxnVYdnnZ6/s6ri9bd1gXy0+q9vh+5s6sf7i4M6vUECnh\nAVXM8rFhg0+MZnnY1OJu6NBU3EXkE49Zs/KAKmaVCn9Zt349LFsGu+5a70jMmo9b7szysanF3ZZb\npp81a/KJx6xZ+QalWaXCp8QLL8CwYTCwITqQmjUWt9yZ5WNTn7kDj5hp1hs+h5lVKnxKLF3qaRDM\n8uIBVczysaktd+Dn7sx6w8WdWaXCp4QHUzHLj7tlmuWjt8WdR8w02zQu7swqFT4lPJiKWX7cLdMs\nH73tlunizmzTeEAVs0qFv6xbssTdMs3y4pY7s3z0puVup51g+fLax2LWzHyD0qxS4VNi6VIYPrze\nUZg1J7fcmeXDxZ1Z//A5zKxS4VNi+XLYeed6R2HWnNxyZ5YPF3dm/cPFnVmlwqfEihXpOQQzqz23\n3JnlozfP3A0bluZ1NbPq+RxmVqnwKbF8uYs7s7x4KgSzfLjlzqx/uLgzq1T4lFixIp3wzKz23C3T\nLB8u7sz6h0fLNKtU6OLujTdg3TrYdtt6R2LWnNwt0ywfvSnu3C3TbNP5HGZWqdApUXrezndlzPLh\nljuzfPR2nrvlyyGi9vGYNSsXd2aVCp0S7pJpli+33Jnlozctd1tvnYrCNWtqH49Zs/I5zKxSoVPC\nI2Wa5csDqpjlozfFHbhrpjUPSUdLmitpvqRzOll/rKTZkmZJelDSh8vWPVu27oHujuPizqzSwHoH\n0J3ly91yZ5anjRt9YjTLQ2+Lu9KgKnvsUdt4zPqTpAHApcARwHPATEk3R8Tcss3uiIhbsu3fB/wC\n2CtbtxFoi4iVPR+rpqGbNbxCX9a55c4sX265M8tHb565A4+YaU3jAODJiFgYEeuB64DjyjeIiLVl\nb7cFyn/zRZXXqL5BaVap0CnhOe7M8uUBVczy4W6Z1uJGAIvL3i/JllWQ9ElJc4BbgbPKVgXwO0kz\nJZ3W3YFc3JlVKnRKeEAVs3x5QBWzfPS1W6ZZK4iImyJiAvAJ4JqyVQdHxBTgGOAMSYd09Rk+h5lV\nKvQzdytWwH771TsKs+blljuzfLi4sxa3FBhV9n5ktqxTEXGPpIGSdoyIFRHxfLZ8maRfkLp53tPZ\nvrfeOo2XXkqv29raaGtrq803MOsn7e3ttLe31+zzCl3ceUAVs3y55c4sH7195m7YMFiwoLaxmNXB\nTGAvSaOB54ETganlG0jaMyKezl5PAYiIFZIGAQMiYo2kbYCPAtO7OtAnPjGN00/P6VuY9YOONyWm\nT+/y170T5coQAAAgAElEQVQqhS7uPKCKWb48oIpZPtxyZ60sIjZIOhO4nfQI0BURMUfS6Wl1zAA+\nLekkYB3wGnBCtvsuwC8kBek69dqIuL2rY/kGpVklF3dmLcxTIZjlY2Avz64u7qxZRMRtwLgOyy4v\ne30RcFEn+z0DVP1Qjs9hZpUKnRLulmmWL7fcmeWjt3NvebRMs03j4s6sUmFTYv16eO01GDy43pGY\nNS8PqGJWLG65M9s0Lu7MKhU2JV5+GYYOddKa5ckDqpgVy9Ch8Oqr8NZb9Y7ErDH4HGZWqbAp4eft\nzPLnljuzYtlss1Tg3XknzJtX72jMiq+3XaDNmlVVxZ2koyXNlTRf0jmdrB8i6UZJsyXdJ2li2bqv\nSXpc0l8kXSupqjHEXNxZq+ljng2WdL2kOVm+/VU1x3TLnVnxHHccfP3raZ5Xt+CZdc/nMLNKPaaE\npAHApcBRwD7AVEnjO2x2HjArIiYBJwOXZPuOBk4DJkfEvqTROU+sJjAPpmKtpC95lrkYuDUiJgCT\ngDldHSvindceUMWseH78Y3jggXQOXNrltM9mBi7uzDqqJiUOAJ6MiIURsR64DjiuwzYTgTsBImIe\nMEbSMGAVaf6SbSQNBAYBz1UTmFvurMX0Os8kbQ8cGhFXZuveiohVXR1o48bK1z4xmhXT2LHwzDP1\njsKs2HwOM6tUTUqMABaXvV+SLSs3GzgeQNIBwChgZESsBL4HLAKWAq9ExB3VBLZiBeywQzVbmjWF\nXucZMBZYLulKSQ9LmiFp664O5JY7s8bg4s6sZy7uzCrVKiUuAIZKehg4A5gFbJC0B/AVYDQwHNhW\n0mer+cBVqzwNglkHneYZqbvzFOCyiJgCrAXO7epDOrbcubgzK6YxY+DZZ+sdhVmxeUAVs0oDq9hm\nKamFoGRktuxtEbEaOLX0XtICYAFwDHBvRLycLb8ROAj4aWcHmjZt2tuv58xp4/DD26oIz6w42tvb\naW9v782uvcmzZ0h5tg2wOCIezFbdALxrQJaSb35zGgOzzF+0qI3hw9t6E69Z3fQhzxrK2LFp1Ewz\n65pb7swqKcr7aHW2gbQZMA84AngeeACYGhFzyrYZDKyNiPWSTgMOjogvSJoE/AT4APAmcCUwMyIu\n6+Q4UR7LKafAoYfCqad23NKscUgiInq8r9iXPMvW3QWcFhHzJZ0PDIqIzkbcjNdeCwYNSu+nT0+t\nd9On9/WbmtVPtXnWXzqez3qrvR2+8Q344x/7HpNZXxUtzyDl2k03Bcd1fELdrIH1Ndd6bLmLiA2S\nzgRuJ3XjvCIi5kg6Pa2OGcAE4CpJG4HHgS9m+86WdDXwEKn72CxgRjWBrV4N22/fm69k1nj6kmeZ\ns4BrJW1Oas07patjlXfL9DN3ZsU1dqy7ZZr1xC13ZpWq6ZZJRNwGjOuw7PKy1/d1XF+27jvAdzY1\nsFWrYLvtNnUvs8bVxzybTWohr+I477zesAG2qGrmSTPrbyNGwEsvwZtvwpZb1jsas2JycWdWqbAp\nsXq1izuzPHgqBLPGMHBgKvAWLap3JGbF5XOYWaXCpoSLO7N8uFumWePwiJlm3fNomWaVXNyZtRhP\nhWDWODzXnVn33HJnVqmwKeEBVczy0bHlzidGs+LyoCpm3fM5zKxSIVMiwgOqmOWlfEAVt9yZFduY\nMW65M+uOizuzSoVMiTfeSA+Sb755vSMxaz5uuTNrHOPHww03wI47Vv7svDP85S/1js6s/nwOM6tU\n1VQI/c3P25nlxwOqmDWO/feHZctSrpb74hfhiSdg333rE5dZUXhAFbNKLu7MWowHVDFrLEOGvHvZ\nmDHw3HP9HopZ4bjlzqxSIVPCxZ1Zftwt06zxDR/u4s4MfA4z66iQKbFqlUfKNMuLB1Qxa3wjRsDS\npfWOwqz+XNyZVSpkSrjlziw/brmzViDpaElzJc2XdE4n64dIulHSbEn3SZpYtu5sSY9mP2eVLR8q\n6XZJ8yT9VtLg/vo+HbnlzizxOcysUiFTwsWdWX78zJ01O0kDgEuBo4B9gKmSxnfY7DxgVkRMAk4G\nLsn23Qf4IrA/sB/wCUl7ZPucC9wREeOAO4Gv5f1duuKWO7PEA6qYVXJxZ9Zi3HJnLeAA4MmIWBgR\n64HrgOM6bDORVKAREfOAMZKGAROA+yPizYjYANwFHJ/tcxxwVfb6KuCT+X6NrpVa7sq7WZu1Ip/D\nzCoVMiVc3Jnlx1MhWAsYASwue78kW1ZuNlnRJukAYBQwEngMODTrgjkIOAbYPdtnl4h4ESAiXgB2\nzu0b9GDbbdNcsK+8Uq8IzIrBxZ1ZpUJOheABVczy4wFVzAC4ALhY0sPAo8AsYENEzJV0IfA7YE1p\neRef0WW72bRp095+3dbWRltbW22iLlNqvRs6tOYfbUZ7ezvt7e31DqNHLu7MKhWyuFu9Gnbbrd5R\nmDUnd8u0FrCU1BJXMjJb9raIWA2cWnov6RlgQbbuSuDKbPm3eacV8AVJu0TEi5J2BV7qKoDy4i4v\nI0ak4m6ffXI/lLWgjjclpk+fXr9guuFzmFmlQqaEu2Wa5ccDqlgLmAnsJWm0pC2AE4FbyjeQNFjS\n5tnr04C7ImJN9n5Y9uco4FPAT7PdbgG+kL0+Gbg55+/RreHDPaiKmYs7s0qFbblzcWeWD7fcWbOL\niA2SzgRuJ93EvCIi5kg6Pa2OGaSBU66StBF4nDRCZsnPJe0ArAe+HBGrsuUXAv8j6VRgIfCZfvpK\nnSq13Jm1Mo+WaVbJxZ1Zi/GAKtYKIuI2YFyHZZeXvb6v4/qydYd1sfxl4Mgahtknw4fD3Ln1jsKs\nc5KOBn7AOzdYLuyw/ljgW8BG0nOt/xQRd5atHwA8CCyJiGO7Oo5vUJpVKmRKrFrl4s4sLx0HVPGJ\n0awxueXOiqrKuSbviIhJETEZOAWY0WH92cATPR3L5zCzSoVMidWrPVqmWV7ccmfWHPzMnRVYj3NN\nRsTasrfbAstLbySNJE1D8uOeDuTizqxSIVPC3TLN8uMBVcyaQ2kqBLMCqmauSSR9UtIc4FbgrLJV\n/w78I91MN1Li4s6skp+5M2sxHlDFrDnsthu89JJb4K1xRcRNwE2SDgWuAcZJ+jjwYkQ8IqkN6HbI\nlEsumfb2XI95zSlplqdazynp4s6sxbjlzqw5bL45TJ4MV18Np5xS72jMKvQ412S5iPijpM0k7Qgc\nDBwr6Rhga2A7SVdHxEmd7fuVr0xjzJjaBW7W32o9p2Th7tm/9Ra8+SYMGlTvSMyaU/mAKm65M2ts\nM2bAOef42TsrnGrmmtyz7PUUQBGxIiLOi4hREbFHtt+dXRV24HOYWUeFS4lSq53nLTHLhwdUMWse\nkybBGWfAl79c70jM3hERG4DSXJOPA9eV5pqU9PfZZp+W9Jikh4GLgRN6cywXd2aVCtct010yzfLV\nsVumT4xmje3cc9Pzd889lwZZMSuCKuaavAi4qIfPuAu4q7ttfA4zq1S4lHBxZ5Yvt9yZNZctt4SP\nfxxuuqnekZj1Pxd3ZpUKlxIu7szy5QFVzJrP8cfDjTfWOwqz/ufHeMwqFa64W7XKxZ1Znjygilnz\nOeoomDkTVqyodyRm/cvnMLNKhUuJtWthm23qHYVZ83LLnVnzGTQIPvIRuOWWnrc1ayYu7swqFS4l\n3ngDttqq3lGYNS9PYm7WnI47Dn7963pHYda/fA4zq1S4lHj9ddh663pHYda8PKCKWXOaOBGefrre\nUZj1Lxd3ZpUKlxIu7szy5akQzJrT7rvD4sX1jsKsf3lAFbNKhbusc7dMs3x1HFDFLXdmzWHYMFiz\nJj27btYqfIPSrFLhUsItd2b58oAqZs1JgpEj3XpnrcXFnVmlwqWEizuzfHlAFbPm5a6Z1mp8DjOr\nVLiUeP11d8s0y5Nb7syal4s7azUu7swqVZUSko6WNFfSfEnndLJ+iKQbJc2WdJ+kidny90iaJenh\n7M9XJZ3V3bHeeMMtd9aaeptnZesHZLnW7UxXbrkza16jRrm4s9biAVXMKvV4WSdpAHApcBSwDzBV\n0vgOm50HzIqIScDJwCUAETE/IiZHxBTg/cBrwC+6O567ZVor6kuelTkbeKKnY3lAFbPm5ZY7azW+\nQWlWqZqUOAB4MiIWRsR64DrguA7bTATuBIiIecAYScM6bHMk8HREdHvacXFnLapPeSZpJHAM8OOe\nDuSpEMyal4s7azU+h5lVqiYlRgDlp4ol2bJys4HjASQdAIwCRnbY5gTgZz0dzFMhWIvqa579O/CP\nQNADT2Ju1rxc3FmrcbdMs0oDa/Q5FwAXS3oYeBSYBWworZS0OXAscG53HzJt2jQefRR+/nPYYYc2\n2traahSeWf9ob2+nvb09r4/vNM8kfRx4MSIekdQGdHuqu/76acybl16vW9fGZpu15RWvWS5yzrOG\ntvvusGhR6n7ti14zs9ajiO5v9Es6EJgWEUdn788FIiIu7GafZ4D3RcSa7P2xwJdLn9HFPhERHH44\nTJ8OruusGUgiInq8xOplni0A9iU9i/e3wFvA1sB2wI0RcVIn+8TVVwef/3x6P2gQLFsG22yzyV/N\nrDCqzbP+Ujqf1UMEbL99ar0bMqQuIViTKlqeQX1zzSwvfc21arplzgT2kjRa0hbAiUDFaHySBmet\nc0g6DbirVNhlplJFl0xwt0xrWb3Js7sjYk1EnBcRoyJij2y/Ozsr7ErKz4OeCsGsuUgeMdPMrJX1\nWNxFxAbgTOB24HHguoiYI+l0SX+fbTYBeEzSHNJof2eX9pc0iDSYyo3VBOQBVawV9TXPNoWnQjBr\nbn7uzsysdVX1zF1E3AaM67Ds8rLX93VcX7ZuLdBx5MwuubizVtWXPCvb5i7gru628YAqZs3NxZ2Z\nWesq3D17d8s0y1epuItIP265M2suLu7MzFpX4S7r3HJnlq/SM3cbN6bnczyinllz2X13WLKk3lGY\nmVk9uLgzazGlljsPpmLWnEaOdHFnZtaqClXcRbhbplneSsWdB1Mxa04u7szMWlehLu3Wr08XmwNr\nNbW6mb2LW+7MmlupuPP0X2ZmradQxZ27ZJrlzy13Zs1tu+3SjZtXX613JGZm1t8KdWnnLplm+Svd\nzfc0CGbNy10zzcxaU6GKO7fcmeXP3TLNmp+LOzOz1uTizqzFuFumWfNzcWdm1poKdWnn4s4sf265\nM2t+Lu7MzFpToYo7P3Nnlj+33Jk1Pxd3ZmatqVCXdm65M8ufB1Qxa34u7qzeJB0taa6k+ZLO6WT9\nsZJmS5ol6UFJH86Wbynp/mz545L+rf+jN2tchZpRzsWdWf7Ku2W65c6sObm4s3qSNAC4FDgCeA6Y\nKenmiJhbttkdEXFLtv37gF8Ae0XEm5I+FBFrJW0G3Cvp4Ii4t7+/h1kjKtSlnbtlmuWvvFumW+7M\nmpOLO6uzA4AnI2JhRKwHrgOOK98gItaWvd0WWN7Jui1J16or8w3XrHkUqrhzy51Z/jygilnzGzIE\n1q+H1avrHYm1qBHA4rL3S7JlFSR9UtIc4FbgrLLlAyTNAl4A2iPiiZzjNWsa7pZp1mI8oIpZ85NS\n693SpTB+fL2jMetcRNwE3CTpEOAaYFy2fCMwWdL2wO2SDo+Iuzr7jGnTpr39uq2tjba2trzDNqup\n9vZ22tvba/Z5hSru3C3TLH+lAVXccmfW3EpdM13cWR0sBUaVvR+ZLetURNwjaaCkHSNiRdnyVZJ+\nDewP9FjcmTWijjclpk+f3qfPK9R9e7fcmeXPLXdmrWH33WHhwnpHYS1qJrCXpNGStgBOBG4p30DS\nnmWvpwBExApJO0kanC3fGvgI8Ei/RW7W4ArVcufizix/HlDFrDWMHQvPPFPvKKwVRcQGSWcCt5Ma\nEq6IiDmSTk+rYwbwaUknAeuA14ATst13A66SpGzfayLi9/3/LcwaU6GKuzfegMGD6x2FWXPzVAjW\nCiQdDfyAdy4sL+ywfgjwX8CewOvAqaVBGyR9DfhbYAPwKHBKRKyTdD5wGvBS9jHnRcRt/fF9emPs\nWPjtb+sdhbWqLDfGdVh2ednri4CLOtnvUWBK7gGaNalCXdq55c4sf265s2ZXNsfWUcA+wFRJHZ88\nOw+YFRGTgJOBS7J9R5MKuMkRsS/pJuiJZft9PyKmZD+FLezALXdmZq3IxZ1Zi/GAKtYCepxjC5gI\n3AkQEfOAMZKGAatI3cS2kTQQGESahLlEeQdfKy7uzMxaT6GKO4+WaZY/D6hiLaCaObZmA8cDSDqA\nNLLfyIhYCXwPWEQa3e+ViLijbL8zJT0i6celQR+KavhwWLky3Tg1M7PWUKhn7txyZ5Y/T2JuBsAF\nwMWSHiY9VzcL2CBpD+ArwGjgVeAGSZ+NiJ8C/wF8MyJC0r8C3we+2NmHF2HurQEDYNQoePZZmDCh\n3w9vDa7Wc2+ZWf9wcWfWYtxyZy2gxzm2ImI1cGrpvaQFwALgGODeiHg5W34jcBDw04hYVvYR/wn8\nsqsAijL31h57pK6ZLu5sU9V67i0z6x+FurRzcWeWPw+oYi2gmjm2BkvaPHt9GnB3RKwB5gEHStoq\nG4r9CGBOtt2uZR9xPPBY/l+lb/zcnZlZaylUy52fuTPLX/mAKm65s2ZU5RxbE0hzaW0EHifrXhkR\nsyVdDTxEmgphFjAj++iLJO0HbASeBU7vx6/VKy7uzMxaS6GKO7fcmeXPLXfWCqqYY+u+juvL1n0H\n+E4ny0+qcZi5GzsW7r+/3lGYmVl/KdR9exd3ZvnzgCpmrWPsWFiwoN5RmJlZfylUcedumWb584Aq\nZq3D3TLNzFpLoS7t3HJnlj+33Jm1jh12SLm+cmW9IzEzs/7g4s6sxZQGVHHLnVnzk2DPPeGpp+od\niZmZ9YdCXdq5W6ZZ/jygillrmTAB5s6tdxRmZtYfClXcrV8PW25Z7yjMmlt5t0y33Jk1vwkT4Ikn\n6h2FmZn1h0Jd2m21VepCYmb5ccudWWuZMAHmzKl3FGZm1h8KV9yZWb48oIpZa5k40cWdmVmrcHFn\n1mI8oIpZa9lrL1i4EN58s96RmJlZ3qq6tJN0tKS5kuZLOqeT9UMk3ShptqT7JE0sWzdY0vWS5kh6\nXNJfdXUcP29nray3eSZppKQ7s/x6VNJZ3R3HLXdmrWWLLWDMGHjyyXpHYmZmeeuxuJM0ALgUOArY\nB5gqaXyHzc4DZkXEJOBk4JKydRcDt0bEBGAS0GXnkM0337TgzZpFH/PsLeCrEbEP8EHgjE72fZsn\nMTdrPe6aaWbWGqq5tDsAeDIiFkbEeuA64LgO20wE7gSIiHnAGEnDJG0PHBoRV2br3oqIVV0daIst\nevMVzJpCr/MsIl6IiEey5WtIN1BGdHUgD6hi1no8YqaZWWuoprgbASwue7+Ed184zgaOB5B0ADAK\nGAmMBZZLulLSw5JmSOpymnK33FkL60uevU3SGGA/4P6uDuRumWatxyNmmpm1hlp1yroAGCrpYeAM\nYBawARgITAEui4gpwFrg3K4+xC13Zt3qKs8AkLQtcANwdtaC1ykPqGLWelzcmZm1hoFVbLOU1EJQ\nMjJb9raIWA2cWnov6RlgAbANsDgiHsxW3QC8a6CIkuefn8a0ael1W1sbbW1tVYRnVhzt7e20t7f3\nZte+5BmSBpLy65qIuLm7Az3+eMqzBx6ALbdsA9p6E69Z3fQhz1rW+PFpQJX1691LxsysmSlKt/G7\n2kDaDJgHHAE8DzwATI2IOWXbDAbWRsR6SacBB0fEF7J1dwGnRcR8SecDgyKis5EA40MfCu68s0bf\nzKwAJBERqmK7vubZ1cDyiPhqD8eJY48Nbr4ZLr4Ynn4aLrmkuz3Miq/aPOsvkqKnc2s9jB8P118P\n73tfvSOxRlS0PIPi5ppZX/Q113psuYuIDZLOBG4ndeO8IiLmSDo9rY4ZwATgKkkbgceBL5Z9xFnA\ntZI2J7UynNLVsXw30VpVX/JM0sHA54BHJc0CAjgvIm7r7Fh+5s6sNU2ZAg895OLOzKyZVdMtk+wi\ncVyHZZeXvb6v4/qydbOBD1RzHD9zZ62st3kWEfcCVZdpngrBrDW9//2puPvCF+odiZmZ5aVQl3Zu\nuTPLX/mAKm65M2sdU6bAww/XOwozM8uTizuzFuNumWatafJkmD073dgxM7PmVKjizt0yzfLnbplm\nrWnIENh1V5g3r96RmJlZXgp1aeeWO7P8ueXOrHWVnrszM7PmVKjizi13Zvlzy51Z6/Jzd9ZfJB0t\naa6k+ZI6mwLrWEmzJc2S9KCkD2fLR0q6U9Ljkh6VdFb/R2/WuAp1aeeWO7P8eUAVs9Y1ZQo88EC9\no7BmJ2kAcClwFLAPMFXS+A6b3RERkyJiMmmarBnZ8reAr0bEPsAHgTM62dfMulCo4s4td2b5K++W\n6ZY7s9Zy6KEwdy4sWpTe/+u/wg031Dcma0oHAE9GxMKIWA9cBxxXvkFErC17uy2wPFv+QkQ8kr1e\nA8wBRvRL1GZNoFCXdm65M8tfebdMt9yZtZattoITToCrr4aVK+GCC9ySZ7kYASwue7+ETgo0SZ+U\nNAe4FXhX90tJY4D9gPtzidKsCVU1iXl/ccudWf48oIpZazvlFJg6FaRU7C1ZUu+IrFVFxE3ATZIO\nAa4BxpXWSdoWuAE4O2vBM7MqFKq4c8udWf48oIpZa9t/f9hyS/j2t+HCC+H66+sdkTWhpcCosvcj\ns2Wdioh7JA2UtGNErJA0kFTYXRMRN3d3oGnTpr39uq2tjba2tr7Ebdbv2tvbaW9vr9nnFaq4c8ud\nWf5KA6q45c6sNUnwD/8Ad9wBH/sY/OAH9Y7ImtBMYC9Jo4HngROBqeUbSNozIp7OXk8BiIgV2er/\nAp6IiIt7OlB5cWfWiDrelJg+fXqfPq9QxZ1b7szy55Y7MzvzTPjSl2D9eli6NN30keodlTWLiNgg\n6UzgdtL4DldExBxJp6fVMQP4tKSTgHXAa8AJAJIOBj4HPCppFhDAeRFxWz2+i1mjKVRx55Y7s/x5\nQBUzk2DgwPSzzTawfDkMG1bvqKyZZMXYuA7LLi97fRFwUSf73Qv47GTWS4W6b++WO7P8eSoEMys3\ncqQHVTEzaxaFurRzy51Z/txyZ2blXNyZmTWPQhV3brkzy58HVDGzci7uzMyaR6GKO7fcmeXPA6qY\nWTkXd2ZmzaNQl3ZuuTPLnycxN7NyI0akETPNzKzxFaq4c8udWf7ccmdm5dxyZ2bWPAp1aeeWO7P8\neUAVMyvn4s7MrHl4njuzFlM+oIpb7sysVNx5IvPmsG4d3HNPmsPwsMPqHY2Z9bdCFXduuTPLn1vu\nzKzc9tunGz2vvgpDhtQ7GuuLV16B8eNh553hjTdg/vx6R2Rm/a1Q9+3dcmeWPw+oYmYdjRjhrpnN\nYNEi2GkneOSRVOgtXFjviMysvxWquHPLnVn+PKCKmXU0fjzMnFnvKKyvVqyAHXdM/7cfeST87nf1\njsjM+luhLu1c3Jnlzy13ZtbR3/0dXHbZO8/klnvzTZg3D554Ai65BD70IXjssf6P0XpWKu4APvIR\nF3dmrahQxZ27ZZrlr3Tx5pY7Myv52MdSN7777nv3uu98Bw45BD71KXjgAZg0Cc48s/NC0OqrvLg7\n8kj4/e/fuaFnZq2hUJd2brkzy58HVDGzjgYMgDPOgEsvffe6WbPghz9MrXc/+Ql897uwciVcf33/\nx2ndKy/udt89PX83a1Z9YzKz/lWo4s4td2b5K++W6ZY7Mys55RT47W9h9OjU9bLk0Ufhfe975/3A\nganY+9KX0nZf/Wr/x2qdKy/uAD76Ufj1r+sXj5n1v0Jd2rnlzix/brmzViDpaElzJc2XdE4n64dI\nulHSbEn3SZpYtu5rkh6X9BdJ10raIls+VNLtkuZJ+q2kwf35nfI2ZAg89RTcfXd6pm7JEli7FhYv\nhve8p3Lbww5LXf7OOw9mzIDXX69PzFapY3H3pS+lQnzZssrtnn++f+Mys/5TqOLOLXdm+fOAKtbs\nJA0ALgWOAvYBpkoa32Gz84BZETEJOBm4JNt3NHAaMDki9iXNB3tits+5wB0RMQ64E/ha3t+lvw0Z\nklruPvhB+POf0yAq73lP5zdf99svDdqxzz4eabMoOhZ3EybA3/4t/PM/p/cRMG1amvrirrvqEqKZ\n5cyTmJu1GA+oYi3gAODJiFgIIOk64Dhgbtk2E4H/DyAi5kkaI2kYsApYB2wjaSMwCFia7XMccHj2\n+iqgnVTwNZ2DDoI//QnWrKnsktmZQw6BP/4xteZZfXUs7gDOPz9NdREBzz0HL74IP/5x6oY7ezZs\nt119YjWzfBTq0s4td2b5c8udtYARwOKy90uyZeVmA8cDSDoAGAWMjIiVwPeARaSi7pWI+H22z84R\n8SJARLwA7JzbN6izUstdx+ftOnPIIXDPPf0Tl3Wvs+JuyJD03N3kyfDJT8If/gCnnpqelzy3KW9N\nmLW2QhV3vtA0y58nMTcD4AJgqKSHgTOAWcAGSXsAXwFGA8OBbSV9tovPaNrJAD7wgVTYzZxZXXH3\npz+l/1Osvjor7gDe/3748pfhtNPeaan7+tfhllv6Nz4zy1+humVK9Y7ArPl5QBVrAUtJLXElI3mn\nayUAEbEaOLX0XtICYAFwDHBvRLycLb8ROAj4KfCipF0i4kVJuwIvdRXAtGnT3n7d1tZGW1tb375R\nPxs0KD2vdc89PRd3w4bBbrulYnC//fonvmbxxhtpYJpjjun7NdDGjWmuwh12qG77UaNg+fI0aM6g\nQe9e397eTnt7e9+CMrN+V6jizszy526Z1gJmAntlg6M8TxoQZWr5BtlIl2sjYr2k04C7I2KNpHnA\nNyRtBbwJHJF9HsAtwBeAC0mDsNzcVQDlxV2jOuigNHrmyJE9b3vooakQdHFXvVdfhWOPhccfT39/\nV16ZulD25fO23TZNVVGNzTaDMWNgwQJ473vfvb7jTYnp06f3Pjgz6zfulGXWYjygijW7iNgAnAnc\nDn4sU5UAABaxSURBVDwOXBcRcySdLunvs80mAI9JmkMaVfPsbN/ZwNXAQ6Tn8gTMyPa5EPhIVgAe\nQera2bQOOQQmTaquRcnP3VXv/PPhiCNSIbzvvmmqiREj4MMfhjff7P3ndtUlszt7750KeDNrHooo\nxiMDkqIosZjViiQiojAdjiXFVlsFr7+eulr99Kc9d7kyK7oi5lkznM82bEgFw85VDBszf36aFmHh\nwvzjanS77w7f/W7qFnnggal4joBPfxrGjoXvfa93n/v/t3fv0VaV5R7Hv89mg4gYKqYmyk3i5lFu\nhqaOEScqoZNbs8EBKk0icpgUBXYgzLLjsLykSaaZpoRYmZQhOLwgMvR4AxQQb1xUzFuAQ0CBTEF5\nzh/v3LLYrPtea8251vp9xliDvedcc81nr83D5FnvO593yRL47ndh6dL8j5k8GQ4/HM4/P/dzk5Zn\nUDu5JpKqtbmmz+1F6owaqohIPtq0ya+wgzACtH17aLUvmW3bBps3w6hRoSNp86ioGdx4I9x+Oyxc\nWNxrFzNy16sXvPBCcecTkWTK6792ZjbCzFab2Vozm5pm/wFmdoeZrTSzxWbWP2XfP6LtK8ysgM+T\nROpLK/Ms67GpdM+diJSaWRiFWrw47kiSbfXqsCh8ug/WOncOI2lz5xb32sUWd5qWKVJbchZ3ZtYA\n/IZwT8LRwFgz69viadOBFe4+gHCT+a9T9u0Chrn7IHcfWpqwRWpLa/Isz2M/opE7ESmH5rXxJLNV\nq0IX0kx69IBXXy3utVXciQjkN3I3FHjB3V9x953AbcBpLZ7TH1gE4O5rgO5m9vFon+V5HpF61po8\ny+fYj6Q2VNHInYiUioq73HIVd127Vra469oVNm4MSzKISG3Ip+jqAryW8v3r0bZUK4EzAMxsKGF9\noebmyQ7cb2ZPRO2mRWRvrcmzfI79iHt4aFqmiJTS0KHw1FOwY0fckSTX6tXQN+O8isKLu127QiOW\nxx8vrrhrbIRu3eDll/fcvnMnjBgB69cX9noiEr9SrXN3KTDDzJYDzwArgA+jfSe5+/pohOF+M1vl\n7mkbJlf7oq8iZV70NVueFeAiLroo3NS/ZMkwuncfVsoYRcpOiysn0/77Q8+esHIlfOpTcUeTTLlG\n7jp3DsshbNsW3s9cbr0V7r4b2rYNH9oVWtzB7qYqqXHNnw/33Rc6Kk+ZUvhrikh8ci6FYGYnABe5\n+4jo+2mAu/tlWY55GTjG3be32P5TYJu7X5XmGLWzlZqTbzvb1uQZ8B/5Hmtm3qaN89574dPapUvD\n+koi1SxpLdrr+Xp27rmhw6bWu97bjh3wsY+Fxcb32Sfz8/r1g7/+FY4+Gt5+O/PC5lu3hlHAmTNh\n9OiwIPkVV4QlKQoxaVK4HkyevHvbKaeExeuXLQujsZC8PIP6zjWpXZVYCuEJoJeZdTOzdsAYYF6L\nIDqZWdvo6wnAQ+6+3cw6mFnHaPt+wBeAZ4sNVqSGFZ1n+RybqqEhfMKrhioiUmpTp8Jvf1vYWmu1\nyn3PKZYvvhimXWYr7GDPqZkDB+69OPz994fFz485BkaODIXYaaeFEdNiRu4++UlYtGj3dNp162D5\ncrjmmjDD45lnCn9NyKsDdFNKN/UnzeyzKftuMrONZvZ0cWcXqV85/2vn7h8CE4EFwHPAbe6+yszO\nMbNvR0/rBzxrZqsIHfsmRdsPBR4xsxXAYmC+uy8o9Q8hUu1ak2eZjs10LrNwn4YaqohIqXXvDtdf\nH0aStmyJO5p4Pf449OkDDz0Uvl+1Kvv9ds2ai7sNG8Ki8Lfeuuf+yy6DCRPgrrvCew3w7egqUUxx\n97WvhT9PPDGstTdtGpx5JnToEPbNnl34a+bZxXmhuw9w90HAOOCGlH0zo2NFpEA5p2VWiobWpRYl\nbRqLmXn79s7mzWHKzZo1cPDBcUcl0jpJzLN6v559//uwdm24dysJHyItWQK33ALXXlu5c86fH6Y8\nbt8epksuWgSHHRaKs2wuvjh0rzzxRPjhD+HNN8Pi8O3ahWJvyBB4/XVo3373Me7wk5/AhReG5xXK\nHW6+GR57LDRZufDCcI14/vnQWOWVV6ChIf88i241+Km7j4y+z3qrgZl9GviVu5+Qsq0bYVDg2Czn\nqftck9pTiWmZIlJDGho0cici5XXFFaFAmTYtFAbNj23b4oln5Uq47jqoZB+et9+Gk06CWbPCwuRb\nt4bpk7l06xbeq2XLoKkJ+veHe+8N+2bNgjFj9izsIMzIuPji4gq75uPHj4ebboLf/S4UdhDO/eST\nYX+B8uribGanR7NR7ga+V0zsIrKnUnXLFJEq0VzcvftumHYjIlJqbdvC7bfDqaeGPyHc09WnT2UL\nrGabN0Pv3qFpyBNPVOaDreZmKCNHhke+mqdlbtsWpkf26BGmZo4YAX/4A8yZU7aQ0zrkkPK9trvP\nBeaa2cnAbKBP+c4mUh9U3InUmYaGME2osTH3jf0iIsU6+OA9FzV/6y046qgwBbCIkaBW2bIFxo0L\nUyU///lwb9rPfx6aiZRLtk6X2TQXd+vWwdVXhw6bF10E++0XpmoOHlzyUMvhDcJarM2OiLal5e6P\nmFmjmXV2902FnEjLaEm1K/XyPrrnTqSMkngvUKdOziOPhA5rb2S81IpUjyTmma5n6XXpAo8+Ghqv\nVNKECWHtvaamcF/ZXXeFounqq8t3zilT4PDDC18n7v33QyHXqVMoiCtdCGdSSJ6ZWRtgDTAcWA8s\nBcamNvsys6Pc/aXo68HAHHc/KmV/d8I9d8dkOY9yTWqO7rkTkYI0NMCmTXDggXFHIiL15thj4ekY\nmttv3gwHHRQampxxRliy4fbbw73HzT74INzrVirFjtzts0+YCjl4cHIKu0Ll2QH6K2b2rJktB2YQ\nlvEBwMz+BDwG9DazV81sXIV/BJGqpWmZInWmubgr5j8dIiKtMWBAKO6amip73ubirlmfPnDoofDw\nwzBsWLgPefx4+POf4eyz4ZxzwtT1vn2Ln75ebHEHoanKcccVd2xSuPu9tLiHzt1/l/L15cDlGY79\nanmjE6ldGrkTqTMNDeE/Ohq5E5FKi2vkbsuWvf/NGz0a/vKXMHr3gx/ASy/Byy9Dx47wzW/CqFFw\n8smwfn1x52xNcXfyyTB8eHHHikh908idSJ3RtEwRicuxx8L//m/lz9ty5A5CcTdkSFh/rkuXcB/e\nAQfAVVeF/e6h6crxx4fGMF32auSfXWuKuyuuKO44ERGN3InUGTMVdyISjz59wn1t775b2fOmK+56\n9IDzzw/r3z3wwN6FmBlccAF8+cvFLX7emuJORKRYKu5E6oxG7kQkLm3bhgLvuecqd84dO0IHyo4d\n9943fXqY/pitccl554XFvd9/v7DzqrgTkTiouBOpMyruRCROzU1VKqX5frtiO0/27h1i/tvf8j/G\nHd55JyxnICJSSSruROqMGqqISJwGDYLFiyt3vnRTMgt17rlw5ZXwi1/AJZfkfv727bDvvqHjpohI\nJam4E6kzGrkTkTg1NcGdd4Z15SohXafMQp16KvTqBRs2hCYru3Zlf76mZIpIXFTcidQZNVQRkTj1\n7Andu4culZVQipG7xsawbMKMGeHfztdey/58FXciEhcVdyJ1RtMyRSRuY8aEYqkSSlHcperTB1av\nzv4cFXciEhcVdyJ1pqEBdu5UcSci8Rk1Cv7+99DJsty2bCltcde3L6xZk/05Ku5EJC4q7kTqTEOU\n9SruRCQuRx4J/fvDPfeU/1ylnqmgkTsRSTIVdyJ1pqEB2rWD9u3jjkRE6tnEiXDZZWHZgHIq9bRM\njdyJSJKpuBOpM2atW/NJRKQURo0KUyYfeCDzc+bMgalTYckSuO02mDABtm4t7DylnpapkTsRSTIV\ndyJ1pqFBUzJFJH5t2sCPfww/+1n60btly+C888K+ceNg1ixYvhzmzg3733svFHyzZ8OLL2Y+T6mn\nZR55ZCjeshWZKu5EJC4q7kTqjIo7EUmK0aNDITRyZBile/DB8Fi0KOy79lq4/HJ4/vlwf96UKbu7\nbM6YERYVnz8fPve58DrplHpaZkMD9O4Na9dmfo6KOxGJS2PcAYhIZam4E5GkaGyEpUtDYXfzzfDv\nf+/ed/bZYepmqlNPhXPPhY0b4Zpr4K67YOBA+M53wijfH/+49zlKPS0Tdk/NPO649PtV3IlIXFTc\nidQZFXcikiT77gtnnRUeuey/fxil+/rXQ4E1cGDY/stfwpAhMHgwdOgAN9wQunFCedb1zNVURcWd\niMRF0zJF6kxzQxURkWo0ejQsXAiTJ+/e1qEDPPooXH89DB++e597GLkr9b95/fvDww9n7vSp4k5E\n4qLiTqTOaORORKrZl74E06eH+/RSHXQQDB0KF1wA69bBfffBtm2h8GvbtrQxnH56GBFMNw0UVNyJ\nSHw0LVOkzqi4E5Fq1qEDXHJJ5v3t2oUmLBMnwimnlOffu3btYOZM+OIXw0jhJz6x534VdyISFxV3\nInVGxZ2I1LrTToNNm+DNN8P9eOUwZAhMmgT9+sGIEaFzZ48eYarmO+9Ap07lOa+ISDYq7kTqjIo7\nEal1ZjB+fPnPM306fOtbcPXVobPnY4/BnXdCt26hE6iISKXpnx6ROtOlC3TvHncUIiK14ZBDwjTR\nZ5+FsWNDo5UFC+KOSkTqlXmmVk8VZmaelFhESsXMcHeLO45myjOpRcozSYK33oJhw+DSS0PTl1qT\ntDwD5ZrUptbmmoo7kTJK2sVQeSa1SHkmUn5JyzNQrkltam2uaSkEERERERGRGqDiTkREREREpAao\nuBMREREREakBKu5ERERERERqgIo7ERERERGRGqDiTkREREREpAbkVdyZ2QgzW21ma81sapr9B5jZ\nHWa20swWm1n/FvsbzGy5mc0rVeAitSaPPOtsZveY2VNm9oyZnZ2y70dm9pyZPW1mfzSzdhUNXiRh\nir1umVlvM1sRXbNWmNk7Zva9aN9Pzez1aN9yMxtR6Z9LpFrkkYNNUf6tMLMnzeyz+R4rIpnlLO7M\nrAH4DXAKcDQw1sz6tnjadGCFuw8AvgH8usX+ScDzrQ+3ch588MG4Q9hD0uKB5MWUtHgKkWeeTQSe\ncveBwH8CV5pZo5l1AyYAg9z9WKARGFO56IuXxN9Z0mJSPIVrzXXL3de6+yB3HwwMAf4F3JFy3FXu\nPjh63Fvun6VUkvZ7Uzy5JTGmfOWZgwvdfYC7DwLGATcUcGwiJe13lrR4IHkxJS2eUshn5G4o8IK7\nv+LuO4HbgNNaPKc/sAjA3dcA3c3s4wBmdgTwReD3JYu6ApL2y05aPJC8mJIWT4HyybMNwP7R1/sD\nm9z9A2ArsAPYz8wagQ7APysTdusk8XeWtJgUT1Fadd1K8TngJXd/PWVbohZxzlfSfm+KJ7ckxlSA\nnDno7u+mfNsReCvfY5Mqab+zpMUDyYspafGUQj7FXRfgtZTvX4+2pVoJnAFgZkOBrsAR0b5fAT8E\nvFWRitS2fPLsRuBoM/snIecmAbj7FuBK4FXgDeBtd19Y9ohFkqu1161mo4E/t9g2MZoa/Xsz61S6\nkEVqSj45iJmdbmargLuB7xVyrIikV6qGKpcCB5rZcuA8YAXwoZn9F7DR3Z8ifNpZlZ94iiTEj4CV\n7n44MAi41sw6mllP4AdAN+BwoKOZfTXGOEWqQdrrVvNOM2sLNAFzUo65DugZTY3eAFxVuXBFao+7\nz3X3foRcmx13PCI1wd2zPoATgHtTvp8GTM1xzDrCEPvPCaMJ64D1wHbglgzHuB561OIjV47lm2eE\nTzZPSvn+AeA44L+BG1O2nwn8RnmmRz09SnDdehnomPJ9U+prpHl+N+Bp5Zke9fTI53rWihx8Cehc\nyLFxvx966FGuR765lu7RSG5PAL2ipg3rCY0axqY+IZqa8q677zSzCcD/uft2wg3r06PnfAaY4u5n\npTuJu2tUT+pZzjwDVhHuAXrUzA4FehM+ONkJXGhm7YH3geHR6+1FeSZ1opjr1kPRdavZWFpMyTSz\nw9x9Q/TtGcCz6U6uPBPJKwePcveXoq8HA7j7JjN7O9exzZRrInvLWdy5+4dmNhFYQJjGeZO7rzKz\nc8JuvwHoB8wys13Ac8D4cgYtUmvyzLNfADPNbCVhivP/uPtmYLOZ3QIsI0wrW0HUdUykHrX2umVm\nHQgfpHy7xUtfbmYDgV3AP4Bzyv7DiFShPHPwK2Z2FqEh2L+IujxnOjaWH0SkClk0rC0iIiIiIiJV\nrFQNVYoW90KVZnaEmS2KFoB+JmWx2gPNbIGZrTGz+yrdFc1aLPyegHg6mdkcM1sVvVfHxxmTpVm0\nu9LxmNlNZrbRzJ5O2ZYxhijmF6L38AvljC1NrLEvCKtcyyuWROVZFFOsuVZNeRadX9e09HElJs+i\n8ycq1+LOsyiGqsk15VnGuJRn2eOpizyLtbizZCxU+QEw2d2PBj4NnBfFMI2wwGYfwlpIP6pwXC0X\nfo87nhnA3R66Wg0AVscVk6VftHtsDPHMJPzdTZU2BjPrT2h80g8YCVxnZhW5VyAheQbKtXwkJs8g\nMblWFXkWnT8JuaY8y09ici0heQZVkmvKs6yUZxnUVZ61phtLax+Ejkj3FNJNqQIxzSXca7EaODTa\ndhiwuoIxHAHcDwwD5kXb4oznY4SFfFtujyUm4MDo3AcSknNeXL8zWnTMyxRDy7/bwD3A8RV6vxKX\nZ1EcyrU9Y0lUnkXnS0SuVUOeRedLXK4pz9LGk6hcS0qeRedJfK4pzzLGoDzLHk/d5Fnc0zITtVCl\nmXUHBgKLCW/yRgAP3dEOqWAo6RZ+jzOeHsBbZjYzGu6/wULDgVhi8r0X7X7Hw6Ldcb5HzQ7JEEPL\nv+tvULm/64nKM1CuZZCoPIvOl9RcS2KepTu/rmlBkvIMEpZrCc4zSGauKc/SU55lUU95Fndxlxhm\n1hH4KzDJQzvslp1mKtJ5xvZe+D2TSnbCaQQGA9e6+2BCV6tpaWKo1HvUctHu/czsa3HFk0MSYkgU\n5VpGicozqKpci/v8iaM8yypRuVZFeQbJiCExlGdZKc+K16oY4i7u3gC6pnx/RLStosyskZCcs939\nzmjzRgtriWFmhwFvViick4AmM1tHWGPps2Y2G9gQUzwQPhV7zd2fjL7/GyFh43qPjgMedffN7v4h\n8HfgxBjjSZUphjeAI1OeV8m/64nIM1Cu5ZC0PIPk5loS86z5/LHnmvIsp6TlWlLzjCwx1P01TXmW\nk/IsfyXNs7iLu48WuTSzdoQ1TubFEMfNwPPuPiNl2zzg7OjrbwB3tjyoHNx9urt3dfeehPdjkbuf\nCcyPI54opo3Aa2bWO9o0nLAuVCzvEbAGOMHM2kc3lg4n3EAcRzzGnp+SZYphHjDGQmemHkAvYGkF\n4oPk5Bko17LFk7Q8g+TkWjXkGSQn15Rn2WNKWq4lJc+gOnJNedaC8iwv9ZNnxd4MWKoHMILwhr8A\nTIvh/CcRFn5+irD48/IopoOAhVFsC4ADYojtM+y+KTbWeAhdjp6I3qc7gE5xxkSYV/4c8DQwC2hb\n6XiAPwH/BN4nzOEeR7hRN20MhO5HLwKrgC9U+PcXa55FMSjXcseRqDyLYoo116opz6Lz65qWObZE\n5Fl0/kTlWtx5FsVQNbmmPMsam/Isczx1kWdaxFxERERERKQGxD0tU0REREREREpAxZ2IiIiIiEgN\nUHEnIiIiIiJSA1TciYiIiIiI1AAVdyIiIiIiIjVAxZ2IiIiIiEgNUHEnIiIiIiJSA1TciYiIiIiI\n1ID/B1wON8aAN0+CAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x10e61bdd0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Plot the cosine similarity with the true W.\n",
    "\n",
    "plt.figure(figsize=(15,5))\n",
    "z = np.arange(num_iter)\n",
    "plt.subplot(1,4,1)\n",
    "plt.plot(z,sim_gd)\n",
    "plt.title('Cosine Sim GD')\n",
    "plt.subplot(1,4,2)\n",
    "plt.plot(z,sim_sgd)\n",
    "plt.title('Cosine Sim SGD')\n",
    "plt.subplot(1,4,3)\n",
    "plt.plot(z,sim_wsgd)\n",
    "plt.title('Cosine Sim WSGD')\n",
    "plt.subplot(1,4,4)\n",
    "plt.plot(z,sim_awsgd)\n",
    "plt.title('Cosine Sim AWSGD')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZEAAAEACAYAAAByG0uxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xu8VHW9//HXB5GHiYooF5XNRUHlUklmSN4avHBNsExF\nLW91Qj2UZZlanZ9geTp68lRKHrOs453QUlFRtxfGtBJQRFE2slFEQIW8oILKZe/P74/v2jIOe+9Z\nezMzay7v5+MxD2bW/n7XfNZi7/nM+t6WuTsiIiLt0SHpAEREpHwpiYiISLspiYiISLspiYiISLsp\niYiISLspiYiISLvFSiJmNtrMFpvZEjO7sIUyV5lZvZktMLOhceua2Q/MrNHMdsvYdnG0rzozG9me\nAxMRkcLLmUTMrAMwDRgFDAFONrOBWWXGAP3dfV9gEnBtnLpmVgMcAyzP2DYIOBEYBIwBrjEz24Zj\nFBGRAolzJTIMqHf35e6+CZgOTMgqMwG4EcDd5wBdzKxnjLq/Ai5oZl/T3X2zu78C1Ef7ERGREhMn\nifQCVmS8Xhlti1OmxbpmNh5Y4e4Lc+xrVTPvJyIiJaBjgfbbavOTmX0K+DGhKUtERMpUnCSyCuiT\n8bom2pZdpnczZTq1ULc/0A94NurvqAHmm9mwmO+HmWnRLxGRdnD3/PUzu3urD2A7YCnQl5AUFgCD\nssqMBe6Lng8HnoxbNyq3DOgaPR8MPBOV3zuqb83UcQkuueSSpEMoGcU+F+vXu596qvuQIe61te6N\njVt+tmaN+/nnu++5p3s6XdSw3F2/F5l0LraIPjtzfvbHfeS8EnH3BjObDNQS+lCud/c6M5sUBXOd\nu88ys7FmthRYD5zZWt3m3oaoCczdF5nZDGARsAk4NzpwkZLywQdw7LHQowfMnQs77vjJn3fvDlde\nCaNHwwknwI03hucilSRWn4i7PwDsn7Xtd1mvJ8et20yZfbJe/wL4RZzYRJLQ2Agnnwx77gk33ADb\nbddy2WOOgbvuggkT4P774aCDihenSKFpxnoFSKVSSYdQMop1Li67DN58E/74x9YTSJNDDoHf/Q6+\n+lX4178KHx/o9yKTzkXhWLm2FJmZWrkkEf/8Z0gGTz8Ne+3Vtro/+hG8/DLcfjtoCq0kwczy2rGu\nKxGRNtiwAb71LfjNb9qeQAAuvRQWLYIZM/Ifm0gSdCUi0gY/+xk89VTo42jvlcSTT4YrmRdfhJ13\nzm98Irnk+0pESUQkpjfegCFDQjNWv37btq9vfAP69oWf/zwvoYnEpiQSURKRYjvnnDCM98ort31f\nK1bA0KHwzDPQp0/u8iL5oiQSURKRYlqyBA49FBYvht13z88+L7oI3nsPrrkmP/sTiUNJJKIkIsX0\nzW+GK4ZLLsnfPtesgYEDYeFC6KUlRqVIlEQiSiJSLKtWwWc+A/X1+bsKafKDH8DmzWG0l0gxKIlE\nlESkWH74Q2hogF/9Kv/7fv11GDwYli7Nf4ISaY6SSERJRIph7VrYZx949lno3Tt3+fY4/fQw6utH\nPyrM/kUyabKhSBHdeCOMGlW4BALw3e/Cb38bmrVEyo2SiEgL3OHaa8PQ3kL6/OehpgZmzizs+4gU\ngpKISAseeyzMSj/88MK/13e+A9OmFf59RPJNSUSkBddeC2efXZyFEr/6VXj++bA4o0g5Uce6SDPW\nrIH994dly2DXXYvznt/7HuyyS1ikUaRQ1LEuUgTTp4e7FhYrgQCceSb83/+F4cQi5UJJRKQZN90E\nX/96cd/zgAOgWzd49NHivq/ItlASEcmyeHGYpX7UUcV/77POgj/9qfjvK9JesZKImY02s8VmtsTM\nLmyhzFVmVm9mC8xsaK66ZnapmT0blX/YzGqi7X3N7AMzmx89tDydFNXNN8Mpp8S77W2+TZwI990H\n69cX/71F2iNnEjGzDsA0YBQwBDjZzAZmlRkD9Hf3fYFJwLUx6l7h7ge4+1DgbmBKxi6XuvuB0ePc\nbTlAkbZobAxJpNhNWU26dQv3Y7/33mTeX6St4lyJDAPq3X25u28CpgMTsspMAG4EcPc5QBcz69la\nXXdfl1G/M/BmxmvdfVoS8fe/h7sNHnBAcjGcdFLo2BcpB3GSSC9gRcbrldG2OGVarWtmPzezV4Ez\ngF9klOsXNWXNNrPDYsQokhczZoQmpWLMDWnJcceFzvV3300uBpG4CtWxHutP0N1/6u59gD8Bv442\nvw70cfcDgR8At5rZToUJU2SLxkb461/ha19LNo5dd4URI8J93EVKXccYZVYBmTfwrIm2ZZfp3UyZ\nTjHqAtwKzAJw943Axuj5fDN7CdgPmJ9dacqUKR8/T6VSpFKpGIcj0rwnn4SuXcMkw6SddFJY/PH0\n05OORMpdOp0mnU4XbP85Z6yb2XbAi8BRhKuEucDJ7l6XUWYs8O/uPs7MhgO/dvfhrdU1swHuvjSq\n/x1gmLt/w8y6AW+7e6OZ7QM8BnzG3ddmxaUZ65JX558f+kOmTk06Eli3DvbaC155BXbbLelopJLk\ne8Z6zisRd28ws8lALaH56/ooCUwKP/br3H2WmY01s6XAeuDM1upGu/4vM9sPaABeBprWSj0CuNTM\nNgKNwKTsBCKSb+6hKeuee5KOJNhpp9CkNWtWciPFROLQ2lkiwFNPhbkhL76YbKd6pj/9KcwZueOO\npCORSqK1s0QK4I474PjjSyeBAHz5y/DQQ/DRR0lHItIyJRERQlPW8ccnHcUnde8e5qs88kjSkYi0\nTElEqt6SJWGZkc9/PulItnbccRrqK6VNSUSq3r33wrhxpdWU1WTChHDbXC0PL6VKSUSq3r33hv6H\nUtS/P/TsGeawiJQiJRGpau++C/PmJbPse1zjx4erEZFSpCQiVa22Fg4/HDp3TjqSlo0dC/ffn3QU\nIs1TEpGqVspNWU0OPjjcJGvlyqQjEdmakohUrYaGMCN83LikI2nddtvBqFG6GpHSpCQiVWvuXNhz\nT+jbN+lIchszRklESpOSiFSt++4r/auQJqNGhXuMbNyYdCQin6QkIlWrHJqymvToEZao//vfk45E\n5JOURKQqrVkDL78cOq3LxZgxIfGJlBIlEalKDz8cllrffvukI4lPQ32lFCmJSFWqrYWRI5OOom0O\nOghWr4ZXX006EpEtlESk6riXZxLp0CHEXFubdCQiWyiJSNV54QXYYYewLlW5OeaY0BQnUiqURKTq\nlONVSJOjjw73F2lsTDoSkUBJRKpOOSeRmhro1g0WLEg6EpEgVhIxs9FmttjMlpjZhS2UucrM6s1s\ngZkNzVXXzC41s2ej8g+bWU3Gzy6O9lVnZmX65y6l6KOPwlyLI49MOpL2O/poNWlJ6ciZRMysAzAN\nGAUMAU42s4FZZcYA/d19X2AScG2Mule4+wHuPhS4G7gkqjMYOBEYBIwBrjErxdsFSTl64gn4zGdg\n112TjqT9jjkm3HtdpBTEuRIZBtS7+3J33wRMByZklZkA3Ajg7nOALmbWs7W67r4uo35n4K3o+Xhg\nurtvdvdXgPpoPyLbrJybspqkUuEmVR9+mHQkIvGSSC9gRcbrldG2OGVarWtmPzezV4EzgF+0sK9V\nzbyfSLtUQhLZZRf47Ge1BIqUho4F2m+s5id3/ynw06iv5NfAmW15kylTpnz8PJVKkUql2lJdqszq\n1bB8OQyrgOvapiato49OOhIpdel0mnQ6XbD9x0kiq4A+Ga9rom3ZZXo3U6ZTjLoAtwJNqwK1tK+t\nZCYRkVxmz4YjjoCOhfrqVERHHw3nnZd0FFIOsr9gT506Na/7j9OcNQ8YYGZ9zawTMBHIvuPzTOA0\nADMbDqx199Wt1TWzARn1jwMWZOxropl1MrO9gQHA3HYdnUiG2bPDelmV4OCD4aWX4M03k45Eql3O\n72Tu3mBmk4FaQtK53t3rzGxS+LFf5+6zzGysmS0F1hM1S7VUN9r1f5nZfkAD8DJwTlRnkZnNABYB\nm4Bz3d3zedBSndJpOPfcpKPIj+23D1dVjz4KJ56YdDRSzaxcP5/NTLlFYnvttTC091//CmtQVYKr\nroKFC+H3v086EiknZoa7523aRIX8OYm0Lp2GL32pchIIhAmTs2cnHYVUuwr6kxJp2ezZYX5FJRky\nBN57D1asyF1WpFCURKQqpNOV06nexCwkRl2NSJKURKTirVwJa9eGb+6VZsQIJRFJlpKIVLxK7A9p\noiQiSavAPyuRT6rE/pAm++8PGzbAsmVJRyLVSklEKl4lTTLMZqarEUmWkohUtOXLYd06GDw46UgK\nR0lEkqQkIhUtnQ5NWZV8R5qmJKK5t5IEJRGpaE1JpJL17x+SZH190pFINVISkYpWyf0hTdQvIklS\nEpGK9cor4Z7qAwfmLFr2lEQkKUoiUrGahvZWcn9IkxEjQtOd+kWk2JREpGJVQ39Ik3794FOfgrq6\nnEVF8kpJRCqSe3X0h2RSk5YkQUlEKtKyZbB5M+y3X9KRFI+SiCRBSUQqUjX1hzRp6hdpbEw6Eqkm\nSiJSkSpx6fdcamqga1d4/vmkI5FqoiQiFaepP6RaOtUzNV2NiBRLrCRiZqPNbLGZLTGzC1soc5WZ\n1ZvZAjMbmquumV1hZnVR+b+Y2S7R9r5m9oGZzY8e12zrQUp1eeml8O+AAcnGkQTdpEqKLWcSMbMO\nwDRgFDAEONnMBmaVGQP0d/d9gUnAtTHq1gJD3H0oUA9cnLHLpe5+YPQ4d1sOUKpPNfaHNEml4G9/\nU7+IFE+cK5FhQL27L3f3TcB0YEJWmQnAjQDuPgfoYmY9W6vr7g+7e9Ov+pNATcb+qvDPX/KlGvtD\nmuy1F3TrBs89l3QkUi3iJJFewIqM1yujbXHKxKkLcBZwf8brflFT1mwzOyxGjCJAdfeHNFG/iBRT\nxwLtN/aVhJn9BNjk7rdGm14D+rj7O2Z2IHCXmQ1293XZdadMmfLx81QqRaqaPzkECCvZduwI++yT\ndCTJSaXgttvge99LOhIpBel0mnQBv1XESSKrgD4Zr2uibdllejdTplNrdc3sDGAscGTTtqjZ653o\n+XwzewnYD5ifHVhmEhGB6u4PaZJKwTnnQEMDbLdd0tFI0rK/YE+dOjWv+4/TnDUPGBCNmuoETARm\nZpWZCZwGYGbDgbXuvrq1umY2GrgAGO/uG5p2ZGbdog55zGwfYADw8jYco1SRau4PabLHHuHx7LNJ\nRyLVIGcScfcGYDJhNNULwHR3rzOzSWb27ajMLGCZmS0Ffgec21rdaNdXAzsBD2UN5T0CeM7M5gMz\ngEnuvjY/hyuVTP0hW6hfRIrFvEzXjjYzL9fYpTAWL4bRo8N9RKrdjBlw001wzz1JRyKlxsxw97w1\n+GrGulQMXYVskUrB44+HfhGRQlISkYqh/pAtevSAXr3gmWeSjkQqnZKIVAT36roJVRxaGl6KQUlE\nKkJdHXTuDH37Jh1J6Uil1LkuhackIhVB/SFbS6XgiSfCzblECkVJRCqC+kO21q1buDKbv9U0XZH8\nURKRstfYqP6QlmhpeCk0JREpey+8AF26QO/euctWG006lEJTEpGyp6aslh1xBPz977BpU9KRSKVS\nEpGyp071lu2+e1jR+Omnk45EKpWSiJS1xkZ47DFdibRG/SJSSEoiUtYWLgyjkPbaK+lISpf6RaSQ\nlESkrKkpK7cjjoB//hM2bkw6EqlESiJS1mbPVlNWLl27woAB8NRTSUcilUhJRMpWQwP87W+6EolD\n/SJSKEoiUrYWLIA99wx38ZPWqV9ECkVJRMqWmrLiO/xwePJJ2LAhd1mRtlASkbKlJBLfrrvC/vvD\nvHlJRyKVRklEytLmzWGF2i99KelIyof6RaQQYiURMxttZovNbImZXdhCmavMrN7MFpjZ0Fx1zewK\nM6uLyv/FzHbJ+NnF0b7qzGzkthygVKb586FPH+jePelIyof6RaQQciYRM+sATANGAUOAk81sYFaZ\nMUB/d98XmARcG6NuLTDE3YcC9cDFUZ3BwInAIGAMcI2Z5e2m8lIZ1JTVdocdBnPnql9E8ivOlcgw\noN7dl7v7JmA6MCGrzATgRgB3nwN0MbOerdV194fdvTGq/yRQEz0fD0x3983u/gohwQxr7wFKZVIS\nabsuXWDQoNDBLpIvcZJIL2BFxuuV0bY4ZeLUBTgLmNXCvla1UEeq1KZN8I9/qD+kPXTLXMm3jgXa\nb+zmJzP7CbDJ3W9r65tMmTLl4+epVIqUZp1VhXnzoH9/2G23pCMpPyNGwOWXwyWXJB2JFEs6nSZd\nwG8OcZLIKqBPxuuaaFt2md7NlOnUWl0zOwMYCxwZY19byUwiUj3UlNV+hx0GJ5wAH30EO+yQdDRS\nDNlfsKdOnZrX/cdpzpoHDDCzvmbWCZgIzMwqMxM4DcDMhgNr3X11a3XNbDRwATDe3Tdk7WuimXUy\ns72BAcDcdh+hVBwtuth+O+8Mn/50WJBRJB9yXom4e4OZTSaMpuoAXO/udWY2KfzYr3P3WWY21syW\nAuuBM1urG+36asKVykPR4Ksn3f1cd19kZjOARcAm4Fx397wetZStDRtgzpywMq20T1O/iK7mJB+s\nXD+fzUy5pQo9/jh8//takXZbPPggXHZZWLxSqo+Z4e55mzahGetSVtQfsu0OPTRM1vzgg6QjkUqg\nJCJl5ZFH4Mgjc5eTlu20E3z2s+oXkfxQEpGysW4dPP20+kPyQfNFJF+URKRsPP44HHQQdO6cdCTl\nb8QILcYo+aEkImXj4Yfh6KOTjqIyHHJIuKnX+vVJRyLlTklEysZDDymJ5EvnzjB0aFg+RmRbKIlI\nWXjjDVixIjRnSX4cdVS4uhPZFkoiUhYefTR0Bncs1GpvVWjkSKitTToKKXdKIlIW1B+Sf8OGwbJl\nsGZN0pFIOVMSkZLnriRSCNtvH67u1KQl20JJREpefX1IJPvtl3QkleeYY8KABZH2UhKRktd0FaKb\nJOdfU7+IlqGT9lISkZKnob2FM2AAdOoEdXW5y4o0R0lEStrmzWF5jqOOSjqSymQWmrQ0SkvaS0lE\nStrTT0Pv3rDHHklHUrlGjlS/iLSfkoiUtNpaNWUV2pFHhnXJNmzIXVYkm5KIlLT774cxY5KOorLt\nthsMGqQlUKR9lESkZL39Njz/PBx+eNKRVD41aUl7KYlIyXrooXDvkB12SDqSyqfOdWmvWEnEzEab\n2WIzW2JmF7ZQ5iozqzezBWY2NFddM/uamT1vZg1mdmDG9r5m9oGZzY8e12zLAUr5euABGD066Siq\nw/DhYVLnm28mHYmUm5xJxMw6ANOAUcAQ4GQzG5hVZgzQ3933BSYB18aouxD4CvBYM2+71N0PjB7n\ntuvIpKw1NoYkov6Q4ujUKSyBoiYtaas4VyLDgHp3X+7um4DpwISsMhOAGwHcfQ7Qxcx6tlbX3V90\n93qguXnImptc5Z57LtwLvH//pCOpHmPHwn33JR2FlJs4SaQXsCLj9cpoW5wyceo2p1/UlDXbzA6L\nUV4qjEZlFd/YseHqr6Eh6UiknBTq7gzbciXxGtDH3d+J+kruMrPB7r4uu+CUKVM+fp5KpUilUtvw\ntlJKHngALmy2900KpXdv2GsvmDsXvvjFpKORfEmn06TT6YLtP04SWQX0yXhdE23LLtO7mTKdYtT9\nhKjZ653o+XwzewnYD5ifXTYziUjlePddmD8/tNFLcY0bB7NmKYlUkuwv2FOnTs3r/uM0Z80DBkSj\npjoBE4GZWWVmAqcBmNlwYK27r45ZFzKuXMysW9Qhj5ntAwwAXm7bYUk5e+QROOQQ2HHHpCOpPuoX\nkbbKeSXi7g1mNhmoJSSd6929zswmhR/7de4+y8zGmtlSYD1wZmt1AczsOOBqoBtwr5ktcPcxwBHA\npWa2EWgEJrn72nwfuJQu9Yck54tfhFdegddeC01bIrmYl+mNBMzMyzV2aVljI9TUhJV7dROqZEyc\nGCYffvObSUcihWBmuHveRsBqxrqUlPnzYZddlECSNG6cmrQkPiURKSkzZ8L48UlHUd1Gj4ZHH4WN\nG5OORMqBkoiUlHvugWOPTTqK6ta9OwwcGJaHF8lFSURKxquvwooVGl5aCjRKS+JSEpGSce+94cOr\nY6GmwEps48fD3XeDxq5ILkoiUjLUH1I6DjggjJRbuDDpSKTUKYlISXj//XBnvVGjko5EAMzguOPg\nrruSjkRKnZKIlITa2tAXsvPOSUciTb7yFbjzzqSjkFKnJCIlQU1ZpefQQ2HVqjCDXaQlSiKSuE2b\nQqe6kkhp2W67MNxaTVrSGiURSdzs2WGGeu/euctKcX3lK0oi0jolEUncHXfA8ccnHYU05+ijYcEC\n+Ne/ko5ESpWSiCRq8+bwTVdJpDTtsENYjPHee5OOREqVkogk6vHHoU8f2HvvpCORlhx3HPzlL0lH\nIaVKSUQSpaas0nfssSHZv/NO0pFIKVISkcQ0NMBf/6okUup22SX0jaiDXZqjJCKJ+cc/oEcP3Tuk\nHJx0EkyfnnQUUoqURCQxt9+uq5ByMW4czJmjUVqyNSURScTmzTBjBpx8ctKRSBydO4f73quDXbLF\nSiJmNtrMFpvZEjO7sIUyV5lZvZktMLOhueqa2dfM7HkzazCzA7P2dXG0rzozG9neg5PS9cgjYVTW\nvvsmHYnEddJJ8Oc/Jx2FlJqcScTMOgDTgFHAEOBkMxuYVWYM0N/d9wUmAdfGqLsQ+ArwWNa+BgEn\nAoOAMcA1Zpa3m8pLabjlFjjllKSjkLYYPTpMPHz99aQjkVIS50pkGFDv7svdfRMwHZiQVWYCcCOA\nu88BuphZz9bquvuL7l4PZCeICcB0d9/s7q8A9dF+pEJ88EFYcHHixKQjkbbYYYewvtnttycdiZSS\nOEmkF7Ai4/XKaFucMnHq5nq/VTHqSBm55x4YNgz22CPpSKStTjkFbrop6SiklBTqRqRFaX6aMmXK\nx89TqRSpVKoYbyvb6JZb4NRTk45C2uPoo+Gss2DRIhg8OOloJI50Ok06nS7Y/uMkkVVAn4zXNdG2\n7DK9mynTKUbd5t6vuX1tJTOJSHl46y147DG4+eakI5H22G47+PrX4YYb4PLLk45G4sj+gj116tS8\n7j9Oc9Y8YICZ9TWzTsBEYGZWmZnAaQBmNhxY6+6rY9aFT165zAQmmlknM9sbGADMbctBSemaPj0M\nFd1ll6QjkfY6/fTQpLV5c9KRSCnImUTcvQGYDNQCLxA6vevMbJKZfTsqMwtYZmZLgd8B57ZWF8DM\njjOzFcBw4F4zuz+qswiYASwCZgHnurvn8ZglQddfH5pDpHwNHgw1NfDww0lHIqXAyvXz2cyUW8rM\nM8+EFWGXLYMOmuZa1n77W3jiCbjttqQjkbYyM9w9b/3WSiJSNJMnQ7duoK6s8vfWW7DPPrB8Oey6\na9LRSFvkO4no+6AUxYcfhm+tZ56ZdCSSD7vvDiNHwq23Jh2JJE1JRIrizjvhoIOgb9+kI5F8Ofts\nuPZaUINAdVMSkaL4wx/gm99MOgrJpxEj4KOPwpL+Ur2URKTg6urC5LQJ2YvlSFnr0GHL1YhUL3Ws\nS8FNngxdu8LPfpZ0JJJvb78N/ftDfX0YNCGlT6OzIkoi5eG996BfP1i4EHppBbSKdMYZMGQIXHBB\n0pFIHBqdJWXlhhvCektKIJWrqUmroSHpSCQJSiJSMI2NMG0afOc7SUcihXTwwaEp6557ko5EkqAk\nIgXz8MPhHhSHHZZ0JFJIZvCDH8Avf5l0JJIEJREpmP/+b/j+98OHjFS2r34VVq2COXOSjkSKTUlE\nCuLpp2HxYt0Ct1p07AjnnQdXXpl0JFJsGp0lBXHiifDFL4YrEakO778fRuI99RTsvXfS0UhLNMQ3\noiRSupYuDQlk2TLYaaeko5FiuuiiMKz7mmuSjkRaoiQSURIpXWefDd27a3JhNVqzBgYOhOeeC/cc\nkdKjJBJREilNy5fDgQeG/pDu3ZOORpJwwQVh1eZp05KORJqjJBJREilN//Zv0KMHXHZZ0pFIUnQ1\nUtqURCJKIqVn6VIYPjyso9S1a9LRSJIuuCCs8Hv11UlHItmURCJKIqXnG9+A/faD//iPpCORpK1Z\nA4MGaaRWKUpk7SwzG21mi81siZld2EKZq8ys3swWmNnQXHXNrKuZ1ZrZi2b2oJl1ibb3NbMPzGx+\n9NA4jzLw/PPw4INhroBIjx7w3e/CT36SdCRSaDmvRMysA7AEOAp4DZgHTHT3xRllxgCT3X2cmR0M\n/Mbdh7dW18wuB95y9yui5NLV3S8ys77APe7+2Rxx6UqkRLjDqFHw5S+HDw4RgPXrw5XpnXfCsGFJ\nRyNNkrgSGQbUu/tyd98ETAeyby80AbgRwN3nAF3MrGeOuhOAG6LnNwDHZexPC2WUkfvugxUr4Jxz\nko5ESknnznDppfDDH+oWupUsThLpBazIeL0y2hanTGt1e7r7agB3fwPokVGuX9SUNdvMtHxfCdu4\nEc4/H/7nf2D77ZOORkrNGWfAO+/AX/+adCRSKB0LtN/2XEk0fVd5Hejj7u+Y2YHAXWY22N3XZVeY\nMmXKx89TqRSpVKodbyvb4uqrw53txoxJOhIpRdttF35HTjsNRo6EnXdOOqLqk06nSafTBdt/nD6R\n4cAUdx8dvb4IcHe/PKPMtcBsd/9z9Hox8CVg75bqmlkdkHL31Wa2R1R/UDPvPxv4gbvPz9quPpGE\nvfIKHHQQ/OMfoe1bpCWnnx7uOaIFGpOXRJ/IPGBANGqqEzARmJlVZiZwWhTgcGBt1FTVWt2ZwBnR\n89OBu6P63aIOecxsH2AA8HL7Dk8KxT30gZx/vhKI5PbLX8LNN8OCBUlHIvmWsznL3RvMbDJQS0g6\n17t7nZlNCj/269x9lpmNNbOlwHrgzNbqRru+HJhhZmcBy4ETo+1HAJea2UagEZjk7mvzdsSSF7fd\nFu4foftqSxzdu8N//id861vwz3+q/6ySaLKhtNlrr4X1se6+O9waVSQO99B3Nnw4ZHRnSpFpxnpE\nSSQZjY0wejQccog+CKTtXnsNPvc5mDlTX0CSksiMdZEmv/51mET2058mHYmUo732gt/+NiyRs26r\n8ZZSjnTcFwyqAAAI1UlEQVQlIrHNmwdjx8LcuVoPSbbNt74Vbl715z+DaWpxUelKRBKxejUcfzz8\n4Q9KILLtpk2Dl18Ok1SlvOlKRHLauBGOOio81A8i+bJ8eegXufVWOPLIpKOpHupYjyiJFIc7nHUW\nvP12WEivg65dJY9mz4aTToJHH4VPfzrpaKqDmrOkqC6+ONzq9tZblUAk/0aMCIM1xo4Ni3hK+SnU\n2llSAa68MgzFfPzxsCKrSCGcckoY+jtqVLgy6dkz6YikLZREpFmXXw6//334o95996SjkUr3wx/C\nBx9AKhWatvbcM+mIJC4lEfkE99B5PmMG/O1vYVy/SDH8v/8XVv1NpaC2Fvr2TToiiUNJRD62cSOc\nfTY88ww89li4xalIMf3kJ7DTTmFFBN0RsTyoq1QAePNNOOaYcAOhxx9XApHknHce/O//wrhxYTKi\nlDYlEeHRR2HoUDjsMPjLX8I3QZEkjR8fmrR+/ONwdfzhh0lHJC1REqliH34IF10U1jH64x/hsss0\njFdKx+c+B/Pnw9q1oVlr3rykI5Lm6COjSj34YJjc9dJLoQ9k5MikIxLZWpcu4d41F14Ixx4bmrre\nfz/pqCSTkkiVefbZ0FRwzjlh/aLbb1f/h5Q2M/j61+H550MC2W+/8Lu7cWPSkQkoiVSNefPC8hKj\nR4c1sBYtCjcIEikX3bqFZtdZs8Jj//3hmmu0pHzSlEQq2Pr14b7Ww4fDCSfAF74AS5eGJoEddkg6\nOpH2+dznQhK5+WZ45BHo1y9MVnzhhaQjq06xkoiZjTazxWa2xMwubKHMVWZWb2YLzGxorrpm1tXM\nas3sRTN70My6ZPzs4mhfdWam1vo2eP99uOsumDgxTBS85Zaw/tVLL4U/NC1fIpXi0EPDaMJ588Ik\nxVGjwijDK64IV9pan7U4cq7ia2YdgCXAUcBrwDxgorsvzigzBpjs7uPM7GDgN+4+vLW6ZnY58Ja7\nXxEll67ufpGZDQZuAb4A1AAPA/tmL9mrVXyD996DP/4xzbp1KR56KIxmGTYsXHkcfzx07550hMWV\nTqdJpVJJh1ESqu1cNDaGVRamT4f77w/bRo2CI44ASHPqqSndAIv8r+IbZ8b6MKDe3ZdHAUwHJgCL\nM8pMAG4EcPc5ZtbFzHoCe7dSdwLwpaj+DUAauAgYD0x3983AK2ZWH8UwZxuOs+ytXx+aourrYckS\nqKuDp56CV1+F3XZLc8IJKS6+GA4/vLqvNqrtg7M11XYuOnQIS6akUuEqZPFieOABuPtuqK1Nc955\nKQ46CAYPhoEDYdCg0K/SvbuGtm+LOEmkF5C5SPNKwod6rjK9ctTt6e6rAdz9DTNrGiPUC/hnRp1V\n0bay19AQRpRs2BCSwnvvhce7737y+Zo18Prr8MYbW/597z3o3x/23TeMThkxAi64IPxB/PznulmU\nSCazkCQGDYLvfz/8fXz72+FKva4u3OL5hhvCF7L334devaCmJjx69oTddoOuXcO/Tc87d4Ydd4RP\nfSo8dtwRtt9et/ct1NpZ7TmtbW6b2v60g9vxNmWkR/Q4ILx8MXrwevRIh+0Nz67kspfvL358JUjn\nYgudiy2aPRcDogfhm+7H33bfih4SS5wksgrok/G6JtqWXaZ3M2U6tVL3DTPr6e6rzWwPYE2OfW1l\n801zY4RfHTY/91rSIZQMnYstdC620LkojDhJZB4wwMz6Er7/TgROziozE/h34M9mNhxYGyWHN1up\nOxM4A7gcOB24O2P7LWb2K0Iz1gBgq2yRz44hERFpn5xJxN0bzGwyUEsYEny9u9eZ2aTwY7/O3WeZ\n2VgzWwqsB85srW6068uBGWZ2FrAcODGqs8jMZgCLgE3AuRqGJSJSmnIO8RUREWlJWQ5sizP5sVKY\nWY2ZPWpmL5jZQjP7brS9aidrmlkHM5tvZjOj11V5LqKh9LdHx/aCmR1cxefi4ugcPGdmt5hZp2o6\nF2Z2vZmtNrPnMra1+fjN7MDoHC4xs1/HenN3L6sHIfEtBfoC2wMLgIFJx1XA490DGBo934kwQGsg\noTnwR9H2C4H/ip4PBp4hNFX2i86VJX0ceT4n3wduBmZGr6vyXAD/B5wZPe8IdKnGcxF9FrwMdIpe\n/5nQz1o15wI4DBgKPJexrc3HT5iP94Xo+SxgVK73LscrkY8nP7r7JqBpAmNFcvc33H1B9HwdUEcY\nsTaBMEmT6N/joucfT9Z091eApsmaFcHMaoCxwB8yNlfduTCzXYDD3f1PANExvksVngvgPWAj0NnM\nOgKfIozorJpz4e5PAO9kbW7T8UejZHd296Y7t9yYUadF5ZhEWprYWPHMrB/h28aTZE3WJMwoga3P\nT8VM1oz8CriAT84rqsZzsTfwppn9KWrau87MdqQKz4W7vwNcCbxKOK533f1hqvBcZOnRxuPvRfg8\nbRLrs7Uck0hVMrOdgDuA86IrkuwRERU/QsLMxgGroyuz1oZ4V/y5IDRFHAj81t0PJIyKvIjq/L3Y\nh9DE2RfYi3BFcipVeC5yKMjxl2MSiTP5saJEl+h3ADe5e9N8mtXR+mS0d7JmGToUGG9mLwO3AUea\n2U1EE1ehqs7FSmCFuz8Vvf4LIalU4+/FQcDf3f1td28A7gQOoTrPRaa2Hn+7zks5JpGPJz+aWSfC\nBMaZCcdUaH8EFrn7bzK2NU3WhK0na06MRqfsTQuTNcuRu//Y3fu4+z6E//dH3f0bwD1U37lYDaww\ns/2iTUcBL1CFvxeEwSbDzWwHMzPCuVhE9Z0L45NX6G06/qjJ610zGxadx9My6rQs6VEF7RyJMJrw\ni1MPXJR0PAU+1kOBBsIotGeA+dHx70ZYJv9FwmTOXTPqXEwYcVEHjEz6GAp0Xr7EltFZVXkuCKuq\nzYt+N/5KGJ1VrefiAkISfY7Qibx9NZ0L4FbC7TY2EPqGzgS6tvX4gc8DC6PP1t/EeW9NNhQRkXYr\nx+YsEREpEUoiIiLSbkoiIiLSbkoiIiLSbkoiIiLSbkoiIiLSbkoiIiLSbkoiIiLSbv8f7pkKrqEt\nRnoAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x10e068610>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Plot the Progression of the sampling distribution\n",
    "for i in range(num_iter):\n",
    "    m = np.linspace(0,N,N*100)\n",
    "    p = norm.pdf(m,taus[i],N/10)\n",
    "    plt.plot(m,p)\n",
    "    plt.hold\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x121fc3c90>]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZMAAAEACAYAAAB27puMAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xt8VPWd//HXBxDxwiLWAgIiughCvTUKutVq6gV02wpY\nsVhdULHrtdtadx/K2gqttbbu6o+uCraVlksVpO7KpaUILMSuVhG8VCsIUQQJclG5aEFIIJ/fH5+T\nZogJRE4yk8y8n4/HeTD5zjkz3znGeed7O8fcHRERkTRa5LoCIiLS/ClMREQkNYWJiIikpjAREZHU\nFCYiIpKawkRERFJr9DAxs1Vm9mcze9nMXkjK2pvZXDNbbmZPmVm7jP1HmlmpmS0zs/4Z5UVm9qqZ\nrTCzMY1dbxERqb9stEwqgWJ3/7y790vKbgfmu3svYAEwEsDM+gCXAb2Bi4CxZmbJMeOAEe7eE+hp\nZgOyUHcREamHbISJ1fI+A4GJyeOJwKDk8cXAVHff5e6rgFKgn5l1Atq6++Jkv0kZx4iISI5lI0wc\nmGdmi83s2qSso7tvAHD39UCHpLwLsCbj2LVJWRegLKO8LCkTEZEmoFUW3uNMd19nZp8F5prZciJg\nMumaLiIizVijh4m7r0v+fc/MpgP9gA1m1tHdNyRdWBuT3dcCR2Uc3jUpq6t8D2amUBIR2Q/ubvve\nq26N2s1lZgeb2aHJ40OA/sBrwEzgqmS34cCM5PFMYKiZtTazY4AewAtJV9hWM+uXDMgPyzhmD+6u\nzZ1Ro0blvA5NZdO50LnQudj71hAau2XSEXgyaTG0Ah5197lmtgSYZmbXAKuJGVy4+1IzmwYsBSqA\nG736k94ETADaALPdfU4j111EROqpUcPE3d8GTqmlfBNwfh3H3APcU0v5i8CJDV1HERFJTyvg81Rx\ncXGuq9Bk6FxU07mopnPRsKyh+suaAjPzfPo8IiLZYGZ4Ux6AFxGRwqAwERGR1BQmIiKSmsJERERS\nU5iIiEhqChMREUlNYSIiIqkpTEREJDWFiYiIpKYwERGR1BQmIiKSmsJERERSU5iIiEhqChMREUlN\nYSIiIqkpTEREJDWFiYiIpKYwERGR1BQmIiKSmsJERERSU5iIiEhqChMREUlNYSIiIqkpTEREJDWF\niYiIpKYwERGR1JpVmJjZhWb2hpmtMLPbcl0fEREJ5u65rkO9mFkLYAVwHvAusBgY6u5vZOzjzeXz\niIg0FWaGu1ua12hOLZN+QKm7r3b3CmAqMLDmTpWVWa+XiEjBa05h0gVYk/FzWVK2h48/zlp9REQk\n0ZzCpF4UJiIi2dcq1xX4FNYC3TJ+7pqU7eHuu0fTrl08Li4upri4OBt1ExFpNkpKSigpKWnQ12xO\nA/AtgeXEAPw64AXgcndflrGPL1vmHH98jiopItIMNcQAfLNpmbj7bjO7GZhLdM+NzwySKtu3Z71q\nIiIFr9mECYC7zwF67W0fhYmISPbl3QC8wkREJPsUJiIiklrehYmmBouIZF/ehYlaJiIi2acwERGR\n1BQmIiKSmsJERERSU5iIiEhqChMREUlNYSIiIqnlXZhonYmISPblXZioZSIikn0KExERSU1hIiIi\nqSlMREQkNYWJiIikpjAREZHU8i5MNDVYRCT78i5M1DIREcm+vAuTnTuhsjLXtRARKSx5FyYHHaSu\nLhGRbMu7MDn4YHV1iYhkW96FyUEHKUxERLIt78JELRMRkexTmIiISGp5GSYagBcRya68DBO1TERE\nskthIiIiqSlMREQktUYLEzMbZWZlZvZSsl2Y8dxIMys1s2Vm1j+jvMjMXjWzFWY2JqO8tZlNTY55\nzsy61fW+mhosIpJ9jd0yud/di5JtDoCZ9QYuA3oDFwFjzcyS/ccBI9y9J9DTzAYk5SOATe5+HDAG\nuLeuN1TLREQk+xo7TKyWsoHAVHff5e6rgFKgn5l1Atq6++Jkv0nAoIxjJiaPnwDOq+sNFSYiItnX\n2GFys5m9YmaPmFm7pKwLsCZjn7VJWRegLKO8LCnb4xh33w1sMbPDa3tDTQ0WEcm+VmkONrN5QMfM\nIsCBO4CxwA/d3c3sR8B9wLVp3q/G+9TqmWdGs20bmEFxcTHFxcUN9JYiIvmhpKSEkpKSBn1Nc/cG\nfcFa38TsaGCWu59kZrcD7u4/TZ6bA4wCVgML3b13Uj4UOMfdb6jax90XmVlLYJ27d6jlffzBB52l\nS+Ghhxr9Y4mI5AUzw93r/CO9PhpzNlenjB8vAf6SPJ4JDE1maB0D9ABecPf1wFYz65cMyA8DZmQc\nMzx5PARYUNf7asxERCT7UnVz7cO9ZnYKUAmsAq4DcPelZjYNWApUADd6dfPoJmAC0AaYXTUDDBgP\nTDazUuADYGhdb6qpwSIi2ZeVbq5sMTOfMcP55S9h1qxc10ZEpHlo0t1cuaJuLhGR7FOYiIhIankZ\nJlpnIiKSXXkZJmqZiIhkl8JERERSy7sw0dRgEZHsy7swUctERCT78i5M2rSB8nLYvTvXNRERKRx5\nFyZm0dWlGV0iItmTd2ECmh4sIpJteRsmGjcREckehYmIiKSWl2Gi6cEiItmVl2GilomISHYpTERE\nJDWFiYiIpKYwERGR1PI2TLTOREQke/I2TNQyERHJnrwME00NFhHJrrwME7VMRESyS2EiIiKpKUxE\nRCQ1hYmIiKSmMBERkdTyNky0zkREJHvyMkw0NVhEJLvyMkzUzSUikl2pwsTMLjWzv5jZbjMrqvHc\nSDMrNbNlZtY/o7zIzF41sxVmNiajvLWZTU2Oec7MumU8NzzZf7mZDdtXvRQmIiLZlbZl8howGHg6\ns9DMegOXAb2Bi4CxZmbJ0+OAEe7eE+hpZgOS8hHAJnc/DhgD3Ju8VnvgTqAvcDowysza7a1SChMR\nkexKFSbuvtzdSwGr8dRAYKq773L3VUAp0M/MOgFt3X1xst8kYFDGMROTx08A5yaPBwBz3X2ru28B\n5gIX7q1eewuT3bvhvvugsrKeH1JERPapscZMugBrMn5em5R1AcoyysuSsj2OcffdwFYzO3wvr1Wn\nvYXJH/8I//qv8Mwz9f0oIiKyL632tYOZzQM6ZhYBDtzh7rMaq2J8srVTL6NHj2bXLvjoIygpKaa4\nuHiP56dMge7d4dFH4eyzG6CWIiLNTElJCSUlJQ36mubu6V/EbCFwq7u/lPx8O+Du/tPk5znAKGA1\nsNDdeyflQ4Fz3P2Gqn3cfZGZtQTWuXuHZJ9id78+Oebh5DUer6Ue7u64Q8uWUFER/1YpL4fOnWH6\ndBg0CN59F1q3Tv3xRUSaNTPD3ffrD/gqDdnNlVmRmcDQZIbWMUAP4AV3X090X/VLBuSHATMyjhme\nPB4CLEgePwVcYGbtksH4C5Kyuitisdak5sLFuXOhd28466z4d86c/f6sIiKSIe3U4EFmtgY4A/id\nmf0BwN2XAtOApcBs4EavbgLdBIwHVgCl7l71lT4eOMLMSoHvALcnr7UZuAtYAiwCfpAMxO9VbeMm\nU6bA5ZfH4yuugMce27/PLSIie2qQbq6moqqbC+Doo+Hpp2N8BCJYOneGFSugQwd4/334+7+HsjJo\n2zZ3dRYRybWm1s3VpNRsmcyaBaefHkECcMQRMQA/fXpu6icikk8KJkwyu7iqfOMbMatLRETS2efU\n4OYqM0y2bIGFC2HixD33ufhiuOEGePxxeOMNWLIETjgB7rkn+/UVEWnO8nbMpH//WOVeXg7Ll8fP\nkyd/8piRI+HFF+G002KG17e+BevXQ5s2Wa68iEiONMSYSd6Gybx58PbbcNxx0KMHdOkCLerRqffF\nL8Ltt8OXv9zIlRURaSIUJjVkhsn+uv9+WLoUHnmkgSolItLEKUxqaIgwefvtmPW1bt2eq+dFRPKV\npgY3gmOOiS6xZ5/NdU1ERJoPhUktBg+GJ5/MdS1ERJoPhUktqsIks8dsxw7dA0VEpC4Kk1qccAK0\nagWvvBI/L10a04a/973c1ktEpKlSmNTCrLp1Mn8+FBfDzTfDz38Oq1fnunYiIk2PZnPV4U9/goED\nY23KtGlwzjlw553w1lu6BIuI5BdNDa6hIcOkshKuuy5u8durV5T99a/QsyfMmAF9+zbI24iI5JzC\npIaGDJO6PPIITJoUl7e35NSXl0NpaVzfa9kyWLkS3nknusRat4Yf/ziuA2ap/lOJiDQOhUkN2QiT\n3bvhlFPg6qtjhtf8+bBoUaxN6d07th494n4q3brBqlVwyy3QtSuMGQN9+jRq9UREPjWFSQ3ZCBOA\nBQui++ucc+D88+O+KHu7wVZFBYwdC3fdFfdV+Yd/aPQqiojUm8KkhmyFyf6aNg1Gj4aXX4YDD8x1\nbUREgi6n0swMGRJXMf7xj3NdExGRhqWWSZatXRtjLgsWwIkn5ro2IiJqmTRLXbrA3XfDiBExmC8i\nkg8UJjlw7bVxW+Hx43NdExGRhqEwyYEWLeC734UpU3JdExGRhqExkxzZvh06dYp1KIcfnuvaiEgh\n05hJM3bwwfClL8Hvf5/rmoiIpKcwyaGBA+M6XyIizZ26uXJo48ZYd7JhA7Rpk+vaiEihUjdXM9eh\nQ6w1Wbgw1zUREUknVZiY2aVm9hcz221mRRnlR5vZdjN7KdnGZjxXZGavmtkKMxuTUd7azKaaWamZ\nPWdm3TKeG57sv9zMhqWpc1Ojri4RyQdpWyavAYOBp2t57k13L0q2GzPKxwEj3L0n0NPMBiTlI4BN\n7n4cMAa4F8DM2gN3An2B04FRZtYuZb2bjIEDYebMuu8vP24c3HdfduskIvJppQoTd1/u7qVAbX1t\nnygzs05AW3dfnBRNAgYljwcCE5PHTwDnJo8HAHPdfau7bwHmAhemqXdT0rMntGsHS5Z88rm33oLv\nfx/uvReefz77dRMRqa/GHDPpnnRxLTSzs5KyLkBZxj5lSVnVc2sA3H03sNXMDs8sT6zNOCYv1NbV\n5Q433gi33QYPPQTDhsXaFBGRpqjVvnYws3lAx8wiwIE73H1WHYe9C3Rz983JWMp0M/u0t4Xar5kF\no0eP/tvj4uJiiouL9+dlsuprX4s7MX71q3DGGVE2dSqsXw/f+Q4ccABMnx7B8sADua2riDR/JSUl\nlJSUNOhrNsjUYDNbCNzq7i/t7XkiZBa6e++kfChwjrvfYGZzgFHuvsjMWgLr3L1Dsk+xu1+fHPNw\n8hqP1/I+zWpqcKbp0+Oe87feGheBPOEEePLJ6nDZvBlOOgl+9Su44ILc1lVE8ktTmxr8t4qY2RFm\n1iJ5fCzQA1jp7uuJ7qt+ZmbAMKCqg2cmMDx5PARYkDx+CrjAzNolg/EXJGV5ZdAgWLw4BuN794ZL\nLqkOEoD27SNIrrgCHn4Ydu3KXV1FRGpK1TIxs0HAA8ARwBbgFXe/yMwuAX4IlAOVwJ3uPjs55lRg\nAtAGmO3u307KDwQmA58HPgCGuvuq5LmrgDuI7rUfufukOurTbFsmVXbtggkT4kZa7WqZs/byy3GR\nyPfeg/vvh/79s15FEckzum1vDfkQJvXhHgP2N98crRUFioikoTCpoVDCpMojj0S32MyZua6JiDRn\nCpMaCi1Mtm2Dbt2i66tbt33vLyJSm6Y2AC9ZdsghMSD/i1/kuiYiUugUJs3c9dfH7X/Ly6vLysrg\nllugoiJ39RKRwqIwaeb69IFevWKdCsDWrXDRRfDEE3DPPbmtm4gUDoVJHrjhhrgg5M6dMHgwFBfH\ntbwefBBeqnUZqYhIw9IAfB4oL48B+BNPhLZt4be/hZYt4Te/gZ/8JC4iqZtviUhdNAAvALRuHa2T\njz+GRx+NIIEYnO/VC0aNym39RCT/qWWSJ6ruh9Kixp8HGzfCySfDpEm6ppeI1E4tE/mbFi0+GSQQ\ntwaeNi1aKS++mP16iUhhUJgUgC9+EX7+87jE/Vtv5bo2IpKP9nk/E8kPgwdHl9eAAfDss9Cx476P\nERGpL7VMCsh118Hw4XFp+wa+L46IFDgNwBeg3/0ugmXIkFjYeNBBua6RiOSSBuBlv3zlK/Dqq9Ht\nddJJsR5FN9sSkTTUMilw8+fDj34Ea9bA7bfDVVfFPedFpHDoEvQ1KEz23zPPRJgcfTRMnlz7NGMR\nyU8KkxoUJul8/HEsbDz9dLjvvlzXRkSyRWMm0qAOOiju2jhnDvznf+a6NiLSnGidiezh8MMjTM48\nE1q1gptu0hiKiOybWibyCUcdBfPmxRTiHj3goYeiC6y0NG7Edc01cP/98NFHua6piDQVChOpVa9e\nMdNr2rQIlsMOg3PPhYUL4bTT4IUX4Nhj4Xvfg/fey3VtRSTXNAAv9fLRR3DooWAZQ3Rvvgn/8R/w\n1FNxM65OnXJXPxHZf5rNVYPCJDd+8AOYPTsu0aLV9CLNj8KkBoVJbrjDlVdCRQVMnao1KiLNjcKk\nBoVJ7uzYAeefHxeRvO66mBV22GHVd30UkaZLYVKDwiS33nsPvvENePtt2LQJPvwQeveOy9737x/3\nVclGN9gHH0Rr6YgjGv+9RPKBwqQGhUnTsmtX3N3xqadie+UV+NznoF8/6Ns3ZoN17QqdO8O2bfDG\nG7BsWdyCeNgwOPDA+r+XOzz9dNwE7A9/iIkCX/0qfOtb8V4iUrech4mZ3Qt8FdgJvAVc7e4fJs+N\nBK4BdgHfdve5SXkRMAFoA8x29+8k5a2BScCpwPvA1939neS54cAdgAN3u/ukOuqjMGnCtm2Dl1+O\nacVLlsDq1bB2Lbz7LrRpE62Y44+Pqxm/+WasZfnKV/acQfbRR/EaL74Ir78OGzbE/mVl0bX2z/8c\n4zfusSbmoYdiltk3vwlf/3rMSBORPTWFMDkfWODulWb2E8DdfaSZ9QEeBfoCXYH5wHHu7ma2CLjZ\n3Reb2WzgZ+7+lJndAJzo7jea2deBwe4+1MzaA0uAIsCAF4Eid99aS30UJs1QZWUERmZozJkDt9wS\nd4T87GcjdNauhfffhxNPhFNPjcvnH3lk3Oe+Y0fo3n3P1wDYvTtmmo0fHy2Xr30N7rorjhORkPMw\nqVGZQcDX3P2fzOx2Ilh+mjz3B2A0sJoInz5J+VDgHHe/wczmAKPcfZGZtQTWuXuHzH2SY8YBJe7+\neC11UJjkkYoKeOKJmB3WpUtsXbvu/+Vd1q2DMWNixtnvfhehVGXZMpg4MVpGffvGv5o8IIWiIcKk\nIa/NdQ0wJXncBXgu47m1SdkuoCyjvCwprzpmDYC77zazrWZ2eGZ5jdeSPHfAAXD55Q33ekceCT/9\nKZxySqzmnzwZvvCFWCczaRJcfXWs9r/77ug6u+km+P7395w0sHo1PPpodMt95jMxyN+nT+2tIpFC\nss8wMbN5QMfMImLs4g53n5XscwdQ4e5TanmJ/aX/NaVRXH45dOsWXV7uMS7z+uvRXValrAxuvRVO\nOAHGjYt/7747WjVDh0bQvfxyzGB77bVoRfXrB+edF5MHPvOZ3H0+kVzYZ5i4+wV7e97MrgL+ETg3\no3gtcFTGz12TsrrKM495N+nm+jt332Rma4HiGscsrKs+o0eP/tvj4uJiiouL69pVCtiZZ8YkgM2b\n9+zuqtK1Kzz+eMwMu/76mOp87bXRHZYZOlXKymJiwfTp8MMfwsUXxzTpTZtg6dKYqQZxbIcO0ZL5\nwhfiQppq0Ui2lZSUUFJS0qCvmXYA/kLgPuBsd/8go7xqAP50oktqHtUD8M8D/wIsBn4P/Je7zzGz\nG4ETkgH4ocCgWgbgWySPT3X3LbXUR2Mm0uB27IirJrdvX7/9338fJkyAJ5+Mac99+lSPwWzcGFtp\nadzdsrwczjoLzj4biosj2HQFAcm2nA/Am1kp0BqoCpLn3f3G5LmRwAiggj2nBp/KnlODv52UHwhM\nBj6fvN5Qd1+VPHcV1VODf6SpwZIv3nkH/u//YqbZ009HEA0dCrfdFl1xzcWHH8KUKTH9+7Ofje3j\nj2MK95Il0TI7+OC4KkK7dtEaKy+PrXPnmNL9pS99+lZaZWV0N7rH1PJDDmmcz5fvch4mTY3CRJq7\nsjJ48EH45S/hkktiUsDu3bB9e2wQLZeWLWMcp3v3nFaXlSvhgQdiJtx558WMu/fei61165jCXVQU\ni1V37oQtW2Dr1vjyb906ttdfj3GpiooIlQsuiP1ra6FVVsLy5fCnP8Vkifnzo9uwdWtYsSLWFPXs\nGZMtjjwyfq66tM9hh8HJJ0Pbttk/T2nt2BH/tmnTOK+vMKlBYSL54oMPYhrz7Nkxm+yQQ6pnlVVW\nxtUFFi+O+85ceSUMGZK9Qf+dO2HGDPjVr6LVcc01cPPN6VpS7tHtN2EC/PGP0UI744wIg9274zNv\n3BifuX37eO788yN4jkpGYXftinArLYX162Mq+Pr1MS62dWuc01WrYvbeiBHVU783boxQ6tcvxrD2\n5t134/0b4rJAO3dGHQ8+OGYF1gzPbdvg97+PewrNnRv7H3BA7HvUURHSp50Wgd2r196nsm/cCLNm\nRSvwlFPivGa2AhUmNShMpJCUl8eXzG9+E186vXpF6+D882P8pSFut/zxx9ECeOON6JJbswaefTa+\nkK65BgYPbpzrrW3YAM89F0FQ1RI77LD4wq9tAkR9vfRSLIbdsiVCeM6c6Io766yYQHH88dEaPOus\n+Fxt2sQkiv/5n/hSX7UqWgnHHRdf4n36xGSNLl2i+2716rh6w8qV8eXfokV1l97WrfG+mzbFAtzN\nm+NLffv2uLJDp05xhYZt26q3s8+Gyy6DQYMixP761wjat9+u7kJcsiRCs0+faHn17h2XKjr22Hiv\nX/wiPmf//hGof/5z1GnsWLj00jgvCpMaFCZSqHbuhEWL4H//N66D9tZbcfmYK66Ibp5nn41t7dq4\nZtmll8ZVA2qzbVsEyLRp0TIqKortqKOq/yLOdfdaGu4RDrNnw5e/DBddFMFRXh6h/OtfR9fbjh1x\nXtu0idl5Q4bEl3tFRUwHX7IkWkFlZdXh0L17tG6OPTZaHJWVsR1wQPV4Ufv2EUAdOlS3JnbsiFbP\ntm0RKIceGt1x9e3W+vDDqNOf/xx1WrkytlatohV25ZXx/lWff926uPZdVWtWYVKDwkQkrFwJjz0W\nCyy3b4+p0GeeGX/9PvlkfGkWFcXssc6d4y/rdeuq75rZt298eV5ySd2hI/lDYVKDwkSkfqq6r958\nM/4iXrs2/mIeMCCuDtAcB6ll/ylMalCYiIh8eg0RJloeJSIiqSlMREQkNYWJiIikpjAREZHUFCYi\nIpKawkRERFJTmIiISGoKExERSU1hIiIiqSlMREQkNYWJiIikpjAREZHUFCYiIpKawkRERFJTmIiI\nSGoKExERSU1hIiIiqSlMREQkNYWJiIikpjAREZHUFCYiIpKawkRERFJLFSZmdq+ZLTOzV8zsv83s\n75Lyo81su5m9lGxjM44pMrNXzWyFmY3JKG9tZlPNrNTMnjOzbhnPDU/2X25mw9LUWUREGl7alslc\n4HPufgpQCozMeO5Ndy9KthszyscBI9y9J9DTzAYk5SOATe5+HDAGuBfAzNoDdwJ9gdOBUWbWLmW9\n815JSUmuq9Bk6FxU07mopnPRsFKFibvPd/fK5Mfnga4ZT1vN/c2sE9DW3RcnRZOAQcnjgcDE5PET\nwLnJ4wHAXHff6u5biAC7ME29C4H+R6mmc1FN56KazkXDasgxk2uAP2T83D3p4lpoZmclZV2Asox9\nypKyqufWALj7bmCrmR2eWZ5Ym3GMiIg0Aa32tYOZzQM6ZhYBDtzh7rOSfe4AKtz9sWSfd4Fu7r7Z\nzIqA6WbW51PW7RMtGxERaaLcPdUGXAU8Cxy4l30WAkVAJ2BZRvlQYFzyeA5wevK4JbAxY5+HM455\nGPh6He/j2rRp06bt029ps2CfLZO9MbMLgX8Dznb3nRnlRxCD6ZVmdizQA1jp7lvMbKuZ9QMWA8OA\n/0oOmwkMBxYBQ4AFSflTwN3JoHsL4ALg9trq4+5qzYiI5ECqMAEeAFoD88wM4Plk5tbZwA/NrByo\nBK5LBs8BbgImAG2A2e4+JykfD0w2s1LgA6JFQtJVdhewhEjQH2S8loiINAGWdA+JiIjst7xZAW9m\nF5rZG8nixttyXZ9sMrOuZrbAzF43s9fM7F+S8vZmNjdZ7PlUoazPMbMWyUzCmcnPBXkeAMysnZn9\nNllc/LqZnV6o58PMRibn4FUzezRZKF0Q58LMxpvZBjN7NaOszs+enKvS5Pemf33eIy/CxMxaAA8S\na1I+B1xuZsfntlZZtQv4rrt/DvgH4Kbk898OzHf3XsQY1Mi9vEY++TawNOPnQj0PAD8jupN7AycD\nb1CA58PMjga+CXze3U8iuvgvp3DOxa+J78dMtX72ZObtZUBv4CJgrCXjGHuTF2EC9ANK3X21u1cA\nU4lFkAXB3de7+yvJ478Cy4gFpJkLQSdSvUA0b5lZV+AfgUcyigvuPAAklzf6orv/GsDdd7n7Vgrz\nfHwIlAOHmFkr4CBizVpBnAt3fwbYXKO4rs9+MTA1+X1ZRVzdpN++3iNfwqTmwsbMxZAFxcy6A6cQ\nVyTo6O4bIAIH6JC7mmXN/yNmGGYOBhbieQA4BnjfzH6ddPv9wswOpgDPh7tvBu4D3iFCZKu7z6cA\nz0WGDnV89v1aKJ4vYSKAmR1KXIrm20kLpebsiryebWFmXwY2JK20vTXL8/o8ZGhFrO96yN2LgG1E\n10ZB/V4AJEsUbgGOBjoTLZQrKMBzsRepPnu+hMlaoFvGz12TsoKRNN2fACa7+4ykeIOZdUye7wRs\nzFX9suRM4GIzWwlMAc41s8nA+gI7D1XKgDXuviT5+b+JcCm03wuA04Bn3X1TcrmmJ4EvUJjnokpd\nn30tcFTGfvX6Ps2XMFkM9Egufd+aWKMyM8d1yrZfAUvd/WcZZTOJKxRALAidUfOgfOLu/+7u3dz9\nWOJ3YIG7/xMwiwI6D1WSLow1ZtYzKToPeJ0C+71ILAfOMLM2yWDyecQkjUI6F8aeLfa6PvtMYGgy\n2+0YYtH5C/t88XxZZ5Ksxv8ZEZDj3f0nOa5S1pjZmcAfgdeovjzCvxO/ANOIvzJWA5cVyoJPMzsH\nuNXdL04NWRXxAAAAgElEQVQuGFqo5+FkYjLCAcBK4GrickUFdz7M7N+IL8/dwMvAtUBbCuBcmNlj\nQDHwGWADMAqYDvyWWj67mY0kbgtSQXSbz93ne+RLmIiISO7kSzeXiIjkkMJERERSU5iIiEhqChMR\nEUlNYSIiIqkpTEREJDWFiYiIpKYwERGR1P4/r5ftm69NOVAAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x10e5be1d0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(taus)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.collections.PathCollection at 0x10e36d0d0>"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZMAAAEACAYAAAB27puMAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3X90VeW95/H3N5gfpwRQJIJFSbSI0B/WxAF7W6cNWpA6\ncytXbyusWVctuR0tXttx1rTCpVa8qG3pWPB2itqWtupSwFvbqjNUgZG0yy4pWLDYgoJtwwAqnFbl\nFhvCr+/8sZ+T7BxOEsI+4ZyTfF5rncXOs88++Z4A+5NnP8+zj7k7IiIiSZQVugARESl9ChMREUlM\nYSIiIokpTEREJDGFiYiIJKYwERGRxPo8TMysxcx+Y2abzGx9aDvNzFaZ2Stm9oyZDYs9f66ZbTez\nrWY2NdbeYGabzWybmS3u67pFROT4nYyeyVGg0d3r3X1SaJsDrHH384FngbkAZvZe4NPABOATwBIz\ns3DMfUCTu48DxpnZ5SehdhEROQ4nI0wsx/e5EngwbD8ITA/bnwSWu/thd28BtgOTzGwUMMTdN4Tn\nPRQ7RkRECuxkhIkDq81sg5n9Y2gb6e57ANz9DeCM0D4a2Bk7dndoGw3sirXvCm0iIlIETjkJ3+Mj\n7v66mdUAq8zsFaKAidM9XURESlifh4m7vx7+TJvZT4FJwB4zG+nue8IlrL3h6buBs2OHnxXaumrv\nxMwUSiIiJ8Ddrednda1PL3OZ2bvMrDpsDwamAi8BTwLXh6ddBzwRtp8EZphZhZmdA4wF1odLYfvM\nbFIYkL82dkwn7l6yj9tvv73gNaj+wteh+kvvUcq1u+fnd/C+7pmMBH4SegynAI+4+yozewF4zMxm\nATuIZnDh7lvM7DFgC3AImO0d7/Qm4IdAFbDS3Z/u49pFROQ49WmYuPsfgQtztL8JfLyLY74KfDVH\n+6+BD+S7RhERSU4r4ItIY2NjoUtIRPUXluovnFKuPV8sX9fLioGZeX96PyIiJ4OZ4cU8AC8iIgOD\nwkRERBJTmIiISGIKExERSUxhIiIiiSlMREQkMYWJiIgkpjAREZHEFCYiIpKYwkRERBJTmIiISGIK\nExERSUxhIiIiiSlMREQkMYWJiIgkpjAREZHEFCYiIpKYwkRERBJTmIiISGIKExERSUxhIiIiiSlM\nREQkMYWJiIgkpjApQul0mg0bNpBOpwtdiojIcVGYFJlly1ZQWzueKVNupLZ2PMuWrSh0SSIiPTJ3\nL3QNeWNmXqrvJ51Os2nTJqZPn0lr61rgAmAzqdRkdux4mZqamkKXKCL9lJnh7pbkNUqqZ2Jm08zs\nZTPbZma3FrqefMn0Rq666vO0tg4nChKACygvr6WlpaWA1YmI9KxkeiZmVgZsAy4DXgM2ADPc/eXY\nc0quZ5JOp6mtHR96I2cC5wPNqGciIidLPnomp+SrmJNgErDd3XcAmNly4Erg5W6PKnItLS1UVNTR\n2prpjdwH/A2DB4/l6NFdLF26REEiIkWvlMJkNLAz9vUuooApaXV1dRw82AJsJuqNTKCqqoIf//gb\n1NfXK0hEpCSU1JhJf1RTU8PSpUtIpSYzdGgDqdRkvv/9+5k6daqCRERKRin1THYDY2JfnxXaOpk/\nf377dmNjI42NjX1dV2IzZ17Dxz9+KS0tLdTV1SlERKRPNTc309zcnNfXLKUB+EHAK0QD8K8D64GZ\n7r419pySG4AXESm0ATUA7+5HzOyfgFVEl+eWxoNEREQKp2R6JsdDPRMRkd4bcIsWRUSkOClMREQk\nMYWJiIgkpjAREZHEFCYiIpKYwkRERBJTmIiISGIKExERSUxhIiIiiSlMREQkMYWJiIgkpjAREZHE\nFCYiIpKYwkRERBJTmIiISGIKExERSUxhIiIiiSlMREQkMYWJiIgkpjAREZHEFCYiIpKYwkRERBJT\nmIiISGIKExERSUxhIiIiiSlMREQkMYWJiIgkpjAREZHEFCYiIpJYn4WJmd1uZrvMbGN4TIvtm2tm\n281sq5lNjbU3mNlmM9tmZotj7RVmtjwc87yZjemrukVEpPf6umfyTXdvCI+nAcxsAvBpYALwCWCJ\nmVl4/n1Ak7uPA8aZ2eWhvQl4093PAxYDC/u4bhER6YW+DhPL0XYlsNzdD7t7C7AdmGRmo4Ah7r4h\nPO8hYHrsmAfD9o+Ay/quZBER6a2+DpN/MrMXzex7ZjYstI0Gdsaeszu0jQZ2xdp3hbZOx7j7EeBt\nMxvep5WLiMhxOyXJwWa2GhgZbwIcmAcsAf7F3d3M7gTuAf4xyffL+j45zZ8/v327sbGRxsbGPH1L\nEZH+obm5mebm5ry+prl7Xl8w5zcxqwWecvcLzGwO4O7+9bDvaeB2YAew1t0nhPYZwMfc/XOZ57j7\nr8xsEPC6u5+R4/v4yXg/IiL9iZnh7l3+kn48+nI216jYl1cBvw3bTwIzwgytc4CxwHp3fwPYZ2aT\nwoD8tcATsWOuC9ufAp7tq7pFRKT3El3m6sFCM7sQOAq0ADcAuPsWM3sM2AIcAmbHuhM3AT8EqoCV\nmRlgwFLgYTPbDvwZmNGHdYuISC+dlMtcJ4suc4mI9F5RX+YSEZGBQ2EiIiKJKUxERCQxhYmIiCSm\nMBERkcQUJiIikpjCREREElOYiIhIYgoTERFJTGEiIiKJKUxERCQxhYmIiCSmMBERkcQUJiIikpjC\nREREElOYiIhIYgoTERFJTGEiIiKJKUxERCQxhYmIiCSmMBERkcQUJiIikpjCREREElOYiIhIYgoT\nERFJTGEiIiKJKUxERCQxhYmIiCSWKEzM7O/N7LdmdsTMGrL2zTWz7Wa21cymxtobzGyzmW0zs8Wx\n9gozWx6Oed7MxsT2XRee/4qZXZuk5lKXTqfZsGED6XS60KWIiLRL2jN5Cfg74OfxRjObAHwamAB8\nAlhiZhZ23wc0ufs4YJyZXR7am4A33f08YDGwMLzWacBXgInAxcDtZjYsYd0ladmyFdTWjmfKlBup\nrR3PsmUrCl2SiAiQMEzc/RV33w5Y1q4rgeXuftjdW4DtwCQzGwUMcfcN4XkPAdNjxzwYtn8EXBq2\nLwdWufs+d38bWAVMS1J3KUqn0zQ1zaa1dS379v2a1ta1NDXNVg9FRIpCX42ZjAZ2xr7eHdpGA7ti\n7btCW6dj3P0IsM/MhnfzWgNKS0sLFRV1wAWh5QLKy2tpaWkpXFEiIsEpPT3BzFYDI+NNgAPz3P2p\nviqMY3s7x2X+/Pnt242NjTQ2NuapnMKqq6vj4MEWYDNRoGzm0KEd1NXVFbQuESk9zc3NNDc35/U1\newwTd59yAq+7Gzg79vVZoa2r9vgxr5nZIGCou79pZruBxqxj1nb1jeNh0p/U1NSwdOkSmpomU15e\ny6FDO1i6dAk1NTWFLk1ESkz2L9p33HFH4tc0d0/+ImZrgf/h7r8OX78XeIRowHw0sBo4z93dzNYB\nnwc2AP8H+Fd3f9rMZgPvd/fZZjYDmO7uM8IA/AtAA9FluReAi8L4SXYdno/3U8zS6TQtLS3U1dUp\nSEQkL8wMdz+hq0Htr5Hk5Gtm04FvASOAt4EX3f0TYd9cohlah4AvuPuq0H4R8EOgCljp7l8I7ZXA\nw0A98GdgRhi8x8yuB+YRXV67090f6qKefh8mIiL5VvAwKTYKExGR3stHmPQ4ZiKFl+vSVrwN0KUv\nESkohUkRS6fTPPDAd7n77nuoqIhmcy1dugSApqbZVFTU8de/bsesjFTqPe37Z868psCVi8hAo8tc\nRWrZshXMmnUjBw4cBJ4nMx24qupjmJXR2roWOBM4H2hu359KTWbHjpfVQxGR45aPy1y60WMRyqx2\nP3Dg28B44gsVBw06g7Kys0NbC3AOWsgoIoWmMClCHavdpxAFxuawZzNHjuzl6NGdoa0O+GOn/VrI\nKCKFoDGTItSx2v11YAnRms3hpFJvsXTp/QDtixdbWw9h9lGqqs7VQkYRKRiNmRSpZctW0NQ0m/Ly\nWg4e/CPz5n2RG274rGZziUjeaZ1Jlv4UJqDV7iJycihMsvS3MDlRCiER6Q3N5pJj6AO0RKQQ1DPp\nR7Zu3Up9/Ydpa/s5WnciIsdLPRNpt2zZCurrP0Rb2xlo3YmInGzqmfQD6XSa2trxtLY+DlxN9HEv\nnXsmwDGzv6qrq9m/f7/GVkQGON3oUYCORY6trY1E61ImA6dTWZlm0aKFne7vlbmX16BBp9Pa+jqp\n1Fhgt+7pJSKJqGfSD3T0TDI9kmYqK6/kzjtv57bbFsTu75W5l9eP6dyDiZ6/adM6JkyYUKi3ISIF\nojETATo+0jeVmszQoQ2kUldz773/k6985a6s+3u1EN3LazDRrVguAFYAV9PWNor6+g9r9peInBD1\nTPqR+PqSlpYWpky5kX37niYKk/hdhjM9k67HWDSGIjJwaMxEOqmpqekUArnu71Vefgiz6ZSVncaB\nA9OAMeSa/aUwEZHe0GWufqrzpa+vU1XlLFgwi927/8CuXdv5xS8e47nn/i+VlWl012ERSUqXufq5\nnm6tEr+hZOauw5rVJTKw6N5cWRQmJ0Z3IBYZ2DSbS/KipqaGiRMnsmbNs7qvl4icEPVMBMi1VkUz\nu0QGCvVMJG86PipY9/USkd5TmAgQ/6hgzewSkd5TmAiQaxX9ZH2evIgcN42ZSCf6lEaRgUdTg7Mo\nTEREeq/gA/Bm9vdm9lszO2JmDbH2WjP7q5ltDI8lsX0NZrbZzLaZ2eJYe4WZLTez7Wb2vJmNie27\nLjz/FTO7NknNcuLS6TQbNmwgnU4XuhQRKTJJx0xeAv4O+HmOfa+6e0N4zI613wc0ufs4YJyZXR7a\nm4A33f08YDGwEMDMTgO+AkwELgZuN7NhCeuWXtJny4tIdxKFibu/4u7bgVzdo2PazGwUMMTdN4Sm\nh4DpYftK4MGw/SPg0rB9ObDK3fe5+9vAKmBakrqlZ/FeSDqdpqlpNq2ta9m379e0tq6lqWm2eigi\n0q4vZ3PVhUtca83sktA2GtgVe86u0JbZtxPA3Y8A+8xseLw92B07RvpAdi/km99cTFnZ2WgNioh0\npcdb0JvZamBkvAlwYJ67P9XFYa8BY9z9rTCW8lMze28vazuhwaD58+e3bzc2NtLY2HgiLzNgxXsh\nra0XAAv52tduJ/q9YzOZT2Zsa/s91dXVBa1VRE5Mc3Mzzc3NeX3NHsPE3af09kXd/RDwVtjeaGa/\nB8YR9SrOjj31rNBGbN9rZjYIGOrub5rZbqIP44gfs7ar7x0PE+m9js+TvwBIA18HfgVsJfprqAT2\nUVY2losuukR3GRYpQdm/aN9xxx2JXzOfl7naexJmNsLMysL2ucBY4A/u/gbR5atJZmbAtcAT4bAn\ngevC9qeAZ8P2M8AUMxsWBuOnhDbpA51XwrcQ5fsFwDXAL4F/B9bR2rpZYyci0i7p1ODpZrYT+BDw\nv83sZ2HXR4HNZrYReAy4IQyeA9wELAW2Advd/enQvhQYYWbbgf8GzAFw97eABcALRL8i3xF7Lcmz\n+Er46upZRH9NmVusbALeTXzsZNCgd7Ny5UoFisgAp0WLklNmJfzGjS9yyy1zKC+v5eDBP3L0qHPw\n4C+IAmUhMJ8hQ8Zz+LA+WEukVGkFfBaFSd+I32JlzZpnaWqazaBB72b//t8D69At60VKm8Iki8Lk\n5Ein06xcuZKbb76Xv/xlY3v70KENrFnzABMnTixgdSLSWwW/nYoMTDU1NVxxxRUcPryD7FvWV1dX\n65YrIgOQwkROSK5b1jc1/QMXXXSJbrkiMgDpMpckkhlPOXjwIJdd9p9pa/s5GkMRKS26zCUFV1NT\nw6uv/oHLLruCtrYz0C1XRAYmhYkkkrn9SlvbE8Cf0Mf+igxMPd5ORaQ7HbdfaQSWAJOB06msTLNo\n0cL2nokudYn0b+qZSCKdb79yDfA4lZV7uPPO27jlljkajBcZIDQAL4ktW7aCpqbZlJfXcujQDhYt\n+hq33DKH1ta1aDBepPhpAF6KwsyZ17Bjx8usWfMAO3a8TEPDhVRU1NExGH8mZWUj2LRpUwGrFJG+\npJ6J5F06naa2dnzomWwFPgecTir1pu7fJVKEdDuVLAqT4rFs2QpmzbqRAwcOAs+TudxVVfUxnnhi\nBfX19brkJVIkdJlLitbMmdfwxBMrGDz4PDoud23lwIGDXHXVlzQoL9LPqGcifabz5a4zgfOBZjIf\n/VtZeSWbNq1jwoQJhSxTZMBTz0SKWvz+XYMHXwKcThQkK4CraWsbRX39h9VDEekH1DORPpdOp9m0\naRPTp8+ktfVx4GpA04ZFioV6JlISampqmDp1KkuXLqGy8kpgBJo2LNK/qGciJ9XWrVupr/9wuLtw\n52nDixZ9jYaGC6murmb//v3U1dWptyJyEmhqcBaFSWnIPW04+jz5yspRtLW9QSo1FtitdSkiJ4HC\nJIvCpHSsWrWKq676Eu+88yKQBsYDGk8RKQSNmUjJqq+v5+jRnUQ3iGwBzgYGA3XEPxNl0KB3s3Ll\nSn0MsEiRU5hIQcSnDVdXzwK2Ae8QBUvmM1EWsn//77n55nu1yFGkyOkylxRU5mN/N258kVtumYP7\nUA4c2ENVVS0HDuwA1qFFjiJ9S2MmWRQmpS0TLNXV1axfv56bb76Xv/xlI9Eix9nACCoq3uC2227l\nhhs+q3EUkTxRmGRRmPQfHbdiiQ/K9zyVGKJPfzzeacXxANN0ZBmoFCZZFCb9y7JlK/jMZ/4rbW2j\ngOeIZnxlZnodO5X48OEWzMpIpd5DW9sfmDfvi516MNnBkbm0BqfS2vo6qdRY3Hcec5xIf6cwyaIw\n6X86Fjl+G7gH+DW5pxLHbyTZ0YOpqvoT8+Z9kZqamk7BUVVVx4EDLcBKuur5aI2LDBT5CBPc/YQf\nRL8ebgVeJPqfPTS2by6wPeyfGmtvIJqusw1YHGuvAJaHY54HxsT2XRee/wpwbTf1uPQ/jz663Kuq\nTnV4l8NvHNY7fDD82eDgse29DsPD85Y7nOZwjkPKYW1sX/ZrxI9zh994VdWp/swzz/jevXvba9m7\nd6+vX7++U5tIqQvnzmR5kOhg+DhQFra/Bnw1bL8X2AScQrRw4FU6ekG/AiaG7ZXA5WH7c8CSsH0N\nsDxsnwb8HhgGnJrZ7qKevvlJS8Ht3bvXFyy4y1Op4V5d/f4c4bA3BMcjOcIhV/hk9mde45HYPg9B\n9C4fPPiDXlV1qi9YcJfff/93PJUa7sOGNXgqNdwffXT5MTUqaKQUFTxMOr0QTAceDttzgFtj+34G\nXAyMArbE2mcA94Xtp4GLw/YgYG/2c8LX9wHXdFFDvn/GUmQyJ+zMib2qqs4h5anU+728vNrLy4eE\nHswj3QRHpvfxdYeUV1bWOVTFej6ZYMrVu8kcu9YrK4f6c88916meYcMa2sNHoSKlIh9hcspxXAk7\nXrOAZWF7NNGlqozdoe0wsCvWviu0Z47ZGRLhiJntM7Ph8fas15IBqKamhpqaGiZOnMhVV00/ZiYW\nwAMPfJe77rop3PtrM9GA/a3AFVRWjqSt7UOkUu8BXmPRonvbZ4Q9/vhPuPvuyZSVjeCdd04nGoeZ\nTDQO0wZ8lo7PY5lNW1s1l1wyJTb+so7W1mjc5bbbfsBdd32jfTAf0Kwx6dd6HIA3s9XAyHgT4MA8\nd38qPGce0ODuV4evvwU87+6Phq+/R3RJawfRpbCpof0S4Evu/kkze4noktdrYd+rwCTgM0Clu98d\n2r8M/NXdv5mjVu/p/cjAkE6neeCB73L33fdQXl7LoUM7juuuxJ0/e+VbdD3oHx/8zwTNajpmnHUM\n5peX78GsjEGDTm+fNQa7dZdkKRr5GIDvsWfi7lN6KOJ64Arg0ljzbqKbLWWcFdq6ao8f85qZDSIa\nzH/TzHYDjVnHrO2qnvnz57dvNzY20tjY2NVTpR+rqanhy1/+Z2644bO9WncS/+yVjjsbx3s304Ba\nOt9HLE3UeV4d2uI9mjM5dOh84MdE4bOO1tZoavONN36hqO6SnD11Otf6nd4GX+Y1FZTFpbm5mebm\n5vy+aJJrZET/s34HnJ7VnhmArwDOofMA/DqiHocR9VamhfbZdAzAzyD3AHxm+9Qu6snTFUSRzoP+\nQ4fWeyo13L/xjXu8svLULsZfMuMu8fGazKB/d4P/ncdhtmzZ0j42tGXLlpMyqP/oo8s9lRruqdS5\nDikvLx8TxqI+4OXlQ7yiYlj7vlTqAz2OC8V/dtnjSCf7vfVEEyeKYACeaBrvDmBjeCyJ7ZsbQiR7\navBFwEvh2Htj7ZXAY6F9HVAX23d9aN+GpgbLSZZ9ssmceOOD/6nUcL///u/4ggV3ZU1jzgzmx4Mj\n1+yy5WH/OB80aPAxJ+/M6+f7pLd3715/5plnPJWKh1u81lz1ZyYljO1UVyYc7r//O1k/g47n5wqm\n+My47k7sSU76XR2b+buMB15PIdcfw6fgYVJsD4WJnCzd/Xad3aMpL6/2ioph7eFTVTXeO09t7unk\nnen5pHzIkPoTni0WPwnGaxw8+HyHsV30orLbstfjZGbEnRPe24Tw3nJN0e7qvUVrem69dW6XPZl4\nLydXgMX/HrL/bnLNtNuyZUssROOBd2bOAO/utfpDqChMFCZSxLJP3rlOSlHAVDqcdxyXxTr/lp/r\n0lFXJ7b4b+Dl5UNiU6hzneS7C7eeplxnel17czw/13vz8J5SXfZkOtd6bIB1d0muI9yODYzKyveE\nEO1p6nh2UB7bO8tec1RqFCYKEylhmQB47rnnwjhMT5fFcodK5gTa1dhE7tfPvUizomJMOEGf7fH1\nOx09q/h6nO4Wg2ZqHNZNaPW02DRXrdkn/u6CL9fPrrvv3dO4Vq6/B3f4jadSw0u6h6IwUZhIP5Hp\nOXR/Way7E253YxOVDuO869vORCfEzO1jerp01P2dCNw7Lsld6FVVp/qnPnVNl++tsvJc79w7yNWT\n6e5uBt1dkvMcr9tdryhXcHT3Wt7+GDq03tevX1/of0YnTGGiMJF+pKfLYh2D2j2dcI/nslU8fN7T\n60s13d2JINdkga7e27HjFrl6MvFa6/zExpsyParsfVGIzpkzt9P7OHZcK9drqWcSfxQ8APL5UJhI\nf5YZiM49W6ynsYnMbLHz2meLDR164oP52XUlmeob75V11ZOJ15odYF1fkjt2pl2u4MueSZZ7XCv3\na2WmjGvMxHULepFSk726v7V1O2AcOnSE6C5GmdvxZxZKriX7o49HjBhRVIsJsxc3xr+GYz/w7EQX\nWPb2w9C6e35/WpCpzzPJojCRgST7hJsdMGaDKCs7jQMH9rTfi0yf0SK5KEyyKExkoMv1G73u/SU9\nUZhkUZiIiPRePsKkLF/FiIjIwKUwERGRxBQmIiKSmMJEREQSU5iIiEhiChMREUlMYSIiIokpTERE\nJDGFiYiIJKYwERGRxBQmIiKSmMJEREQSU5iIiEhiChMREUlMYSIiIokpTEREJDGFiYiIJKYwERGR\nxBQmIiKSWKIwMbOFZrbVzF40s8fNbGhorzWzv5rZxvBYEjumwcw2m9k2M1sca68ws+Vmtt3Mnjez\nMbF914Xnv2Jm1yapWURE8i9pz2QV8D53vxDYDsyN7XvV3RvCY3as/T6gyd3HAePM7PLQ3gS86e7n\nAYuBhQBmdhrwFWAicDFwu5kNS1h3UWpubi50CYmo/sJS/YVTyrXnS6Iwcfc17n40fLkOOCu227Kf\nb2ajgCHuviE0PQRMD9tXAg+G7R8Bl4bty4FV7r7P3d8mCrBpSeouVqX+D1L1F5bqL5xSrj1f8jlm\nMgv4WezrunCJa62ZXRLaRgO7Ys/ZFdoy+3YCuPsRYJ+ZDY+3B7tjx4iISBE4pacnmNlqYGS8CXBg\nnrs/FZ4zDzjk7o+G57wGjHH3t8ysAfipmb23l7Ud07MREZEi5e6JHsD1wC+Bym6esxZoAEYBW2Pt\nM4D7wvbTwMVhexCwN/ac+2PH3A9c08X3cT300EMPPXr/SJoFPfZMumNm04AvAh9197ZY+wiiwfSj\nZnYuMBb4g7u/bWb7zGwSsAG4FvjXcNiTwHXAr4BPAc+G9meAu8KgexkwBZiTqx53V29GRKQAEoUJ\n8C2gAlhtZgDrwsytjwL/YmYHgaPADWHwHOAm4IdAFbDS3Z8O7UuBh81sO/Bnoh4J4VLZAuAFogS9\nI/ZaIiJSBCxcHhIRETlh/WIFfFeLJ8O+uWEh5FYzm1rIOrtjZtPM7OWwOPPWQtfTHTM7y8yeNbPf\nmdlLZvb50H6ama0Ki0ufKfb1QGZWFmYcPhm+Lpn6zWyYmf1b+Hf9OzO7uMTqnxvq3mxmj4RFy0Vb\nv5ktNbM9ZrY51tZlvcV23umi/ryeN/tFmNDF4skwg+zTwATgE8ASC9fjiomZlQH/i2hNzfuAmWY2\nvrBVdesw8N/d/X3A3wA3hXrnAGvc/XyiMa+53bxGMfgCsCX2dSnVfy/RZeIJwAeBlymR+s2sFvgs\nUO/uFxBdbp9Jcdf/A6L/n3E56y3S806u+vN63uwXYdLN4slPAsvd/bC7txD9wCYVoMSeTAK2u/sO\ndz8ELCdaxFmU3P0Nd38xbO8HthL9zOMLTx+kY0Fq0TGzs4ArgO/Fmkui/vAb5H909x8AhH/f+yiR\n+oF/Bw4Cg83sFCBFtH6saOt39+eAt7Kau6q36M47uerP93mzX4RJllnAyrBdKgses+uML+YsamZW\nB1xI9I9xpLvvgShwgDMKV1mPFhHNRIwPGpZK/ecAfzKzH4TLdN8xs3dRIvW7+1vAPcD/I/o/uc/d\n11Ai9cec0UW9pXLeiUt83iyZMDGz1eH6aubxUvjzb2PPySyeXFbAUgcMM6smuvXNF0IPJXs2R1HO\n7jCz/wTsCb2r7rrvRVk/0WWhBuDb7t4AvEN0yaVUfv7nArcAtcC7iXoo/4USqb8bpVYvkL/zZtKp\nwSeNu0/pbr+ZXU902eLSWPNu4OzY12eFtmKzGxgT+7pY62wXLk/8CHjY3Z8IzXvMbKS777HoPmx7\nC1dhtz4CfNLMriC6xDLEzB4G3iiR+ncBO939hfD140RhUio///8A/NLd3wQws58AH6Z06s/oqt5S\nOe/k9byA0rclAAABPElEQVRZMj2T7ljH4slPxhdPEi2EnBFmipxDtHhyfSFq7MEGYKxFt+6vIFpj\n82SBa+rJ94Et7n5vrO1JojsiQLQA9Ynsg4qBu/+zu49x93OJftbPuvs/AE9RGvXvAXaa2bjQdBnw\nO0rk5w+8AnzIzKrCwO5lRBMhir1+o3NPtqt6i/W806n+vJ83ky6hL4YH0QDRDmBjeCyJ7ZsLvEo0\nSDy10LV28x6mEf0n2w7MKXQ9PdT6EeAI8CKwKfzMpwHDgTXhfawCTi10rcfxXj4GPBm2S6Z+ohlc\nG8LfwY+BYSVW/xeJAnAz0eB1eTHXDzxKdM/BNqKxns8Ap3VVb7Gdd7qoP6/nTS1aFBGRxPrFZS4R\nESkshYmIiCSmMBERkcQUJiIikpjCREREElOYiIhIYgoTERFJTGEiIiKJ/X+Lq9+M2Itu9wAAAABJ\nRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x121d8b750>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.scatter(np.arange(num_iter),zeds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 280,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 1.12925219],\n",
       "       [ 1.18696124]])"
      ]
     },
     "execution_count": 280,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Calculate the cosine similarties of the points and see which is best\n",
    "\n",
    "similarities = np.zeros(N)\n",
    "\n",
    "true_grad = lin_reg_grad(X,y,w_init)\n",
    "\n",
    "for i in range(N):\n",
    "    grad_estimate = lin_reg_grad(X[i:i+1,:],y[i],w_init)\n",
    "\n",
    "cosine_similarity(grad_estimate,true_grad)\n",
    "    \n",
    "\n",
    "true_grad\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2,)"
      ]
     },
     "execution_count": 132,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.shape(X[10,:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:downhill.dataset:train: 1 of 1 mini-batches from (20, 5)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I 2016-07-07 15:49:51 downhill.dataset:174 train: 1 of 1 mini-batches from (20, 5)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:downhill.base:-- patience = 0\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I 2016-07-07 15:49:51 downhill.base:389 -- patience = 0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:downhill.base:-- validate_every = 10\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I 2016-07-07 15:49:51 downhill.base:390 -- validate_every = 10\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:downhill.base:-- max_updates = None\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I 2016-07-07 15:49:51 downhill.base:391 -- max_updates = None\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:downhill.base:-- min_improvement = 0\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I 2016-07-07 15:49:51 downhill.base:392 -- min_improvement = 0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:downhill.base:-- max_gradient_norm = 1\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I 2016-07-07 15:49:51 downhill.base:393 -- max_gradient_norm = 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:downhill.base:-- max_gradient_elem = 0\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I 2016-07-07 15:49:51 downhill.base:394 -- max_gradient_elem = 0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:downhill.base:-- learning_rate = 0.1\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I 2016-07-07 15:49:51 downhill.base:395 -- learning_rate = 0.1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:downhill.base:-- momentum = 0\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I 2016-07-07 15:49:51 downhill.base:396 -- momentum = 0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:downhill.base:-- nesterov = False\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I 2016-07-07 15:49:51 downhill.base:397 -- nesterov = False\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:downhill.adaptive:-- rms_halflife = 14\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I 2016-07-07 15:49:51 downhill.adaptive:220 -- rms_halflife = 14\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:downhill.adaptive:-- rms_regularizer = 1e-08\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I 2016-07-07 15:49:51 downhill.adaptive:221 -- rms_regularizer = 1e-08\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:downhill.base:compiling evaluation function\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I 2016-07-07 15:49:51 downhill.base:118 compiling evaluation function\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:downhill.base:compiling RMSProp function\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I 2016-07-07 15:50:03 downhill.base:124 compiling RMSProp function\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:downhill.base:validation 0 loss=5.875907 err=4.540981 |u|<0.1=0.066667 |v|<0.1=0.133333 grad(v)=1.766186 grad(u)=0.778860 *\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I 2016-07-07 15:50:19 downhill.base:232 validation 0 loss=5.875907 err=4.540981 |u|<0.1=0.066667 |v|<0.1=0.133333 grad(v)=1.766186 grad(u)=0.778860 *\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:downhill.base:RMSProp 1 loss=5.875907 err=4.540981 |u|<0.1=0.066667 |v|<0.1=0.133333 grad(v)=1.766186 grad(u)=0.778860\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I 2016-07-07 15:50:19 downhill.base:232 RMSProp 1 loss=5.875907 err=4.540981 |u|<0.1=0.066667 |v|<0.1=0.133333 grad(v)=1.766186 grad(u)=0.778860\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:downhill.base:RMSProp 2 loss=3.599935 err=2.770614 |u|<0.1=0.083333 |v|<0.1=0.200000 grad(v)=0.161211 grad(u)=0.146888\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I 2016-07-07 15:50:19 downhill.base:232 RMSProp 2 loss=3.599935 err=2.770614 |u|<0.1=0.083333 |v|<0.1=0.200000 grad(v)=0.161211 grad(u)=0.146888\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:downhill.base:RMSProp 3 loss=3.163241 err=2.489223 |u|<0.1=0.166667 |v|<0.1=0.133333 grad(v)=0.063558 grad(u)=0.060468\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I 2016-07-07 15:50:19 downhill.base:232 RMSProp 3 loss=3.163241 err=2.489223 |u|<0.1=0.166667 |v|<0.1=0.133333 grad(v)=0.063558 grad(u)=0.060468\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:downhill.base:RMSProp 4 loss=2.925698 err=2.273789 |u|<0.1=0.250000 |v|<0.1=0.066667 grad(v)=0.011586 grad(u)=0.050720\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I 2016-07-07 15:50:19 downhill.base:232 RMSProp 4 loss=2.925698 err=2.273789 |u|<0.1=0.250000 |v|<0.1=0.066667 grad(v)=0.011586 grad(u)=0.050720\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:downhill.base:RMSProp 5 loss=2.771293 err=2.088483 |u|<0.1=0.200000 |v|<0.1=0.066667 grad(v)=0.022198 grad(u)=0.031960\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I 2016-07-07 15:50:19 downhill.base:232 RMSProp 5 loss=2.771293 err=2.088483 |u|<0.1=0.200000 |v|<0.1=0.066667 grad(v)=0.022198 grad(u)=0.031960\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:downhill.base:RMSProp 6 loss=2.650094 err=1.896720 |u|<0.1=0.250000 |v|<0.1=0.066667 grad(v)=0.012511 grad(u)=0.030314\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I 2016-07-07 15:50:19 downhill.base:232 RMSProp 6 loss=2.650094 err=1.896720 |u|<0.1=0.250000 |v|<0.1=0.066667 grad(v)=0.012511 grad(u)=0.030314\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:downhill.base:RMSProp 7 loss=2.564841 err=1.758069 |u|<0.1=0.150000 |v|<0.1=0.133333 grad(v)=0.007610 grad(u)=0.027490\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I 2016-07-07 15:50:19 downhill.base:232 RMSProp 7 loss=2.564841 err=1.758069 |u|<0.1=0.150000 |v|<0.1=0.133333 grad(v)=0.007610 grad(u)=0.027490\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:downhill.base:RMSProp 8 loss=2.486564 err=1.627358 |u|<0.1=0.183333 |v|<0.1=0.133333 grad(v)=0.006564 grad(u)=0.025311\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I 2016-07-07 15:50:19 downhill.base:232 RMSProp 8 loss=2.486564 err=1.627358 |u|<0.1=0.183333 |v|<0.1=0.133333 grad(v)=0.006564 grad(u)=0.025311\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:downhill.base:RMSProp 9 loss=2.415072 err=1.524319 |u|<0.1=0.150000 |v|<0.1=0.200000 grad(v)=0.006541 grad(u)=0.023743\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I 2016-07-07 15:50:19 downhill.base:232 RMSProp 9 loss=2.415072 err=1.524319 |u|<0.1=0.150000 |v|<0.1=0.200000 grad(v)=0.006541 grad(u)=0.023743\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:downhill.base:RMSProp 10 loss=2.343810 err=1.417606 |u|<0.1=0.233333 |v|<0.1=0.200000 grad(v)=0.007307 grad(u)=0.023718\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I 2016-07-07 15:50:19 downhill.base:232 RMSProp 10 loss=2.343810 err=1.417606 |u|<0.1=0.233333 |v|<0.1=0.200000 grad(v)=0.007307 grad(u)=0.023718\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:downhill.base:validation 1 loss=2.280968 err=1.322145 |u|<0.1=0.200000 |v|<0.1=0.200000 grad(v)=0.005309 grad(u)=0.026911 *\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I 2016-07-07 15:50:19 downhill.base:232 validation 1 loss=2.280968 err=1.322145 |u|<0.1=0.200000 |v|<0.1=0.200000 grad(v)=0.005309 grad(u)=0.026911 *\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:downhill.base:RMSProp 11 loss=2.280968 err=1.322145 |u|<0.1=0.200000 |v|<0.1=0.200000 grad(v)=0.005309 grad(u)=0.026911\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I 2016-07-07 15:50:19 downhill.base:232 RMSProp 11 loss=2.280968 err=1.322145 |u|<0.1=0.200000 |v|<0.1=0.200000 grad(v)=0.005309 grad(u)=0.026911\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:downhill.base:RMSProp 12 loss=2.206108 err=1.212760 |u|<0.1=0.183333 |v|<0.1=0.200000 grad(v)=0.007822 grad(u)=0.023253\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I 2016-07-07 15:50:19 downhill.base:232 RMSProp 12 loss=2.206108 err=1.212760 |u|<0.1=0.183333 |v|<0.1=0.200000 grad(v)=0.007822 grad(u)=0.023253\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:downhill.base:RMSProp 13 loss=2.143547 err=1.108200 |u|<0.1=0.166667 |v|<0.1=0.066667 grad(v)=0.004180 grad(u)=0.023786\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I 2016-07-07 15:50:19 downhill.base:232 RMSProp 13 loss=2.143547 err=1.108200 |u|<0.1=0.166667 |v|<0.1=0.066667 grad(v)=0.004180 grad(u)=0.023786\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:downhill.base:RMSProp 14 loss=2.085339 err=1.029058 |u|<0.1=0.216667 |v|<0.1=0.133333 grad(v)=0.005305 grad(u)=0.019959\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I 2016-07-07 15:50:19 downhill.base:232 RMSProp 14 loss=2.085339 err=1.029058 |u|<0.1=0.216667 |v|<0.1=0.133333 grad(v)=0.005305 grad(u)=0.019959\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:downhill.base:RMSProp 15 loss=2.038661 err=0.950677 |u|<0.1=0.216667 |v|<0.1=0.133333 grad(v)=0.003796 grad(u)=0.019180\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I 2016-07-07 15:50:19 downhill.base:232 RMSProp 15 loss=2.038661 err=0.950677 |u|<0.1=0.216667 |v|<0.1=0.133333 grad(v)=0.003796 grad(u)=0.019180\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:downhill.base:RMSProp 16 loss=2.002167 err=0.903375 |u|<0.1=0.266667 |v|<0.1=0.066667 grad(v)=0.004247 grad(u)=0.015912\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I 2016-07-07 15:50:19 downhill.base:232 RMSProp 16 loss=2.002167 err=0.903375 |u|<0.1=0.266667 |v|<0.1=0.066667 grad(v)=0.004247 grad(u)=0.015912\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:downhill.base:RMSProp 17 loss=1.970705 err=0.843304 |u|<0.1=0.266667 |v|<0.1=0.133333 grad(v)=0.004924 grad(u)=0.012568\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I 2016-07-07 15:50:19 downhill.base:232 RMSProp 17 loss=1.970705 err=0.843304 |u|<0.1=0.266667 |v|<0.1=0.133333 grad(v)=0.004924 grad(u)=0.012568\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:downhill.base:RMSProp 18 loss=1.945026 err=0.828127 |u|<0.1=0.250000 |v|<0.1=0.066667 grad(v)=0.004590 grad(u)=0.013993\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I 2016-07-07 15:50:19 downhill.base:232 RMSProp 18 loss=1.945026 err=0.828127 |u|<0.1=0.250000 |v|<0.1=0.066667 grad(v)=0.004590 grad(u)=0.013993\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:downhill.base:RMSProp 19 loss=1.929324 err=0.787570 |u|<0.1=0.266667 |v|<0.1=0.066667 grad(v)=0.007736 grad(u)=0.012078\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I 2016-07-07 15:50:19 downhill.base:232 RMSProp 19 loss=1.929324 err=0.787570 |u|<0.1=0.266667 |v|<0.1=0.066667 grad(v)=0.007736 grad(u)=0.012078\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:downhill.base:RMSProp 20 loss=1.910094 err=0.774437 |u|<0.1=0.300000 |v|<0.1=0.000000 grad(v)=0.008011 grad(u)=0.010532\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I 2016-07-07 15:50:19 downhill.base:232 RMSProp 20 loss=1.910094 err=0.774437 |u|<0.1=0.300000 |v|<0.1=0.000000 grad(v)=0.008011 grad(u)=0.010532\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:downhill.base:validation 2 loss=1.900761 err=0.753212 |u|<0.1=0.250000 |v|<0.1=0.066667 grad(v)=0.012624 grad(u)=0.008483 *\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I 2016-07-07 15:50:19 downhill.base:232 validation 2 loss=1.900761 err=0.753212 |u|<0.1=0.250000 |v|<0.1=0.066667 grad(v)=0.012624 grad(u)=0.008483 *\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:downhill.base:RMSProp 21 loss=1.900761 err=0.753212 |u|<0.1=0.250000 |v|<0.1=0.066667 grad(v)=0.012624 grad(u)=0.008483\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I 2016-07-07 15:50:19 downhill.base:232 RMSProp 21 loss=1.900761 err=0.753212 |u|<0.1=0.250000 |v|<0.1=0.066667 grad(v)=0.012624 grad(u)=0.008483\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:downhill.base:RMSProp 22 loss=1.884520 err=0.750528 |u|<0.1=0.283333 |v|<0.1=0.000000 grad(v)=0.012071 grad(u)=0.008434\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I 2016-07-07 15:50:19 downhill.base:232 RMSProp 22 loss=1.884520 err=0.750528 |u|<0.1=0.283333 |v|<0.1=0.000000 grad(v)=0.012071 grad(u)=0.008434\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:downhill.base:RMSProp 23 loss=1.875121 err=0.724898 |u|<0.1=0.266667 |v|<0.1=0.066667 grad(v)=0.011941 grad(u)=0.009880\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I 2016-07-07 15:50:19 downhill.base:232 RMSProp 23 loss=1.875121 err=0.724898 |u|<0.1=0.266667 |v|<0.1=0.066667 grad(v)=0.011941 grad(u)=0.009880\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:downhill.base:RMSProp 24 loss=1.871716 err=0.740554 |u|<0.1=0.300000 |v|<0.1=0.066667 grad(v)=0.014710 grad(u)=0.007642\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I 2016-07-07 15:50:19 downhill.base:232 RMSProp 24 loss=1.871716 err=0.740554 |u|<0.1=0.300000 |v|<0.1=0.066667 grad(v)=0.014710 grad(u)=0.007642\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:downhill.base:RMSProp 25 loss=1.861975 err=0.702337 |u|<0.1=0.300000 |v|<0.1=0.133333 grad(v)=0.019385 grad(u)=0.006563\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I 2016-07-07 15:50:19 downhill.base:232 RMSProp 25 loss=1.861975 err=0.702337 |u|<0.1=0.300000 |v|<0.1=0.133333 grad(v)=0.019385 grad(u)=0.006563\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:downhill.base:RMSProp 26 loss=1.866143 err=0.747450 |u|<0.1=0.350000 |v|<0.1=0.066667 grad(v)=0.027147 grad(u)=0.011754\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I 2016-07-07 15:50:19 downhill.base:232 RMSProp 26 loss=1.866143 err=0.747450 |u|<0.1=0.350000 |v|<0.1=0.066667 grad(v)=0.027147 grad(u)=0.011754\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:downhill.base:RMSProp 27 loss=1.876302 err=0.716036 |u|<0.1=0.300000 |v|<0.1=0.133333 grad(v)=0.039911 grad(u)=0.011448\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I 2016-07-07 15:50:19 downhill.base:232 RMSProp 27 loss=1.876302 err=0.716036 |u|<0.1=0.300000 |v|<0.1=0.133333 grad(v)=0.039911 grad(u)=0.011448\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:downhill.base:RMSProp 28 loss=1.868788 err=0.735541 |u|<0.1=0.283333 |v|<0.1=0.066667 grad(v)=0.039201 grad(u)=0.008732\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I 2016-07-07 15:50:19 downhill.base:232 RMSProp 28 loss=1.868788 err=0.735541 |u|<0.1=0.283333 |v|<0.1=0.066667 grad(v)=0.039201 grad(u)=0.008732\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:downhill.base:RMSProp 29 loss=1.856367 err=0.705306 |u|<0.1=0.350000 |v|<0.1=0.200000 grad(v)=0.031814 grad(u)=0.011359\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I 2016-07-07 15:50:19 downhill.base:232 RMSProp 29 loss=1.856367 err=0.705306 |u|<0.1=0.350000 |v|<0.1=0.200000 grad(v)=0.031814 grad(u)=0.011359\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:downhill.base:RMSProp 30 loss=1.858760 err=0.727562 |u|<0.1=0.283333 |v|<0.1=0.066667 grad(v)=0.031196 grad(u)=0.009496\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I 2016-07-07 15:50:19 downhill.base:232 RMSProp 30 loss=1.858760 err=0.727562 |u|<0.1=0.283333 |v|<0.1=0.066667 grad(v)=0.031196 grad(u)=0.009496\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:downhill.base:validation 3 loss=1.854054 err=0.682883 |u|<0.1=0.333333 |v|<0.1=0.200000 grad(v)=0.031582 grad(u)=0.008366 *\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I 2016-07-07 15:50:19 downhill.base:232 validation 3 loss=1.854054 err=0.682883 |u|<0.1=0.333333 |v|<0.1=0.200000 grad(v)=0.031582 grad(u)=0.008366 *\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:downhill.base:RMSProp 31 loss=1.854054 err=0.682883 |u|<0.1=0.333333 |v|<0.1=0.200000 grad(v)=0.031582 grad(u)=0.008366\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I 2016-07-07 15:50:19 downhill.base:232 RMSProp 31 loss=1.854054 err=0.682883 |u|<0.1=0.333333 |v|<0.1=0.200000 grad(v)=0.031582 grad(u)=0.008366\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:downhill.base:RMSProp 32 loss=1.850058 err=0.730499 |u|<0.1=0.316667 |v|<0.1=0.066667 grad(v)=0.032111 grad(u)=0.008037\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I 2016-07-07 15:50:19 downhill.base:232 RMSProp 32 loss=1.850058 err=0.730499 |u|<0.1=0.316667 |v|<0.1=0.066667 grad(v)=0.032111 grad(u)=0.008037\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:downhill.base:RMSProp 33 loss=1.846542 err=0.683443 |u|<0.1=0.333333 |v|<0.1=0.200000 grad(v)=0.029244 grad(u)=0.010265\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I 2016-07-07 15:50:19 downhill.base:232 RMSProp 33 loss=1.846542 err=0.683443 |u|<0.1=0.333333 |v|<0.1=0.200000 grad(v)=0.029244 grad(u)=0.010265\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:downhill.base:RMSProp 34 loss=1.845850 err=0.724608 |u|<0.1=0.316667 |v|<0.1=0.066667 grad(v)=0.027795 grad(u)=0.006626\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I 2016-07-07 15:50:19 downhill.base:232 RMSProp 34 loss=1.845850 err=0.724608 |u|<0.1=0.316667 |v|<0.1=0.066667 grad(v)=0.027795 grad(u)=0.006626\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:downhill.base:RMSProp 35 loss=1.835608 err=0.679016 |u|<0.1=0.400000 |v|<0.1=0.200000 grad(v)=0.024380 grad(u)=0.008580\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I 2016-07-07 15:50:19 downhill.base:232 RMSProp 35 loss=1.835608 err=0.679016 |u|<0.1=0.400000 |v|<0.1=0.200000 grad(v)=0.024380 grad(u)=0.008580\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:downhill.base:RMSProp 36 loss=1.841233 err=0.720096 |u|<0.1=0.300000 |v|<0.1=0.066667 grad(v)=0.025601 grad(u)=0.008978\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I 2016-07-07 15:50:19 downhill.base:232 RMSProp 36 loss=1.841233 err=0.720096 |u|<0.1=0.300000 |v|<0.1=0.066667 grad(v)=0.025601 grad(u)=0.008978\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:downhill.base:RMSProp 37 loss=1.838217 err=0.679684 |u|<0.1=0.350000 |v|<0.1=0.200000 grad(v)=0.025318 grad(u)=0.011007\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I 2016-07-07 15:50:19 downhill.base:232 RMSProp 37 loss=1.838217 err=0.679684 |u|<0.1=0.350000 |v|<0.1=0.200000 grad(v)=0.025318 grad(u)=0.011007\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:downhill.base:RMSProp 38 loss=1.840841 err=0.705487 |u|<0.1=0.283333 |v|<0.1=0.133333 grad(v)=0.024713 grad(u)=0.005012\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I 2016-07-07 15:50:19 downhill.base:232 RMSProp 38 loss=1.840841 err=0.705487 |u|<0.1=0.283333 |v|<0.1=0.133333 grad(v)=0.024713 grad(u)=0.005012\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:downhill.base:RMSProp 39 loss=1.825896 err=0.680124 |u|<0.1=0.383333 |v|<0.1=0.200000 grad(v)=0.021076 grad(u)=0.004992\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I 2016-07-07 15:50:19 downhill.base:232 RMSProp 39 loss=1.825896 err=0.680124 |u|<0.1=0.383333 |v|<0.1=0.200000 grad(v)=0.021076 grad(u)=0.004992\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:downhill.base:RMSProp 40 loss=1.829243 err=0.701609 |u|<0.1=0.350000 |v|<0.1=0.133333 grad(v)=0.018113 grad(u)=0.008736\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I 2016-07-07 15:50:19 downhill.base:232 RMSProp 40 loss=1.829243 err=0.701609 |u|<0.1=0.350000 |v|<0.1=0.133333 grad(v)=0.018113 grad(u)=0.008736\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:downhill.base:validation 4 loss=1.832844 err=0.678388 |u|<0.1=0.316667 |v|<0.1=0.200000 grad(v)=0.016386 grad(u)=0.007855 *\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I 2016-07-07 15:50:19 downhill.base:232 validation 4 loss=1.832844 err=0.678388 |u|<0.1=0.316667 |v|<0.1=0.200000 grad(v)=0.016386 grad(u)=0.007855 *\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:downhill.base:RMSProp 41 loss=1.832844 err=0.678388 |u|<0.1=0.316667 |v|<0.1=0.200000 grad(v)=0.016386 grad(u)=0.007855\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I 2016-07-07 15:50:19 downhill.base:232 RMSProp 41 loss=1.832844 err=0.678388 |u|<0.1=0.316667 |v|<0.1=0.200000 grad(v)=0.016386 grad(u)=0.007855\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:downhill.base:RMSProp 42 loss=1.827903 err=0.685325 |u|<0.1=0.316667 |v|<0.1=0.066667 grad(v)=0.019330 grad(u)=0.004481\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I 2016-07-07 15:50:19 downhill.base:232 RMSProp 42 loss=1.827903 err=0.685325 |u|<0.1=0.316667 |v|<0.1=0.066667 grad(v)=0.019330 grad(u)=0.004481\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:downhill.base:RMSProp 43 loss=1.829233 err=0.687731 |u|<0.1=0.383333 |v|<0.1=0.200000 grad(v)=0.021139 grad(u)=0.006803\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I 2016-07-07 15:50:19 downhill.base:232 RMSProp 43 loss=1.829233 err=0.687731 |u|<0.1=0.383333 |v|<0.1=0.200000 grad(v)=0.021139 grad(u)=0.006803\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:downhill.base:RMSProp 44 loss=1.831514 err=0.693295 |u|<0.1=0.366667 |v|<0.1=0.133333 grad(v)=0.022987 grad(u)=0.007882\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I 2016-07-07 15:50:19 downhill.base:232 RMSProp 44 loss=1.831514 err=0.693295 |u|<0.1=0.366667 |v|<0.1=0.133333 grad(v)=0.022987 grad(u)=0.007882\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:downhill.base:RMSProp 45 loss=1.832809 err=0.689536 |u|<0.1=0.333333 |v|<0.1=0.200000 grad(v)=0.024340 grad(u)=0.006671\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I 2016-07-07 15:50:19 downhill.base:232 RMSProp 45 loss=1.832809 err=0.689536 |u|<0.1=0.333333 |v|<0.1=0.200000 grad(v)=0.024340 grad(u)=0.006671\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:downhill.base:RMSProp 46 loss=1.830783 err=0.694585 |u|<0.1=0.333333 |v|<0.1=0.133333 grad(v)=0.022856 grad(u)=0.007902\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I 2016-07-07 15:50:19 downhill.base:232 RMSProp 46 loss=1.830783 err=0.694585 |u|<0.1=0.333333 |v|<0.1=0.133333 grad(v)=0.022856 grad(u)=0.007902\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:downhill.base:RMSProp 47 loss=1.827568 err=0.679975 |u|<0.1=0.366667 |v|<0.1=0.200000 grad(v)=0.018615 grad(u)=0.005444\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I 2016-07-07 15:50:19 downhill.base:232 RMSProp 47 loss=1.827568 err=0.679975 |u|<0.1=0.366667 |v|<0.1=0.200000 grad(v)=0.018615 grad(u)=0.005444\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:downhill.base:RMSProp 48 loss=1.828022 err=0.694670 |u|<0.1=0.350000 |v|<0.1=0.200000 grad(v)=0.019892 grad(u)=0.009820\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I 2016-07-07 15:50:19 downhill.base:232 RMSProp 48 loss=1.828022 err=0.694670 |u|<0.1=0.350000 |v|<0.1=0.200000 grad(v)=0.019892 grad(u)=0.009820\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:downhill.base:RMSProp 49 loss=1.838223 err=0.681714 |u|<0.1=0.283333 |v|<0.1=0.200000 grad(v)=0.025465 grad(u)=0.005411\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I 2016-07-07 15:50:19 downhill.base:232 RMSProp 49 loss=1.838223 err=0.681714 |u|<0.1=0.283333 |v|<0.1=0.200000 grad(v)=0.025465 grad(u)=0.005411\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:downhill.base:RMSProp 50 loss=1.830096 err=0.693359 |u|<0.1=0.366667 |v|<0.1=0.200000 grad(v)=0.027129 grad(u)=0.006829\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I 2016-07-07 15:50:19 downhill.base:232 RMSProp 50 loss=1.830096 err=0.693359 |u|<0.1=0.366667 |v|<0.1=0.200000 grad(v)=0.027129 grad(u)=0.006829\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:downhill.base:validation 5 loss=1.836166 err=0.689789 |u|<0.1=0.366667 |v|<0.1=0.200000 grad(v)=0.027648 grad(u)=0.008700\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I 2016-07-07 15:50:19 downhill.base:232 validation 5 loss=1.836166 err=0.689789 |u|<0.1=0.366667 |v|<0.1=0.200000 grad(v)=0.027648 grad(u)=0.008700\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:downhill.base:patience elapsed!\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I 2016-07-07 15:50:19 downhill.base:414 patience elapsed!\n",
      "('u =', array([[ -3.18959022e+00,  -1.55650043e+00,   1.25228256e-01],\n",
      "       [  1.49464583e+00,  -1.10656023e-03,   2.36193347e+00],\n",
      "       [  5.02728939e-01,   1.96595931e+00,   4.50359657e-02],\n",
      "       [ -6.33716285e-01,   1.26184070e+00,  -1.32035410e+00],\n",
      "       [ -1.70202881e-01,   5.56546152e-02,  -9.29425538e-01],\n",
      "       [  1.27949774e+00,   5.59897572e-02,  -8.53152946e-03],\n",
      "       [  3.15853208e-01,   1.38277352e+00,  -7.25604407e-03],\n",
      "       [ -2.00586963e+00,   1.13515997e+00,  -3.91623974e-02],\n",
      "       [ -1.68593359e+00,   5.92828631e-01,   9.57718119e-02],\n",
      "       [ -6.82622194e-04,   9.23861042e-02,  -4.97543395e-01],\n",
      "       [ -1.60177350e-01,  -1.41539332e-02,   3.28974843e+00],\n",
      "       [ -1.80730358e-01,   1.38905203e+00,   1.74537867e-01],\n",
      "       [  6.27478600e-01,  -6.43601865e-02,   9.45688367e-01],\n",
      "       [ -1.97225690e+00,  -3.65263447e-02,   1.92878380e-01],\n",
      "       [ -8.17843601e-02,  -2.79370403e+00,   7.44865835e-02],\n",
      "       [ -1.66340023e-02,  -1.03492409e-01,  -1.96018827e+00],\n",
      "       [ -1.68918669e-01,   2.19809368e-01,  -7.61018574e-01],\n",
      "       [ -8.02401900e-02,  -1.75611269e+00,  -1.87505573e-01],\n",
      "       [ -4.45972025e-01,   9.66961563e-01,   9.41383690e-02],\n",
      "       [  1.20289445e+00,  -7.30863586e-02,  -2.12380573e-01]], dtype=float32))\n",
      "('v =', array([[ 1.14945495, -0.43622887, -0.23185308, -0.8556546 ,  0.01746497],\n",
      "       [ 0.35558021,  0.84514546,  0.51486415, -0.2253741 , -0.91127837],\n",
      "       [-0.04443723, -0.93642706,  0.81109631, -0.00206804, -0.48095405]], dtype=float32))\n"
     ]
    }
   ],
   "source": [
    "import climate\n",
    "import downhill\n",
    "import numpy as np\n",
    "import theano\n",
    "import theano.tensor as TT\n",
    "\n",
    "climate.enable_default_logging()\n",
    "\n",
    "def rand(a, b): return np.random.randn(a, b).astype('f')\n",
    "\n",
    "A, B, K = 20, 5, 3\n",
    "\n",
    "# Set up a matrix factorization problem to optimize.\n",
    "u = theano.shared(rand(A, K), name='u')\n",
    "v = theano.shared(rand(K, B), name='v')\n",
    "e = TT.sqr(TT.matrix() - TT.dot(u, v))\n",
    "\n",
    "# Minimize the regularized loss with respect to a data matrix.\n",
    "y = np.dot(rand(A, K), rand(K, B)) + rand(A, B)\n",
    "\n",
    "downhill.minimize(\n",
    "    loss=e.mean() + abs(u).mean() + (v * v).mean(),\n",
    "    train=[y],\n",
    "    patience=0,\n",
    "    batch_size=A,                 # Process y as a single batch.\n",
    "    max_gradient_norm=1,          # Prevent gradient explosion!\n",
    "    learning_rate=0.1,\n",
    "    monitors=(('err', e.mean()),  # Monitor during optimization.\n",
    "              ('|u|<0.1', (abs(u) < 0.1).mean()),\n",
    "              ('|v|<0.1', (abs(v) < 0.1).mean())),\n",
    "    monitor_gradients=True)\n",
    "\n",
    "# Print out the optimized coefficients u and basis v.\n",
    "print('u =', u.get_value())\n",
    "print('v =', v.get_value())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "class discrete_distribution(object):\n",
    "    \"\"\" Discrete probability distribuiton on the range [1,N]\n",
    "    \"\"\"\n",
    "    \n",
    "    def __init__(self,probs):\n",
    "        \"\"\"Probs is a np array of probabilites\"\"\"\n",
    "        self.Z = float(np.sum(probs))\n",
    "        self.probs = probs/self.Z\n",
    "        self.cumsum = np.cumsum(self.probs)\n",
    "    \n",
    "    def probability(self,index):\n",
    "        return self.probs[index]\n",
    "    \n",
    "    def sample(self):\n",
    "        seed = np.random.random()\n",
    "        smp = 0\n",
    "        while(seed > self.cumsum[smp]):\n",
    "            smp = smp + 1\n",
    "        return smp\n",
    "    \n",
    "    def update_prob(self,index,prob):\n",
    "        self.probs = self.probs*self.Z\n",
    "        self.Z = self.Z - self.probs[index] + prob\n",
    "        self.probs[index] = prob\n",
    "        self.probs = self.probs/float(self.Z)\n",
    "        self.cumsum = np.cumsum(self.probs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "np.random.normal?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}