Logistic_Regression_Toy_Ex.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Logistic Regression\n",
    "For a first pass at Logistic Regression, we will run it on some simulated data that we create.\n",
    "\n",
    "This is useful to see how well Logistic Regression performs in a situation\n",
    "where we know what the answer should be."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "%matplotlib inline\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.linear_model import LogisticRegression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# We will create some fake data\n",
    "\n",
    "num_trials = 10000\n",
    "np.random.seed(seed = 142)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([5, 3, 4, 5, 1, 1, 5, 3, 3, 5, 5, 1, 1, 3, 4, 5, 4, 3, 5, 5, 5, 5, 1,\n",
       "       1, 5, 3, 5, 5, 3, 3, 5, 3, 5, 3, 1, 5, 5, 5, 5, 3, 1, 5, 5, 3, 1, 1,\n",
       "       1, 1, 5, 3, 3, 3, 3, 3, 1, 5, 3, 2, 3, 5, 3, 2, 3, 5, 3, 3, 3, 3, 5,\n",
       "       2, 5, 3, 2, 3, 3, 3, 3, 1, 3, 5, 5, 3, 5, 2, 1, 5, 4, 1, 4, 5, 1, 5,\n",
       "       5, 1, 3, 3, 3, 3, 5, 3])"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# First we create a \"clean\" x variable\n",
    "# \"Clean\" in this case means that if we knew x_clean exactly, then the prob(y=1|x) would be fixed\n",
    "# The \"clean\" value of x is either 1,2,3,4, or 5\n",
    "# Each one corresponds to a particular probability which we will set below\n",
    "\n",
    "\n",
    "x_clean = np.random.choice(a=[1,2,3,4,5],p=[.2,.1,.3,.1,.3], size=num_trials)\n",
    "x_clean[:100]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Now we create a \"noisy\" version of x.  This is the value we actually observe and learn from\n",
    "\n",
    "x_noisy = x_clean + np.random.normal(size=num_trials)#+np.random.uniform(size=num_trials)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([ 2042.,     0.,  1002.,     0.,     0.,  2926.,     0.,  1002.,\n",
       "            0.,  3028.]),\n",
       " array([ 1. ,  1.4,  1.8,  2.2,  2.6,  3. ,  3.4,  3.8,  4.2,  4.6,  5. ]),\n",
       " <a list of 10 Patch objects>)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYEAAAEACAYAAABVtcpZAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAFadJREFUeJzt3X2QXfV93/H3B2SBbYhCSWETyTF4CERQUsw0qlOm9U1j\n85CmoPEfjGLPAMWecQ1+mLrTKXLa0SqTGdmd2oW2AzOJDYiMGSLTJMgdhQcNvn/YU5CMRZAtBTTj\nSEaKtU5ql/hhxkjWt3/cI3y97Grv7t7du6vzfs3c4ez3POz3iN37Oed3zzmbqkKS1E5njLoBSdLo\nGAKS1GKGgCS1mCEgSS1mCEhSixkCktRiM4ZAkrOSPJtkT5K9STY19U1JDif5WvO6vm+djUkOJNmf\n5Nq++tVJXkjyUpK7F2aXJEmDyiD3CSR5U1X9KMmZwFeAjwI3AN+vqs9MWnYt8DDw68AaYCfwK1VV\nSZ4FPlxVu5PsAO6pqieGu0uSpEENNBxUVT9qJs8CVgAnkyNTLH4T8EhVHa+qg8ABYF2SMeDcqtrd\nLPcQsH6ujUuS5m+gEEhyRpI9wFHgqb438g8neT7JZ5OsamqrgZf7Vj/S1FYDh/vqh5uaJGlEBj0T\nOFFVb6c3vLMuyeXAvcDbquoqeuHw6YVrU5K0EFbMZuGq+vskXeD6SZ8F/BHwxWb6CPCWvnlrmtp0\n9ddJ4gONJGkOqmqqYfppDXJ10C+cHOpJ8kbg3cBfNWP8J70H+HozvR3YkGRlkouBS4BdVXUUeCXJ\nuiQBbgEeO8WOLPnXpk2bRt7D6dCjfdrnUn8tlz7nYpAzgV8EtiY5g15o/ElV7UjyUJKrgBPAQeCD\nzZv3viTbgH3AMeCO+ml3dwIPAmcDO6rq8Tl1LUkaihlDoKr2AldPUb/lFOtsAbZMUX8OuHKWPUqS\nFoh3DM9Dp9MZdQszWg49gn0Om30O13Lpcy4GullssSWppdiXJC1lSahhfzAsSTp9GQKS1GKGgCS1\nmCEgSS1mCEhSixkCktRihoAktZghIEktZghIUosZApLUYoaAJLWYISBJLWYISFKLGQKS1GKGgCS1\nmCEgSS1mCEhSixkCktRihoAktZghIEktZghIUovNGAJJzkrybJI9SfYm2dTUz0vyZJIXkzyRZFXf\nOhuTHEiyP8m1ffWrk7yQ5KUkdy/MLkmSBjVjCFTVj4HfrKq3A1cBNyRZB9wF7Kyqy4CngY0ASS4H\nbgbWAjcA9yZJs7n7gPdX1aXApUmuG/YOSZIGN9BwUFX9qJk8C1gBFHATsLWpbwXWN9M3Ao9U1fGq\nOggcANYlGQPOrardzXIP9a0jSRqBgUIgyRlJ9gBHgaeaN/ILq2oCoKqOAhc0i68GXu5b/UhTWw0c\n7qsfbmqSloCxNWMkmfY1tmZs1C1qAawYZKGqOgG8PcnPAX+W5Ap6ZwM/s9gwGxsfH39tutPp0Ol0\nhrl5SZNMHJmA8VPMH59YtF40mG63S7fbndc2UjW79+4k/xn4EfABoFNVE81Qz5eqam2Su4Cqqk81\nyz8ObAIOnVymqW8A3llVH5rie9Rs+5I0P0lOGQKMg7+XS1sSqiozL/lTg1wd9Asnr/xJ8kbg3cB+\nYDtwW7PYrcBjzfR2YEOSlUkuBi4BdjVDRq8kWdd8UHxL3zqSpBEYZDjoF4GtSc6gFxp/UlU7kjwD\nbEtyO72j/JsBqmpfkm3APuAYcEffYf2dwIPA2cCOqnp8qHsjSZqVWQ8HLQaHg6TF53DQ8rcgw0GS\nFpZX5WiUBro6SNLC8aocjZJnApLUYoaAJLWYISBJLWYISFKLGQKS1GKGgCS1mCEgSS1mCEhSixkC\nktRihoAktZghIEktZghIUosZApLUYoaAJLWYISBJLWYISFKLGQKS1GKGgCS1mCEgSS1mCEhSixkC\nktRiM4ZAkjVJnk7yjSR7k3ykqW9KcjjJ15rX9X3rbExyIMn+JNf21a9O8kKSl5LcvTC7JEka1IoB\nljkOfLyqnk9yDvBckqeaeZ+pqs/0L5xkLXAzsBZYA+xM8itVVcB9wPuraneSHUmuq6onhrc7kqTZ\nmPFMoKqOVtXzzfQPgP3A6mZ2pljlJuCRqjpeVQeBA8C6JGPAuVW1u1nuIWD9PPuXJM3DrD4TSHIR\ncBXwbFP6cJLnk3w2yaqmthp4uW+1I01tNXC4r36Yn4aJJGkEBhkOAqAZCnoU+FhV/SDJvcDvV1Ul\n+QPg08AHhtXY+Pj4a9OdTodOpzOsTUvSaaHb7dLtdue1jfSG6mdYKFkB/G/gL6rqninmvxX4YlX9\nWpK7gKqqTzXzHgc2AYeAL1XV2qa+AXhnVX1oiu3VIH1Jp4MkMH6KBcZhMX4flkofmrskVNVUw/TT\nGnQ46H5gX38ANGP8J70H+HozvR3YkGRlkouBS4BdVXUUeCXJuiQBbgEem02zkqThmnE4KMk1wPuA\nvUn2AAV8AnhvkquAE8BB4IMAVbUvyTZgH3AMuKPvsP5O4EHgbGBHVT0+1L2RJM3KjCFQVV8Bzpxi\n1rRv4FW1BdgyRf054MrZNChJWjjeMSxJLWYISFKLGQKS1GKGgCS1mCEgSS1mCEhSixkCktRihoAk\ntZghIEktZghIUosZApLUYoaAJLWYISBJLWYISFKLGQKS1GKGgCS1mCEgSS1mCEhSixkCktRihoAk\ntZghIEktZghIUosZApLUYjOGQJI1SZ5O8o0ke5N8tKmfl+TJJC8meSLJqr51NiY5kGR/kmv76lcn\neSHJS0nuXphdkiQNapAzgePAx6vqCuA3gDuT/CpwF7Czqi4DngY2AiS5HLgZWAvcANybJM227gPe\nX1WXApcmuW6oeyM19uzZw6rzV3HOz58z7euStZfw6quvjrpVaaRWzLRAVR0FjjbTP0iyH1gD3AS8\ns1lsK9ClFww3Ao9U1XHgYJIDwLokh4Bzq2p3s85DwHrgieHtjtTzzW9+k/ql4oc3/HDaZQ7990O8\n+uqrrFy5chE7k5aWGUOgX5KLgKuAZ4ALq2oCekGR5IJmsdXA/+lb7UhTOw4c7qsfburSgsiKwJtP\nMf+1E1SpvQYOgSTnAI8CH2vOCGrSIpO/npfx8fHXpjudDp1OZ5ibl6Rlr9vt0u1257WNgUIgyQp6\nAfDHVfVYU55IcmFVTSQZA77T1I8Ab+lbfU1Tm64+pf4QkCS93uQD5M2bN896G4NeIno/sK+q7umr\nbQdua6ZvBR7rq29IsjLJxcAlwK7ms4VXkqxrPii+pW8dSdIIzHgmkOQa4H3A3iR76A37fAL4FLAt\nye3AIXpXBFFV+5JsA/YBx4A7qurkUNGdwIPA2cCOqnp8uLsjSZqNQa4O+gpw5jSz3zXNOluALVPU\nnwOunE2DkqSF4x3DktRihoAktZghIEktZghIp6GxsYtIMquX2mlWdwxLGo25vUnP9v5Ng6CNDAFp\nWfANXQvD4SBpFuYyzOIwjJYyzwSkWZiYOMSQH5OFR+0aJc8EJKnFDAFJajFDQJJazBCQpBYzBCSp\nxQwBSWoxQ0CSWswQkKQWMwQkqcUMAUlqMUNAklrMEJCkFjMEJKnFDAFJajFDQJJabMYQSPK5JBNJ\nXuirbUpyOMnXmtf1ffM2JjmQZH+Sa/vqVyd5IclLSe4e/q5IkmZrkDOBB4Drpqh/pqqubl6PAyRZ\nC9wMrAVuAO7NT/900n3A+6vqUuDSJFNtU5K0iGYMgar6MvC9KWZN9eeQbgIeqarjVXUQOACsSzIG\nnFtVu5vlHgLWz61lSdKwzOczgQ8neT7JZ5OsamqrgZf7ljnS1FYDh/vqh5uaJGmE5vo3hu8Ffr+q\nKskfAJ8GPjC8tmB8fPy16U6nQ6fTGebmJWnZ63a7dLvdeW1jTiFQVX/b9+UfAV9spo8Ab+mbt6ap\nTVefVn8ISJJeb/IB8ubNm2e9jUGHg0LfZwDNGP9J7wG+3kxvBzYkWZnkYuASYFdVHQVeSbKu+aD4\nFuCxWXcrzcL3v//9U84/duwY5557LkkGfkmnmxnPBJI8DHSA85N8C9gE/GaSq4ATwEHggwBVtS/J\nNmAfcAy4o6qq2dSdwIPA2cCOk1cUSQulTtQMS7wB+C5wziy2ahDo9DJjCFTVe6coP3CK5bcAW6ao\nPwdcOavuJEkLyjuGJanFDAFJajFDQJJazBCQpBYzBCSpxQwBSWoxQ0CSWswQkKQWMwQkqcUMAUlq\nMUNAklrMEJA0sNk8cXWQ19jYRaPepdab6x+VkdRKMz2ZdXYmJnwq66h5JiBJLWYISFKLGQKS1GKG\ngCS1mCEgSS1mCEhSixkCktRihoAktZghIEktZghIUovNGAJJPpdkIskLfbXzkjyZ5MUkTyRZ1Tdv\nY5IDSfYnubavfnWSF5K8lOTu4e+KJGm2BjkTeAC4blLtLmBnVV0GPA1sBEhyOXAzsBa4Abg3ycmH\ng9wHvL+qLgUuTTJ5m5KkRTZjCFTVl4HvTSrfBGxtprcC65vpG4FHqup4VR0EDgDrkowB51bV7ma5\nh/rWkSSNyFw/E7igqiYAquoocEFTXw283Lfckaa2GjjcVz/c1CRJIzSsR0kP9/mywPj4+GvTnU6H\nTqcz7G8hzezMMfjJxKRi3+OPz7wQfnJ0UVua0nLpU0PV7Xbpdrvz2sZcQ2AiyYVVNdEM9XynqR8B\n3tK33JqmNl19Wv0hII3MTyZg/BTzxye/8Y7IculTQzX5AHnz5s2z3sagw0HhZw4r2A7c1kzfCjzW\nV9+QZGWSi4FLgF3NkNErSdY1HxTf0reOJGlEZjwTSPIw0AHOT/ItYBPwSeALSW4HDtG7Ioiq2pdk\nG7APOAbcUVUnh4ruBB4EzgZ2VNXjw90VSdJszRgCVfXeaWa9a5rltwBbpqg/B1w5q+4kSQvKO4Yl\nqcUMAUlqMUNAklpsWPcJSPMyNnYRExOHhrvRi4e7Oel05JmAloReANQQX48u7g5Iy5QhIEktZghI\nUosZApLUYoaAJLWYISBJLWYISFKLGQKS1GKGgCS1mCEgSS1mCEhSixkCktRihoAktZghIEktZghI\nUosZApLUYoaAJLWYISBJLWYISFKLzSsEkhxM8pdJ9iTZ1dTOS/JkkheTPJFkVd/yG5McSLI/ybXz\nbV6SND/zPRM4AXSq6u1Vta6p3QXsrKrLgKeBjQBJLgduBtYCNwD3Jsk8v78kaR7mGwKZYhs3AVub\n6a3A+mb6RuCRqjpeVQeBA8A6JEkjM98QKOCpJLuTfKCpXVhVEwBVdRS4oKmvBl7uW/dIU5MkjciK\nea5/TVV9O8k/BJ5M8iK9YOg3+euBjI+Pvzbd6XTodDpz7VGSTkvdbpdutzuvbcwrBKrq281//zbJ\nn9Mb3plIcmFVTSQZA77TLH4EeEvf6mua2pT6Q0CS9HqTD5A3b948623MeTgoyZuSnNNMvxm4FtgL\nbAduaxa7FXismd4ObEiyMsnFwCXArrl+f0nS/M3nTOBC4M+SVLOdz1fVk0m+CmxLcjtwiN4VQVTV\nviTbgH3AMeCOqprTUJEkaTjmHAJV9dfAVVPUvwu8a5p1tgBb5vo9JUnD5R3DkkboLJIM9TU2dtGo\nd2pZme/VQZI0Dz9mjhcQTmtiwntQZ8MzAUlqMUNAklrMEJCkFjMEJKnFDAFJajFDQJJazBDQrI2N\nXTT0a7sljYb3CWjWJiYOMexru3t/mkLSYvNMQJJazBCQpBYzBCSpxQwBSWoxQ0CSWswQkKQWMwQk\nqcUMAUlqMUNAklrMEJCkFjMEJKnFDAFJarFFD4Ek1yf5qyQvJfmPi/39JZ3uzhr6U27Hxi4a9U4t\nmEUNgSRnAP8TuA64AvjdJL+6mD0MU7fbHXULM1oOPS4rfz3qBga0XPpcED+m95Tb4b16T849PS32\nmcA64EBVHaqqY8AjwE2L3MPQLIc32N/5nfU++3+YDo66gQEdHHUDWi4WOwRWAy/3fX24qWmB/PCH\nrzDsoyJJp49W/FGZnTt3cs899wx1m+vXrx/q9jRsb4C/Ae7/uekXOfFDvDZCbZeqxTuyS/IOYLyq\nrm++vguoqvrUpOU83JSkOaiqWY3ZLnYInAm8CPwW8G1gF/C7VbV/0ZqQJL1mUYeDquonST4MPEnv\nPPxzBoAkjc6inglIkpaWJfupWJKPJNmfZG+ST466n1NJ8u+TnEjyD0bdy1SS/Jfm3/L5JP8rySk+\nLV18y+EGwiRrkjyd5BvNz+RHR93TdJKckeRrSbaPupfpJFmV5AvNz+U3kvzTUfc0lST/LsnXk7yQ\n5PNJVo66J4Akn0sykeSFvtp5SZ5M8mKSJ5KsGmRbSzIEknSAfw1cWVVXAv91tB1NL8ka4N3AUr6b\n5Engiqq6CjgAbBxxP69ZRjcQHgc+XlVXAL8B3LlE+wT4GLBv1E3M4B5gR1WtBf4xsOSGhZP8EvAR\n4Oqq+jV6w+cbRtvVax6g9zvT7y5gZ1VdBjzNgL/nSzIEgA8Bn6yq4wBV9Xcj7udU/hvwH0bdxKlU\n1c6qOtF8+QywZpT9TLIsbiCsqqNV9Xwz/QN6b1pL7h6X5qDkt4HPjrqX6TRnov+8qh4AqKrjVfX3\nI25rOmcCb06yAngTvQuPR66qvgx8b1L5JmBrM70VGOg69qUaApcC/yLJM0m+lOSfjLqhqSS5EXi5\nqvaOupdZuB34i1E30WfZ3UCY5CLgKuDZ0XYypZMHJUv5w76Lgb9L8kAzbPWHSd446qYmq6q/AT4N\nfAs4Avy/qto52q5O6YKqmoDeQQtwwSArjexmsSRPARf2l+j94P4nen2dV1XvSPLrwDbgbYvf5Yx9\nfoLeUFD/vJE4RZ+/V1VfbJb5PeBYVT08ghZPC0nOAR4FPtacESwZSf4VMFFVzzdDqkv1GR8rgKuB\nO6vqq0nupjeUsWm0bf2sJD9P7+j6rcArwKNJ3ruMfn8GOhAYWQhU1bunm5fk3wJ/2iy3u/nQ9fyq\n+r+L1mBjuj6T/CPgIuAv03ugzhrguSTrquo7i9gicOp/T4Akt9EbJviXi9LQ4I4Av9z39ZqmtuQ0\nQwKPAn9cVY+Nup8pXAPcmOS3gTcC5yZ5qKpuGXFfkx2mdwb91ebrR4GleEHAu4BvVtV3AZL8KfDP\ngKUaAhNJLqyqiSRjwEDvQ0t1OOjPad6sklwKvGEUAXAqVfX1qhqrqrdV1cX0frDfPooAmEmS6+kN\nEdxYVT8edT+T7AYuSfLW5sqLDcBSvarlfmBfVQ33GSRDUlWfqKpfrqq30ft3fHoJBgDNkMXLze82\n9G4eXYofZH8LeEeSs5sDvd9iaX2AHX72bG87cFszfSsw0IHKUn120APA/Un20nsu7JL7QZ5CsXRP\nv/8HsBJ4qnkK6DNVdcdoW+pZLjcQJrkGeB+wN8keev+/P1FVj4+2s2Xro8Dnk7wB+Cbwb0bcz+tU\n1a4kjwJ7gGPNf/9wtF31JHkY6ADnJ/kWvaG0TwJfSHI7vasVbx5oW94sJknttVSHgyRJi8AQkKQW\nMwQkqcUMAUlqMUNAklrMEJCkFjMEJKnFDAFJarH/D/v7QMpgm0GxAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x109f6fa58>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Plot histograms of the two versions: X_clean and x_noisy \n",
    "# Just to understand them better\n",
    "\n",
    "plt.hist(x_noisy, bins=15, range = [-5,10])\n",
    "plt.hist(x_clean)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.01,  0.04,  0.09,  0.26,  0.4 ])"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Here we set the \"true probability\".  This will be unknown to us, but instead we will see realizations\n",
    "# of binary 0/1 variables based on these probabilities\n",
    "\n",
    "true_probs = (x_clean/10)**2  + .1*(x_clean>=4)+ .05*(x_clean>=5)\n",
    "real_prob_vec = np.unique(true_probs)\n",
    "real_prob_vec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Generate our target y using the probabilities in \"true_probs\"\n",
    "\n",
    "y = np.random.binomial(n=1, p=true_probs, size=num_trials)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.17749999999999999"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.mean(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(7000, 1)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Divide into train and test sets\n",
    "\n",
    "X_train = x_noisy[:7000].reshape(-1,1)  #sklearn likes it better to have a 2-D array for X\n",
    "X_test = x_noisy[7000:].reshape(-1,1)\n",
    "y_train = y[:7000]\n",
    "y_test = y[7000:]\n",
    "\n",
    "X_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "          intercept_scaling=1, max_iter=100, multi_class='ovr', n_jobs=1,\n",
       "          penalty='l2', random_state=None, solver='liblinear', tol=0.0001,\n",
       "          verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Fit a default Logistic Regression Model\n",
    "## Note, unlike LinearRegression, the LogisticRegression automatically does a penalized regression\n",
    "\n",
    "\n",
    "lr_model_1 = LogisticRegression()\n",
    "lr_model_1.fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.94967126,  0.05032874],\n",
       "       [ 0.95046901,  0.04953099],\n",
       "       [ 0.77172597,  0.22827403],\n",
       "       ..., \n",
       "       [ 0.78946898,  0.21053102],\n",
       "       [ 0.96633847,  0.03366153],\n",
       "       [ 0.77502759,  0.22497241]])"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Use the model to predict *Probabilities* for the test set\n",
    "\n",
    "y_prob_pred_mat = lr_model_1.predict_proba(X_test)\n",
    "y_prob_pred_mat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# what comes out of predict_proba is a matrix\n",
    "# for binary classification, we typically look at the probability of being 1\n",
    "# So we will take the second column (:1)\n",
    "y_prob_pred = y_prob_pred_mat[:,1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([ 434.,  630.,  417.,  242.,  218.,  158.,  132.,   91.,   48.,\n",
       "          34.,   15.,   10.,    2.,    2.,    0.,    0.,    0.,    0.,\n",
       "           0.,    0.]),\n",
       " array([ 0.  ,  0.05,  0.1 ,  0.15,  0.2 ,  0.25,  0.3 ,  0.35,  0.4 ,\n",
       "         0.45,  0.5 ,  0.55,  0.6 ,  0.65,  0.7 ,  0.75,  0.8 ,  0.85,\n",
       "         0.9 ,  0.95,  1.  ]),\n",
       " <a list of 20 Patch objects>)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlsAAAHfCAYAAABnDB0iAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3X+M3Hd95/HnK5ikhILPQLN7TSAJ5EicqkARDZzK6aZA\nE0J1ccSdciktTYhoK6Ut6E5qsZFab6W2IZVOLRWXShGUujq4nAGhuBSISZO5CokkUBKSYic1UJtg\nbhfxUwqIyIb3/THfRItZe8ez85nZ2X0+pBHf+e73O+/P8M2sX/v5fObzTVUhSZKkNs6YdgMkSZI2\nMsOWJElSQ4YtSZKkhgxbkiRJDRm2JEmSGjJsSZIkNbRq2EryniRLSR5ctm9bkv1JHklyR5Kty362\nK8mhJAeTXN6q4ZIkSbNgmJ6t9wJXnLBvJ3BnVV0M3AXsAkhyKXANsB24ErglScbXXEmSpNmyatiq\nqk8C3zph9w5gT7e9B7i6274KuK2qjlfVYeAQcNl4mipJkjR7Rp2zdU5VLQFU1SJwTrf/XODRZccd\n7fZJkiRtSlvG9Dqnfc+fJN4nSJIkzYyqGmlq1KhhaynJXFUtJZkHvtbtPwo8d9lx53X7VuR9GWfX\nwsICCwsL026GRuT1m11eu9nm9Ztda5mCPuwwYrrHE/YB13fb1wG3L9t/bZIzk1wIXATcN3LrJEmS\nZtyqPVtJ3g/0gGcn+TKwG3gH8IEkNwBHGHwDkao6kGQvcAA4BtxYdl9JkqRNbNWwVVVvOMmPXnOS\n428CblpLo7T+9Xq9aTdBa+D1m11eu9nm9ducMq2OpyR2ekmSpJmQZOQJ8t6uR5IkqSHDliRJUkOG\nLUmSpIYMW5IkSQ0ZtiRJkhoybEmSJDVk2JIkSWrIsCVJktSQYUuSJKkhw5YkSVJDhi1JkqSGDFsa\nyfz8BSSZ2GN+/oJpv2VJkkbijag1kiTAJK9f8L8XSdK0eCNqSZKkdcqwJUmS1NCWaTdAGs5Z3dDl\nZMzNnc/i4uGJ1ZMkbVzO2dJIpjFnyzlikqRpcc6WJEnSOmXYkiRJasiwJUmS1JBhS5IkqSHDliRJ\nUkOGLUmSpIYMW5IkSQ0ZtiRJkhoybEmSJDVk2JIkSWrIsCVJktSQYUuSJKkhw5YkSVJDhi1JkqSG\n1hS2kvy3JP+c5MEk70tyZpJtSfYneSTJHUm2jquxkiRJs2bksJXkp4HfBV5aVS8CtgC/AuwE7qyq\ni4G7gF3jaKgkSdIsWusw4lOApyfZAjwNOArsAPZ0P98DXL3GGpIkSTNr5LBVVV8F/gfwZQYh6ztV\ndScwV1VL3TGLwDnjaKgkSdIs2jLqiUn+DYNerPOB7wAfSPKrQJ1w6InPn7SwsPDkdq/Xo9frjdoc\nSZKksen3+/T7/bG8VqpOmoVOfWLyX4Arquo3uudvBF4BvAroVdVSknng7qravsL5NWptTV8STpGj\nW1SceD3/+5QkPSEJVZVRzl3LnK0vA69I8hMZ/Mv7auAAsA+4vjvmOuD2NdSQpuQskkzsMT9/wbTf\nsCSpkZF7tgCS7AauBY4B9wNvBp4B7AWeCxwBrqmqb69wrj1bM2wz9GzZkyZJesJaerbWFLbWwrA1\n2wxb46/n50GS1q9pDSNKkiRpFYYtSZKkhgxbkiRJDRm2JEmSGjJsSZIkNWTYkiRJasiwJUmS1JBh\nS5IkqSHDliRJUkOGLUmSpIYMW5IkSQ0ZtiRJkhoybEmSJDVk2JIkSWrIsCVJktSQYUuSJKkhw5Yk\nSVJDhi1JkqSGDFuSJEkNGbYkSZIaMmxJkiQ1ZNhqZH7+ApJM7DE/f8G037IkSVpBqmo6hZOaVu1J\nSAJM8v2FSf7/OY33t9HrbeTPgyTNuiRUVUY5154tSZKkhgxbkiRJDRm2JEmSGjJsSZIkNWTYkiRJ\nasiwJUmS1NCWaTdA43JWtxyDJElaTwxbG8bjTH4dKkmStJo1DSMm2ZrkA0kOJvl8kpcn2ZZkf5JH\nktyRZOu4GitJkjRr1jpn653AR6tqO/Bi4GFgJ3BnVV0M3AXsWmMNSZKkmTXy7XqSPBO4v6pecML+\nh4H/WFVLSeaBflVdssL53q5nvBWtN+P1NvLnQZJm3bRu13Mh8PUk703y2SS3JjkbmKuqJYCqWgTO\nWUMNSZKkmbaWsLUFeCnwP6vqpcB3GQwhnvjnuX+uS5KkTWst30b8CvBoVX2me/4hBmFrKcncsmHE\nr53sBRYWFp7c7vV69Hq9NTRHkiRpPPr9Pv1+fyyvNfKcLYAk/xf4jar6lyS7gbO7H32zqm5O8jZg\nW1XtXOFc52yNt6L1ZrzeRv48SNKsW8ucrbWGrRcD7waeCnwJeBPwFGAv8FzgCHBNVX17hXMNW+Ot\naL0Zr7eRPw+SNOumFrbWwrA19orWm/F6G/nzIEmzblrfRpQkSdIqDFuSJEkNGbYkSZIaMmxJkiQ1\nZNiSJElqyLAlSZLUkGFLkiSpIcOWJElSQ2u5N6KksTmrWwh3Mubmzmdx8fDE6knSZuYK8o24grz1\n1nu9jfz5k6RxcwV5SZKkdcqwJUmS1JBhS5IkqSHDliRJUkOGLUmSpIYMW5IkSQ0ZtiRJkhoybEmS\nJDVk2JIkSWrIsCVJktSQYUuSJKkhw5YkSVJDhi1JkqSGDFuSJEkNGbYkSZIaMmxJkiQ1ZNiSJElq\nyLAlSZLUkGFLkiSpIcOWJElSQ4YtSZKkhgxbkiRJDa05bCU5I8lnk+zrnm9Lsj/JI0nuSLJ17c2U\nJEmaTePo2XorcGDZ853AnVV1MXAXsGsMNSRJkmbSmsJWkvOA1wHvXrZ7B7Cn294DXL2WGpIkSbNs\nrT1bfw78HlDL9s1V1RJAVS0C56yxhiRJ0szaMuqJSX4ZWKqqB5L0TnFonewHCwsLT273ej16vVO9\nzNrMz1/A0tKRZq8vSZI2jn6/T7/fH8trpeqkWejUJyZ/CvwacBx4GvAM4MPAy4BeVS0lmQfurqrt\nK5xfo9Yesb2cIve1qGg9663repP8/EnSrEtCVWWUc0ceRqyqt1fV86rq+cC1wF1V9Ubg74Dru8Ou\nA24ftYYkSdKsa7HO1juAX0ryCPDq7rkkSdKmNPIw4poLO4xoPetNtZ7DiJI0vKkMI0qSJGl1hi1J\nkqSGDFuSJEkNGbYkSZIaMmxJkiQ1ZNiSJElqyLAlSZLUkGFLkiSpIcOWJElSQ4YtSZKkhgxbkiRJ\nDRm2JEmSGjJsSZIkNWTYkjals0gykcf8/AXTfrOSNFWpqukUTmqStZMAk3yv1rOe9Z6oNa3fM5I0\nLkmoqoxyrj1bkiRJDRm2JEmSGjJsSZIkNWTYkiRJasiwJUmS1JBhS5IkqSHDliRJUkOGLUmSpIYM\nW5IkSQ0ZtiRJkhoybEmSJDVk2JIkSWrIsCVJktSQYUuSJKkhw5YkSVJDhi1JkqSGDFuSJEkNjRy2\nkpyX5K4kn0/yUJK3dPu3Jdmf5JEkdyTZOr7mSpIkzZZU1WgnJvPAfFU9kOQngX8CdgBvAr5RVX+W\n5G3AtqraucL5NWrtEdsLTK4eWM961nui1iQ/65LUQhKqKqOcO3LPVlUtVtUD3fZjwEHgPAaBa093\n2B7g6lFrSJIkzbqxzNlKcgHwEuAeYK6qlmAQyIBzxlFDkiRpFm1Z6wt0Q4gfBN5aVY8lOXG84KTj\nBwsLC09u93o9er3eWpsjSZK0Zv1+n36/P5bXGnnOFkCSLcBHgI9V1Tu7fQeBXlUtdfO67q6q7Suc\n65wt61lvU9Rzzpak2TeVOVudvwYOPBG0OvuA67vt64Db11hDkiRpZq3l24i/APwj8BCDP5ELeDtw\nH7AXeC5wBLimqr69wvn2bFnPepuinj1bkmbfWnq21jSMuBaGLetZb7PU+wng8QnVgrm581lcPDyx\nepI2B8PWcPXYuP+YWc9667ne5N+bPWmSxm2ac7YkSZJ0CoYtSZKkhgxbkiRJDRm2JEmSGjJsSZIk\nNWTYkiRJasiwJUmS1JBhS5IkqSHDliRJUkOGLUmSpIYMW5IkSQ0ZtiRJkhoybEmSJDW0ZZrF/+RP\n/mSa5SVJkppLVU2ncFLJ2ydU7UtU3QZM8r3Getaz3sRrAfwE8PjEqs3Nnc/i4uGJ1ZM0HUmoqox0\n7jTD1uR+Ae8HrmDj/mNmPeut53ob+b0N6k3r96ikyVlL2HLOliRJUkOGLUmSpIamOkFekmbfWSQj\njSyMxDli0uwxbEnSmjzOJOeILS1NLthJGg+HESVJkhoybEmSJDVk2JIkSWrIsCVJktSQYUuSJKkh\nw5YkSVJDLv0gSTPFdb2kWWPYkqSZ4rpe0qxxGFGSJKkhw5YkSVJDzcJWktcmeTjJvyR5W6s6kiTN\nin6/P+0maAqahK0kZwDvAq4Afgb4lSSXtKglSdKsMGxtTq16ti4DDlXVkao6BtwG7GhUS5LUzODb\nj5N6zM9fMO03LI1dq28jngs8uuz5VxgEsBPc36j8ib4woTqStNH47Udpraa89MNLJ1xv0h9i61nP\nepOvZb1ZrzfJdcSm4Y/+6I+m3QRNWKuwdRR43rLn53X7nlRVG/vTJEmSRLs5W58GLkpyfpIzgWuB\nfY1qSZIkrVtNeraq6gdJfgfYzyDQvaeqDraoJUmStJ6lanITHyVJkjab5ivID7O4aZK/THIoyQNJ\nXtK6TRreatcvyRuSfK57fDLJz06jnfpxwy4snOTnkxxL8vpJtk+nNuTvzl6S+5P8c5K7J91GrWyI\n35vPTLKv+zfvoSTXT6GZWkGS9yRZSvLgKY45/cxSVc0eDMLcF4DzgacCDwCXnHDMlcDfd9svB+5p\n2SYfY79+rwC2dtuv9fqtj8cw127Zcf8AfAR4/bTb7WP46wdsBT4PnNs9f8602+1j6Gu3C7jpiesG\nfAPYMu22+yiAVwIvAR48yc9Hyiyte7aGWdx0B/C3AFV1L7A1yVzjdmk4q16/qrqnqr7TPb2HwRpr\nmr5hFxb+XeCDwNcm2Titapjr9wbgQ1V1FKCqvj7hNmplw1y7Ap7RbT8D+EZVHZ9gG3USVfVJ4Fun\nOGSkzNI6bK20uOmJ/xifeMzRFY7RdAxz/ZZ7M/Cxpi3SsFa9dkl+Gri6qv6KyS/cpFMb5rP3QuBZ\nSe5O8ukkb5xY63Qqw1y7dwGXJvkq8DngrRNqm9ZupMwy5UVNtVEk+UXgTQy6YDUb/gJYPp/EwDVb\ntjBYGfpVwNOBTyX5VFV5y4z17wrg/qp6VZIXAJ9I8qKqemzaDVMbrcPWqoubds+fu8oxmo5hrh9J\nXgTcCry2qk7V/arJGebavQy4LYPlup8DXJnkWFW5Jt70DXP9vgJ8vaq+D3w/yT8CL8b7k03bMNfu\nTcBNAFX1xST/ClwCfGYiLdRajJRZWg8jDrO46T7g1wGSvAL4dlUtNW6XhrPq9UvyPOBDwBur6otT\naKNWtuq1q6rnd48LGczbutGgtW4M87vzduCVSZ6S5GwGk3Vdz3D6hrl2R4DXAHTzfV4IfGmirdSp\nhJP39I+UWZr2bNVJFjdN8luDH9etVfXRJK9L8gXguwwSv9aBYa4f8AfAs4Bbuh6SY1W1wk3HNUlD\nXrsfOWXijdRJDfm78+EkdwAPAj8Abq2qA1Nsthj6s/fHwN8sW17g96vqm1NqspZJ8n6gBzw7yZeB\n3cCZrDGzuKipJElSQ80XNZUkSdrMDFuSJEkNGbYkSZIaMmxJkiQ1ZNiSJElqyLAlSZLUkGFLkiSp\nIcOWJElSQ4YtSZKkhgxbkiRJDRm2JEmSGjJsSZIkNWTYkiRJasiwJUmS1JBhS5IkqSHDliRJUkOG\nLUmSpIYMW5IkSQ0ZtiRJkhoybEmSJDVk2JIkSWrIsCVJktSQYUuSJKmhVcNWkhcmuT/JZ7v//U6S\ntyTZlmR/kkeS3JFk67JzdiU5lORgksvbvgVJkqT1K1U1/MHJGcBXgJcDvwN8o6r+LMnbgG1VtTPJ\npcD7gJ8HzgPuBP5dnU4hSZKkDeJ0hxFfA3yxqh4FdgB7uv17gKu77auA26rqeFUdBg4Bl42hrZIk\nSTPndMPWfwXe323PVdUSQFUtAud0+88FHl12ztFunyRJ0qazZdgDkzyVQa/V27pdJw4LntYwYRKH\nFSVJ0syoqoxy3tBhC7gS+Keq+nr3fCnJXFUtJZkHvtbtPwo8d9l553X7fozTuGbXwsICCwsL026G\nRuT1m11eu9nm9ZtdyUg5Czi9YcRfAf73suf7gOu77euA25ftvzbJmUkuBC4C7hu5hZIkSTNsqJ6t\nJGczmBz/m8t23wzsTXIDcAS4BqCqDiTZCxwAjgE3+k1ESZK0WQ0Vtqrqe8BPnbDvmwwC2ErH3wTc\ntObWad3q9XrTboLWwOs3u7x2s83rtzmd1jpbYy2c2OElSZJmQpKRJ8h7ux5JkqSGDFuSJEkNGbYk\nSZIaMmxJkiQ1ZNiSJElqyLAlSZLUkGFLkiSpIcOWJElSQ4YtSZKkhgxbkiRJDRm2JEmSGjJsSZIk\nNWTYkiRJasiwJUmS1NCmCVvz8xeQZGKP+fkLpv2WJUnSOpCqmk7hpCZZOwkwyfcapvX/rSRJGq8k\nVFVGOXfT9GxJkiRNg2FLkiSpIcOWJElSQ0OFrSRbk3wgycEkn0/y8iTbkuxP8kiSO5JsXXb8riSH\nuuMvb9d8SZKk9W3Ynq13Ah+tqu3Ai4GHgZ3AnVV1MXAXsAsgyaXANcB24Erglgxmp0uSJG06q4at\nJM8E/kNVvRegqo5X1XeAHcCe7rA9wNXd9lXAbd1xh4FDwGXjbrgkSdIsGKZn60Lg60nem+SzSW5N\ncjYwV1VLAFW1CJzTHX8u8Oiy8492+yRJkjadLUMe81Lgt6vqM0n+nMEQ4omLSJ32olILCwtPbvd6\nPXq93um+hCRJ0tj1+336/f5YXmvVRU2TzAGfqqrnd89fySBsvQDoVdVSknng7qranmQnUFV1c3f8\nx4HdVXXvCa/roqaSJGkmNF3UtBsqfDTJC7tdrwY+D+wDru/2XQfc3m3vA65NcmaSC4GLgPtGaZwk\nSdKsG2YYEeAtwPuSPBX4EvAm4CnA3iQ3AEcYfAORqjqQZC9wADgG3DjRLixJkqR1xHsjtqvoMKIk\nSRuE90aUJElapwxbkiRJDRm2JEmSGjJsSZIkNWTYkiRJasiwJUmS1JBhS5IkqSHDliRJUkOGLUmS\npIYMW5IkSQ0ZtiRJkhoybEmSJDVk2JIkSWrIsCVJktSQYUuSJKkhw5YkSVJDhi1JkqSGDFuSJEkN\nGbYkSZIaMmxJkiQ1NFTYSnI4yeeS3J/kvm7ftiT7kzyS5I4kW5cdvyvJoSQHk1zeqvGSJEnr3bA9\nWz8EelX1c1V1WbdvJ3BnVV0M3AXsAkhyKXANsB24ErglScbbbEmSpNkwbNjKCsfuAPZ023uAq7vt\nq4Dbqup4VR0GDgGXIUmStAkNG7YK+ESSTyd5c7dvrqqWAKpqETin238u8Oiyc492+yRJkjadLUMe\n9wtV9f+S/BSwP8kjDALYcic+X9XCwsKT271ej16vd7ovIUmSNHb9fp9+vz+W10rV6WWkJLuBx4A3\nM5jHtZRkHri7qrYn2QlUVd3cHf9xYHdV3XvC69Tp1l6LwbSxydWDMMn3J0mS2klCVY00B33VYcQk\nZyf5yW776cDlwEPAPuD67rDrgNu77X3AtUnOTHIhcBFw3yiNkyRJmnXDDCPOAR9OUt3x76uq/Uk+\nA+xNcgNwhME3EKmqA0n2AgeAY8CNE+3CkiRJWkdOexhxbIUdRpQkSTOi6TCiJEmSRmfYkiRJasiw\nJUmS1JBhS5IkqSHDliRJUkOGLUmSpIYMW5IkSQ0ZtiRJkhoybEmSJDVk2JIkSWrIsCVJktSQYUuS\nJKkhw5YkSVJDW6ZZ/A//cPc0y0uSJDWXqppO4aRgYULVvgD8L2CS7zVM6/9bSZI0Xkmoqox07nTD\n1qRq7weuwLAlSZJGsZaw5ZwtSZKkhgxbkiRJDRm2JEmSGprqtxE3trNIRhraHcnc3PksLh6eWD1J\nkjScoXu2kpyR5LNJ9nXPtyXZn+SRJHck2brs2F1JDiU5mOTyFg1f/x5nMCF/Mo+lpSMTel+SJOl0\nnM4w4luBA8ue7wTurKqLgbuAXQBJLgWuAbYDVwK3ZJJdPJIkSevIUGEryXnA64B3L9u9A9jTbe8B\nru62rwJuq6rjVXUYOARcNpbWSpIkzZhhe7b+HPg9fnShqrmqWgKoqkXgnG7/ucCjy4472u2TJEna\ndFadIJ/kl4GlqnogSe8Uh46wgufCsu1e95AkSZqufr9Pv98fy2utuoJ8kj8Ffg04DjwNeAbwYeBl\nQK+qlpLMA3dX1fYkO4Gqqpu78z8O7K6qe0943Q2/grwr1kuStDE0XUG+qt5eVc+rqucD1wJ3VdUb\ngb8Dru8Ouw64vdveB1yb5MwkFwIXAfeN0jhJkqRZt5Z1tt4B7E1yA3CEwTcQqaoDSfYy+ObiMeDG\nsstFkiRtUt6IuhmHESVJ2ii8EbUkSdI6ZdiSJElqyLAlSZLUkGFLkiSpIcOWJElSQ4YtSZKkhgxb\nkiRJDRm2JEmSGjJsSZIkNWTYkiRJasiwJUmS1JBhS5IkqSHDliRJUkOGLUmSpIYMW5IkSQ0ZtiRJ\nkhoybEmSJDVk2JIkSWrIsCVJktSQYUuSJKmhVcNWkrOS3Jvk/iQPJdnd7d+WZH+SR5LckWTrsnN2\nJTmU5GCSy1u+AUmSpPVs1bBVVY8Dv1hVPwe8BLgyyWXATuDOqroYuAvYBZDkUuAaYDtwJXBLkjRq\nvyRJ0ro21DBiVX2v2zwL2AIUsAPY0+3fA1zdbV8F3FZVx6vqMHAIuGxcDZYkSZolQ4WtJGckuR9Y\nBD5RVZ8G5qpqCaCqFoFzusPPBR5ddvrRbp8kSdKms2WYg6rqh8DPJXkm8OEkP8Ogd+tHDjv98gvL\ntnvdQ5Ikabr6/T79fn8sr5Wq08tISf4A+B7wZqBXVUtJ5oG7q2p7kp1AVdXN3fEfB3ZX1b0nvE6N\nlM9Gsh+4gsnVA8jE653utZQkScNJQlWNNAd9mG8jPueJbxomeRrwS8BBYB9wfXfYdcDt3fY+4Nok\nZya5ELgIuG+UxkmSJM26YYYR/y2wJ8kZDMLZ/6mqjya5B9ib5AbgCINvIFJVB5LsBQ4Ax4Abyy4X\nSZK0SZ32MOLYCjuMOPZ6ZlpJktpoOowoSZKk0Rm2JEmSGjJsSZIkNWTYkiRJasiwJUmS1JBhS5Ik\nqSHDliRJUkOGLUmSpIYMW5IkSQ0ZtiRJkhoybEmSJDVk2JIkSWpoy7QboHE5i2Sk+2OOZG7ufBYX\nD0+sniRJs8qwtWE8DtTEqi0tTS7YSZI0yxxGlCRJasiwJUmS1JBhS5IkqSHDliRJUkOGLUmSpIb8\nNqJG5FITkiQNY9WerSTnJbkryeeTPJTkLd3+bUn2J3kkyR1Jti47Z1eSQ0kOJrm85RvQtDyx1MRk\nHktLRyb0viRJGq9hhhGPA/+9qn4G+PfAbye5BNgJ3FlVFwN3AbsAklwKXANsB64Ebskku0AkSZLW\nkVXDVlUtVtUD3fZjwEHgPGAHsKc7bA9wdbd9FXBbVR2vqsPAIeCyMbdbkiRpJpzWBPkkFwAvAe4B\n5qpqCQaBDDinO+xc4NFlpx3t9kmSJG06Q0+QT/KTwAeBt1bVY0lOvDfMCPeKWVi23esekiRJ09Xv\n9+n3+2N5rVStnpGSbAE+Anysqt7Z7TsI9KpqKck8cHdVbU+yE6iqurk77uPA7qq694TXrMndy28/\ncAWTvHcgxHpjrjfMf6uSJLWQhKoaaQ76sMOIfw0ceCJodfYB13fb1wG3L9t/bZIzk1wIXATcN0rj\nJEmSZt2qw4hJfgH4VeChJPcz6M54O3AzsDfJDcARBt9ApKoOJNkLHACOATeWXRKSJGmTGmoYsUlh\nhxGtd5r1zOySpGmZxDCiJEmSRmDYkiRJasiwJUmS1JBhS5IkqSHDliRJUkOGLUmSpIYMW5IkSQ0N\nfW9EabrOIhlpeZORzM2dz+Li4YnVkyRtXIYtzYjHmeQiqktLkwt2kqSNzWFESZKkhgxbkiRJDRm2\nJEmSGjJsSZIkNWTYkiRJasiwJUmS1JBLP0grcl0vSdJ4GLakFbmulyRpPBxGlCRJasiwJUmS1JBh\nS5IkqaFVw1aS9yRZSvLgsn3bkuxP8kiSO5JsXfazXUkOJTmY5PJWDZckSZoFw/RsvRe44oR9O4E7\nq+pi4C5gF0CSS4FrgO3AlcAtmeRXuiRJktaZVcNWVX0S+NYJu3cAe7rtPcDV3fZVwG1VdbyqDgOH\ngMvG01RJkqTZM+qcrXOqagmgqhaBc7r95wKPLjvuaLdPkiRpUxrXOlsjLki0sGy71z0kSZKmq9/v\n0+/3x/JaqVo9JyU5H/i7qnpR9/wg0KuqpSTzwN1VtT3JTqCq6ubuuI8Du6vq3hVesya3aOR+BtPO\nJrdIJcR61jutesN8FiVJ05GEqhppHvqww4jpHk/YB1zfbV8H3L5s/7VJzkxyIXARcN8oDZM2l8Ht\ngSb1mJ+/YNpvWJI2jVWHEZO8n8H43rOTfBnYDbwD+ECSG4AjDL6BSFUdSLIXOAAcA24s/1yXhuDt\ngSRpoxpqGLFJYYcRrWe9qdbz7yBJGt4khhElSZI0AsOWJElSQ4YtSZKkhgxbkiRJDRm2JEmSGjJs\nSZIkNWTYkiRJasiwJUmS1JBhS5IkqSHDliRJUkOGLWlTmtyNr73ptaTNbtUbUUvaiCZ342tvei1p\ns7NnS5IkqSHDliRJUkMOI0pqbDA/bFLm5s5ncfHwxOpJ0moMW5Iam9z8MHCOmKT1x2FESZKkhgxb\nkiRJDRm2JEmSGnLOlqQNxgn5ktaXZj1bSV6b5OEk/5Lkba3qSNKPemJC/mQeS0tHJvS+Bvr9/kTr\naby8fptTk56tJGcA7wJeDXwV+HSS26vq4Rb1JGl6JtuT9vSnb+Wxx749sXoar36/T6/Xm3YzNGGt\nhhEvAw5V1RGAJLcBOwDDlqQNZrJLW3z3uy5tIc2aVmHrXODRZc+/wiCA/YhnPvM/NSr/o44f/xrf\n+95ESkn764JaAAADw0lEQVRSY0+ZaE/aGWeczQ9/OLlfoM6B00aUqvH/RZbkPwNXVNVvds9/Dbis\nqt6y7JjJ/SkoSZK0RlU10l86rXq2jgLPW/b8vG7fk0ZtsCRJ0ixp9W3ETwMXJTk/yZnAtcC+RrUk\nSZLWrSY9W1X1gyS/A+xnEOjeU1UHW9SSJElaz5rM2ZIkSdJA89v1DLO4aZK/THIoyQNJXtK6TRre\natcvyRuSfK57fDLJz06jnfpxwy4snOTnkxxL8vpJtk+nNuTvzl6S+5P8c5K7J91GrWyI35vPTLKv\n+zfvoSTXT6GZWkGS9yRZSvLgKY45/cxSVc0eDMLcF4DzgacCDwCXnHDMlcDfd9svB+5p2SYfY79+\nrwC2dtuv9fqtj8cw127Zcf8AfAR4/bTb7WP46wdsBT4PnNs9f8602+1j6Gu3C7jpiesGfAPYMu22\n+yiAVwIvAR48yc9Hyiyte7aeXNy0qo4BTyxuutwO4G8BqupeYGuSucbt0nBWvX5VdU9Vfad7eg+D\nNdY0fcN89gB+F/gg8LVJNk6rGub6vQH4UFUdBaiqr0+4jVrZMNeugGd0288AvlFVxyfYRp1EVX0S\n+NYpDhkps7QOWystbnriP8YnHnN0hWM0HcNcv+XeDHysaYs0rFWvXZKfBq6uqr8CXIplfRnms/dC\n4FlJ7k7y6SRvnFjrdCrDXLt3AZcm+SrwOeCtE2qb1m6kzNJqnS1tMkl+EXgTgy5YzYa/AJbPJzFw\nzZYtwEuBVwFPBz6V5FNV9YXpNktDuAK4v6peleQFwCeSvKiqHpt2w9RG67C16uKm3fPnrnKMpmOY\n60eSFwG3Aq+tqlN1v2pyhrl2LwNuy+DeL88BrkxyrKpcE2/6hrl+XwG+XlXfB76f5B+BFzOYL6Tp\nGebavQm4CaCqvpjkX4FLgM9MpIVai5EyS+thxGEWN90H/DpAklcA366qpcbt0nBWvX5Jngd8CHhj\nVX1xCm3Uyla9dlX1/O5xIYN5WzcatNaNYX533g68MslTkpzNYLKu6xlO3zDX7gjwGoBuvs8LgS9N\ntJU6lXDynv6RMkvTnq06yeKmSX5r8OO6tao+muR1Sb4AfJdB4tc6MMz1A/4AeBZwS9dDcqyqfuym\n45qsIa/dj5wy8UbqpIb83flwkjuAB4EfALdW1YEpNlsM/dn7Y+Bvli0v8PtV9c0pNVnLJHk/0AOe\nneTLwG7gTNaYWVzUVJIkqaHmi5pKkiRtZoYtSZKkhgxbkiRJDRm2JEmSGjJsSZIkNWTYkiRJasiw\nJUmS1ND/B95FJYiBVGCrAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1104fa6d8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#Let's visualize - is there a difference in how we predict when the true answer is 1 versus 0?\n",
    "fig, ax = plt.subplots(2, 1, figsize=(10, 8))  # 2 Rows, 1 Col\n",
    "\n",
    "ax[0].hist(y_prob_pred[np.where(y_test==1)], bins=20, range=(0,1))\n",
    "ax[1].hist(y_prob_pred[np.where(y_test==0)], bins=20, range=(0,1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "## How \"good\" is this?\n",
    "## First, lets evaluate the log-likelihood\n",
    "## sklearn has a \"log_loss\" function which computes log-likelihood, negates it,\n",
    "## and then averages by the number of data points\n",
    "from sklearn.metrics import log_loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.42308852639501532"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "log_loss(y_test,y_prob_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.42308852639501532"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## Alternatively, compute it yourself\n",
    "pred_prob_of_data = y_test*y_prob_pred + (1-y_test)*(1-y_prob_pred)\n",
    "np.mean(-np.log(pred_prob_of_data))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x11053cf98>]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXkAAAEACAYAAABWLgY0AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAFc1JREFUeJzt3W+MXXWdx/HPBys2uyLMLAmJRch2CsE1tsQo9gHJXsRd\nyj4Zg3EtJFhIjI0U6TPRB4YhMXF5ZlwibJOGxm1M6comdLX8MYarYbVYYxFbWlvU1LaYjmg1oRuS\n2nz3wb23c3qZO/fMvefc8+/9Sm4yd+bMmR8nM79++J7f73scEQIA1NMlRQ8AAJAfJnkAqDEmeQCo\nMSZ5AKgxJnkAqDEmeQCosaGTvO3ttk/bfmWJY75h+5jtl23fmO0QAQCjSpPkn5B026Av2r5d0kxE\nXCdps6THMxobAGBMQyf5iHhR0pklDpmV9K3usS9Jutz2VdkMDwAwjixq8qsknUi8P9X9HACgYNx4\nBYAaW5HBOU5Jel/i/dXdz72NbRrlAMAIIsKjfF/aJO/uazF7JH1Gkmyvl/TniDg96EQRwStCDz30\nUOFjKMuLa8G1KOu1mJoKSfm8pqYW/5mn3zytTz75Sb3/0fdr34l9ihgvG6dZQvltST+WdL3t39m+\n1/Zm25/rTtp7Jf3W9muS/kPSfWONCAByND0t2elekhSRz+tPf7p4XBGhJw8+qbWPrdWa6TX6+eaf\n66NXf3Ts/96h5ZqIuCvFMfePPRIAGMH0tHRmqfV/faamOpNsmcyfndd937tPr/7hVT298elMJvce\nbrwWpNVqFT2E0uBaLOBaLOhdi2HJWxovQRdt96HdWvvYWs1MzWSW3pM8br1nWT/Mjkn+PADVMSiR\nT02Vb2LOwvzZeW3Zu0UH5w9qx+yOJSd324qcb7wCQGrLqXsPS+R1nOB76X31Fat1YPOBzNN7UhZL\nKAHUxHLr24OUse5dBsn0nnXtfRCSPABJnQleymflSNNFxETTexJJHmio/tRe19p30XorZw794dDE\n0nsSSR6osaVq4xLpO0/Jde8zUzMTTe9JJHmghnopndp4MYpO70kkeaAmkqldIp0XIVl7LzK9J5Hk\ngQpZavULqb1YZUrvSSR5oCKGrX4htRejLLX3QUjyQMkl6+tM5OVS1vSeRJIHSqpXY5dI6mXT3zGy\nbOk9iSQPlMRi69apsZdPnh0j80CSByZkud0USe7lkle/97yR5IEcJdM5yby6qlB7H4QkD+SkfzUM\nybyayrbufblI8sCIhnVsZDVMtRXRMTIPJHlgmfpXvbBuvX6K6hiZB5I8sAzJEgzqpy7pPYkkDySk\nWQFDQq+f5MqZOqT3JJI8Go216ajyypk0SPJopEF1dVJ6c5S950xWSPJoHOrqqNqu1XGQ5NEovQme\nxN5MVd21Og6SPBqDCb7ZmpTek0jyqLX+pyUxwTdPE9N7EkketcQzTiHVf+VMGiR51E7yxirJvbmq\n3nMmKyR51AZPUIJUz12r4yDJo/J4ghJ66tRzJiskeVQaa94hkd6XQpJH5bBiBj117jmTFZI8KoEn\nLKEfK2fSIcmj9HjCEpKa0nMmKyR5lN6ZMyR3dJDely9Vkre9wfYR20dtP7jI199je4/tl23/0vY9\nmY8UjdOrvU9NFT0SFI30PjrHkIhk+xJJRyXdKul1SfslbYyII4ljvizpPRHxZdtXSvqVpKsi4q99\n54phPw+Q6DODBcn0vmN2RyMnd9uKCI/yvWmS/E2SjkXE8Yg4J2mXpNm+Y0LSZd2PL5P0x/4JHkgj\nueadCb7ZSO/ZSFOTXyXpROL9SXUm/qRHJe2x/bqkd0v6dDbDQ9NQf4fU3I6Recjqxuttkg5ExMds\nz0j6vu21EfFm/4Fzc3MXPm61Wmq1WhkNAVWWbEmA5ooI7T60W1uf3ap7brxHO+/YqZUrVhY9rIlr\nt9tqt9uZnCtNTX69pLmI2NB9/yVJERGPJI75rqSvRcT/dt//QNKDEfGzvnNRk4ekxZ+tSnmm2ZLp\n/YnZJ0jvCXnX5PdLWmP7WtuXStooaU/fMcclfbw7mKskXS/pN6MMCPXGs1WxmF7PmSb2e8/b0HJN\nRJy3fb+k59X5R2F7RBy2vbnz5dgm6auSdth+pfttX4wI/mxxEfrMoB89Z/I3tFyT6Q+jXNNYLIlE\nv92HduuBZx7QpnWb9PAtDzey9p7WOOUadrwid0zwSCK9Txa9a5ArJngk0e998kjyyAVPaUIS6b04\nJHlkiqc0IYl+78UjySMTyeTOvXVIdIwsC5I8xkJyR79kel8zvYb0XjCSPJaNpzRhEHrOlA9JHsvW\nayJGckdPf3pn12p5kOSxLNPTNBHDxUjv5UaSR2qseUe/3rr3makZ0ntJkeSRGr3e0cO69+ogySMV\nyjToYddqtZDkMRRlGkik96oiyWMgnrcKaeFpTaT3aiLJY1H0fofErtU6IMljUWfOkN6bLLnufWZq\nhvReYSR5XIQHaoP0Xi8keUiiBw0urr2T3uuDJA/q7yC91xhJvqF6yZ3VM81G7b3+SPINxe5VkN6b\ngSTfQOxebTb6vTcLSb5h2L3abKffPK0te7fQMbJBSPINwgTfXL30vu7xdfR7bxiSfEMwwTdXL71T\ne28mknxDsIO1eZLpnZUzzUWSB2qIlTPoIck3AKtpmoVdq0giyTcAa+KbgX7vWAxJvsZ6u1pJ8fVG\nv3cshSRfU/SjaQZq7xiGJF9TrKapN3rOIC2SfA1xo7XeeumdXatIgyRfM2x6qq/+njPsWkUaJPma\nSD7RiQm+fkjvGFWqJG97g+0jto/afnDAMS3bB2wftP1CtsPEML1lkkzw9UJ6x7gcQ5Zf2L5E0lFJ\nt0p6XdJ+SRsj4kjimMsl/VjSP0fEKdtXRsQbi5wrhv08LA8Jvr6SK2d2zO5gcm8w24oIj/K9aZL8\nTZKORcTxiDgnaZek2b5j7pL0VESckqTFJnjkgwRfP/R7R5bS1ORXSTqReH9SnYk/6XpJ7+yWad4t\n6RsR8Z/ZDBGDsIqmfqi9I2tZra5ZIelDkm6XtEHSV2yvyejcGIC18PVB7R15SZPkT0m6JvH+6u7n\nkk5KeiMi3pL0lu0fSVon6bX+k83NzV34uNVqqdVqLW/EuKgOj+qj5wz6tdtttdvtTM6V5sbrOyT9\nSp0br7+X9FNJd0bE4cQxN0j6d3VS/LskvSTp0xHxat+5uPGaAZt2BXXQ6zmz9dmt2rRukx6+5WGt\nXLGy6GGhhMa58To0yUfEedv3S3penfLO9og4bHtz58uxLSKO2H5O0iuSzkva1j/BA1hAzxlMytAk\nn+kPI8mPheWS1Ud6xyhyTfIoB7pKVh/pHUWgd00F0I+m2ugYiSKR5EuOCb7aSO8oGkm+pHpPdZKY\n4KuI9I6yIMmXFM9lrS52raJMSPIl0kvvPJe1mti1ijIiyZcI6b26Tr95WvftvU+H/3CY9I5SIcmX\nBM3GqqmX3tc9vk7XTV9HekfpkORLghRfPfNn5/X5732e9I5SI8mXACm+enYf2q21j60lvaP0SPIF\nYx18tdAxElVDki8QE3y19NL76itWs+4dlUGSLxB1+GogvaPKSPLAAMl176R3VBVJviDcbC03es6g\nLkjyBeH5rOVEzxnUDUl+wng+a3nRcwZ1RJKfsN7NVlJ8edBzBnVGkp8QEnw5kd5RdyT5CUg+uo8E\nXw79tXfSO+qKJD8BrIcvF9I7moQknzOWSpYHtXc0EUk+R7QtKA/SO5qKJJ8j1sIXLyIu9JwhvaOJ\nSPI5oUxTPHatAiT5XFCmKRa7VoEFJPkcsJqmOKR34GIk+QxNT0s2ZZoiJGvvpHdgAUk+QyT4YpDe\ngcFI8hnhRuvkUXsHhiPJZ4AbrZNHegfSIclngPXwk9O/a5X0DiyNJD8myjSTw65VYPlI8mOgTDMZ\n9JwBRkeSHwOrafJHegfGQ5JHaSXXvZPegdGkSvK2N0j6ujr/KGyPiEcGHPcRST+W9OmI+O/MRllC\n1OLzM392Xlv2btHB+YOkd2BMQ5O87UskPSrpNkkfkHSn7RsGHPdvkp7LepBlxIqafPTS++orVrNy\nBshAmiR/k6RjEXFckmzvkjQr6UjfcV+Q9B1JH8l0hCVEis8e6R3IR5qa/CpJJxLvT3Y/d4Ht90r6\nREQ8JsnZDa+cSPHZSfacIb0D2ctqdc3XJT2YeF/biZ4Unx12rQL5SzPJn5J0TeL91d3PJX1Y0i7b\nlnSlpNttn4uIPf0nm5ubu/Bxq9VSq9Va5pCLxbLJ8fXS+9Znt2rTuk3aecdOrVyxsuhhAaXRbrfV\nbrczOZdjyIxl+x2SfiXpVkm/l/RTSXdGxOEBxz8h6X8WW11jO4b9vLKzmeTHkUzvO2Z3kN6BFGwr\nIkaqkAytyUfEeUn3S3pe0iFJuyLisO3Ntj+32LeMMhDUG/3egWIMTfKZ/rCKJ3naGIyG9A6MJ9ck\njwWsqlke+r0DxaN3TUqsqlkeVs4A5UCST4kUnw793oFyIcmnQIpPh46RQPmQ5FMgxS+Nfu9AeZHk\nhyDFL42eM0C5keSHIMUPRs8ZoPxI8ksgxS+O9A5UB0l+CaT4tyO9A9VCkh+AFH8x0jtQTST5RdC+\nYEFy5QzpHageknwfJvgF7FoFqo8kn8AE30HPGaA+SPJdTPAd7FoF6oUkLyZ4iV2rQF01PskzwZPe\ngTprfJJv8lp40jtQf41P8k3FyhmgGRqd5Ju64YlnrQLN0egkf+aMVOFHzi4bu1aB5mlskm9aiqfn\nDNBMjUzyTVpRQ3oHmq2RSb4JK2roOQNAamCSb0KZhpUzAHoaleTrXqah5wyAfo1J8nWf4EnvABbT\nmCRf1zo86R3AUhqT5OuI9A5gmMYk+TohvQNIiyRfMXSMBLAcjUjydVg2ScdIAKNoRJKveo8a0juA\nUTUiyVdZr+cM6R3AKBqR5KuInjMAskCSLyE6RgLISq2T/PR0px5flZuupHcAWUuV5G1vsH3E9lHb\nDy7y9bts/6L7etH2B7MfanrT05Ld+TiiGjtdSe8A8uAYsuzE9iWSjkq6VdLrkvZL2hgRRxLHrJd0\nOCL+YnuDpLmIWL/IuWLYz8uCXZ3VNMn0vmN2B5M7gLexrYjwKN+bJsnfJOlYRByPiHOSdkmaTR4Q\nEfsi4i/dt/skrRplMFmoypp4+r0DmIQ0NflVkk4k3p9UZ+If5LOSnhlnUOOowpp4es4AmJRMb7za\nvkXSvZJuHnTM3NzchY9brZZarVaWQyi1iNDuQ7u19dmtuufGe7Tzjp1auWJl0cMCUDLtdlvtdjuT\nc6Wpya9Xp8a+ofv+S5IiIh7pO26tpKckbYiIXw84V641+TL3jE/uWn1i9gnSO4DU8q7J75e0xva1\nti+VtFHSnr4BXKPOBH/3oAl+EsrYM56eMwCKNLRcExHnbd8v6Xl1/lHYHhGHbW/ufDm2SfqKpGlJ\n37RtSeciYqm6febKeMOVnjMAija0XJPpD8uxXFO2ZZO7D+3WA888oE3rNunhWx6m9g5gZOOUa2qx\n47VMKZ5dqwDKpPK9a8p0s5VdqwDKptJJviwTPOkdQFlVOskXvZqmt+6d9A6grCqb5Iuuw7NrFUAV\nVDLJF1mmSa57n5maIb0DKLXKJfkiJ3jSO4CqqVySL6IOn6y9k94BVEmlknwRdXjSO4Aqq1SSn2SK\np/YOoA4qleQnhfQOoC4qk+QnUarp7xhJegdQdZVJ8nk/8en0m6e1Ze8WOkYCqJXKJPm89NL7usfX\n0e8dQO1UIsnnVarppXdq7wDqqvRJPo/NT8n0zsoZAHVW+iSfdS2elTMAmqTUST7LMg3r3gE0UamT\nfFYpnvQOoKlKneTHRXoH0HSlTfLjlmpI7wBQ0iQ/zooa0jsALChdkh9ngu+ld3atAkBH6ZL8KJ0m\n+3vOsGsVADpKleRHqcOT3gFgsNIk+eWWaUjvADBcKZL8cid4Vs4AQDqFJ/nlTPD0eweA5Sk8yafd\n1UrtHQCWr9Akn+ZGK7V3ABhdoUl+WIqfPzuvLXu36OD8QdI7AIyg8Jr8YpLpffUVq6m9A8CICkvy\ng0o1rJwBgOwUkuQXW1FDzxkAyF4hSb6/Fk96B4B8pErytjfYPmL7qO0HBxzzDdvHbL9s+8Y05yW9\nA0C+hk7yti+R9Kik2yR9QNKdtm/oO+Z2STMRcZ2kzZIeH3S+Xi1+/uy8PvVfn9LcD+f09Man9cg/\nPaKVK1aO9R9TJe12u+ghlAbXYgHXYgHXIhtpkvxNko5FxPGIOCdpl6TZvmNmJX1LkiLiJUmX275q\nsZOdORN67Eekd36BF3AtFnAtFnAtspGmJr9K0onE+5PqTPxLHXOq+7nTbzvbv35Kcz+k9g4AkzDx\nG6/v+r8ZHdi8s1GlGQAoimNI4xjb6yXNRcSG7vsvSYqIeCRxzOOSXoiIJ7vvj0j6x4g43XeuFF1q\nAAD9IsKjfF+aJL9f0hrb10r6vaSNku7sO2aPpC2Snuz+o/Dn/gl+nEECAEYzdJKPiPO275f0vDo3\nardHxGHbmztfjm0Rsdf2v9h+TdJZSffmO2wAQBpDyzUAgOrKpa1BXpunqmjYtbB9l+1fdF8v2v5g\nEeOchDS/F93jPmL7nO07Jjm+SUr5N9KyfcD2QdsvTHqMk5Lib+Q9tvd054pf2r6ngGHmzvZ226dt\nv7LEMcufNyMi05c6/3C8JulaSe+U9LKkG/qOuV3S97off1TSvqzHUYZXymuxXtLl3Y83NPlaJI77\ngaTvSrqj6HEX+HtxuaRDklZ1319Z9LgLvBZflvS13nWQ9EdJK4oeew7X4mZJN0p6ZcDXR5o380jy\nmW6eqrih1yIi9kXEX7pv96mzv6CO0vxeSNIXJH1H0vwkBzdhaa7FXZKeiohTkhQRb0x4jJOS5lqE\npMu6H18m6Y8R8dcJjnEiIuJFSWeWOGSkeTOPSX6xzVP9E9egzVN1k+ZaJH1W0jO5jqg4Q6+F7fdK\n+kREPCapziux0vxeXC9p2vYLtvfbvntio5usNNfiUUn/YPt1Sb+QtHVCYyubkebNwp/xig7bt6iz\nKunmosdSoK9LStZk6zzRD7NC0ockfUzS30r6ie2fRMRrxQ6rELdJOhARH7M9I+n7ttdGxJtFD6wK\n8pjkT0m6JvH+6u7n+o9535Bj6iDNtZDttZK2SdoQEUv971qVpbkWH5a0y7bVqb3ebvtcROyZ0Bgn\nJc21OCnpjYh4S9Jbtn8kaZ069es6SXMt7pX0NUmKiF/b/q2kGyT9bCIjLI+R5s08yjUXNk/ZvlSd\nzVP9f6R7JH1GurCjdtHNUzUw9FrYvkbSU5LujohfFzDGSRl6LSJidff19+rU5e+r4QQvpfsbeVrS\nzbbfYftv1LnRdnjC45yENNfiuKSPS1K3Bn29pN9MdJSTYw3+P9iR5s3Mk3yweeqCNNdC0lckTUv6\nZjfBnouI/gZwlZfyWlz0LRMf5ISk/Bs5Yvs5Sa9IOi9pW0S8WuCwc5Hy9+KrknYklhZ+MSL+NOCU\nlWX725Jakv7O9u8kPSTpUo05b7IZCgBqrJBnvAIAJoNJHgBqjEkeAGqMSR4AaoxJHgBqjEkeAGqM\nSR4AaoxJHgBq7P8B3RSNhz5fQSQAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x11054a710>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "## Next let's evaluate how it does as a *Ranking Classifier*\n",
    "from sklearn.metrics import roc_auc_score, roc_curve\n",
    "\n",
    "fpr, tpr, thresholds = roc_curve(y_test,y_prob_pred)\n",
    "plt.plot(fpr, tpr)\n",
    "plt.plot(np.linspace(0,1,100),np.linspace(0,1,100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.75462247129598814"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## Get the area under the curve\n",
    "roc_auc_score(y_test,y_prob_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x110c37b00>]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXkAAAEACAYAAABWLgY0AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAHPFJREFUeJzt3XmU1NWd9/H3l6URBDGAEgU0CqLiCEZkiSHHHvWJMMHg\nmqBJJsG4zaPRc6KjcSaOZEbGmNEZHh80xgmPWyYSosYQFbeRPgwYVllE2UVlUyOrIC29fJ8/bjXV\n3VTT1V1VfWv5vM6p87u/ql8Xn1PdfPv2/d3f/Zm7IyIixald7AAiIpI7KvIiIkVMRV5EpIipyIuI\nFDEVeRGRIqYiLyJSxJot8mY21cw+MrPlhzjmATNba2ZLzeyM7EYUEZHWSqcn/yhwQVMvmtkYoL+7\nnwRcBzycpWwiIpKhZou8u88BdhzikHHAE4lj5wPdzax3duKJiEgmsjEm3wfYWG9/c+I5ERGJTCde\nRUSKWIcsvMdmoF+9/b6J5w5iZlooR0SkFdzdWvN16fbkLfFIZQbwtwBmNhLY6e4fNfVG7h798fHH\nzsyZ4TF06F2AA8748c6ePfHzuTt33XVX9AzKpEylmCsfM2UinSmUvwXeAAaa2QdmNsHMrjOzaxNF\n+0Vgg5mtA34F/O+MErWBo46C0aPDY+xYqKmB666DadOga1f4znfguecgw89WRCS6Zodr3P3KNI65\nMTtx4mjXDh5+GH7xC7j3Xnj7bbj4YjjrLJgzBzp1ip1QRKR1Sv7Ea3l5+YH2EUfApEmhF/+nP8Gi\nRXDYYfDTn8bNlS+UKT3KlL58zJWPmTJhmY73tOgfM/O2/Pcy9Ze/wGOPwW23wTXXhOdGjoSrrooa\nS0RKjJnhrTzxqiLfjNpa+K//gn37YMECmDo1PD9vHpxxBphBWVncjCJS3FTk28j+/fDOOzBiRGiX\nlYXt9Olw+eWx04lIsVKRj8gSH/uSJaFnLyKSbZkU+ZI/8ZqpvXvD9hvfCMM5IiL5REU+Q126wK9/\nHXr0I0bA+vWxE4mIJGm4JovMwtj8U09B+/ax04hIsdBwTZ6YNAl+/3vo0AF+9avYaURE1JPPuk8/\nhSuvhOefD8sltNOvURHJkGbX5Jlt26BXL9i6Fb74xdhpRKTQabgmz3TtGrYTJsTNISKiIp8DnTrB\nz34GL70UFjoTEYlFwzU59KMfwZQpcP/9MGhQWNpYRKSlNCafp9zh+9+H996D//kfGDUq9O4PPzx2\nMhEpJBqTz1Nm8MQTMHt2WK54zhx46KHYqUSklKgn34bq1rm57LKwsNkzz4Q59SIih6KefIFYuBCe\nfDIM28yYEYZwRERyST35SOp69fo4RKQ56skXoKeeClszWLECdu+Om0dEipN68pG4w+bN0K9f8rmT\nTgrFf+jQeLlEJP9oCmWBq66G11+HCy4I+/qIRKQ+Ffki8dZbMHgwfP657hsrIkkaky8SRx0VtqtX\nx80hIsVDRT6PdOsWtr17x80hIsVDRT6PvPde2P7zP0eNISJFREU+j/TtCzfdBA8+CDfcEDuNiBQD\nnXjNM+5hxcpVqzTLRkQCnXgtImbw5S+H9kcfxc0iIoVPPfk8tGwZnHFGOAH74Yex04hIbOrJF5kh\nQ2DSpNCTnzUrdhoRKWQq8nnqttvCdv78sPyBiEhraLgmT7nDiBFheWIIPfry8qiRRCQSLWtQxKqq\nwhIH3bpppUqRUqUx+SLWsSO0bx8eIiItpSJfAE49FXbujJ1CRApRWkXezEab2SozW2Nmt6d4/Qgz\nm2FmS83sLTP7QdaTlrALLwzb6uq4OUSk8DQ7Jm9m7YA1wHnAFmAhMN7dV9U75g7gCHe/w8x6AauB\n3u5e3ei9NCbfCnVLEB95JOzYETuNiLS1XI/JDwfWuvv77l4FTAPGNTrGgcQainQDtjUu8NJ6p58O\nN9+sIRsRabl0inwfYGO9/U2J5+qbAgwysy3AMuDm7MSTOj17hu3Pfx43h4gUlg5Zep8LgCXufq6Z\n9QdeNbPB7r6n8YETJ0480C4vL6dck7/TcsstMH063HEHjB0Lf/VXsROJSK5UVFRQUVGRlfdKZ0x+\nJDDR3Ucn9n8CuLvfW++Y54F73H1uYv+/gdvdfVGj99KYfAYefRSuuiq5v3MndO8eL4+ItI1cj8kv\nBAaY2fFmVgaMB2Y0OuZ94PxEmN7AQODd1gSSpk2YEK6EXZT41bliRdw8IpL/mi3y7l4D3Ai8ArwN\nTHP3lWZ2nZldmzjsbuBsM1sOvArc5u7bcxW61A0dGra6F6yINEfLGhQoMxg2DBYsiJ1ERHJNyxqU\nqHXrYicQkXynIl+gLr44XBi1eHHsJCKSzzRcU6D27YMuXUJ73Tro3z9uHhHJHQ3XlKDOncMyxAAD\nBmhsXkRSU5EvYB06wPbEHKYRI2DKlLh5RCT/aLimCNTdWATCPHoRKS4arilxHTvCc8+F9h13xM0i\nIvlFRb5IjEusC7psWdwcIpJfVOSLSM+e8OGHsVOISD5RkS8i1dXQqVPsFCKST1Tki8iuXVqVUkQa\nUpEvErW1YXvCCXFziEh+UZEvEmYwZAg8/DDMaLwQtIiULM2TLyK1tdC+fWjrYxYpHponLwC0awfP\nPx/aO3bEzSIi+UFFvsj06hW2dVfAikhpU5EvMqecErZdu8Ly5XGziEh8KvJFpnt3mDs3tIcMgT17\n4uYRkbhU5IvQ2WfDG2+EdrduMHUqVFbGzSQicWh2TRF74w348Y9h/nz4y1+S4/UiUlgymV3TIdth\nJH+cfTZ85SthPRsVeJHSpJ58kbPE73597CKFS/PkpUnXXx87gYjEpCJf5Hr3DttXX42bQ0TiUJEv\ncv/0T2G7d2/cHCISh4p8kZs5M2wvvDBuDhGJQ0W+yG3bFrbV1XFziEgcKvJFbuDA2AlEJCYV+SJ3\n111hqwXLREqTinyRu/fesH3iibg5RCQOXQxV5NzDOvO9e4crX0Wk8GhZA2mSWbjv66WXxk4iIjFo\nuKbIbdsGGzbAggVa2kCkFKnIF7mePeHaa2H2bPjoo9hpRKStqciXgGHDwvaYY2D79rhZRKRtpVXk\nzWy0ma0yszVmdnsTx5Sb2RIzW2Fms7IbUzJx9dXJG3zv3h03i4i0rWZn15hZO2ANcB6wBVgIjHf3\nVfWO6Q68AXzd3TebWS93/yTFe2l2TSSbNkG/fuGWgEuXxk4jIi2R66WGhwNr3f19d68CpgHjGh1z\nJfCMu28GSFXgJa6+fcP23Xfj5hCRtpVOke8DbKy3vynxXH0DgR5mNsvMFprZ97IVULKnZ0+oqYmd\nQkTaUrZOvHYAzgTGAKOBO81sQJbeW7KkfXv47DNYtar5Y0WkOKRzMdRm4Lh6+30Tz9W3CfjE3SuB\nSjObDQwB1jV+s4kTJx5ol5eXU15e3rLE0mrz5sGJJ8ZOISLNqaiooKKiIivvlc6J1/bAasKJ163A\nAuAKd19Z75hTgP9L6MV3AuYD33b3dxq9l068RrRkCZx5Jqxfr2IvUkhyeuLV3WuAG4FXgLeBae6+\n0syuM7NrE8esAl4GlgPzgEcaF3iJr+4GItdeGzeHiLQdLVBWQj7/PMyZ/81v4NRT4R39GhYpCJn0\n5FXkS5AlflT0rRApDLmeJy9F5pZbwnbFirg5RCT31JMvQRs2JE+8fvBBuBJWRPKXevLSIiecAIsW\nhfYDD8TNIiK5pZ58iaquho4dQ7tvX3jrLTjyyLiZRCQ19eSlxTp0gE8/hSlTwuJlF10UO5GI5IJ6\n8iWushI6d4aXX4avfz12GhFJRT15abW66ZS6ybdIcVKRL3GdOoXt9Olxc4hIbmi4RnRxlEie03CN\nZGTChLCtro6bQ0SyT0Ve+OIXw7Z9+7g5RCT7VORLnDvccw/89KfJYRsRKR4q8iVu//6wLSuLm0NE\nckNFvsTVza6ZNStuDhHJDc2uKXGffw6HHRban34KXbvGzSMiB9PsGmm1Tp3g1VdD+7HHokYRkRxQ\nT17Ytw+6dAnt/fuTC5eJSH5QT14y0rkzTJ0a2mVlMH9+3Dwikj3qycsBa9fCwIGhrW+TSP5QT16y\nYsCAsL322rg5RCR7VOTlgMrKsD3mmLg5RCR7NFwjB1RVJS+K2rs3eTJWROLScI1kRceO8LOfhfZD\nD8XNIiLZoZ68NLBzJ3zhC/DnP8PIkbHTiAioJy9Z9NprYXv66XFziEh2qMhLA3VLHHTtGlamFJHC\npiIvDYwdCytWhPYRR8TNIiKZ05i8HOSkk2DdOti+PYzPi0hcGpOXrKobj+/RA37967hZRCQzKvJy\nkGefhXffDW3dElCksGm4Rg6yY0foxV9zDTzySOw0IpLJcI2KvBxk/frkOjYbN0LfvnHziJQ6jclL\nVvXvDy+9FNpjx8bNIiKZUZGXlE49NWwfeCBuDhHJjIq8pPTcc2G7cWPcHCKSmbSKvJmNNrNVZrbG\nzG4/xHHDzKzKzC7JXkSJ4dJLw7buClgRKUzNnng1s3bAGuA8YAuwEBjv7qtSHPcqsA/4f+7+bIr3\n0onXAmGJUzz6donEl+sTr8OBte7+vrtXAdOAcSmO+xHwNPBxa4JI/qipCdujjoqbQ0Qyl06R7wPU\nH5ndlHjuADM7FrjI3X8JtOq3jeSP2bPD9skn4+YQkcxl68TrZKD+WL0KfQFbsyZstW6NSOHrkMYx\nm4Hj6u33TTxX31nANDMzoBcwxsyq3H1G4zebOHHigXZ5eTnl5eUtjCy5VFsL118f2sOHx80iUqoq\nKiqoqKjIynulc+K1PbCacOJ1K7AAuMLdVzZx/KPAn3TitTBt3QrHHguXXw7Tp8dOIyKQ2YnXZnvy\n7l5jZjcCrxCGd6a6+0ozuy687I1XN1EVL2DHHhu2KvAixSGd4Rrc/SXg5EbP/aqJY6/KQi6JYMuW\nsL3zzrg5RCR7tECZHDB2LLzwAmzbFlahFJH8oAXKJCtGjQrbHTvi5hCR7FGRlwOWLw/b3r3j5hCR\n7FGRFwCqq+Gpp+Cqq6Br19hpRCRbNCYvAOzeDd27h6UMPtbCFCJ5RWPykrEOiXlWF18cN4eIZJeK\nvABwe2JRikmT4uYQkexSkRcAZs4M2z174uYQkexSkRcA7r47bC+6CPbvj5tFRLJHRV4AGD0aTjwR\nli0LM2xEpDhodo0cUFUFZWWhvWwZDB4cN4+IBJpdI1mxezf0SdwOZsgQeO21uHlEJHPqyctBduxI\nrl3z7LNhnN50GxiRaNSTl6z6whfg6adD+5JLwoJlIlKYVOQlpUsvhauvDu3TToubRURaT0VemvSv\n/xq2ffoc+jgRyV8q8pLSli3wta+F9ty5cbOISOupyEtKe/fC6tWhXVkZN4uItJ6KvKR00knJtmbW\niBQuFXlJqbo6bDt3DksQi0hhUpGXlB59NGxHjoSamrhZRKT1dDGUpLRtG/TqFdrr1kH//nHziJQy\nXQwlWffee8n2gAGwfn20KCKSARV5SenMM8M0yu9+N+z/4Q9hiYOdO+PmEpGW0XCNHNLTT8Pll4d2\nWRm8/jp89atxM4mUGg3XSM5cdhncf39o33OPCrxIoVFPXg5p/fowJg+gb51IHOrJS9ZVV8O//RuM\nGBF68FVVsROJSGt0iB1A8s+bb4YVKHv2hAULwm0BRaQwqchLA3PnwqhRob19e1hbXkQKl8bkpYGa\nGuiQ+NVfWQmdOsXNIyIak5csqa0NQzQAt96qAi9SDDRcIwCsWAE33AC7dsHSpeFG3iJS+NSTL3Hu\ncPjhcPrp8O1vh1k0KvAixUM9+RJ3333w2WfQvj306wczZ8Lu3XDWWXDyybHTiUimdOK1xG3ZEu7h\n2qMHnH12eO7555Ovv/02DBoUJ5uIBJmceE2ryJvZaGAyYXhnqrvf2+j1K4HbE7ufAn/n7m+leB8V\n+QKwaxccd1zo0QMsWQJnnBE3k0gpy2mRN7N2wBrgPGALsBAY7+6r6h0zEljp7rsSvxAmuvvIFO+l\nIl8gPvwQjjkmtHfvhm7d4uYRKWW5nkI5HFjr7u+7exUwDRhX/wB3n+fuuxK784A+rQkj+WPlymR7\nyZJ4OUQkM+kU+T7Axnr7mzh0Eb8amJlJKImv/pWu06bFyyEimcnq7Boz+2tgAjCqqWMmTpx4oF1e\nXk55eXk2I0iGampg4kSYPDnsL1wYZtqISNupqKigoqIiK++Vzpj8SMIY++jE/k8AT3HydTDwDDDa\n3VPeLE5j8vnv88/D1a5TpoT9MWPgxRfjZhIpdbkek18IDDCz482sDBgPzGgU4DhCgf9eUwVeCsPc\nuaHAH3EEPPkkzJjR/NeISP5qdrjG3WvM7EbgFZJTKFea2XXhZX8EuBPoATxkZgZUufvwXAaX3Hj8\n8bDdvTt5f1cRKVy6GEoOqKyEzp2T+wsWwLBh8fKISKBVKCUrampg8ODkFa7Dh8Mll4S1bXTyVaQw\nqScvKT3xBJSVwRVXhH2zsBSxiLS9nC9rkC0q8oWluho6dgzt66+HX/4ybh6RUqUiLzlh9X6k9G0T\niUdj8pJ1GzY03O/QIRT9f/gHmDo1TiYRaTn15CWl/fvDHPkjjwz7kyfDCy8kX6+tbdjTF5Hc0XCN\n5JR7mGmzaFHY/+EP4bbbYODAuLlESoWKvOSUO3z5y/Dxx7B1a/L5IUPC/WBFJLc0Ji85ZRaKed2d\nowDuuAPeeCNeJhFJj+7xKs2aPRvOOSe0u3cPPfqysriZRCQ96slLk+bMCb34ugL/3e/C9u0q8CKF\nREVemvS1rzXcHzIE2uknRqSg6L+spFRZCQ8+mNzv2lU38xYpRJpdIw3s2pWcGz9qFIwcCRdf3PCk\nq4i0Lc2ukaypK/AA550XVqB85RW46Sbo2RPuvz9eNhFpOfXkpYEXX4RvfQu+8x045hjYty/cLWru\n3OQxH3wA/frFyyhSanQxlGSVO4weHXrwEIZqzjsPzj8/DN9odo1I28qkyGuevDSwfz906pTcX7o0\nzKoRkcKkMXk5wL1hgb/nnnCnKBEpXCryckD9OfBlZfDHP4ahGrPw+NKXokUTkVbSmLwcsHgx7NgR\nZtRAmCt/7rkNj/n3f4eTToLy8jB3XkRyTydeJSeqq+HWW+Hpp2Hz5oav/e53YRaOiOSeTrxKTnTo\nAHv2NCzw778fpk/qhiEihUFFXlI655yw+mR9XbuGXn1j1dWwfj28/TZccw18//ttk1FEmqciLynV\nzbI55RQYMyb5/KZNyfZ//MfBX7drF3zySSj0vXrlNqOINE9j8tJqM2bAuHGHPuab34SVK6GqKlxN\n27Nn8rWjj85tPpFioROvkhc2bYKHH4ZJk+Coo8LjnXeaPn7atDC2v2ED9O0bllIQkYOpyEve2707\n3FXqUPSjIZKaZtdI3rrvPnj8cXjvveaPNYPf/hY6dgxz9DdvDo/zzw/DPiLScurJS06tWxdm3tRx\nD7cQ3LwZFi2C6dObf48LLoCXXspdRpF8p+EaySu1tWH9+YceSg7BnH02fPRRKO6HHQZ9+oRx+D59\nGj7qnuvVS3PxReqoyEs0ixbBsGHNH3f88WE6Zp8+0KVLdv7tysowFFRV1fB5rXcvxUZj8hJNjx4w\ndGjoeffuHR5HH91wNctcuemm1M8fd1xYY2fHjvDYvj1s+/eHyZOhffvcZxPJF+rJS8Fwh88+Cxdc\n7doF//iP8Ic/pP/1XbqEYq+bnkih0T1epaD953+GYZyuXZPLGqd6tGsXjunTBwYNSl3g27WDE04I\ns3rcGz727lWBl9KT1nCNmY0GJhN+KUx193tTHPMAMAbYC/zA3ZdmM6gUr7VrYcuW0DYLSx3v2ZPe\n15pBRUXq1xqvvZNKVVUY29+3Lzzq2pWV8I1vwGmnpZdDJG+5+yEfhMK+Djge6AgsBU5pdMwY4IVE\newQwr4n38nwza9as2BFSysdcbZXp3Xcb98EP9ZjVgmNb/hg61P2009wHDHDv18/96KPdu3d379LF\nfcEC9x07wmPbNvetW90/+MD95Zfb5nNqiXz8eXLPz1z5mClRO5ut16ke6fTkhwNr3f19ADObBowD\nVtU7ZhzwRKKKzzez7mbW290/yvSXUK5VVFRQXl4eO8ZB8jFXtjK5h5Urq6uhpibZrttv1y7Mr2/q\n9br24sXw939fAWSeqSmLFzf92vDhqZ8fOLCC1avLcxGn1fLx5wnyM1c+ZspEOkW+D7Cx3v4mQuE/\n1DGbE8/lfZHPlpoaWLYsFLDa2oaP1jz3zjvhQqHWfv2hnmvt18+fH4ZVamqSj9rahu3q6rBwWVsa\nNiyMtXfsmNzWb7fFa+3bJ+f133cffPhh234GzdmzJ/8yQX7mysdMmdAUyiz54x/h0kuz+56//312\n3y8bDtWzjaFz5zAvPp/s2QO/+U3sFA3lYybIz1z5mCkTzU6hNLORwER3H53Y/wlhfOjeesc8DMxy\n998l9lcB5zQerjEzzZ8UEWkFz+HFUAuBAWZ2PLAVGA9c0eiYGcANwO8SvxR2phqPb21IERFpnWaL\nvLvXmNmNwCskp1CuNLPrwsv+iLu/aGZ/Y2brCFMoJ+Q2toiIpKNNr3gVEZG2lZMrXs1stJmtMrM1\nZnZ7itdPNrM3zKzSzH6ciwytyHSlmS1LPOaY2el5kOmbiTxLzGyBmX01dqZ6xw0zsyozuyTXmdLJ\nZWbnmNlOM3sz8fhp7EyJY8oT378VZjYrdiYzuzWR500ze8vMqs3syMiZjjCzGWa2NJHpB7nMk2am\nI83s2cT/v3lmNqgNMk01s4/MbPkhjnnAzNYmPqsz0nrj1k6wb+pBehdP9QKGAv8C/DjbGVqZaSTQ\nPdEeTRMXdLVxpi712qcDK2NnqnfcfwPPA5fkyffvHGBGrrO0MFN34G2gT2K/V+xMjY4fC7wWOxNw\nB3BP3WcEbAM6RM70C+DORPvkXH9OiX9nFHAGsLyJ19O66LTxIxc9+QMXT7l7FVB38dQB7v6Juy8G\nqnPw77c20zx335XYnUeY5x8702f1drsCtbEzJfwIeBr4OMd5WpqrLU/sp5PpSuAZd98M4ec+DzLV\ndwXwVB5kcqBbot0N2ObuuawN6WQaBLwO4O6rgS+Z2VE5zIS7zwF2HOKQBhedAt3NrHdz75uLIp/q\n4qlcF8zmtDTT1cDMnCZKM5OZXWRmK4E/AVfFzmRmxwIXufsvabuimu737yuJP2NfaIM/r9PJNBDo\nYWazzGyhmX0vDzIBYGadCX+xPpMHmaYAg8xsC7AMuDkPMi0DLgEws+HAcUDfHOdqTlMXnR6SLoZq\nxMz+mjA7aFTsLADu/hzwnJmNAu4G/lfkSJOB+mOY+TItdjFwnLt/ZmZjgOcIRTamDsCZwLnA4cCf\nzezP7r4ubiwALgTmuPvO2EGAC4Al7n6umfUHXjWzwe6e5jJ1OfFz4P+Y2ZvAW8ASoCZinlbLRZHf\nTPitV6dv4rmY0spkZoOBR4DR7n6oP5vaLFMdd59jZieaWQ933x4x01nANDMzwvjpGDOrcvdcLmbQ\nbK76BcHdZ5rZQ3nwWW0CPnH3SqDSzGYDQwjjwbEy1RlP7odqIL1ME4B7ANx9vZltAE4BFsXK5O6f\nUu8v50Smd3OUJ12bgfr3PEuvtubg5EF7kic1yggnNU5t4ti7gFva4IRGs5kI3/S1wMhc52lBpv71\n2mcCG2NnanT8o7TNidd0Pqve9drDgffyINMpwKuJY7sQeoSDYn//CCeEtwGd8+R79yBwV933kTAk\n0SNypu5Ax0T7GuCxXH9WiX/rS8BbTbz2NyRPvI4kzROvWe/JexoXTyVOFiwinGSpNbObCT/8Ofnz\nLJ1MwJ1AD+ChRC+1yt2bWGewzTJdamZ/C+wH9gHfylWeFmRq8CW5zNPCXJeZ2d8BVYTP6tuxM7n7\nKjN7GVhO+FP/EXd/J2amxKEXAS+7+75cZWlhpruBx+pNHbzNc/cXWLqZTgUeN7NawgypH+YqTx0z\n+y1hSdWeZvYBoRNcRvLnqVUXnepiKBGRIqbb/4mIFDEVeRGRIqYiLyJSxFTkRUSKmIq8iEgRU5EX\nESliKvIiIkVMRV5EpIj9f/XYjAHpNPhTAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1109b6780>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "## Plot a precision-recall curve\n",
    "from sklearn.metrics import precision_recall_curve\n",
    "prec, recall, thresholds = precision_recall_curve(y_test,y_prob_pred)\n",
    "plt.plot(prec[0:-1], recall[0:-1])\n",
    "\n",
    "## Note: sklearn incorrectly adds a point at precision=1 and recall = 0\n",
    "## For this reason, I remove the last point when I plot\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x110d21978>]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXkAAAEACAYAAABWLgY0AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAHw9JREFUeJzt3XuUFOWZx/HvM+LgJQiLGkQEohJFjIKoiIraKoYBjbgm\nMUIk6sZILmZ3zeYEjNk4MRh1s4lRiasY1GzEsEl0FUXRjdp4w4CCN+4KIgyK3IziBYeZZ/94e+hm\nmO6umenLTM3vc06f7qp6u+qZYvhVzVs3c3dERCSeKspdgIiIFI9CXkQkxhTyIiIxppAXEYkxhbyI\nSIwp5EVEYixvyJvZVDNbZ2av5Ghzk5ktN7OXzGxQYUsUEZGWirInfycwIttEMxsJHOzunwfGA7cW\nqDYREWmlvCHv7s8Am3M0GQ38d6rt34CuZtajMOWJiEhrFKJPvhewOmO4JjVORETKTAdeRURirFMB\n5lED9M4YPiA1bidmphvliIi0gLtbS74XdU/eUq+mzAC+AWBmQ4H33H1dthnNnOm463XVVVeVvYa2\n8tK60LrQusj9ao28e/Jmdg+QAPY2s7eAq4BKwN19irs/bGajzOx14EPg4lZVJCIiBZM35N19bIQ2\nlxWmHBERKSQdeC2TRCJR7hLaDK2LNK2LNK2LwrDW9vc0a2FmPnOmM2pUyRYpItLumRle5AOvIiLS\nDinkRURiTCEvIhJjCnkRkRhTyIuIxJhCXkQkxhTyIiIxppAXEYkxhbyISIwp5EVEYkwhLyISYwp5\nEZEYU8iLiMSYQl5EJMYihbyZVZnZEjNbZmYTmpjezczuM7OXzex5MxtQ+FJFRKS58oa8mVUAk4ER\nwOHAGDPr36jZj4EF7j4QuBC4qdCFiohI80XZkx8CLHf3Ve5eC0wHRjdqMwB4AsDdlwKfM7N9C1qp\niIg0W5SQ7wWszhhekxqX6WXgXAAzGwL0AQ4oRIEiItJyeR/kHdF1wI1mNh94FVgA1DXVcNq0aubO\nDZ8TiYSe4ygi0kgymSSZTBZkXnmf8WpmQ4Fqd69KDU8E3N2vz/GdlcAR7r6l0Xg941VEpJmK/YzX\neUA/M+trZpXA+cCMRgV0NbNdU5+/BcxuHPAiIlJ6ebtr3L3OzC4DHiNsFKa6+2IzGx8m+xTgMOD3\nZlYPLAS+WcyiRUQkmrzdNQVdmLprRESardjdNSIi0k4p5EVEYkwhLyISYwp5EZEYU8iLiMSYQl5E\nJMYU8iIiMaaQFxGJMYW8iEiMKeRFRGJMIS8iEmMKeRGRGFPIi4jEmEJeRCTGFPIiIjGmkBcRibFI\nIW9mVWa2xMyWmdmEJqbvZWYzzOwlM3vVzC4qeKUiItJseUPezCqAycAI4HBgjJn1b9Tse8BCdx8E\nnAr8yszyPlpQRESKK8qe/BBgubuvcvdaYDowulEbB7qkPncBNrr7tsKVKSIiLREl5HsBqzOG16TG\nZZoMDDCztcDLwL8UpjwREWmNQnWpjAAWuPtpZnYw8H9mdqS7b2nccNq0aubODZ8TiQSJRKJAJYiI\nxEMymSSZTBZkXubuuRuYDQWq3b0qNTwRcHe/PqPNQ8C17v5savhxYIK7v9BoXj5zpjNqVEFqFxHp\nEMwMd7eWfDdKd808oJ+Z9TWzSuB8YEajNquA4aliegCHACtaUpCIiBRO3u4ad68zs8uAxwgbhanu\nvtjMxofJPgWYBNxlZq+kvvYjd99UtKpFRCSSvN01BV2YumtERJqt2N01IiLSTinkRURiTCEvIhJj\nCnkRkRhTyIuIxJhCXkQkxhTyIiIxppAXEYkxhbyISIwp5EVEYkwhLyISYwp5EZEYU8iLiMSYQl5E\nJMYU8iIiMaaQFxGJsUghb2ZVZrbEzJaZ2YQmpv/QzBaY2Xwze9XMtplZt8KXKyIizZE35M2sApgM\njAAOB8aYWf/MNu7+n+5+lLsPBq4Aku7+XjEKFhGR6KLsyQ8Blrv7KnevBaYDo3O0HwP8sRDFiYhI\n60QJ+V7A6ozhNalxOzGz3YEq4N7WlyYiIq3VqcDz+xLwTK6ummnTqpk7N3xOJBIkEokClyAi0r4l\nk0mSyWRB5mXunruB2VCg2t2rUsMTAXf365toex/wJ3efnmVePnOmM2pU6wsXEekozAx3t5Z8N0p3\nzTygn5n1NbNK4HxgRhNFdAVOAR5oSSEiIlJ4ebtr3L3OzC4DHiNsFKa6+2IzGx8m+5RU03OAR939\n4+KVKyIizZG3u6agC1N3jYhIsxW7u0ZERNophbyISIwp5EVEYkwhLyISYwp5EZEYU8iLiMSYQl5E\nJMYU8iIiMaaQFxGJMYW8iEiMKeRFRGJMIS8iEmMKeRGRGFPIi4jEmEJeRCTGIoW8mVWZ2RIzW2Zm\nE7K0SZjZAjN7zcyeLGyZIiLSEnmfDGVmFcBk4HRgLTDPzB5w9yUZbboCvwW+6O41ZrZPsQoWEZHo\nouzJDwGWu/sqd68FpgOjG7UZC9zr7jUA7r6hsGWKiEhLRAn5XsDqjOE1qXGZDgG6m9mTZjbPzMYV\nqkAREWm5Qh147QQMBkYCVcC/m1m/XF/4/vdh+fICLV1ERJqUt08eqAH6ZAwfkBqXaQ2wwd0/AT4x\ns6eAgcDrjWc2bVo1c+fC5MlQV5fgllsSLatcRCSmkskkyWSyIPMyd8/dwGwXYCnhwOvbwFxgjLsv\nzmjTH7iZsBffGfgb8DV3X9RoXj5zpjNqFJjBlCnwrW8V5OcQEYktM8PdrSXfzbsn7+51ZnYZ8Bih\ne2equy82s/Fhsk9x9yVm9ijwClAHTGkc8E3Za6+WlCwiIlHl3ZMv6MIa7cmfcw787/+WbPEiIu1S\na/bky3LF67Zt4f3++8uxdBGRjqMsIf/BB+VYqohIx1OWkH///XIsVUSk49GevIhIjJV9T/6ZZ8pR\ngYhIx1D2kD/ppPSBWBERKayyh7yIiBRPWUJ+y5ZyLFVEpOMpS8h/+GE5lioi0vEo5EVEYqxsIf/V\nr5ZjySIiHUtZQv6jj2DPPcuxZBGRjqVse/LZQn7NGrjjjtLWIyISV20u5B9/HKZNK209IiJx1eZC\nfunS0tYiIhJnbSrkt26FBQtKX4+ISFxFCnkzqzKzJWa2zMwmNDH9FDN7z8zmp14/yTW/bCFfVQWz\nZsG8eTB/PpTweSYiIrGUN+TNrAKYDIwADgfGpJ7p2thT7j449ZqUa57ZQr7hubUffABHHw2jR+er\nTkREcomyJz8EWO7uq9y9FpgONBW/kR9N1RDyFRXhMYDZrFwZdY4iItKUKCHfC1idMbwmNa6x483s\nJTObaWYDcs2wIeS7dw9Bn82gQRGqExGRrAp14PVFoI+7DyJ07WR9euull8LatbDvvtCnD9TVwa67\nNr1H/7nPFag6EZEOqlOENjVAn4zhA1LjtnP3LRmfHzGzW8ysu7tv2mlmNdUA3HcfXH99gjPOSGyf\n1qcPvPEG1NbCNdfk7soREYmrZDJJsuEgZSuZ5zmFxcx2AZYCpwNvA3OBMe6+OKNND3dfl/o8BPiT\nu3+uiXk5hOXV14cQzwzy004LF0MBXH11eJjI1Ve34qcTEYkBM8PdW7Tbm3dP3t3rzOwy4DFC985U\nd19sZuPDZJ8CfMXMvgPUAh8DX8tf9M7jDjqoecWLiEhuUbprcPdZwKGNxt2W8fm3wG9bW8zBB2ef\ntnYtrF4Nxx3X2qWIiHQcZbniNZtcB1p/9zu46aaSlSIiEgttKuT79s0+7amnSleHiEhclD3kL78c\nnnsufM4W8p9+mm4jIiLRlT3kf/3rEOIA++3XdJsXX4SPP4Z77oF160pXm4hIe1f2kIdwQBWyX/36\n1FPQrVv4/PTT4cycCy8sTW0iIu1Zmwj5M86An/0s+/TZs2GPPcLnmtRlWPdnvaZWREQatImQ79ED\nfvrTpqfV1cGzz8Kf/hSGH3kkvA8fDosWlaY+EZH2qk2EfC6vvAI9e8KJJ4bX7Nnwj/8Ybotw+OHl\nrk5EpG1r8yH//PNwwgnpYTP47GfLV4+ISHvS5kN+zhw4/vj08Kmn7jj8xBOlr0lEpL1o0yF/ww3w\nhz/sGOojR4Yza55+OgzPnl2e2kRE2oNI964ply2pGxgfdlh4Hz8eRowIn3fdNby/9lrp6xIRaS/a\n9J58g112Ce/jxqX74487Lny+7z7tzYuIZFOWkM91Tnxjb72VfdqGDeF9zBj4299g006PKBER6djK\nEvJXXBGtXZ8+0Lt39ulvvhne334bzj4bCvQgFRGR2ChLyHeKeCTgmGNyT+/dG7797fD53XdbV5OI\nSBxFCnkzqzKzJWa2zMwm5Gh3rJnVmtm5ueeXf5mDB8PYsfnb7btv/jaFUldXumWJiBRC3n1qM6sA\nJhOe8boWmGdmD7j7kibaXQc8WojCzjorWrurrw7Pi23cVbNxYzgDZ6+9ClENvPoqDBsWDvIOGlSY\neYqIFFuUPfkhwHJ3X+XutcB0YHQT7b4P/AUoecfJpEk7XwX79a/D1Km5v/fii/Dee/nnf/fdcOSR\n8P77cNRRsHBh+Gtk3LiW1ywiUgpRQr4XsDpjeE1q3HZmtj9wjrv/F9CiJ4oXym9/CxMmwKOPwg9+\nEO4//+CDO7dbvz70+f/DP+w4fuNGWLEifJ49OzyScNy4cGFWwwHjL3whvN99d+6zf0qtvh6WLw8/\n93/8R7mrEZG2oFAXQ/0GyOyrzxH01VRXh0+JRIJEIlGgEmDBgrBXn6nhQSTuO45v6gyfv/8dRo2C\nAQPgmmugobT774fRo0Po33EHVFWF4B8+HJYuDWcBvfxyCNfPfz4E7fTp+Y8X1Ndnv4d+c7z/fqiv\nocvq858PyzaDgw6CL3852nxqa9MXmYlI+SSTSZKFOl3Q3XO+gKHArIzhicCERm1WpF4rgQ+Ad4Cz\nm5iXgxfFqae6hygPr9dfT3+urEy3q693f+YZ95493RctCu/u7lu37vh9cP/ud0ObzO9+8kl6OJHY\n+TvgXlHhvnjxzjVu3uz+6qthPk8/7d61q/s997h//evuV1zh/rOfub/zTvN+7qeecj/wwLDcCRPc\nX3klzDOznptvdu/SJdS0ZUuY/sUvun/nO+6XXuo+a5b7gAHp9iee6D5kSGhbLvX17hs2uK9eHYa3\nbHF/8EH3det2bLd0qXtNTenrEymlENW5szrbK0rI7wK8DvQFKoGXgMNytL8TODfLtKKFfENAzZkT\nAsLd/c473Z97zn3gwBAS1dXpdr//fQiH/fZz37bN/d/+zf3YY0OQNLTZti33Mk8/Pd32wQfDhmLR\novS4lSvdP/44tF29Oj3+Rz9y33ffpjcQ4P7AA00vr7bWfe7csJFavz6Ees+e7jNm7Nzugw/CegD3\n/ffPvqw99nDfay/3k092v+++ML/M6SNHNr3Baq1t29zffdd90yb3G290v/tu90mT3Hv0cN9tt+z1\ngvvee7vfcIP7ccelxz36aNgo3Hab+/HHuy9cWPiaRcqlqCEf5k8VsBRYDkxMjRsPXNpE2zvKEfKX\nXx7CorEFC5oOivr6EPKZ4xr2CBs2Evm89daOe/YNBg4M8zvkkB3n/9OfhoDq2TPUtXp1COytW0PY\n3Xlnuu2mTen51deHvwAa/wyjR4egzOX550Po19S433GH+/LlO05/+eWd947d3f/8Z/ddd00v6957\n3Z98MmxcmqOuLtRfXx+WdfXV7ldemZ5vt27pz5WV7oceGv6SmDUrbCCnT3d/+OGw0dq82f2ii0Lb\nk05yf+ihsP4HD07P4+ST059POCH8tVRZGYYvuMB9/Hj3994LfzHNmRM2CkuXNu9nEim1ood8oV7F\nDPls5s7dMRgz9/C2bEmPf/zxwi63f/8w3zPOCO+/+EUYP2uW+6pV2b/35ps7bnC2bHH/0pfCuMmT\nw3fvuit0wUTdGLXGRx+FDVPjbqzrrnN/6aVQ7w9+4D52bAjssWPdv/GN3Hvi48e7L1kSNmo1Ne6f\nfho2Bi21cmXonnv77TC8aFH4a2DAAPfhw93/9V/djzrKvV+/7DVdcEHzN2AipdKakLfw/dIwMwen\nhItkwwa46CKYNg122w06d95xunu0i7Oaa/Nm2HNPqKxs/nc7dw5nB1VWwiWXwJAh4WyZcj0sZcWK\ncHD5o4/CweZ583Zus88+6XsJnXdeuJjtyCNhzZow/qijwh1Ei7Gum+Ptt8N6dYe99w4XuP3wh3Dj\njTu2GzcOfvzjcPbU4MHQvTs8/ni4R9J998FJJ4Xfp9NOg379wtXXn3wSDlwvWBB+3t13L8/PKPFj\nZrh7i/73xD7k26PMILzzzrCRaksa9n83bgy17rNPuStqPXeYNStswB58EF54oel2gwfD/PlhA7Fx\nY+55vvhiuHCuEGdQScemkI+Z9ethxoywR9ylS7mr6ZjWrQt/OdXWhjDv1GnnU2K3bQsbua1bw2vj\nxvBQ+trasBFo0L073HsvHH20/j2lZRTyIm3M+vWh22bOHLZfF9Kgf/9wLcNvfhOuYxDJRyEv0oZ9\n+CHU1IQuoEceCXv6//M/YdqgQaFPf8WK0Mf/2mth2ptvhr8KBgwIT0g76qjQXl0/HZNCXqSdcYdf\n/xr+8hc45ZRwD6X99ks/UKdLF/jggxDq9fXp7511FkycGG68d8QRYdy2baGdNgDx1a5C3sx3+KUV\nkaZt3RrOtFq2LJyttWgR/NM/pacffDB85jPhlhoAvXqFM7pGjoRu3eCxx6Bv33DcoGtXuO22cMZP\ndXWY9/r14ZYXTz4ZnqrWu3c400jannYV8pWVztatJVukSGy4h737vfaCyZPhnXfCnn3PnqH//8kn\nQ5B36wZf/Wq4l1GPHuEZySNHhmC/+ebQfXTIITB3bjhAfPjhcOihcPvt4R5IkyeHDUa5T3eVtHYV\n8nvs4Xz4YckWKSIRfPopfO978Lvfpce99BIMHFi8a0kkutaEfMl78aI++k9ESqeyMuzJ//3voZsH\nwkFhs9DXbxZew4eHA8iZ+4b19elhHW9re0q+J9+9u+e9iEREym/x4nBW0O67h66e228Pt9CG9IHh\nTLvtFq76hdCldOWVoZvo4INhjz1KW3vctKvums9+1lm3rmSLFJECe/bZcIpn167hNg4DB4YNQq9e\n4eDwhg1w003hkZkQLiJr6O+XlmlXIb///k5NTckWKSJlUl8f7vUzZgysWhXG/eQn4TRRne7ZPO0q\n5Pv08e3/4CISf+4h7P/5n9M3tzvxxLDH369fCPyzzw5n/eyzD3zlKzvfSLCja1chf9BBzhtvlGyR\nItKGbN4cbuewZQssWRJO3/zlL0O/vRm8/no44Dt/vs7oydSuQv6QQ5ylS0u2SBFpR2bPDs9WPuEE\nOOAAuPzy8N65c/5nJsdZ0UPezKoID+uuAKa6+/WNpp8N/ByoB2qBy9392Sbm4wMGOAsXtqRUEekI\nbr0Vli6F3/8+7PlnmjgxnNkzZEi41//ee4czd4YNi/f9+4sa8mZWASwDTgfWAvOA8919SUabPdz9\no9TnI4A/ufthTczLjzzSt1+GLSKSTcOV8Z07h66diy8Ot23+61/D7Rw++ijc7K2uLrR76CE488zy\n1VtMxb4Yagiw3N1XuXstMB3Y4WSohoBP+Qxhj75JuhhKRKLo3Dl9ALZ//3Db5pkz0/fu//jjcHO2\n2lo49thwi4dTTw3377nhBnj6aV2cBRAlcnsBqzOG1xCCfwdmdg5wLbAvkHV7qpAXkULq1AkeeCAE\n+6BB4YEvV14ZNgIQ7tNjFrp4rrgCDtupjyHeCha57n4/cL+ZDQMmAWc01W7t2urtD1FIJBIkEolC\nlSAiHVTPnuE5yABjx4b3+fPDs5LNwnn6t94Kf/hDuJPnpk3hPP716+HAA2H1avjud+Gcc8KVu+WW\nTCZJJpMFmVeUPvmhQLW7V6WGJxKeHH59ju+8ARzr7psajfeTT3Zmz2594SIizXXllfD+++Eg7X77\nwcKFoU9/1qzwF8CBB4ZbO7e1HofW9MlH+VHmAf3MrC/wNnA+MKZRAQe7+xupz4OBysYBv32BbWzl\niUjHcc012ae9+CIcc0zY+4/TAdy8B17dvQ64DHgMWAhMd/fFZjbezC5NNfuymb1mZvOBm4Hzss1P\nIS8ibdHRR0NVVTiAe9NN4eydOCj5xVBnnuk89FDJFikiEll9fbi/zrXXhuFddgkPW7/1VjjttPLV\npfvJi4gUQEUF/OIX4dTLefPg5z+Hd9+FO+8sd2UtV/LIVciLSHtwzDHhtd9+4YycuXNh5cpw64Wh\nQ9vPvXW0Jy8iksNXvgJ//jPcdVfYwz/hhHD1bXuhkBcRyaFLlxD0xx8frq695ZZwX52zzgoPUGnr\n1F0jItIMF1wQnoR1883hNgsHHBCeejV8eHhSVr9+cOih5a4yreRn13zzm77DE+FFRNqr2bN3fPZt\np07h3jrHHBMelFKoJ2Dp7BoRkTI45RS4++5wo7Rt28K59XPmwAsvQO/e4Qrb224LN1Qrl5KH/Be+\nUOolioiURkVFOPPmhRdCH/4TT8C3vx3ukrl5c7ilQqmVvLumvt7bzalHIiKtNWdOOCOnwQ9/COed\nF4I/qnb1+L9SLk9EpK2or4dLLw0PQNl773B75KgU8iIi7cQDD4RbGj/8MIwcGe077erAq4hIRzZs\nWDjNctQo+OMfi//0Ku3Ji4iU2JYtcOGFYa9+8+ZwwVUu6q4REWmHzOBb34KvfQ1OPz1XO3XXiIi0\nOxMmhCdRDR8OixaFg7OFpj15EZEy+vDD0Ef/zjvwmc+E581267Zjm6LvyZtZlZktMbNlZjahielj\nzezl1OsZMzuiJcWIiHQ0e+4JNTWwYkXoq580qbDzzxvyZlYBTAZGAIcDY8ysf6NmK4CT3X0gMAm4\nvbBliojEV0VFeIj41VfDr34Fl10G771XmHnn7a4xs6HAVe4+MjU8EXB3vz5L+27Aq+7eu4lp6q4R\nEcnhuuvgiitg991h/fqwp1/s7ppewOqM4TWpcdlcAjzSkmJERDq6iRPDE6g+/hguuaT18yvoPSHN\n7FTgYmBYtjbV1dXbPycSCRKJRCFLEBFp9958M8nZZyd59lnIiMwWidpdU+3uVanhJrtrzOxI4F6g\nyt3fyDIvddeIiETw9NNw8snQowesW1fc7pp5QD8z62tmlcD5wIzMBmbWhxDw47IFvIiIRDdsGPz1\nr3Dmma2bT6Tz5M2sCriRsFGY6u7Xmdl4wh79FDO7HTgXWAUYUOvuQ5qYj/bkRUSaSbc1EBGJMd3W\nQEREmqSQFxGJMYW8iEiMKeRFRGJMIS8iEmMKeRGRGFPIi4jEmEJeRCTGFPIiIjGmkBcRiTGFvIhI\njCnkRURiTCEvIhJjCnkRkRhTyIuIxFikkDezKjNbYmbLzGxCE9MPNbPnzOwTM/tB4csUEZGWyBvy\nZlYBTAZGAIcDY8ysf6NmG4HvA78seIUxlUwmy11Cm6F1kaZ1kaZ1URhR9uSHAMvdfZW71wLTgdGZ\nDdx9g7u/CGwrQo2xpF/gNK2LNK2LNK2LwogS8r2A1RnDa1LjRESkjdOBVxGRGMv7IG8zGwpUu3tV\nangi4O5+fRNtrwI+cPdfZ5mXnuItItICLX2Qd6cIbeYB/cysL/A2cD4wJkf7rIW0tEgREWmZvHvy\nEE6hBG4kdO9MdffrzGw8YY9+ipn1AF4AugD1wBZggLtvKV7pIiKST6SQFxGR9qkoB17zXTyVanOT\nmS03s5fMbFAx6mgLIlxINtbMXk69njGzI8pRZylE+b1ItTvWzGrN7NxS1ldKEf+PJMxsgZm9ZmZP\nlrrGUonwf2QvM5uRyopXzeyiMpRZdGY21czWmdkrOdo0PzfdvaAvwobjdaAvsCvwEtC/UZuRwMzU\n5+OA5wtdR1t4RVwXQ4Guqc9VHXldZLR7HHgIOLfcdZfx96IrsBDolRrep9x1l3FdXAFc27AeCBdf\ndip37UVYF8OAQcArWaa3KDeLsSef9+Kp1PB/A7j734CuqX79uIlyIdnz7v731ODzxPcahCi/FxCu\nnP4L8G4piyuxKOtiLHCvu9dAuOCwxDWWSpR14YTjfaTeN7p77C68dPdngM05mrQoN4sR8lEunmrc\npqaJNnHQ3AvJLgEeKWpF5ZN3XZjZ/sA57v5f5DhLKwai/F4cAnQ3syfNbJ6ZjStZdaUVZV1MBgaY\n2VrgZeBfSlRbW9Oi3IxyCqWUgJmdClxM+JOto/oNkNknG+egz6cTMBg4DdgTmGNmc9z99fKWVRYj\ngAXufpqZHQz8n5kd6Tp7L5JihHwN0Cdj+IDUuMZteudpEwdR1gVmdiQwBahy91x/rrVnUdbFMcB0\nMzNC3+tIM6t19xklqrFUoqyLNcAGd/8E+MTMngIGEvqv4yTKurgYuBbA3d8ws5VAf8Jp2x1Ji3Kz\nGN012y+eMrNKwsVTjf+TzgC+AduvqH3P3dcVoZZyy7suzKwPcC8wzt3fKEONpZJ3Xbj7QanXgYR+\n+e/GMOAh2v+RB4BhZraLme1BONC2uMR1lkKUdbEKGA6Q6oM+BFhR0ipLx8j+F2yLcrPge/LuXmdm\nlwGPkb54anHmxVPu/rCZjTKz14EPCVvq2ImyLoB/B7oDt6T2YGvdfUj5qi6OiOtih6+UvMgSifh/\nZImZPQq8AtQBU9x9URnLLoqIvxeTgLsyTi38kbtvKlPJRWNm9wAJYG8zewu4Cqiklbmpi6FERGJM\nd6EUEYkxhbyISIwp5EVEYkwhLyISYwp5EZEYU8iLiMSYQl5EJMYU8iIiMfb/jd76Sc4ftt0AAAAA\nSUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x110532550>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "## Some people plot them the other way\n",
    "plt.plot(recall[0:-1], prec[0:-1])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 0, 0, ..., 0, 0, 0])"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## What if we just use sklearn.predict?\n",
    "from sklearn.metrics import confusion_matrix, precision_score, recall_score\n",
    "y_pred_hard = lr_model_1.predict(X_test)\n",
    "y_pred_hard"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[2404,   29],\n",
       "       [ 544,   23]])"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# rows are actual 0/1 columns are predicted 0/1\n",
    "\n",
    "confusion_matrix(y_test, y_pred_hard)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.44230769230769229, 0.040564373897707229)"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# This model is terrible: It predicts almost all zeros!\n",
    "# My recall is really low.\n",
    "# But is the model bad?  Or just a bad threshold?\n",
    "\n",
    "precision_score(y_test, y_pred_hard),recall_score(y_test, y_pred_hard)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.38033395176252321, 0.36155202821869487)"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Let's try another threshold - similar precision, but much higher recall!\n",
    "\n",
    "y_pred_hard_2 = (y_prob_pred>.3).astype(int)\n",
    "precision_score(y_test, y_pred_hard_2),recall_score(y_test, y_pred_hard_2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x110f0a940>]"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAEACAYAAACj0I2EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJztnXt4XOV95z/vzGhmJI2utizfL7GDL9iNoawhwRQRkqwT\nUsDZtEuSJynbbpamKA3pbhI2oYud9AJsG0RrSnCXFnph2ZaUS2kghIahJdxMwAGMDb7buliWrevo\nMjMavfvHmXN0NBpJI+mcM6PR7/M8fvSe91zed2z5e975vb+L0lojCIIgzH18+Z6AIAiC4Awi6IIg\nCEWCCLogCEKRIIIuCIJQJIigC4IgFAki6IIgCEVCToKulNqhlDqklHpfKfWtLOf/h1LqTaXUG0qp\nt5VSw0qpauenKwiCIEyEmsoPXSnlA94HrgZagX3ADVrrQxNc/2ngFq31xxyeqyAIgjAJuazQtwGH\ntdYntdZJ4BHgukmu/xzwf52YnCAIgpA7uQj6MuC07bg53TcOpVQpsAP44eynJgiCIEwHpzdFfxV4\nUWvd7fBzBUEQhCkI5HBNC7DSdrw83ZeNG5jE3KKUksQxgiAIM0Brraa6JpcV+j5gnVJqlVIqiCHa\nT2ZepJSqAq4EnphiUkX75/bbb8/7HOTzyeebb59tPny+XJlyha61TimlGoFnMV4AD2itDyqlbjJO\n673pS68Hfqy1Hsx5dEEQipL4cJxQIJTvaRQH0WjOl+ZickFr/QywPqPv/ozjh4CHch5ZEISiJZFK\niKA7xTQEXSJFHaShoSHfU3AV+XxzF68/W3w47ul4RftvNw0xhxwCi5xEKaW9HE8QBO+ID8dJpBIA\ntPW1saRiCQBBf1BW69MlGh3988ILKHLbFBVBFwTBcVp7W1lauTTf05j77NoFgNq9OydBz8mGLgiC\nMBX2FXosGaMv3gfICn3amCtzgN274corc75VBF0QBEcIBUKWcMeH41SEKvI8ozlKQ4Pxx3581VU5\n3SqbooIgCIVE5kboNDZ8RdAFQRAKCbugT9N7RwRdEATHEZu5Q0xT0MWGLgiCI9g3RfsSfVa/bIrm\nQOZGqEmmPX0KRNAFQXAE+6YoIJui0yFTuNPuitNFTC6CIAhFggi6IAiOE/QH8z2Fucss0hhIpKgg\nCEKBo5TKKVJUVuiCIAhFggi6IAiO43W2xTnHNLMo5ooIuiAIjmO6LwoTIIIuCIIgTIb4oQuC4AgS\nWDQFDgUPTYYIuiAIjiCBRVPgUPDQZIjJRRAEoUgQQRcEwXEk3iSDWaTEnQ4i6IIgOI5ZrUjAEPMH\nHxzbJ4IuCMJcoT/Zn+8pFA7RKJw44clQOW2KKqV2AE0YL4AHtNZ3ZrmmAbgbKAE6tNa51UwSBKEo\nsHu5dA51Sk1Rc2V+4gS88MLoJqiDXi2ZTCnoSikfsAe4GmgF9imlntBaH7JdUwXcC3xCa92ilFro\nymwFQShY4sNxYokYAB39HfRVGoIeCUbmn6BHo3DHHXDoEJw8Odq3erWrw+ayQt8GHNZanwRQSj0C\nXAccsl3zeeCHWusWAK31OacnKghCYVMZrqQyXAnA0c6jLK1cmucZ5ZGGBkPAL7vMOLb7oLtILoK+\nDDhtO27GEHk7FwAlSqnngQjwZ1rrv3VmioIgzAXsJpfkSHL+mlxM8Y5GDVPLlVdCd7dx7JKpxcSp\nwKIAcDHwUaAceFkp9bLW+kjmhbtszvQNDQ00uPwBBUHwnoCapzGLpqnlsstGxXz1ati6dVpiHo1G\nic5gRT9lPnSl1GXALq31jvTxrYC2b4wqpb4FhLXWu9PH/wd4Wmv9w4xnST50QZgH/PuJf+eK1Vfk\nexr5Yd06OHJkdBPUgYhQJ/Oh7wPWKaVWKaWCwA3AkxnXPAFsV0r5lVJlwKXAwelOWhCE4qBrsCvf\nU8gPTU1wzraF6LEFYsrvRVrrlFKqEXiWUbfFg0qpm4zTeq/W+pBS6sfAW0AK2Ku1ftfVmQuCUFD0\nDvVaXi6Hzh3ikt5LAMPLxdwsLVqamuB//2/o7IShIUPIu7uhutpTUc/J0KW1fgZYn9F3f8bxnwB/\n4tzUBEGYq5jCPq84fx6Gh0ePq6s9n4JEigqC4Djx5DyqWNTUBHv2GO1UCvx+o3399XDLLZ5OZZ5u\nRQuC4DQdsQ5aY60AdA52crTzKABLI0uL1+TS2AgPP2yYV0yHj1QKDh/Oy3RE0AVhHhAfjrvuC752\n4VrWLlwLwL6WffPDy2XdOsNu3tBghPh3dxteLR6vzE1E0AVhHpBIJVwXdPumaHw4TmuvsVov2k3R\npia47TZ4/HHD5zwUAjWlZ6GrTOmH7uhg4ocuCHmhL97naQWhJw8+ybUbr/VsvLyxerWxMm9ogM2b\njRW7C6vzXP3QZYUuCEWKPRT/3MCob7Rbofj28cpD5cUb+t/UNJrf/ORJQ8z378/LJmgmIuiCUKTY\na3x6vUIP+oKejeU5W7eOdUlsaIBw2OjPMyLogiA4gv0FUh+pL94i0fZ85idOuFLseaaIoAtCkWI3\ngcRTcddNIPbxEqlE8ZlcMgtWANTUwI03Gn8KINGgCLogFCn2FXN8OO76itk+XkAFim+F3tAA99xj\nCLsH1YdmgkSKCsI8wFw5e0V3vNvT8TwhGoXnnx/bV0BiDrJCF4R5QdDv7SalIr/+2K5gz09eYEJu\nIoIuCEXKGJv2iPs27Y5YB11DRtrcEz0nqCmtAaAmXENdpM7x8Txj5074yU+MxFvx+KiHy1VXFZyw\ni6ALQpFit2kDrtu0K8OVhEvCABzvOs6SiiWA998OHCUahQ99CLq6RqNBb7ml4GznJhIpKgjzAK/9\n0IuuYpG5CdrUZORr8RiJFBUEwcKLVbLdxFPiK5n7botmoWeA3buN+qCLF3tS7HmmiKALwjzAC0G1\nm3hWVK2Y+26LpmjbfxaokJuI26IgCI7TOdiZ7ynMnsbGOeHZYkcEXRAEx5nzRaKjUXjqqdHjOSDm\nICYXQRAcwm5D70n0zF0bejRqeLJ0dxu2czsFLuyyQhcEwRF6h3pp62ujra+NA+0HrHbvUG++p5Y7\njY3wiU/AoUPQ0wNVVUb+lnPnCl7MQVbogiA4hN0PPZ6Kzz0/9GgUYjEYGYFk0hDzrVsNl8U5IOYg\nK3RBmBfEh+Oejhcbink6nmPY7eZgVCSaQ+S0QldK7QCaMF4AD2it78w4fyXwBHAs3fVPWus/cHKi\ngiDMHC9qijZ3N9MaM+qItsRa2N+2H4ClkaVW8eiCZft2eOUVSKWMY6Wgrw+OHJkzq3PIQdCVUj5g\nD3A10ArsU0o9obU+lHHpv2mt50ERQUEQsrG8ejmLKhYZ7YrlbF1iVPApeJNLNAof+5gh3u3t4Pcb\n9UGbmuaUmENuK/RtwGGt9UkApdQjwHVApqAXYXo1QZi72L1O+hJ9Vr9bXie9Q71Wcq5zg+do62sD\n5kByrnvugX/5FyP5Fhir9CNHjP4iFPRlwGnbcTOGyGfyYaXUfqAF+IbW+l0H5icIwgzxOjlXKBAi\nEowARk1Rs13QLotNTfDaa+DzgdaGqaWiwnBXzHPB55nglJfLz4GVWusBpdQngceBC7JduMtWf6+h\noYGGOfYGFAQhO6FACKWML+pKKesFUtAmlwcegP5+o8hzPA6BAHz963kv+ByNRonao1RzJBdBbwFW\n2o6Xp/sstNYxW/tppdRfKKVqtdbj4n93FVBBVUGYL3iR5dRucmmPtRe+yaWxEd55x7CZp1LGz0Cg\nIHzOMxe7uzMDnCYgF7fFfcA6pdQqpVQQuAF40n6BUqre1t6GkZa3CJI5CEJxYK6cvaLEV+LpeNOm\nsREefdRom2JeUgK/+ZuwZ09+5zYLplyha61TSqlG4FlG3RYPKqVuMk7rvcBnlVJfAZLAIPCf3Zy0\nIAiFR+9QL+2xdgD6h/utth9/Ya3Qt2+H11+HhK3OaioFGzfOaTGHHG3oWutngPUZfffb2vcC9zo7\nNUEQZoPXXi6V4UpSGH7cYV+Y+ki91V9Q1NUZ0aAmShmmlnXr8jcnh5DQf0EoUuxeLvHhuKcl6MpD\n5YUZ+h+NGiH9fr8R3g9wySVw1115t5s7gYT+C0Ie8DoU31ypu0l8OE5fvI++eB9Bf9Bqe/1ZJ+XR\nR+HJJw2PFpOyMti/P39zchBZoQtCHvAiFN+OVyXoYgnD4S0YCFrtgtkgbWw03BSTScPn3NwI3bx5\nTvqcZ0MEXRCKlEwbuvkCccuGbg8sCqhAYQUWRaNGANHQ0Gifzwfr1xeF7dxEBF0QPMLrTUq7Db0v\n3udpjc/OgQLyWo5G4UtfgjNnxvbX1cGNNxbN6hxE0AXBM7wOxc8ng8nBfE9hlP37oaXF8GYBCIUM\nL5dvfKOoxBxE0AWhaLF/I4in4q6XhLPb0AeGBwrDhh6Nwg9+MNZNMZk0NkKPHMnbtNxCBF0Q8oAX\nm5RefyPo6O/gdK+Rx68r3sXxruMADFcO5yewqLERHnrIqEJkJxCA3/iNOR9ElA0RdEHIAwWxUegw\ndeV1BPyGpAT9QdbUrAGMXC55o8T27cA0uXz5y0Up5iCCLgjzAq/dFgcTg1a7LFDm+tjjiEbhueeg\nq2u0T2sjq2IRI4IuCPMAL74R2N0W/cqfX7fFRx+Fo0fH9kUiRWtqMZFIUUEQHKfEn8eN0KYmeOSR\n0QpEJpdfDp/9bH7m5BGyQheEeUB8OO76StmebbFvqC8/2RajUbjtNqNohR2/H0pLiyJfy2SIoAvC\nPMCLVAOhQIjyYDkAw3rYantqcrnnnvFi7vPBr/4qPPaYd/PIEyLogiA4wpjQf18eQv8bG+Hll8f3\nX3vtvBBzEEEXhKLF61QDx7uOc6TTCNYZGB7g9ZbXAVhXu859k8vOnfDEE4YnSyYdHe6OXUCIoAtC\nkeJ1YNHSiqWWH3pVsIoNdRsAWFS2yNVxLbQ2fM1NUff7542pxUQEXRDmAV4UuKiL1FFVWgWA3+fn\ngws+CHjgAx+NGrlaYOwK/bnnin4TNBMRdEGYB3hRZKJ3qJeuISOQJ56K09bXBhiRoq6ZXMxMiqdP\nj+03i1aIoAuCUGwkR5Kuj2F3W0ylUt64Ld5223gxVwouuqjoMinmggi6IBQp9k3RUz2nWFC2AHBv\nU9ROUrv/AqGxEV56aXx/eTls3er++AWICLogzAO8yE8eT8UZSA4AEEvErHY85YK5p7ER7r9/vFeL\nzwcf+1hRh/dPhgi6IAiOEPKHKCsxEnFpra12yO/Ct4EXXhgf2g+wadO88mrJJKdcLkqpHUqpQ0qp\n95VS35rkuv+glEoqpT7j3BQFQZgJ8WGjqEVfvI/ESMJqu7VBakaKlgfLUT5ltR0372zfDu+8M76/\nvBz+/M+dHWuOMeUKXSnlA/YAVwOtwD6l1BNa60NZrrsD+LEbExWEYsKL3CqV4Uoqw5UAdA52srRy\nqavjtfa18tJbL/OP9+6n/b0L+Oq//jm/dvNWPvJLH2Z59XJnBmlqgtdfz35uzZp559WSSS4r9G3A\nYa31Sa11EngEuC7LdV8FHgXOOjg/QShKzM1KrzDD8N3E1xPg+41HeenpPSSOPcdLT+/h+41H8fU4\naNk9cgTiGd8wlIKbb4a333ZunDlKLoK+DLD7BTWn+yyUUkuB67XW9wHKuekJguAEtaW1ro+x6/a/\n4/SJPwbK0z3lnD7xx+y6/e+cGWDnTti7d3x/RQWsW+fMGHMcp16dTYDdtj6hqO/atctqNzQ00DDP\nvyIJ8wevc6vYMU0vbtLdEWJUzE3K6T7nwGczo0GTGe6Qfj988YtF53MejUaJRqPTvk/pbMls7Bco\ndRmwS2u9I318K6C11nfarjlmNoGFQD/w37TWT2Y8S081niDMB/rifa6H4tvpHep1XdR37PwqP378\nDsaKej//8fpbeeaxWW5WLl4M7e3j++vr4cyZ2T17DqCUQms9pfUjF5PLPmCdUmqVUioI3ACMEWqt\n9QfSf9Zg2NF/J1PMBUHIH2YYvpv8r11fYPmqWzHWcwD9LF91K/9r1xdm9+BoFM5m2ZqLRIq+AtF0\nmVLQtdYpoBF4FjgAPKK1PqiUukkp9d+y3eLwHAWh6PCiaLMdMwzfTcrqwnxn72auuOZr+FZfyRXX\nfI3v7N1MWd0sCzN/5jPZ0+IuXDhvA4gmIicbutb6GWB9Rt/9E1z7mw7MSxCKGi+KPvQO9RJLxABo\n7m2mtbcVMDxe3DC/nImdob8sxnW3buTlf32I666+ln5inInNwiQSCEAqNb7f74fjx2f+3CJFIkUF\nIQ94YdO240VyrkgwQnW4GgCNttozdpmsqMgu5koZqXGFcYigC0IeOD9w3lNBN1fqbrIksgSdtrgG\nCHDBwgus/mnT1GTkZcnGRz4y7wOIJkIEXRDygBcCa48UjZREXI8UPT94ntM9RsiKD5/VDvlDrGXt\n9B52xx3Q2zu+v74eXnxxtlMtWkTQBcEj7Dbt1lgrC3qNdLZu2bTtfu8aTV/c8H13y+89kUpYWR0D\nKmC1px0VW1oKQ0Pj+30+eOSR2U6zqBFBFwSPCAVCKGW4EpeVlFl+6G55vMSH49YLpMRfYgl6JBhx\nRdDXL1zPoohRPzTgD3DF6isAo2LRtAhMIEubNompZQpE0AUhH3jg3Gt/gZSWlLr+ArFXLBoeHp5Z\nxaI1ayCWxRy1erXkaskBEXRB8IhQIGStjBeVL/I0UtQLLxe7DR2Yvg29tha6usb3RyLiopgjIuiC\nkAe8yH5of4FUllS6/gKx29CHGJqeDX3nzuxiDvChDzk1xaJHBF0QihT7Jmz7QLvrgUW1pbUsrzLy\nnidJWu2cMj1O5FcuXi3TQgRdEPKAadt2E7vbYtgfdt1tsXOwk+aeZuvYbJeXZGZgzKC2NrvdvKZm\nXiTechIRdEHwCLsbYSwRc92N0L5CPz9w3vUVeiKVYGh41N3QbE9qcqmoyC7mAJ//vJPTmxeIoAtC\nPvC4DIxvoqhLBznccZg32960js12gAAfXfvR8Tc0NU0s5iCJt2aACLogeIR9k/L8wHlPvVyGU8Ou\nj7EwspClFaNmHbO9MLIw+w2/93vZ+5WCkRGnpzcvEEEXhDzQG88S1u4wY0L/Q+6H/tdH6llVvco6\nNtv1kfrxF9fWZk+JW1ICCW/rrRYTIuiCkAd8yn0TiN1mD7hus+8c7ByTKtds15VnBBU1Nk7sohhy\nP61wMSOCLggeYRfYxEjCdYG1h/4rpVwP/V8cWTwm6diamjVW/xh+8IPsDwiHoa8v+zkhJ0TQBcEj\n7ALbn+h3XWDtof8lqsT10P+cKC3NnuM8HIbBQe/nU2SIoAuCR9ht2rFEzHWbtp3OwU7PxpqQxsbs\nWRQB1q3zdi5Figi6IOSBoeQEwuYSg8Pur37tof/A2ND/pia4997sN9bXS+IthxBBF4Q8MO0c4TPA\nbuI5P3jedRNPe6ydk90nrWOzXR2uhq9/PftNSkk0qIOIoAtCHvAi0Mcu6L2J0ajREl+JK+OtqFpB\nPBW3jjfUbTD6P/nZiW8Sf3NHEUEXBI8YU7Gor5XFvYb3h1uh+KFAyMrqODIyYrXdWJ2DsS/QNTjq\njtg12AW3fYfYRLnfV692ZR7zGRF0QcgDXuQn7x3q5ezAWaM92Gv5hS8qW+TKC6RrsMsqagHQ/pPH\nYbCHxRM51UiOc8fJSdCVUjuAJsAHPKC1vjPj/LXA94ARIAl8XWv9M4fnKghzGruXS+mZUte9XOoi\ndVSVVgFQW1bLBxd8EHDPbbHEXzJm9R/60XOgoSTTquL3w7D7qQjmI1MKulLKB+wBrgZagX1KqSe0\n1odslz2ntX4yff0W4B+AjS7MVxDmLPbAojN9ZzzJttg1ZJhA+hP9tPW1AUaNz5xLwk2DlVUrSem0\nj3k3bDqX7h/IuPC3f9vxsQWDXFbo24DDWuuTAEqpR4DrAEvQtdb2f7IIxkpdEAQb9uRcfp/f9eRc\ndht6RajCdRv6qZ5TvHv2XfjBfbAM3l1g9PuTsNW8qKZGsii6SC6Cvgw4bTtuxhD5MSilrgf+GKgD\nrnFkdoJQRNg3RXsTva7nJ7fjlmeLnbaeNo6/8AR0tcByOG5Ye1hoz0PWWQABTkWMY5uiWuvHgceV\nUtuBPwA+nu26Xbt2We2GhgYaGhqcmoIgFDR2N8K+RJ/rboT28QaSA66Pd+nKS1n0978NGh7YBNcc\nM/rX9KQvyJZdUchKNBolGo1O+75cBL0FWGk7Xp7uy4rW+kWl1AeUUrVa63GvY7ugC8J8wm4CGRoe\nct0EUhmuJFwSBoy6nksqlgDubYrGPnwRneYXDQ2dxtDU9QPPP+/KmMVK5mJ39+7dOd2Xi6DvA9Yp\npVYBbcANwOfsFyil1mqtj6bbFwPBbGIuCPMZu5fLSGrEdS8X+wo9loy5Gym6ZYvh32Zij5u6cCPI\nN3FPmFLQtdYppVQj8CyjbosHlVI3Gaf1XuA/KaW+BCSAQeDX3Zy0IMxF7Db0JEnXbehjTDxxF008\nTU3wzjsE66HU5o1YOgwEIfhXDzo7njAhOdnQtdbPAOsz+u63te8C7nJ2aoJQXNgFNplMum7Ttptc\nwoGweyaXdJ6WM+HRjVBIt+/4AyIxydXiFRIpKgh5oH+43/UxOmIdVqTo8a7jHD5/GDAiRZdXL3dm\nEDVa7ToyDNWjqVyo3nY5hKutvQLBfUTQBcEj6iJ1VkBPqb+UCxZe4Op4YzZhUy5swpaWjjnsCkJ7\nWfogAO2f+ijE2sdXLBJcQwRdEDyiubvZWjE39zTzRusbgMMr5gmIJ+NTXzQd1qwZV6yiJgH1A0DI\nB+UjVnHomtIaZ8cWJkQEXRA8ojJcaaXN9fv81srVLZNES28Lx7uNBFj9iX4OnD0AwJrqNbML/d+5\nE06cGNd9IgIHFgC/exO8eZ81Xqm/FNbOfDghd0TQBcEjxgT6DLsf6LOwbCEj6Swc5aFyVlWvsvpn\nxeOPZ+1eOgBrd/2Jdby21lDxpVXeldqb74igC4JH2AV9KDlktcsCZZPdNitaTrXxF3c9y7sHFvCd\nZx7id775CRZtWDTzB9o2QTNZ+Rd/japfYh1vrt8MwIrKFTMfT5gWIuiCkAfUJMLoFP0dA3ztC//K\n8WN/BJTzzPv9vPeL3+fpZz4I1TN44OJJNjcjEY6sreXtltetrtfT7fhwnE31m2YwoDBdRNAFwSO8\ntqHf+u0HOH7se0B5uqec48e+x63f/i6P/eOdk906nu3bob09+7n6ejhzhpWt+0fT5wKbFhkivrJq\nZfb7BMcRQRcEj7CbXHz4XLehd3YEGRVzk3K6zk0zsGjnTvjZBPVq/H6ryLOVPjeN2fYrP1uXbs16\nu+AsIuiCkAcqQ+6mywVYvEQD/YwV9X7qF08j62E0OuEmKDCm8tBAYoDeodFcuWZ7IJFZ4UJwCxF0\nQfAI+wodH65vit7xR1/m5/tu5+jR3Rii3s/atbdzxx99NfeHXHXVxOcy0uFuqNtAqCQdtPQqXLby\nMsBwkxS8QQRdEDzCXuMzHAi7XuNzwZIa/u6ffp27/vC7vHX4LL/0wUV88zu/zoIlOQb6TLZxmyUd\n7pnYGY51HrOOzXZpoHTctYI7iKALQh4423fW9TFCgRAXrt/IQ39zG/e9dh9f2fYVIMcXiN8/8bm7\n786aDjfoDxIOhK1js+3WC0sYjwi6IHiEvWhz91C360Wbx/i9Dw/lng/d74eRCcoCr14Nt9yS9VTX\nYBftsVFPGLMtuVy8QwRdEDwiU2DdtqHbx2vtbc3Nq6a0dGIx9/ng+HGnpyk4iAi6IHjE++ff52DH\nQQDaYm28fOplADbWbXQlOZc922Jfom/qbIs7d45LuDWGVGric8CKqhXEU6NJwDbUbbD6BW8QQReE\nNPHhuGv1PQG21G9heZUh3N9/6ft8fJ1RR70m7E42wlAgZEWkxofjVIQqgAls2o2Nk7sn5lDguXOw\n0zIjAVa7trR2GrMWZoMIuiCkSaQSrgr6mIpFOum6yeXo+aOc7j0NQG+8l5dOvQQYuVXGhOLv3Dlr\nMRcKAxF0QfCI3ngvHf0dAOgRbbXD/vBkt82YylAldeXGZuvIyIjVHhfU5JCY9yf66RnqsY7Ndn/C\n/epMgoEIujCviQ/HSaQSgGFnNgn6g46v1kP+EGUlxmo8XBK22iG/O98K7Db03kRvdhv6NH3NJ6Nn\nsIdzA+esY7PdM9gz0S2Cw4igC/OaUCBkCZzdzuw2/XH3V632kncKNb7k3WRifvPNWX3NJ+PiZRdT\nU27sB9z35n3WHoFEinqHCLogpIklYiwsn2XxhxwZSk7iTeIQr51+jSOdRwBjw/LhXzwMwLradWxb\neenEN95994S+5pNREawYUzzDbFcEvXlJCiLogmChXd78CwVClAeNRFmpVMpqu7URGwlGWFC2AACf\n32e1I1snEfNweEZiDvDSqZf4edvPreMfHvghAL+85JdZu1Bq0HlBToKulNoBNAE+4AGt9Z0Z5z8P\nfCt92Ad8RWv9tpMTFQQ3sNvQkzppRVO6YUN/o+0N3m43/lsMMcTT7z8NpN0Z3fBDt9nsR0ZGjPb2\nXyE0kTu5UjA4OOPxllUtozcxmm3RzFWzrGrZjJ8pTI8pBV0p5QP2AFcDrcA+pdQTWutDtsuOAb+i\nte5Ji/9fApe5MWFBcA2XvfNqS2utMHg/owUu3PLTrgxXksJQ71J/KfUX/4rRn8hycUkJJLKdyJ1E\nKsHQ8KgpyWybL0zBfXJZoW8DDmutTwIopR4BrgMsQddav2K7/hVAXsnCnMC+KdoX73N1U7S2tNYK\nLPIrv9V2S9B7h3qtfCrBV9+mPe3u7h+BusyF+CzFHCQ5VyGQi6AvA07bjpsxRH4i/ivw9GwmJQjg\nfuRmJl4Kz4ieIF+Kg1gr9AvWU/EJqE/XmRi3Qndo76C2tJallUsBoyKT2ZZIUe9wdFNUKXUV8F+A\n7RNds2vXLqvd0NBAwzRdo4T5g9uRm5m47bJot2kHCHjjh75mPQDlQxBJpvtNG7rfP6bi0GwJ+oPW\nZyql1GrLCn36RKNRotHotO/LRdBbAHuV1+XpvjEopX4J2Avs0Fp3TfQwu6ALQiHh5ctjGOeEdCJ6\n66o4mzY+gsCrAAAY2ElEQVSzxINwJt1eRHqV7qCYg+H2eX7gPAA+5bPabuWqKWYyF7u7d+/O6b5c\nBH0fsE4ptQpoA24APme/QCm1Evgh8EWt9dHcpiwI4/EycjPb2G6OcbLnpJVtcYgh3jrzFgDJuqTz\nbn1KESodXZWXpGwrdJ8P9OSZE2fC0oqlBPyGpARLgla2xUVlixwfS8jOlIKutU4ppRqBZxl1Wzyo\nlLrJOK33Ar8P1AJ/oYz0bkmt9WR2dkHISr4iN8HYFHVT0CPBCDWlxmrVh89qmyH5jpGOAA3ZRHxI\npdtVVYSOnHJ2vGxTYJIoVME1crKha62fAdZn9N1va38Z+LKzUxPmO33xPs8iN8F99zp7etlhht1J\nL2sL5+8ohdb0u+JsBI5WAS/+M0tjHVSGK7PfPwta+1qtyFSlFIc6DEe44dphV/zshfFIpKhQsKjJ\nco04hN3Ec37w/Jic4U6v1iPBCNXhauvYbDuyQo9G4aqrxnRVJiCV9mxRGur3v2f0uyDmYPydlZYY\nBaF9yme1ZVPUO0TQhYLCLrCJkYSrkZuZdA12sbp6tWvPXxJZgk5HL5VQYiXLWhJZMrsHZxFzgJYI\nHK8CgoruLcs4cPYAYCTLcqOGqT2Xix+/5HLJAyLoQuHiQV0Fu80+mUq6arM/P3ie0z1GSMcII1Y7\n5A+xlhluim7ZAu+8M/H5mkp48EFG/rlxZs+fBvbI1FAgRH2k3uoXvEEEXSgovIzchLHfCLrj3Z59\nI9BOvK0qKiAWm/D0wk2/zMjDewEoC5axqnqV0V/mzr5ES28Lx7uNItIjjLj+jUAYjwi6ULB4YXu1\nl4UbSgxZgh4JRhwXdLuNGZidjbm0dPKCzpdfzrmH93AyLbCJVIKT3ScBw8PGjU3KZZXLKAsazu6V\nwUouXHQhIH7oXiKCLhQsXrgs2gspBwKByQspz5IzsTMc6zwGGCtYs10aKJ3stvFMtVn8/PPQ0MAH\nhnpZFDF8wBeEFnDpciNtruNukmnsuWNiyZjV9uOXFbpHiKALBYuXkZsAg4mZp47NBUe8XHIU80zK\nwu4UorZjz/c+wojr+d6F8YigC0IaR+zak9A52MmZ2Bnr2GybxZsnZefOyYs5w7gkW809zZzuNTZe\nlVZWLvYVlSvYFN40jZlPHx8+V58vZEcEXRBMXPaqmXF62Vz88bNkTKwMVVovi2BJ0GpXhtzxOnn/\n/PtWaoNYIsbLp14GYGPdRgks8ggRdGFeY98UHUgOuLopOiOTS3AKsa+pgc7OrKfqInVUlVYBRsUi\ns4KQW5vNW+q3WDne7375bqtItGyKeocIulCweJEP3b4pOsKIq5uisUSM7qFu69hsmy+UMTQ1wde/\nPvkD/f4JxRzGvqxiiZirLyuAjv4Oy8QzPDLM8S7Dw2a4clg2RT1CBF0oWLzIh24XvZaeFldFrz/R\nT89Qj3VstvsT/WMvXLwY2tsnf9j118Njj016ybHOY5ZfeNdQF682vwoYfuFbl26d5uynxm7iiZRE\nXDfxCOMRQRcKFi+yLdpX6P2JfldX6PWReiu4B7DaZkQlkJu9/O674ZZbprzMHorvUz7XQ/FDgZBl\nPgoHwlZbvFy8QwRdKCjskZtnYmcsMXArcrN3qJeuoS5rPDMDYk24xnEzQXus3QruAax2dbga1qyB\nEyemfsg0ysXFU3EGkkZ2rsHkoNWOp+K5T3qGmCkABG8RQRcKinyE/psml3MD56x2WcB5v+1jncd4\n9+y71rHZrvjG9+DEFDdHItDXN8VFEzMy4n4NUzuSYTE/iKALBYV9hX6s8xib6zcD3mRbNFewbrE4\nstjyAgFYfrgdoi+yuH+SmwA2b4a33572eHabdlmozHWb9vGu41Y+9Fg8xustrwOwrnadbIp6hAi6\nULD0JHqmvshBEiPuFrgYwwDw2otTXzcNE0sm8eG4teFa4iux2vGgOyaXNTVrqC5Nu2KWRLhk2SWA\nuC16iQi6UFDYbdpn+8+6atMG6I330tHfARjJucx22B+e7LYZUVVaRV17H+z9K7gS6tKZBqomeo/M\nQsxhbCh+gICnofiDw+6mURCyI4IuFBR2T4l4Mu66p4TdLIHCVbNE8j9dR7wG8Btjxf3p/swoeZ8P\nUrPfVLT/Xfrweep14lMS+p8P5G9dKFja+6fwxXYAU/QiwQgjjFhtR0Vv505QilMBOFxt/CE12j5l\nDxS9+WZHxHwcfucfmYmZbbE91k5JSYnV7h3qdX9wAZAVulBg2P3Cffhc9QuHsSlfUyMp51O++v2Q\n9jBZMgArTUcVNdpeMoBjq3I79oIT8ZG46wUn7G6SOqU9dZMUDETQhYKiI9bB2YGzAPQO9nL4/GEA\nFpUtciXBk12EkqmkcyJUWwtdXWO6BgLQZ76X1Gh7YNViONI2u/GyYA8sCquw64FFdvNVaahUIkXz\ngAi6UFDYPTNiiZjrnhl2HPFy2b4dXn7ZWpXbaS+DlvL0gUq3f+1aViy/bPbjZsFe47MiXOF6jU97\nMrClZUtdTwYmjCcnQVdK7QCaMGzuD2it78w4vx74a+Bi4Nta6+87PVFhfrC8ejmLKowqO8FAkK1L\njJwjbomCfVU5khqZ3arS55vUM6V+AJaZPucalv3WV4z+8voJ73GK6mD11BfNEruHUm15reseSsJ4\nphR0pZQP2ANcDbQC+5RST2itD9kuOw98FbjelVkK8wa7KPQke1wXhda+VisYJp6Kc6jD+LUerh3O\n3cQTCORk/z5aDe/VAJ/8KJz+Ke+dew/wxk97YcSdwtB2KsOVhEsMd8+NdRtZUrEEkBW6l+SyQt8G\nHNZanwRQSj0CXAdYgq61PgecU0p92pVZCvMGeyj+YGLQ1VB8MLIqLihbABgbsmY7p7JwuaS4tbE2\nqej64m8D8NPTP2X9wvVGf83aac46N+oiddZLcHX1ai5YeIEr42SjKlzl2VjCKLkI+jLgtO24GUPk\nBcFVxqWVdYFEKmFthA6nhq22mX4gK1u2wDvv5D6I3w/Dw5x87naONr9idR/tPApAXZk75gh7GoVI\nKGKlBnYrjYI9D8/yyuWeFPkWxuL5puiuXbusdkNDAw1ZCtoKAhgZAt3G7gkCTO4JMl0hB1i9Go4b\nroPhQJhUp+bQo0lo286BJQk2fLaE8Grno1JhrPlqMDnoftTtUK/1jSo5kqS1txUwvu24tRFbrESj\nUaLR6LTvy0XQW4CVtuPl6b4ZYRd0Qcjk3MA5K63s4PCg1fbhc8Vt0e4JorTK7gmyfTv87Ge5P9Tn\ng7o6OHNmTHfs7ACv3rWIWPv9QDktJ/rpOXYTl/6ZO0nB7Dbt6nC16zbtynCl9ffWOdjJ0sqlrowz\nH8hc7O7evTun+3IR9H3AOqXUKqANuAH43CTX55ChXxBywIPfJHuGwMHU4NgMgVu25ZajPJMJNkgf\nu+8tYu0PA6bvYjmx9vt57L7P84e/Pv1hpsJuAhkeGfbUBCIboflhSkHXWqeUUo3As4y6LR5USt1k\nnNZ7lVL1wOtABTCilPoasElrnaVYoiBMjN0EkkgmXA+GCfqDlJaUAkZRhtKSUvjMZwh2Ah3TfNjz\nz8MkJsTB7mpGxdyknKFud1wKm7ubrSCt5t5m3mh9A3AxSMtms9dau26zF8aTkw1da/0MsD6j735b\nux1Y4ezUhPmIPUNgMpl0PUNg52CnZeuNd8Vp/cJnoBSqprPArK8fZ17JRt2iFCfoZ6yo97NwkTvV\nfZZXL7eE+42WN7h46cWujGNi/0awoGyBbIrmAYkUFQoKe6ToAAOuR4om7v5Thl79kXFwBQyl/0ck\ncklbN838K5d/qZ539n+ZwbN/iSHq/ZQu+jKXf2nJdKedEx2xDmtTtL2/nffPvQ94E+hzfuA8C8vd\n930XxiKCLhQUhzsPW0mkkiR5rfk1AC5cdCFrFzror71mDZw7R6QiRvWC0e7qIeNnZHiSe5XKGto/\nFT2hLpbd9HM6ftRAz/kwVQuGqPtULz2hy6f9rFywb4qurFnpaaCP6e0ieIsIujC/yIjq7FwAZ2wW\nELNdl618Z46mlYn4xAc/YayMPwJ3vXoXN136TQAuWnLRjJ+ZK4vKFrk+ht2GPpAcEBt6HhBBFwqK\njlgHp3tG49jM9qwEqbbW+JmR/RAgOAJhU9/1aDtoX4CXlEBi9om72vraONVzyjo224sji2f97GzY\nbdpLKpZ4WnDbLuiO55cXJkQEXSgoqkqrRisIMVpByMzilzNr1uTkcri5CxaZ5vkUXJXW20Wma/gs\ny8DZWVm90rJpA1Y2wpXVKye6xTGWVrjvE27PZR/wB1zPZS+MRwRdKCje63iPN1vftI7NdtAXhI1T\n3JyjiNt5pwYOmMkO/fD8SuDDl3Jh4y6WX7BjWs+aivJg+ZgcJ2bb9ORxEy88TuzfCCIlEfFyyQMi\n6EJBURmstCrHA1a7MjhB6Lgp4j7fjDYqV/aDOpc+ULD54WcAWFFZXF64Xps8JDlXfhBBFwqKN8+8\nyRstb1jHZjugbL+qSkE4nf9kKO2WMgMxJxLhTHmMI/93j3H840aOnDeiRkt8JWyq3zT9Z06CK/sD\nBYR9UzQUCMmmaB4QQRcKim3LtuHzGU7gb55/k8tWGtV8LvnmfbDz/tELTSGfLn6/4eVSUgJ9fbS8\n8bccbvu5ddoseTfhN4JZUBYsG1M4w2yXBd1JDew1dpMLeGPmEcYigi4UFO+df8/wQ3/u32A1HPj7\n+8APFbWzfHBNjeHlMjzWwXzL4i1Ewkbu83tev4cr11wJGIWUnSaZShIfHg2QMtvJVNLxsYT5iQi6\nUBisWQOnTtFyzQhtCwEjvQpt6cVry2wyzE7iPx5LxOgc6LSOzbYbOcqrS6tZFBk1r5ht+55BsSCe\nLflBBF3ID2vWwNat8MQTRrBP0lil9sRhwPZbOVBi/OwpmebzL78cXnxxyssSqQTNp1p56v5D8P52\nfvDST/n0TRtYU+P8Cv1k50kOdhy0js12JJBDdaQ5htjM84MIuuA+TU3w3e9CT49hw/b7DRt4V5fh\n5510wOTg88HKlVYxiVxpO93Ont89xbmWvUA5r5/o58SBW1j3d+vA4cpwFy27iIpSw678wFsP8NEP\nfBQwUvUKghOIoAvOU1sLmzbBK69AZSXEYqOiPTIy2u7pGX9rAipsQZlmuzYzUNP0ckkkppUgK5NH\n793PuZYm7DnKz7U08ei93+ULDZOl/Z8+SyuWEvAb/+VChNhQtwEoHi8XIf+IoAszJxqF/fvhgQfg\nyBFDWLdtM1ber79uHGcJt5+M6iRU2RbsZrvavogPh2HQmfJ0R0/1kS1HudHvLPZMkqWUup5JUph/\n5JIkVJjvNDUZNu/GRmP17fcb7W9+Ex5/HA4eNEwoyaQh5DDOmyRXmkvhTPoPpNtl0HzTZwzzjNaO\niTnA4iUKyCxG3Z/ud5Z4Ks5AcoCB5AALqhZY7XhKBF1wBlmhCwZmQdpHH4V/+RejvWyZYRZpa4PO\nTnjqqdEV91NPQXe3sbFpJ54Wp1zMIFVVY80uPh/+8hFjlbGyHnQ7vk3G5qRf+Wf6ySZl1+4vcnD/\nrTSfvAMzR/nyVbeya/cXHR8r5A9RVmK47SwsXWi1Q37ZQBScQQR9PtHYaPxsaYErDX9rXngBOjog\nEoHLLjOE+lQ6Q1Vb28Qr7ZMnR++fCT4fXHXVqJfL+vXw9tuU/sNvEGozcqDT3W6JXWmwdGbjTMHm\n9Zv4xye/yF1/+F1+8ubbfPyiLXzzO19k0wUbHB/LXpB6ScWS7AWpBWEWiKAXC01No+2tW42V9mc/\na9S4NG3dTz1lnO/uHl1p799vmEsWLzYE3UTr0dW2iSnikxEKGfeZP01qasZ6ufzWb8GePeNu7xjo\noCc+umo32x0D0y3wmRuhQIgL12/kob+5jcYnG9lz7W2AO37U9oITm+o2eVpwQpgfiKDPFaJRQ5yb\nmuCWW0b7wBDlBx+E6nSASne3Id4LF44Kunmtne5uGBgwbN8nT8Lu3bnPZ9WqUZPLiy+ONbHU1Iz1\ncgkGcy4MsTiymNpeIyy0bbCN2nCt1e8GvUO9VkrbFbUraOtrA9wp02YPjb90+aUSGi84jgh6vsgU\nWLNafFOTIZKZ1eNNQX/88fGCHo0a4lqdJeKwsRH27h3v6z2VqUQpQ4hNk8vIiOHnbV+lh8Nw/fVw\n/vyol8sll+QU0DMRK6pX0BJrAeBA1wGWVS6z+t0gFAgRCRqBPaurVltttwNjNi/a7OrzhfmJCHou\nmGI6VV8u5+zX2DGvf/xxQ5ynuh+MFfBrrxnXaz3erp254vb7Ddu13w+XXmr02U0uN95orPRPnRp7\nX0mJcf6aa+Dhhw3Tyac/bZy75ZbRF4wDbKzbyJnms/z7Q61wYjunV4e54jeWsrFuqmToM6MyXGnZ\nsD9Q+wGWVrpfCAJgWdUyT8YR5hc5CbpSagfQhOHm+IDW+s4s1/wZ8EkMH7Abtdb7nZxoXnFD0HMd\nNxo1hPvHPzbE9uRJw3VwcHB8xsGSEmNz89prjftuvBF27TL+RKOjxR+6u8ePVV1tzPncufFeLuvW\nwWOPGX1Z7N5OEoqV8vjvj3C2+WGgnIMn+jl/5PfY8Yg7m6J23MjfMhESGi+4wZSCrpTyAXuAq4FW\nYJ9S6gmt9SHbNZ8E1mqtP6iUuhT4AXBZ1gcWMdFolIbJhDwaNUR5/374xS/GnrvzTmPl3N9vrLAf\nfBA2bIBbbx0V5V27Rm3iYBybzz1xAlavNo7Nn5mYq2q7l0tNjeHlYm6gNjRMKNpTfj4H+Ms//Sln\nm7+PPXLzbPP3+cs//TY7L7/O1bFPvXWKLYu3uDpGvvDi3y6fFPvny5VcVujbgMNa65MASqlHgOuA\nQ7ZrrgP+BkBr/apSqkopVa+1bnd6wp5h30g0TRfmCnf16rHmjPQvUnTXLuOXKvOc+Ytmb5tibGIe\nm9dknp+KDRtgR7pk2tatxkrbPm51dXbTyDTMJV78p+k9V0a2yM3e8+7nDN/30j6u+cQ1ro+TD4pd\n8Ir98+VKLoK+DDhtO27GEPnJrmlJ981dQbeLL2QX2Mw+uxBPV5BzmQ8Ym5CZfdXV4zdSM9tz5Jd9\n9cowP6OfsaLez+oVs8mfKwjzA9kUzRcTCez114+PvrRfb19R21fgRcL3vncjr7xyO0eP7saM3Fy7\n9na+972v5nlmglD4KK315BcodRmwS2u9I318K6DtG6NKqR8Az2ut/1/6+BBwZabJRSk1+WCCIAhC\nVrTWUyYYymWFvg9Yp5RaBbQBNwCZeUWfBG4G/l/6BdCdzX6ey4QEQRCEmTGloGutU0qpRuBZRt0W\nDyqlbjJO671a6x8ppT6llDqC4bb4X9ydtiAIgpDJlCYXQRAEYW6Qt3zoSqn/rpQaUUrNtp57QaGU\nukspdVAptV8p9UOl1JxPpaeU2qGUOqSUel8p9a18z8dJlFLLlVI/VUodUEq9rZT63XzPyQ2UUj6l\n1BtKqSfzPRenSbtJ/2P6/92BdCxMUaCU+rpS6h2l1FtKqb9XSk2ayS0vgq6UWg58HMghfd+c41ng\nQq31VuAw8D/zPJ9ZYQss+4/AhcDnlFLO55bNH8PA72mtLwQ+DNxcZJ/P5GvAu/mehEvcA/xIa70R\n+BBwcIrr5wRKqaXAV4GLtda/hGEiv2Gye/K1Qr8b+EaexnYVrfVzWuuR9OErwPJ8zscBrMAyrXUS\nMAPLigKt9RkzTYXWOoYhBkWVaCW9gPoU8H/yPRenSX8DvkJr/dcAWuthrXVvnqflJH6gXCkVAMow\novUnxHNBV0pdC5zWWr/t9dh54DeBp/M9iVmSLbCsqATPRCm1GtgKvJrfmTiOuYAqxg2zNcA5pdRf\np01Ke5VS7if+8QCtdSvwp8ApjGDNbq31c5Pd44qgK6V+krb5mH/eTv+8Fvg2cLv9cjfm4CaTfL5f\ntV3zHSCptX44j1MVckQpFQEeBb6WXqkXBUqpa4D29LcQxRz8/zYFAeBi4F6t9cXAAHBrfqfkDEqp\naoxvw6uApUBEKfX5ye5xJVJUa/3xbP1Kqc3AauAXSimFYY74uVJqm9b6rBtzcYOJPp+JUupGjK+4\nH/VkQu7SAqy0HS9P9xUN6a+zjwJ/q7V+It/zcZjLgWuVUp8CSoEKpdTfaK2/lOd5OUUzxjf+dHVy\nHgWKZeP+Y8AxrXUngFLqn4CPABMuEj01uWit39FaL9Zaf0BrvQbjH+OiuSTmU5FONfwN4FqtdTGU\nc7cCy9I77DdgBJIVE38FvKu1viffE3EarfW3tdYrtdYfwPi3+2kRiTnpAMbTSqkL0l1XUzybv6eA\ny5RS4fQC+Gqm2PDNdy4XTfF9BfxzIAj8xPg34BWt9e/kd0ozZ6LAsjxPyzGUUpcDXwDeVkq9ifE7\n+W2t9TP5nZkwDX4X+HulVAlwjCIJbNRav6aUehR4E0imf+6d7B4JLBIEQSgS8hZYJAiCIDiLCLog\nCEKRIIIuCIJQJIigC4IgFAki6IIgCEWCCLogCEKRIIIuCIJQJIigC4IgFAn/H7Z5EbZTG3PsAAAA\nAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x110df09e8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# How do our probabilistic predictions compare with what the \"right\" answer is?\n",
    "\n",
    "plt.plot(X_test,y_prob_pred, 'r+')\n",
    "plt.plot(x_clean[7000:],y_prob_pred, 'g+', alpha=.1)\n",
    "plt.plot(range(1,6),real_prob_vec,'bo')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}