deep_neural_net.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Deep Neural Net with L2 Regularization\n",
    "\n",
    "Creating a variable-layer neural network from memory as a learning exercise.\n",
    "\n",
    "Uses the cat v non-cat dataset from the [coursera course](https://www.coursera.org/specializations/deep-learning) on Deep Learning by Andrew Ng.\n",
    "\n",
    "Hyperparameters are number of layers and hidden units, lambda for L2 regularisation, learning rate alpha and number of epochs."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Import Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import libraries\n",
    "import numpy as np\n",
    "import h5py\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.datasets import make_circles, make_moons, make_classification\n",
    "import time\n",
    "\n",
    "\n",
    "# load data\n",
    "train_data = h5py.File('C:/Users/leahy/Google Drive/Freelance/own_projects/ml_from_scratch/datasets/train_catvnoncat.h5', 'r')\n",
    "test_data = h5py.File('C:/Users/leahy/Google Drive/Freelance/own_projects/ml_from_scratch/datasets/test_catvnoncat.h5', 'r')\n",
    "X_train = np.array(train_data['train_set_x'])\n",
    "y_train = np.array(train_data['train_set_y'])\n",
    "X_test = np.array(test_data['test_set_x'])\n",
    "y_test = np.array(test_data['test_set_y'])\n",
    "\n",
    "# X_train and X_test contain samples of [m, 64, 64, 3], representing image\n",
    "# hight x width x RGB channels. Reshape to be single vector of size\n",
    "# n * m. Also rescale by 255\n",
    "X_train = X_train.reshape(X_train.shape[0], -1).T / 255\n",
    "X_test = X_test.reshape(X_test.shape[0], -1).T / 255\n",
    "\n",
    "# reshape the y's\n",
    "y_train = y_train.reshape(1, len(y_train))\n",
    "y_test = y_test.reshape(1, len(y_test))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Set up Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sigmoid(Z):\n",
    "    A = 1 / (1 + np.exp(-Z))\n",
    "    return A\n",
    "\n",
    "def initialise_params(layer_dims):\n",
    "    init_params = dict()\n",
    "    L = len(layer_dims)\n",
    "    for i in range(1, L):\n",
    "        cur_n = layer_dims[i]\n",
    "        # set W initial weights to be the square root of nodes in prev. layer\n",
    "        # ie using \"He 2012\" initialisiation\n",
    "        init_params[\"W\" + str(i)] = np.random.randn(\n",
    "            layer_dims[i], layer_dims[i - 1]) *\\\n",
    "                (np.sqrt(2 / layer_dims[i - 1]))\n",
    "        init_params[\"b\" + str(i)] = np.zeros([layer_dims[i], 1])\n",
    "    \n",
    "        assert(init_params['W' + str(i)].shape == (\n",
    "            layer_dims[i], layer_dims[i - 1]))\n",
    "        assert(init_params['b' + str(i)].shape == (layer_dims[i], 1))\n",
    "    return init_params\n",
    "\n",
    "\n",
    "def forward_prop(X, params):\n",
    "    L = len(params) // 2\n",
    "    m = X.shape [1]\n",
    "    preds = {}\n",
    "    preds['A0'] = X\n",
    "    A_prev = X\n",
    "    # For layers 1:L-1 we use relu\n",
    "    for i in range(1, L):\n",
    "        W = params['W' + str(i)]\n",
    "        b = params['b' + str(i)]\n",
    "        Z = np.dot(W, A_prev) + b\n",
    "        A = np.maximum(0, Z)\n",
    "        \n",
    "        preds['Z' + str(i)] = Z\n",
    "        preds['A' + str(i)] = A\n",
    "\n",
    "        assert(preds['Z' + str(i)].shape == (layer_dims[i], m))\n",
    "        assert(preds['A' + str(i)].shape == (layer_dims[i], m))\n",
    "        \n",
    "        A_prev = A\n",
    "\n",
    "    # For final layer use sigmoid\n",
    "    W = params['W' + str(L)]\n",
    "    b = params['b' + str(L)]\n",
    "    Z = np.dot(W, A_prev) + b\n",
    "    A = sigmoid(Z)\n",
    "    \n",
    "    preds['Z' + str(L)] = Z\n",
    "    preds['A' + str(L)] = A\n",
    "    \n",
    "    assert(preds['Z' + str(L)].shape == (layer_dims[L], m))\n",
    "    assert(preds['A' + str(L)].shape == (layer_dims[L], m))\n",
    "    \n",
    "    return preds\n",
    "\n",
    "\n",
    "def compute_cost(AL, y, params, l2_lambda=None):\n",
    "    m = y.shape[1]\n",
    "    L = len(params) // 2\n",
    "        \n",
    "    log_probs = np.multiply(y, -np.log(AL)) + np.multiply(\n",
    "        (1 - y), -np.log(1 - AL))\n",
    "    cost = np.nansum(log_probs) / m\n",
    "\n",
    "    # calc L2 reg cost\n",
    "    reg_term = 0\n",
    "    for i in range(1, L + 1):\n",
    "        W = params['W' + str(i)]\n",
    "        #  we divide by 2m simply by convention so that the derivative\n",
    "        # constant in backprop is lambda/m rather than 2 * lambda/m\n",
    "        reg_term += (l2_lambda / (2 * m)) * (np.linalg.norm(W, ord='fro'))\n",
    "\n",
    "    cost += reg_term\n",
    "\n",
    "    assert cost.shape == ()\n",
    "    \n",
    "    return cost\n",
    "\n",
    "\n",
    "def back_prop(y, preds, params, l2_lambda=None):\n",
    "    L = len(params) // 2\n",
    "    grads = {}\n",
    "    m = y.shape[1]\n",
    "\n",
    "    for i in reversed(range(1, L + 1)):\n",
    "        if i == L:\n",
    "            # for final case, dLdZL = dLdAL*dALdZL = AL - y\n",
    "            AL = preds['A' + str(L)]\n",
    "            dZ = AL - y\n",
    "        else:\n",
    "            # for other cases\n",
    "            # _plus represent values at i + 1\n",
    "            dZ_plus = grads['dZ' + str(i + 1)]\n",
    "            W_plus = params['W' + str(i + 1)]\n",
    "            Z = preds['Z' + str(i)]\n",
    "            # gradient of relu fn\n",
    "            relu_grad = np.zeros(Z.shape)\n",
    "            relu_grad[Z > 0] = 1\n",
    "            dZ = np.dot(W_plus.T, dZ_plus) * relu_grad\n",
    "        \n",
    "        grads['dZ' + str(i)] = dZ\n",
    "        \n",
    "        A_prev = preds['A' + str(i - 1)]\n",
    "        \n",
    "        # including L2 regularization!\n",
    "        W = params['W' + str(i)]\n",
    "        grads['dW' + str(i)] = (np.dot(dZ, A_prev.T) + (l2_lambda * W)) / m\n",
    "        grads['db' + str(i)] = np.sum(dZ, axis=1, keepdims=True) / m\n",
    "        \n",
    "    return grads\n",
    "\n",
    "\n",
    "def update_params(grads, params, alpha):\n",
    "    L = len(params) // 2\n",
    "    for i in range(1, L + 1):\n",
    "        params['W' + str(i)] -= (alpha * grads['dW' + str(i)])\n",
    "        params['b' + str(i)] -= (alpha * grads['db' + str(i)])\n",
    "        \n",
    "    return params\n",
    "\n",
    "\n",
    "def fit(X, y, layer_dims, num_iterations, alpha, l2_lambda=None,\n",
    "        cost_interval=200):\n",
    "    params = initialise_params(layer_dims)\n",
    "    L = len(params) // 2\n",
    "    costs = []\n",
    "    for i in range(num_iterations):\n",
    "        preds = forward_prop(X, params)\n",
    "        J = compute_cost(preds['A' + str(L)], y, params, l2_lambda)\n",
    "        if i % cost_interval == 0:\n",
    "            print(f'cost after iter {i}: {J}')\n",
    "        if i % 100 == 0:\n",
    "            costs += [J]\n",
    "        grads = back_prop(y, preds, params, l2_lambda)\n",
    "        params = update_params(grads, params, alpha)\n",
    "        \n",
    "    plt.plot(costs)\n",
    "    plt.show()\n",
    "    return params, costs\n",
    "\n",
    "def predict(X, params):\n",
    "    L = len(params) // 2\n",
    "    preds = forward_prop(X, params)\n",
    "    y_pred = preds['A' + str(L)]\n",
    "    y_pred[y_pred > 0.5] = 1\n",
    "    y_pred[y_pred <= 0.5] = 0\n",
    "    return y_pred\n",
    "\n",
    "\n",
    "def accuracy_score(y_test, y_pred):\n",
    "    m = y_test.shape[1]\n",
    "    score = np.sum(y_test == y_pred) / m\n",
    "    print(f'accuracy: {score}')\n",
    "    return score\n",
    "        "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Fit, Predict, Score\n",
    "\n",
    "Let's see effects of regularisation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "lambda=0\n",
      "cost after iter 0: 0.7095350929383001\n",
      "cost after iter 200: 0.5531215775179099\n",
      "cost after iter 400: 0.20616499168258476\n",
      "cost after iter 600: 0.3972160327329145\n",
      "cost after iter 800: 0.04453866422900078\n",
      "cost after iter 1000: 0.019518166851579973\n",
      "cost after iter 1200: 0.011426204115048692\n",
      "cost after iter 1400: 0.00766872922979565\n",
      "cost after iter 1600: 0.005609708390514454\n",
      "cost after iter 1800: 0.0043353010112505185\n",
      "cost after iter 2000: 0.003490820023837182\n",
      "cost after iter 2200: 0.002898564319892692\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3de3Scd33n8fd3ZnSxLjOOrbEtyXZsObJjxYakEU42FEoK5JhbAlsoztLTZBeatltDC3TPpl1IOaHdQ2GBXkgpgbKEbsFNoSUuNQ0hpKVtSLAIgUhyfI0dW4pk2Y51sazLaL77x8zIY1myx9JcPDOf1zk6M88zv3nmm4ny0ZPf7/f8HnN3RESk+AUKXYCIiGSHAl1EpEQo0EVESoQCXUSkRCjQRURKRKhQH9zQ0OBr1qwp1MeLiBSlH//4xyfcPTrbawUL9DVr1tDR0VGojxcRKUpmdmSu19TlIiJSIhToIiIlQoEuIlIiFOgiIiVCgS4iUiIU6CIiJUKBLiJSIoou0DsOn+KP//l5tOyviMj5Mgp0M9tqZnvN7ICZ3TvL6581s2eTP/vM7HT2S03o7Bnk8/9ykL6hsVx9hIhIUbrklaJmFgQeAN4IHAN2m9lOd+9OtXH3D6a1fz9wQw5qBWDzyggAnT1DNEYW5epjRESKTiZn6FuAA+5+yN0ngB3AHRdpfyfw9WwUN5uNjWECBs/1DObqI0REilImgd4MHE3bPpbcdwEzuxpYC3x/jtfvMbMOM+sYGBi43FoBqKkMsS5aR6cCXUTkPJkEus2yb64RyW3AN9x9arYX3f1Bd2939/ZodNbFwjKyuTmiM3QRkRkyCfRjwKq07ZVA7xxtt5HD7paUTc0RBobHOa6BURGRaZkE+m6g1czWmlklidDeObORmW0ArgJ+mN0SL5QaGNVZuojIOZcMdHePAduBR4E9wMPu3mVm95vZ7WlN7wR2eB4miLc1hjENjIqInCejG1y4+y5g14x9983Y/lj2yrq42qoQLQ21GhgVEUlTdFeKpmxujtDZM1ToMkRErhhFG+ibmiP0DY0xMDxe6FJERK4IRR3ogLpdRESSijbQr2sKAxoYFRFJKdpAr6+u0MCoiEiaog10SHS7KNBFRBKKPNDD9A6OcXJEA6MiIkUe6LpiVEQkpSQCXd0uIiJFHujh6grWLK3RBUYiIhR5oANcp6V0RUSAEgj0zc0Rek6f5eUzE4UuRUSkoEoi0EEDoyIiRR/om5qSA6O9CnQRKW9FH+iRmgpWLVmkmS4iUvaKPtBB9xgVEYESCfRNzRGOnjrL6VENjIpI+SqJQE8NjHb1aj66iJSvkgj01MCoul1EpJxlFOhmttXM9prZATO7d442v2xm3WbWZWZfy26ZF3dVbSXNixcp0EWkrF3yJtFmFgQeAN4IHAN2m9lOd+9Oa9MK/B7wand/2cyW5arguWzWUroiUuYyOUPfAhxw90PuPgHsAO6Y0ebXgAfc/WUAdz+e3TIvbfPKCEdOjjJ4djLfHy0ickXIJNCbgaNp28eS+9KtB9ab2X+Y2VNmtnW2A5nZPWbWYWYdAwMD86t4Dqlb0nXpAiMRKVOZBLrNss9nbIeAVuB1wJ3Al8xs8QVvcn/Q3dvdvT0ajV5urRe1WUvpikiZyyTQjwGr0rZXAr2ztHnE3Sfd/QVgL4mAz5uldVU0Rap5TkvpikiZyiTQdwOtZrbWzCqBbcDOGW2+BdwKYGYNJLpgDmWz0Exsao7QpTN0ESlTlwx0d48B24FHgT3Aw+7eZWb3m9ntyWaPAifNrBt4Avgf7n4yV0XPZVNzhEMnzjA8poFRESk/l5y2CODuu4BdM/bdl/bcgQ8lfwom/YrRm1uWFrIUEZG8K4krRVN0j1ERKWclFejR+ipWhKsV6CJSlkoq0AE2NYe1BICIlKUSDPTEwOjIeKzQpYiI5FXJBfrm5gju0K2ldEWkzJRkoIOW0hWR8lNygb4sXE20vmrBFxjFpuL8wSOdPHUo79PpRUTmpeQCHbJzj9Edu4/y0A+PcO83f8bkVDxLlYmI5E5JBvqm5ggHB0YYnZjfwOjg2Uk+89g+VoSrOXxylL9/5liWKxQRyb6SDPTNzRHiCxgY/bPH9/Py6ARfuqud61ct5s8eP8B4bCrLVYqIZFdJBvqm5sTa6PO5wOjA8REeevIw2161ik3NEX73tg30nD7L159+MdtliohkVUkG+opwNQ11lfNaSveP/qmbRRVBPnzbBgBefc1Sbm5ZwueeOMjZCZ2li8iVqyQD3czYNI97jP7L3uM8sXeA97/+GhrqqqaP9bu3beDEyDgP/fBw9osVEcmSkgx0SPSj7z8+nPFZ9eRUnI9/u5s1S2u4+5a1573WvmYJr9sQ5S//9aCW5hWRK1bJBvp1TYmB0T19mXW7/L+njnBw4AwfeUsblaELv5YPv3EDp0cn+at/fyHbpYqIZEXJBvrmlZkvpXvqzASffWwfr2lt4PUbl815vK3XreBL//YCL5+ZyGqtIiLZULKB3hSpZkltJc8du3Sgf/axfZyZmOKjb23DbLZ7Yid86Lb1nJmI8YUf5P3ueiIil1SygZ4aGL3UFaN7+4b5m6eP8J6bVrN+ef1F265fXs8dr2ziK0++wPHhsWyWKyKyYCUb6ACbmsLsPz7C2OTsA6Puzse/3U19dQUffMP6jI75O29Yz+SU8xdPHMxmqSIiC5ZRoJvZVjPba2YHzOzeWV6/28wGzOzZ5M/7sl/q5dvcHGEq7jzfNzzr69/bc5x/P3CCD76hlatqKzM65pqGWt5140q+9vSL9Jw+m81yRUQW5JKBbmZB4AHgTUAbcKeZtc3S9G/d/frkz5eyXOe8bLrIUrrjsSn+8J+6uWZZHe+5+erLOu77X98KwJ8/vn/hRYqIZEkmZ+hbgAPufsjdJ4AdwB25LSs7Vl61iMU1FXTOMjD6lf84zJGTo3z0rW1UBC+v56l58SL+y02r+bsfH+PwiTPZKldEZEEySbJm4Gja9rHkvpl+ycx+ZmbfMLNVsx3IzO4xsw4z6xgYGJhHuZfHzNjUFKGz9/xAHxge58+/f4BfvHYZv7A+Oq9j//db11ERNP7ke/uyUaqIyIJlEuizzePzGdv/CKxx91cA3wMemu1A7v6gu7e7e3s0Or8gvVybmiPs6x8+b7XET393L2OTU3zkLRvnfdxl9dXcfctaHvlpL/v6Z++jFxHJp0wC/RiQfsa9EuhNb+DuJ919PLn5ReDG7JS3cJubI0xOOXuTA6OdPYP8bcdR7r5lDS3RugUd+9df20JdZYjPfFdn6SJSeJkE+m6g1czWmlklsA3Ymd7AzBrTNm8H9mSvxIVJv8eou3P/P3ZzVU3l9MDmQlxVW8l7X7OWf+7qy+gCJhGRXLpkoLt7DNgOPEoiqB929y4zu9/Mbk82+4CZdZnZT4EPAHfnquDLtWrJIsLVITp7htj1XB8/OnyKD9+2nsiiiqwc/70/v5bFNRV8+rG9WTmeiMh8hTJp5O67gF0z9t2X9vz3gN/LbmnZkbpi9JkjL/ODfQNcu6Keba9anbXj11dX8Bu/sI5PfOd5Og6fon3NkqwdW0TkcpT0laIpm5sj7O0fpuf0We57WxvBwNzrtczHXf9pDQ11VXzq0b24zxwvFhHJj7II9NQFRluvW8Et6xqyfvxFlUG237qOp184xX8cOJn144uIZKIsAv21rVHefn0TH33bbBe4ZsedN62mKVLNp76rs3QRKYyyCPRITQV/su0GmhcvytlnVIWCfOD1rfz06Gke33M8Z58jIjKXsgj0fPmlG1eyekkN//dJ3dVIRPJPgZ5FFcEAP9/awHPHBtXtIiJ5p0DPsrbGMENjMXoHdQMMEckvBXqWbWwMA9Ddm9nNqUVEskWBnmXXrqjHDPa8pEAXkfxSoGdZbVWINUtrdYYuInmnQM+BjY317OlToItIfinQc6CtMcyRk6OMjMcKXYqIlBEFeg6kBkafVz+6iOSRAj0HUoGugVERyScFeg40RqpZXFNBtwJdRPJIgZ4DZsbGFWG6X9K9RkUkfxToOdLWFGZv3xBTcS0BICL5oUDPkY2NYcYm47xw4kyhSxGRMqFAz5GNjfWABkZFJH8yCnQz22pme83sgJnde5F27zQzN7P27JVYnFqX1VMRNA2MikjeXDLQzSwIPAC8CWgD7jSzC279Y2b1wAeAp7NdZDGqDAVYF63TGbqI5E0mZ+hbgAPufsjdJ4AdwB2ztPs48ElA68YmtTWFr7hAHzw7SVwDtSIlKZNAbwaOpm0fS+6bZmY3AKvc/dsXO5CZ3WNmHWbWMTAwcNnFFpu2xjD9Q+OcHBkvdCkADI1N8upPfJ9vPHOs0KWISA5kEug2y77pUzwzCwCfBT58qQO5+4Pu3u7u7dFoNPMqi9S5K0avjPnoXT1DjIzH+OnR04UuRURyIJNAPwasStteCfSmbdcDm4B/MbPDwM3ATg2Mpt3s4qXBAleS0NWbqOPQgKZSipSiTAJ9N9BqZmvNrBLYBuxMvejug+7e4O5r3H0N8BRwu7t35KTiIrKktpIV4eor5gy9sycZ6CdGClyJiOTCJQPd3WPAduBRYA/wsLt3mdn9ZnZ7rgssdhsb66+YgdHO5E03+ofGtbSvSAkKZdLI3XcBu2bsu2+Otq9beFmlo60pzL/tP8F4bIqqULBgdYxOxDg4MMK1K+p5vm+YFwbOsHllpGD1iEj26UrRHNvYGCYWd/b3F7abY89Lw7jD7dc3Aep2ESlFCvQca5seGC1st0tqQPQtmxsJGBzUwKhIyVGg59jVS2tZVBEseD96Z88gS2srWb2khpVX1XBoQGfoIqVGgZ5jwYCxYUXhB0Y7e4a4rjmCmdESrdXURZESpEDPg7amMN29Q7gX5pL78dgU+48Pc11TovunpaGOF06c0RIAIiVGgZ4HGxvDDI3F6B0szDI3+/tHmJxyNjUlZrW0RGs5OzlF35CW3REpJQr0PJgeGO0tTLdL6oKiTc3JM/RoLaArRkVKjQI9D65dUY9Z4W520dk7SH11iNVLagBYF60D4KAGRkVKigI9D2qrQly9pKZggd7VO0RbYxizxDpry+qrqK0MaqaLSIlRoOdJW1O4IHPRY1Nx9rw0xKbmc1eFJma61HFI9zsVKSkK9DzZuCLMkZOjeV9D5dCJM4xNxqf7z1M0dVGk9CjQ86QtOWXw+TyfpU8PiDadv25LS0MdPafPcnZiKq/1iEjuKNDz5NzNLvIb6F29Q1RXBGhJDoSmpGa6vKBuF5GSoUDPk8ZINZFFFXTneW30zp5BNjaGCQbOv/HU9NRFLdIlUjIU6HliZrQ15ndgNB53unuHLuhuAVjboLnoIqVGgZ5HGxvD7O0bYipPl9y/eGqU4fHY9CX/6WoqQzRFqjV1UaSEKNDzqK0pzNhkPG/91l3JK1PTpyym09RFkdKiQM+jjY31QP4GRjt7B6kIGq3L62Z9PTV1sVCLholIdinQ8+iaZXWEApa/QO8ZZP3y+jlvfdfSUMvIeIyB4fG81CMiuZVRoJvZVjPba2YHzOzeWV7/DTN7zsyeNbN/N7O27Jda/KpCQa5ZVpeXgVF3p6t3aNb+85SW6TVd1O0iUgouGehmFgQeAN4EtAF3zhLYX3P3ze5+PfBJ4DNZr7REtDWG83KG3jc0xqkzE3P2n4OmLoqUmkzO0LcAB9z9kLtPADuAO9IbuHt6QtUC6pSdw8bGMP1D45wcyW03R2dP4l/JdbNMWUxpiiyiuiKgqYsiJSKTQG8GjqZtH0vuO4+Z/ZaZHSRxhv6B2Q5kZveYWYeZdQwMDMyn3qKXWgJgT44vMOrsGSRg5wZiZxMIGGuW1mrqokiJyCTQbZZ9F5yBu/sD7r4O+J/AR2Y7kLs/6O7t7t4ejUYvr9ISka8lALp6B2mJ1lFTGbpou3WauihSMjIJ9GPAqrTtlUDvRdrvAN6+kKJK2ZLaSlaEq3M+MNrZM8SmiwyIprREazl6apTxmBbpEil2mQT6bqDVzNaaWSWwDdiZ3sDMWtM23wLsz16JpWdjY31Oz9BPjIzTNzR20QHRlJZoLXGHF0+O5qweEcmPSwa6u8eA7cCjwB7gYXfvMrP7zez2ZLPtZtZlZs8CHwLuylnFJWBjY5gDx0dydlacukL0YgOiKS0NmrooUiou3sGa5O67gF0z9t2X9vy3s1xXSWtrChOLO/v7RzI6i75cqTXQ2zLscgFNXRQpBbpStAByPTDa1TvI6iU1RBZVXLJtfXUF0foqTV0UKQEK9AJYs7SWRRXBnA2MdvUOXXDLuYtpadDURZFSoEAvgGDA2LAiNwOjg2cnOXJyNKP+85R1yzR1UaQUKNALZGNjmO7eoayvdNg9PSB6eWfop0cnOXVmIqu1iEh+KdALpK0pzNBYjN7Bsawet6s3MSB6WWfoyUW61O0iUtwU6AXSllobvTe73S5dvUOsCFcTra/K+D3TM100MCpS1BToBbJhRRgzsj4w2tkzeFkDogArr6qhMhjgoKYuihQ1BXqB1FWFuHpJTVYHRkcnYhwcGKHtMrpbIDFIe/XSGp2hixQ5BXoBbWwMZ/UMfc9Lw8SdjNZwmSlxOzqdoYsUMwV6AbU1hjlycpSR8VhWjtedHBCdz9WnLdE6Xjw1SmwqnpVaRCT/FOgFlLpidG9fds7SO3uGWFJbSWOk+rLf29JQy+SUc/Tls1mpRUTyT4FeQKm1VrqzNNOls3eQ65rCmM22hP3FtWjqokjRU6AXUGOkmsiiCrqzcPeiiVicff3DlzX/PN06TV0UKXoK9AIyMzY21mdlYHRf/zCTU37ZUxZTFtdUsqS2UqsuihQxBXqBtTVG2Ns3xFR8YUsApK4Q3TTPM3RI9KNrXXSR4qVAL7CNjfWMTcY5fHJhQdrZM0RdVYjVS2rmfYzE1EUFukixUqAXWGqmy0IHRjt7B2lrChMIXP6AaEpLtI4TI+MMjU0uqBYRKQwFeoG1Lq8jFLAFXTE6FXf2vDS0oO4WSHS5gAZGRYqVAr3AqkJBrllWt6CB0UMDI4xNxuc9IJqiqYsixS2jQDezrWa218wOmNm9s7z+ITPrNrOfmdnjZnZ19kstXW2NYTp7hhibnN9NozvnsWTubFYvqSEYMJ2hixSpSwa6mQWBB4A3AW3AnWbWNqPZT4B2d38F8A3gk9kutJS99ZWNnBgZ54N/++y8Zrt09gxRFQpMzyWfr8pQgNVLajR1UaRIZXKGvgU44O6H3H0C2AHckd7A3Z9w99Hk5lPAyuyWWdp+8drlfOQtG/lOZx8f29l12Xcx6uodZGNjmFBw4T1oifuL6gxdpBhlkgDNwNG07WPJfXN5L/Cd2V4ws3vMrMPMOgYGBjKvsgy87zUt3PPaFv76qSN87vsHMn5fPO509VzeTaEvpiVaywsnzix4XryI5F8mgT7bPLhZ/2s3s18B2oFPzfa6uz/o7u3u3h6NRjOvskzcu/Va3nFDM59+bB87fvRiRu85+vIow+OxBfefp7RE6xiPxek9rUW6RIpNKIM2x4BVadsrgd6ZjczsDcD/An7B3cezU155CQSMT77zFZw8M8Hv/8NzLK2r4o1tyy/6ns6exOyYhU5ZTElNXTw4MMKqBVykJCL5l8kZ+m6g1czWmlklsA3Ymd7AzG4AvgDc7u7Hs19m+agIBvj8e36Ozc0Rtn/tGToOn7po+87eQUIBY/2Kuqx8/rmpi+pHFyk2lwx0d48B24FHgT3Aw+7eZWb3m9ntyWafAuqAvzOzZ81s5xyHkwzUVoX48t2vomnxIt77UAf7++dejbGrd4jW5fVUhYJZ+eyGukrqq0Oa6SJShDKaFuHuu9x9vbuvc/c/Su67z913Jp+/wd2Xu/v1yZ/bL35EuZSldVV89b9toTIU4Fe//CNeGrywT9vd6eoZnNct5+ZiZrRE63SGLlKEdKXoFWzVkhq+8l9fxfBYjLu+/CMGR89fY6VvaIyTZybmdcu5i1mnqYsiRUmBfoW7rinCg796I4dPjPK+r+4+72rS6QHRLE1ZTGmJ1tI3NMaZLN3rVETyQ4FeBG5Z18Bn3309HUde5v1f/8n0jZy7egcxg2tXZDvQEwOjL5zQWbpIMVGgF4m3vKKRj73tOh7r7uejj3Ti7nT2DNHSUEttVSazTzPXEj03dVFEikd2k0By6q5b1nB8eIwHnjhItL6art5BtqxdkvXPWbO0FjNNXRQpNgr0IvO7t21gYHicP3t8P5C9C4rSVVcEaV68iEPqchEpKupyKTJmxv9+x2Zef+0ygKzPcElJTF1Ul4tIMdEZehEKBQM88J6f4/vPH+emHHS5QGIJgI7Dp3B3zOZ/WzsRyR+doRep6oogb97cuKB7iF7MumV1jE5M0Tc0lpPji0j2KdBlVut0f1GRoqNAl1np/qIixUeBLrNaHq6itjLIQZ2hixQNBbrMysxYG63V1EWRIqJAlzm1NGjqokgxUaDLnFqitfScPnvegmAicuVSoMucWqJ1uMPhk+p2ESkGCnSZU4umLooUFQW6zCm16qL60UWKgwJd5lRTGaIxUq0zdJEikVGgm9lWM9trZgfM7N5ZXn+tmT1jZjEze2f2y5RCaYnWclBTF0WKwiUD3cyCwAPAm4A24E4za5vR7EXgbuBr2S5QCis1ddHdC12KiFxCJmfoW4AD7n7I3SeAHcAd6Q3c/bC7/wyI56BGKaCWaC3DYzFOjEwUuhQRuYRMAr0ZOJq2fSy577KZ2T1m1mFmHQMDA/M5hOSZ1nQRKR6ZBPps67PO6/+/3f1Bd2939/ZoNDqfQ0ieTU9dVD+6yBUvk0A/BqxK214J9OamHLnSNC9eRFUooDN0kSKQSaDvBlrNbK2ZVQLbgJ25LUuuFIGAsak5wo7dR3nq0MlClyMiF3HJQHf3GLAdeBTYAzzs7l1mdr+Z3Q5gZq8ys2PAu4AvmFlXLouW/PrTbdezrL6KX/2rH7HruZcKXY6IzMEKNR2tvb3dOzo6CvLZcvlOj07w3oc6eObFl/mDt7Zx96vXFrokkbJkZj929/bZXtOVopKRxTWV/M37buKNG5fzsX/s5hPfeZ54XHPTRa4kCnTJWHVFkM//yo2856bV/OW/HuTDf/dTJmK69EDkShEqdAFSXIIB4w/fvonGSDX/57v7ODEyzud/5UbqqvSrJFJoOkOXy2ZmbP/FVj75zlfw5MGTvPsLP+T48FihyxIpewp0mbdfbl/Fl+5q59DAGf7zXzypueoiBaZAlwW5dcMydtxzM2cnpvilzz/JT158udAliZQtBbos2CtXLeabv3kL4UUV3PnFp3h8T3+hSxIpSwp0yYo1DbV88zdvYf3yen7tqx3s+NGLhS5JpOwo0CVrGuqq+Pqv3cxrWqPc+/fP8Yff7qbn9NlClyVSNnSlqGTd5FScj36rkx27E6sub1m7hHfc0MybNzUSqakocHUixe1iV4oq0CVnjp4a5ZFne/iHn/RwcOAMlcEAt14b5R03NHPrtcuoCgULXaJI0VGgS0G5O509Q3zr2R52/rSXgeFxwtUh3ry5kbff0MyWNUsIBGZbdl9EZlKgyxUjNhXnyYMn+dazPfxzZx+jE1M0Raq544Zm3n59MxtW1Be6RJErmgJdrkijEzEe6+7nWz/p4Qf7TzAVd9Yvr2NTc4QNy+tZv6KeDcvraYxUY6YzeBFQoEsRODEyzj/97CW+t6efvX3DHB8en36tvipE6/I61i+vZ/3yejasqKd1eR3RuioFvZQdBboUndOjE+zrH2Ff/zD7+ofZ25d4fHl0crrNVTUV0yHffNUiloerWB6uZnm4mhXhamq1YJiUoIsFun7j5Yq0uKaSLWuXsGXtkul97s6JkQn29w+zNxn0+/pHeOTZHobGYhcco64qNB3yK8LVLI9Us7y+ihWRapaFq1laW0lkUQX11RUENSgrJUCBLkXDzIjWVxGtr+KWaxrOe+3MeIz+oTH6hsboHxqjf2icvsExjg+P0Tc4xtMvnOL48BiTUxf+H6lZolsnUlNBZFH6T+WM7QrCi0LUVIaorQpSUxGipipIbWWI6oqAun+k4BToUhJqq0K0ROtoidbN2SYed06NTiQDf4xTZyYZPJv8GZ049/zsJH2DY9PPZ/sjMJMZ1FaGqKkMUluVeEz8JJ5XVwSpCgWmH6tmbF/weihAZeonGKAimHheEUxsJ54bwYDpD4lMyyjQzWwr8KdAEPiSu39ixutVwFeBG4GTwLvd/XB2SxVZmEDAaKiroqGuiuuaIhm9x905Ozk1He5DZ2OMTsQ4OzHFmYkpRidinBlPPI7Osn367CS9p88yHoszNjl13mM2mJEI+GCAilCAUMCoCAYIBe2858FAgIqAEQom9wWM0IzHYMAImhFMvjcYSDwGAqnttHbJtoGAEbTEjU8CaftS7QJ2/mMwkPg/raAl9gUCpLVJPE9tW+q4lnjN7NzzgCVeD9jM1xPvMQwLcO41zrU/9z5K7o/hJQPdzILAA8AbgWPAbjPb6e7dac3eC7zs7teY2Tbgj4F356JgkXwys+RZdojGyKKsHdfdGY/FEz8zgn5scoqxyTiTU3EmppKPsXOPE1N+/r7U/licqbgzOeXE4nFiaY+TcSc2FScWd86Mx4gl200lX59yTzzGnVg8uT/uxKe3E4+l6LyATwb/dPiTeGTGHwMj8buR2E57X9r+C9omP8vM+O3Xt/K2VzZl/Z8lkzP0LcABdz+U+Ie3HcAdQHqg3wF8LPn8G8DnzMy8UFNoRK5wZkZ1RaIrhkXFsb6NuxN3iMUTfzim4k48DlOefJ58nP7xxB+E6dfjJNq44+5MJbfj8cRxp9zP304e0z3Z7rznJLfPPU9/nyfrTW/vJLeTr0/vT7abSj6m2vn064m2wHk1zPyMxHuZfq+nfWfTbZOPkRz9O88k0JuBo2nbx4Cb5mrj7jEzGwSWAifSG5nZPcA9AKtXr55nySJSCImuEggGtAbPlSqT5XNn62SaeeadSRvc/UF3b3f39mg0mkl9IiKSoUwC/RiwKm17JdA7VxszCwER4FQ2ChQRkcxkEui7gVYzW2tmlcA2YOeMNjuBu5LP3wl8X/3nIiL5dck+9MMJxPoAAAMZSURBVGSf+HbgURLTFr/s7l1mdj/Q4e47gb8C/trMDpA4M9+Wy6JFRORCGc1Dd/ddwK4Z++5Lez4GvCu7pYmIyOXQPUVFREqEAl1EpEQo0EVESkTB1kM3swHgyDzf3sCMi5bKlL6Hc/RdJOh7SCjl7+Fqd5/1Qp6CBfpCmFnHXAu8lxN9D+fou0jQ95BQrt+DulxEREqEAl1EpEQUa6A/WOgCrhD6Hs7Rd5Gg7yGhLL+HouxDFxGRCxXrGbqIiMygQBcRKRFFF+hmttXM9prZATO7t9D1FIqZHTaz58zsWTPrKHQ9+WJmXzaz42bWmbZviZk9Zmb7k49XFbLGfJnju/iYmfUkfy+eNbM3F7LGXDOzVWb2hJntMbMuM/vt5P6y/J0oqkBPu7/pm4A24E4zaytsVQV1q7tfX2bzbb8CbJ2x717gcXdvBR5PbpeDr3DhdwHw2eTvxfXJhfVKWQz4sLtvBG4GfiuZCWX5O1FUgU7a/U3dfQJI3d9UyoS7/4ALb55yB/BQ8vlDwNvzWlSBzPFdlBV3f8ndn0k+Hwb2kLglZln+ThRboM92f9PmAtVSaA5818x+nLxXazlb7u4vQeI/cGBZgesptO1m9rNkl0xZdDUAmNka4Abgacr0d6LYAj2je5eWiVe7+8+R6H76LTN7baELkivC54F1wPXAS8CnC1tOfphZHfBN4HfcfajQ9RRKsQV6Jvc3LQvu3pt8PA78A4nuqHLVb2aNAMnH4wWup2Dcvd/dp9w9DnyRMvi9MLMKEmH+N+7+98ndZfk7UWyBnsn9TUuemdWaWX3qOXAb0Hnxd5W09Hva3gU8UsBaCioVYknvoMR/L8zMSNwCc4+7fybtpbL8nSi6K0WT07D+hHP3N/2jApeUd2bWQuKsHBK3EfxauXwPZvZ14HUklkftB/4A+BbwMLAaeBF4l7uX/GDhHN/F60h0tzhwGPj1VF9yKTKznwf+DXgOiCd3/z6JfvTy+50otkAXEZHZFVuXi4iIzEGBLiJSIhToIiIlQoEuIlIiFOgiIiVCgS4iUiIU6CIiJeL/Azw1DipAmuErAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train set:\n",
      "accuracy: 1.0\n",
      "Test Set:\n",
      "accuracy: 0.72\n",
      "took 42.47967529296875 seconds\n",
      "\n",
      "\n",
      "lambda=0.001\n",
      "cost after iter 0: 0.7167000400948975\n",
      "cost after iter 200: 0.542840414784643\n",
      "cost after iter 400: 0.4283448245457164\n",
      "cost after iter 600: 0.3103674353245235\n",
      "cost after iter 800: 0.12612176897562521\n",
      "cost after iter 1000: 0.049329710445128586\n",
      "cost after iter 1200: 0.02068344396949862\n",
      "cost after iter 1400: 0.01204235353584038\n",
      "cost after iter 1600: 0.0080872037882747\n",
      "cost after iter 1800: 0.005902712843186399\n",
      "cost after iter 2000: 0.004562085184142841\n",
      "cost after iter 2200: 0.0036763618888465627\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAf6klEQVR4nO3deXiU5b3/8fd3JjtCAAkIIQFksYRVCGirP5efG7hA3UFrq13oIq1tra1tT621p6cubbW2dKF1qT1aiktbPIdK69IWq9YEhCqbBEQTQAgIQSQhy3zPHzPBMSQywIQnM/N5XVeumed57nnmew2TT27uZ7nN3RERkdQXCroAERFJDgW6iEiaUKCLiKQJBbqISJpQoIuIpImsoN64T58+Pnjw4KDeXkQkJS1ZsmSbuxe1ty2wQB88eDCVlZVBvb2ISEoys9c72qYhFxGRNKFAFxFJEwp0EZE0oUAXEUkTCnQRkTShQBcRSRMKdBGRNJFygf7Kxjpue2I1uu2viMh7pVygL31jBz//2zqeX7c96FJERLqUlAv0y8pLOKZHHnc9tVa9dBGROAkFuplNMbM1ZlZlZje2s/1OM1sW+3nVzHYmv9SovOwwnz1tKC++9hbPr1cvXUSk1QED3czCwBxgKlAGzDSzsvg27v4ldx/v7uOBnwCPdUaxrS6fVEK/Hrnc9eTaznwbEZGUkkgPfTJQ5e7r3b0RmAdMf5/2M4HfJaO4juRlh/ncacOivXSNpYuIAIkFejFQHbdcE1u3HzMbBAwBnu5g+ywzqzSzytra2oOt9T3e7aW/elj7ERFJF4kEurWzrqOjkTOAR9y9pb2N7j7X3cvdvbyoqN3b+SYsLzvMZ08dyr/USxcRARIL9BqgJG55ILCpg7Yz6OThlve82eRS+nbP5cdPqZcuIpJIoFcAw81siJnlEA3tBW0bmdlxQC/g+eSW2LHWM15eWK9euojIAQPd3ZuB2cAiYBUw391XmNktZjYtrulMYJ4f4ZPDZ6qXLiICJDgFnbsvBBa2WXdTm+Wbk1dW4lp76d95fCUvrN/OicceHUQZIiKBS7krRdszc3IpRd1z+bHOSxeRDJYWgd56xsvz67fzL109KiIZKi0CHeCKE2K99KfUSxeRzJQ2gZ6XHeYzpw7luXXqpYtIZkqbQAe4Ur10EclgaRXo8b30F197K+hyRESOqLQKdIj20vscpfPSRSTzpF2gR3vpx/LPqu1UbFAvXUQyR9oFOsCVJwyK9tJ1XrqIZJC0DPT8nGgv/dmqbeqli0jGSMtAh9Zeeo566SKSMdI20PNzwnz6lKE8W7WNSvXSRSQDpG2gA1x5Ymm0l67z0kUkA6R1oBfkZPHpU4ayeO02lryuXrqIpLe0DnSI9tKP7pbDXRpLF5E0l/aBXpCTxadPPZbFa7fxgu7xIiJpLO0DHeCqEwdT3DOfm/70Ck0tkaDLERHpFBkR6Pk5YW6eNopXt+zm3mdfC7ocEZFOkVCgm9kUM1tjZlVmdmMHbS4zs5VmtsLMHkpumYfvrLJ+nDmyH3c9uZZNO+uDLkdEJOkOGOhmFgbmAFOBMmCmmZW1aTMc+DpwkruPAr7YCbUetpunleE433l8RdCliIgkXSI99MlAlbuvd/dGYB4wvU2bTwFz3H0HgLtvTW6ZyTGwVwFfOGM4i1Zs4enVW4IuR0QkqRIJ9GKgOm65JrYu3ghghJn908xeMLMpySow2T558rEM63sU316wgvrGlqDLERFJmkQC3dpZ522Ws4DhwGnATODXZtZzvx2ZzTKzSjOrrK2tPdhakyInK8R3p4+m+q165jxTFUgNIiKdIZFArwFK4pYHApvaafMnd29y99eANUQD/j3cfa67l7t7eVFR0aHWfNg+OPRoLjq+mF/+Yx3rancHVoeISDIlEugVwHAzG2JmOcAMYEGbNn8ETgcwsz5Eh2DWJ7PQZPv6uSPJzw7zrT++gnvb/3CIiKSeAwa6uzcDs4FFwCpgvruvMLNbzGxarNkiYLuZrQSeAW5w9y59WWZR91xumPIBnlu3nQXL2/6HQ0Qk9VhQvdPy8nKvrKwM5L1btUScC3/2TzbXNfDU9afSIy870HpERA7EzJa4e3l72zLiStGOhEPG9z48hm279/Kjv2hSaRFJbRkd6ABjBhZy1YmDeOD5DbyysS7ockREDlnGBzrA9WcfR+9uuXzzDy/TEtEBUhFJTQp0oDA/m/84byTLa+r43YtvBF2OiMghUaDHTB8/gA8NPZrbn1hN7dt7gy5HROSgKdBjzIxbpo+mvqmF7/95VdDliIgcNAV6nGF9j2LWKcfy2NKNmt1IRFKOAr2N2acPZ2CvfL71x1dobNbsRiKSOhTobeTnhPnOtFGs3bqbezS7kYikEAV6O84Y2Y+zy/px91NrqdmxJ+hyREQSokDvwLenjcIMvjhvmYZeRCQlKNA7UNwzn9suHkvl6zu4WVPWiUgKyAq6gK7sgnEDWLFpF7/4+zpGDejBlScMCrokEZEOqYd+ADeccxynjiji5gUrqNzwVtDliIh0SIF+AOGQcfeM4ynumc9n/nspm+vqgy5JRKRdCvQEFBZkM/ej5dQ3NvOZ3y6hoUmTS4tI16NAT9CIft350eXjWV5Txzf/oGnrRKTrUaAfhHNGHcN1Zwzn0aU13P/chqDLERF5DwX6QbrujOGcVdaP//zfVTy3blvQ5YiI7JNQoJvZFDNbY2ZVZnZjO9uvNrNaM1sW+/lk8kvtGkIh40eXjWNIn25c++BSqt/SlaQi0jUcMNDNLAzMAaYCZcBMMytrp+nv3X187OfXSa6zS+mel83cqybSHHE+/dsl1DfqIKmIBC+RHvpkoMrd17t7IzAPmN65ZXV9xxYdxd0zjmfVm7v46qP/1kFSEQlcIoFeDFTHLdfE1rV1sZn928weMbOS9nZkZrPMrNLMKmtraw+h3K7l9A/05YZzjuPx5ZuY+4/1QZcjIhkukUC3dta17Y4+Dgx297HAk8Bv2tuRu89193J3Ly8qKjq4Sruoz546lPPG9Oe2J1bz91dT/4+UiKSuRAK9BojvcQ8ENsU3cPft7t46EeevgInJKa/rMzPuuHQsI/p15/MPLWXDtneCLklEMlQigV4BDDezIWaWA8wAFsQ3MLP+cYvTgIyalLMgJ4tffbScUMiY9dtKdu9tDrokEclABwx0d28GZgOLiAb1fHdfYWa3mNm0WLMvmNkKM1sOfAG4urMK7qpKehcw54oJVG3dzZd+v4yWiA6SisiRZUGdnVFeXu6VlZWBvHdnuv+fr3Hz4yv5xMlD+Nb57Z3dKSJy6MxsibuXt7dN90NPsqtPGsKG7Xu459nXKO1dwMc+NDjokkQkQyjQO8G3zi+jZkc933l8BQN75XPGyH5BlyQiGUD3cukE4ZBx98zxjBpQyOyHXuKVjXVBlyQiGUCB3kkKcrK45+pyenfL4eP3V7BxpybGEJHOpUDvRH2753HfNZOob2zh4/dVsKuhKeiSRCSNKdA72Yh+3fn5RyayrnY31z64lKaWSNAliUiaUqAfAScP78N/XTSGxWu38a0/arYjEekcOsvlCLmsvIQ3tu/hp89UUXp0AZ87bVjQJYlImlGgH0HXnz2CN97aw+1PrGFgrwKmjRsQdEkikkYU6EdQ64283qxr4CsPL2dAYR7lg3sHXZaIpAmNoR9huVlhfnnVRIp75vOpByp1d0YRSRoFegB6dcvhvqsnYWZcfd+LvPVOY9AliUgaUKAHZHCfbvzqoxPZVNfArAcqaWjSvKQicngU6AGaOKg3d142nsrXd/DVRzQvqYgcHgV6wM4b258bzjmOBcs38ZvnNgRdjoikMAV6F/C504Zy5si+fG/hKpZX7wy6HBFJUQr0LsDM+MGl4+jbPY9rH1pKXb3u+SIiB0+B3kX0LMjhJ1ccz5t1DXz1keUaTxeRg6ZA70ImlPbixqkfYNGKLdz3zw1BlyMiKSahQDezKWa2xsyqzOzG92l3iZm5mbU7350c2CdOHsKZI/vx/T+vYpnG00XkIBww0M0sDMwBpgJlwEwz22/2YzPrDnwB+Feyi8wk0fH0sfTtnsfsh5ZSt0fj6SKSmER66JOBKndf7+6NwDxgejvtvgvcDjQksb6M1LMgh5/GxtO/ovF0EUlQIoFeDFTHLdfE1u1jZscDJe7+P++3IzObZWaVZlZZW1t70MVmkuNj4+l/XbmFezWeLiIJSCTQrZ11+7qMZhYC7gSuP9CO3H2uu5e7e3lRUVHiVWaoT5w8hLPK+nGrxtNFJAGJBHoNUBK3PBDYFLfcHRgN/M3MNgAnAgt0YPTwmRk/uCR2fvqDS9m5RzfxEpGOJRLoFcBwMxtiZjnADGBB60Z3r3P3Pu4+2N0HAy8A09y9slMqzjCFBdnMuXICW99u4CsP634vItKxAwa6uzcDs4FFwCpgvruvMLNbzGxaZxcoML6kJ1+fOpInV23hnmdfC7ocEemiEpqxyN0XAgvbrLupg7anHX5Z0tY1Jw3mhfXbufXPq5kwqBcTSnsFXZKIdDG6UjRFmBl3XDKOYwrz+PxDL2k8XUT2o0BPIYUF2cy5onU8Xeeni8h7KdBTzLiSnnzj3JE8uWorv16s8XQReZcCPQVd/aHBnDOqH7c9sZrn1m0LuhwR6SIU6CnIzLj9knEM6dONT/92CWvefDvokkSkC1Cgp6jC/Gzu//hk8rPDXHPfi2zZpVvoiGQ6BXoKK+6Zz71XT6Kuvomr76tg997moEsSkQAp0FPc6OJCfvaRiby65W0++99LaGqJBF2SiAREgZ4GTh1RxH9dOJrFa7fxjcde1umMIhkqoStFpeu7fFIpG3fUc/fTVRT3yueLZ44IuiQROcIU6GnkS2eNYOPOBu56ci3FPfO5tLzkwC8SkbShQE8jZsb3LxrDll0NfP2xl+nXI49TRui+8yKZQmPoaSYnK8TPPzKBYX2P4nMPLmXlpl1BlyQiR4gCPQ11z8vmvmsmcVRuFtfc/yKbdtYHXZKIHAEK9DTVvzCf+z8+iT17W7jmvgrq6puCLklEOpkCPY194Jge/OKqiazftpvP/HYJjc06R10knSnQ09xJw/pw28VjeX79dr72qKawE0lnOsslA1w0YSAbd9Tzw7++SnHPfL5yznFBlyQinUCBniFm//9hbNxZz0+fqWJ0cQ+mjO4fdEkikmQJDbmY2RQzW2NmVWZ2YzvbP2NmL5vZMjN71szKkl+qHA4z4z8/PJohfbppommRNHXAQDezMDAHmAqUATPbCeyH3H2Mu48Hbgd+lPRK5bBlhUNcPqmEig07qNqqe6iLpJtEeuiTgSp3X+/ujcA8YHp8A3ePv3qlG6Ajb13UJRMHkh025r1YHXQpIpJkiQR6MRD/218TW/ceZnatma0j2kP/Qns7MrNZZlZpZpW1tbWHUq8cpj5H5XJWWT8eXVrD3uaWoMsRkSRKJNCtnXX79cDdfY67DwW+BvxHezty97nuXu7u5UVFusdIUGZMKmXHniYWrdgSdCkikkSJBHoNEH/bvoHApvdpPw/48OEUJZ3r5GF9GNgrn3kvvhF0KSKSRIkEegUw3MyGmFkOMANYEN/AzIbHLZ4HrE1eiZJsoZAxY1IJz63bzoZt7wRdjogkyQED3d2bgdnAImAVMN/dV5jZLWY2LdZstpmtMLNlwJeBj3VaxZIUl5aXEA4Z8yp0cFQkXSR0YZG7LwQWtll3U9zz65Jcl3Syfj3yOP24vjyypJovnzWCnCzdBUIk1em3OINdcUIJ23Y38tQqHRwVSQcK9Ax26oi+9C/M43cadhFJCwr0DBYOGZeWl7B4bS3Vb+0JuhwROUwK9Ax3+aToGanzK9VLF0l1CvQMV9wzn1NHFDG/sprmFk2AIZLKFOjCjEmlbNm1l7+t0e0YRFKZAl04Y2RfirrnMq9CV46KpDIFupAdDnHpxIE8vXorm+vqgy5HRA6RAl2A6MHRiMPDlTVBlyIih0iBLgAMOrobJw07mt9XVNMS0e3sRVKRAl32mTm5lI0761m8VgdHRVKRAl32OausH7275Wg2I5EUpUCXfXKzwlw8oZgnV21h69sNQZcjIgdJgS7vMWNyKc0R55ElOjgqkmoU6PIeQ4uOYvKQ3vy+opqIDo6KpBQFuuxn5uQSXt++hxfWbw+6FBE5CAp02c/U0f0pzM/mIc05KpJSFOiyn7zsMBceX8xfVmzhrXcagy5HRBKUUKCb2RQzW2NmVWZ2Yzvbv2xmK83s32b2lJkNSn6pciTNnFxKY0uEx5bq4KhIqjhgoJtZGJgDTAXKgJlmVtam2UtAubuPBR4Bbk92oXJkHXdMd44v7cnvXnwDdx0cFUkFifTQJwNV7r7e3RuBecD0+Abu/oy7t0558wIwMLllShBmTi5lXe07VGzYEXQpIpKARAK9GIi/dLAmtq4jnwD+3N4GM5tlZpVmVllbq8vLu7rzx/ane24W83RwVCQlJBLo1s66dv8PbmYfAcqBO9rb7u5z3b3c3cuLiooSr1ICUZCTxbTxA/jflzdTt6cp6HJE5AASCfQaoCRueSCwqW0jMzsT+CYwzd33Jqc8CdrMyaXsbY7w2Es6OCrS1SUS6BXAcDMbYmY5wAxgQXwDMzse+CXRMN+a/DIlKKOLC5k4qBd3P7WWTTs1+YVIV3bAQHf3ZmA2sAhYBcx39xVmdouZTYs1uwM4CnjYzJaZ2YIOdicp6I5LxtLYHOHzv3uJJk0kLdJlWVCnpJWXl3tlZWUg7y0H70/LNnLdvGV85tSh3Dj1A0GXI5KxzGyJu5e3t01XikpCpo8v5ooTSvnF39fx9OotQZcjIu1QoEvCbjq/jJH9e/Dl+cs1ni7SBSnQJWF52WF+duUEmjSeLtIlKdDloAzp043vXzyWJa/v4AeL1gRdjojEUaDLQZs2bgBXnlDKL/+xnqdWaTxdpKtQoMsh+db5ZZT178H1Dy9no8bTRboEBbockrzsMHOunEBzi/P5h5ZqPF2kC1CgyyEb0qcbt148hqVv7OQOjaeLBE6BLofl/LEDuOrEQcz9x3qeXKnxdJEgKdDlsH3zvJGMGqDxdJGgKdDlsOVlh5lzxQRaIs7sh5bS2KzxdJEgKNAlKQb36cZtF4/lpTd2csei1UGXI5KRFOiSNOeN7c9HPziIXy1+jb9qPF3kiFOgS1J949yRjC7uwVceXk7Njj0HfoGIJI0CXZKqdTw9EnE+9+BSdu9tDrokkYyhQJekG3R0N+68fDwrNu3i4/dVsKdRoS5yJCjQpVOcWdaPuy4fT+Xrb/HJ31TS0NQSdEkiaU+BLp3mgnED+OFl43h+/XY+9YBCXaSzKdClU114/EBuu2gsi9du43MP6hx1kc6UUKCb2RQzW2NmVWZ2YzvbTzGzpWbWbGaXJL9MSWWXTSrhexeO5unVW5mtG3mJdJoDBrqZhYE5wFSgDJhpZmVtmr0BXA08lOwCJT1cecIgbr6gjL+s3MIX5y2jWaEuknRZCbSZDFS5+3oAM5sHTAdWtjZw9w2xbfotlQ5dfdIQmlqc7y1cRXbY+OFl4wmHLOiyRNJGIoFeDFTHLdcAJxzKm5nZLGAWQGlp6aHsQlLcp045lsaWCHcsWkNWOMTtF48lpFAXSYpEAr293zY/lDdz97nAXIDy8vJD2oekvmtPH0ZTS4S7nlxLdtj43ofHKNRFkiCRQK8BSuKWBwKbOqccyRTXnTGcxuYIP/vbOrLDIb4zbRRmCnWRw5FIoFcAw81sCLARmAFc0alVSdozM2445ziaWiL8avFrZIdD/Md5IxXqIofhgIHu7s1mNhtYBISBe919hZndAlS6+wIzmwT8AegFXGBm33H3UZ1auaQ8M+Mb546kqcW559loqH9tynEKdZFDlEgPHXdfCCxss+6muOcVRIdiRA6KmfHtC8poaonwi7+vIztsfPmsEQp1kUOQUKCLdCYz47vTR9PUEuEnT1fxwvrtfPuCUYwuLgy6NJGUokv/pUsIhYxbLxrLrReNYX3tO1zw02f5+mP/ZvvuvUGXJpIyFOjSZYRCxozJpTz9ldO45kNDeLiyhtN+8DfuffY13S5AJAEKdOlyCvOzuemCMp744v9jfElPbvmflZz748UsXlsbdGkiXZoCXbqsYX2788DHJzP3qonsbY5w1T0vMuuBSt7YrqntRNqjQJcuzcw4e9Qx/OVLp3DDOcfxbNU2zrzz79yxaDXvaHo7kfdQoEtKyMsOc+3pw3j6+tM4b0x/5jyzjjN++Hf+tGwj7rqLhAgo0CXFHFOYx52Xj+fRz36Qou65XDdvGZf+4nmeXr1Ft+SVjGdB9W7Ky8u9srIykPeW9BCJOA8vqeYHf3mV2rf30q9HLhdPGMhl5SUM7tMt6PJEOoWZLXH38na3KdAl1TW1RHh69VbmV1TzzJqtRBxOGNKby8pLOHdMf/JzwkGXKJI0CnTJGFt2NfDIkhoerqxmw/Y9dM/N4oLxA7i8vISxAwt1SwFJeQp0yTjuzouvvcXvK6tZ+PJmGpoifOCY7lxaXsKFxxfTu1tO0CWKHBIFumS0XQ1NPL58E/MrqlleU0d22Di77BjOKuvH+JKeDDq6QD13SRkKdJGY1W/uYn5FDX94qYYde5oA6FWQzbiSnoyP/Ywb2JNe6sFLF6VAF2mjuSXCmi1vs7y6jmXVO1hWvZO1W3fT+usw+OiCfQE/vrQXI/t3JzdLB1cleAp0kQS83dDEyxvrWFa9k2Vv7GRZ9U62vh2922NOOMTIAT0o69+d/oX5HFOYR//CPPoX5tO/MI9uuboTtRwZ7xfo+haKxHTPy+ZDQ/vwoaF99q3bXFcfDfeaaMj/deUWtu1ubOe1WQxoJ+iPKcyjX488ehVkU1iQrV6+dCoFusj76F+YT/8x+Uwd03/fur3NLWzdtZdNO+t5c1cDm3Y28GZdPZvrGthc18CKTbvY1sF93AtywvTMz6awIIee+dn0LIj+FObnRJ/H1vXIz+ao3Cy65WbRLSeLgtww3XKyCId08FY6pkAXOUi5WWFKehdQ0rugwzatob+5roEtuxrYWd9E3Z5Gdu5pYmd9Ezv3NFFX30jV1t2xbU00JnDrgrzsEN1yokFfkBOOBn5uFt1ywhTkZJGfEyIvK0xedpi87BC5WbHH7Ni6rFBsW5jc2PPcrBA5WSGyw9HH3Nhz/fFIPQkFuplNAX5MdJLoX7v7rW225wIPABOB7cDl7r4huaWKpI5EQj+eu9PQFGFnfSz09zTxzt5m3mls5p29LeyJPUaXYz+N0fV19U1s3lnPO3ubaWiO0NDUQkNTC5HDPDwWDhk54RDZYSMnKxwLetsX/lnhENkhIyts0eVQ9DG6zcgKRdvHPw+Hou3CIdtvOSscewy9uz4UWw5ZdFs4RNxzIxx7Hop73voTsmjb1vZm7HuNWVyb1v1btE0obluqnc56wEA3szAwBzgLqAEqzGyBu6+Ma/YJYIe7DzOzGcBtwOWdUbBIOjIz8nPC5Ofk078w/7D35+40tTgNzdFw39sUYW9zCw1NrYEfe2xuobE5QmNzhKaWCHubIzS1+L7lxpbottbHprjHphanqSVCc4uzu7mZ5tbliNMcvz3ybruWiNMciRz2H5sjad8fhjZ/JFr/QBjEllv/CLy3TXuP150xnAvGDUh6rYn00CcDVe6+HsDM5gHTgfhAnw7cHHv+CPBTMzPXfU1FAmFm5GRFe9M98rKDLmc/kYjT4q0B77S0RIN+3/K+xwgtEaJ/BCLse00k9rjvxz26z0jr9mhb93eXW98zEmsbcfbtK7o/cN7dFl3Pe/bhra+Pe6177HWt2yPvLrdub/tYmN85/yaJBHoxUB23XAOc0FEbd282szrgaGBbfCMzmwXMAigtLT3EkkUk1YVCRggjWyf9JFUi90NvbxCpbc87kTa4+1x3L3f38qKiokTqExGRBCUS6DVASdzyQGBTR23MLAsoBN5KRoEiIpKYRAK9AhhuZkPMLAeYASxo02YB8LHY80uApzV+LiJyZB1wDD02Jj4bWET0tMV73X2Fmd0CVLr7AuAe4LdmVkW0Zz6jM4sWEZH9JXQeursvBBa2WXdT3PMG4NLkliYiIgdDk0SLiKQJBbqISJpQoIuIpInA7oduZrXA64f48j60uWgpQ+lzeJc+iyh9DlHp/DkMcvd2L+QJLNAPh5lVdnSD90yiz+Fd+iyi9DlEZernoCEXEZE0oUAXEUkTqRroc4MuoIvQ5/AufRZR+hyiMvJzSMkxdBER2V+q9tBFRKQNBbqISJpIuUA3sylmtsbMqszsxqDrCYqZbTCzl81smZlVBl3PkWJm95rZVjN7JW5dbzP7q5mtjT32CrLGI6WDz+JmM9sY+14sM7Nzg6yxs5lZiZk9Y2arzGyFmV0XW5+R34mUCvS4+U2nAmXATDMrC7aqQJ3u7uMz7Hzb+4EpbdbdCDzl7sOBp2LLmeB+9v8sAO6MfS/Gx26sl86agevdfSRwInBtLBMy8juRUoFO3Pym7t4ItM5vKhnC3f/B/pOnTAd+E3v+G+DDR7SogHTwWWQUd9/s7ktjz98GVhGdEjMjvxOpFujtzW9aHFAtQXPgL2a2JDZXaybr5+6bIfoLDvQNuJ6gzTazf8eGZDJiqAHAzAYDxwP/IkO/E6kW6AnNXZohTnL3CUSHn641s1OCLki6hJ8DQ4HxwGbgh8GWc2SY2VHAo8AX3X1X0PUEJdUCPZH5TTOCu2+KPW4F/kB0OCpTbTGz/gCxx60B1xMYd9/i7i3uHgF+RQZ8L8wsm2iYP+juj8VWZ+R3ItUCPZH5TdOemXUzs+6tz4GzgVfe/1VpLX5O248BfwqwlkC1hljMhaT598LMjOgUmKvc/UdxmzLyO5FyV4rGTsO6i3fnN/1ewCUdcWZ2LNFeOUSnEXwoUz4HM/sdcBrR26NuAb4N/BGYD5QCbwCXunvaHyzs4LM4jehwiwMbgE+3jiWnIzM7GVgMvAxEYqu/QXQcPfO+E6kW6CIi0r5UG3IREZEOKNBFRNKEAl1EJE0o0EVE0oQCXUQkTSjQRUTShAJdRCRN/B/orCKPmhaQJgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train set:\n",
      "accuracy: 1.0\n",
      "Test Set:\n",
      "accuracy: 0.7\n",
      "took 36.07852053642273 seconds\n",
      "\n",
      "\n",
      "lambda=0.01\n",
      "cost after iter 0: 0.6791798599988482\n",
      "cost after iter 200: 0.4294197808564114\n",
      "cost after iter 400: 0.2686086231309208\n",
      "cost after iter 600: 0.09628875219808145\n",
      "cost after iter 800: 0.03728987608693348\n",
      "cost after iter 1000: 0.0182768342707358\n",
      "cost after iter 1200: 0.011562224275459934\n",
      "cost after iter 1400: 0.008213025327934977\n",
      "cost after iter 1600: 0.006259425816006024\n",
      "cost after iter 1800: 0.005010924886780197\n",
      "cost after iter 2000: 0.004155486964256444\n",
      "cost after iter 2200: 0.0035424934716190653\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD5CAYAAAA3Os7hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAexklEQVR4nO3deZSU9Z3v8fe3qnqhWYVuQOmW7tZGWTQqLSpGjGNEMBMwERO5iaPXJJi5cZK5yblzzSTXZEhyTyY5M7m5NxxHYjIaE0PcQ4wZyGgMbhgaxQWRrdmatRtkp5fq+t4/qhqLplsKqOqnls/rnD79LL96+nPK8tMPTz+LuTsiIpL7QkEHEBGR9FChi4jkCRW6iEieUKGLiOQJFbqISJ5QoYuI5IlIKoPMbBrwYyAM3O/u3++2/kfANYnZMmC4uw/5oG2Wl5d7dXX1SQcWESlky5cvb3H3ip7WnbDQzSwMzAOuA5qAZWa20N3f6Rrj7v89afzfARefaLvV1dU0NDSkEF9ERLqY2abe1qVyyGUSsM7dG929HVgAzPyA8bOBX59cRBEROV2pFPooYEvSfFNi2XHMbDRQAzx3+tFERORkpFLo1sOy3u4XcAvwmLt39rghszlm1mBmDc3NzalmFBGRFKRS6E1AVdJ8JbCtl7G38AGHW9x9vrvXu3t9RUWPx/RFROQUpVLoy4A6M6sxs2Lipb2w+yAzOw84A3glvRFFRCQVJyx0d48CdwGLgFXAI+6+0szmmtmMpKGzgQWu2zeKiAQipfPQ3f0Z4Jluy+7pNv/t9MUSEZGTlXNXir6xZS///B/vBh1DRCTr5F6hN+3l3ufX8/rm94KOIiKSVXKu0D95SSUDSiI8+PLGoKOIiGSVnCv0ASURZk2s5PdvbWfXgdag44iIZI2cK3SAv7liNB2dzsOvbg46iohI1sjJQq+tGMDVYyr41aubaY/Ggo4jIpIVcrLQAW6fXE3zgTb+8Pb2oKOIiGSFnC30q8dUUD2sTH8cFRFJyNlCD4WMW6+o5rXNe3mraV/QcUREApezhQ5wc30lZcVhHtBeuohIbhf6oNIiPnnJKH735jZ2H2wLOo6ISKByutABbruimvZojAXLtpx4sIhIHsv5Qq8bMZArzx3GL5duItqpUxhFpHDlfKFDfC99+75WFr+zM+goIiKByYtCv3bsCCrP6Kc/jopIQcuLQg+HjFsvH81fNuxh1fb9QccREQlEXhQ6wKcvraK0KKQLjUSkYOVNoQ8pK+YTF4/iqRVb2Xu4Peg4IiJ9Lm8KHeC2ydW0dsT4jU5hFJEClFeFfv7IQVxWM5SHlm6iM6ZnVYtIYUmp0M1smpmtNrN1ZnZ3L2M+ZWbvmNlKM3s4vTFTd/vkapreO8Kzq3QKo4gUlhMWupmFgXnAdGAcMNvMxnUbUwd8HbjS3ccDf5+BrCm5btwIzhpcyoOvbAwqgohIIFLZQ58ErHP3RndvBxYAM7uN+QIwz93fA3D3XemNmbpIOMRnLh/NS+t2s3bngaBiiIj0uVQKfRSQ/FfGpsSyZGOAMWb2kpktNbNp6Qp4KmZPOpviSEh76SJSUFIpdOthWfe/OEaAOuAjwGzgfjMbctyGzOaYWYOZNTQ3N59s1pQN7V/MjA+dxROvbWV/a0fGfo6ISDZJpdCbgKqk+UpgWw9jfuvuHe6+AVhNvOCP4e7z3b3e3esrKipONXNKbp9czeH2Th5taMrozxERyRapFPoyoM7MasysGLgFWNhtzFPANQBmVk78EExjOoOerAmjBjNx9Bk89MpGYjqFUUQKwAkL3d2jwF3AImAV8Ii7rzSzuWY2IzFsEbDbzN4B/gT8D3ffnanQqbptcjUbdx/mz2syd3hHRCRbmHswe6/19fXe0NCQ0Z/R0Rnjyu8/x9gzB/HgHZMy+rNERPqCmS139/qe1uXVlaLdFYVDfOay0fx5TTONzQeDjiMiklF5XegAsy+roihs/OKVTUFHERHJqLwv9OEDS/nYBWfy2PImDrZFg44jIpIxeV/oALdeUc3Btij/8faOoKOIiGRMQRT6JWcP4czBpSxaqUIXkfxVEIVuZkwdN4Ila5o53K7DLiKSnwqi0AGmjh9JWzTGkjUtQUcREcmIgin0STVDGdyviMU67CIieapgCr0oHOLascP5z1U76eiMBR1HRCTtCqbQAaaOG8n+1iivNu4JOoqISNoVVKFfPaaC0qIQi9/RYRcRyT8FVej9isNMqatg8cqdugOjiOSdgip0gOvHj2TH/lbe3Lov6CgiImlVcIV+7djhhEOms11EJO8UXKEPKSvmspqhumpURPJOwRU6xA+7rG8+xLpduqWuiOSPgiz068aNANDZLiKSVwqy0M8a0o8LKwezaOXOoKOIiKRNQRY6xA+7vLFlLzv2tQYdRUQkLQq20KfqsIuI5JmCLfRzhw+gtrw/i3XYRUTyREqFbmbTzGy1ma0zs7t7WH+7mTWb2YrE1+fTHzW9zIyp40eytHE3+w53BB1HROS0nbDQzSwMzAOmA+OA2WY2roehv3H3ixJf96c5Z0ZcP34E0Zjz7LvaSxeR3JfKHvokYJ27N7p7O7AAmJnZWH3jQ5VDGD6wRIddRCQvpFLoo4AtSfNNiWXd3WRmb5rZY2ZWlZZ0GRYKGVPHj+DPa5pp7egMOo6IyGlJpdCth2Xdb1X4O6Da3S8E/hN4sMcNmc0xswYza2hubj65pBly/fiRHOnoZMma7MgjInKqUin0JiB5j7sS2JY8wN13u3tbYvanwMSeNuTu89293t3rKyoqTiVv2l1WM4yBpRFdZCQiOS+VQl8G1JlZjZkVA7cAC5MHmNmZSbMzgFXpi5hZxZEQ154/nGff3UlUj6YTkRx2wkJ39yhwF7CIeFE/4u4rzWyumc1IDPuyma00szeALwO3ZypwJlw/fiR7D3fwl416NJ2I5K5IKoPc/RngmW7L7kma/jrw9fRG6ztXn1dBSSTE4pU7mXxOedBxREROScFeKZqsrDjCVXXlLF65A3c9mk5EcpMKPWHq+JFs29fK21v3Bx1FROSUqNATPjp2BCFDTzISkZylQk8Y2r+YS6uH6u6LIpKzVOhJrh8/kjU7D7Kh5VDQUURETpoKPcnU8fF7pOuwi4jkIhV6ksozypgwapAKXURykgq9m6njRvL65r3s2q9H04lIblGhd3P9+JEALH5H93YRkdyiQu9mzIgBVA8r02EXEck5KvRuuh5N98r63ew7okfTiUjuUKH3oOvRdM+v3hV0FBGRlKnQe3Bx1RmUDyjRYRcRySkq9B6EQsZ140bw/Go9mk5EcocKvRfXjx/B4XY9mk5EcocKvReTzyln+MAS7n7iLV7f/F7QcURETkiF3oviSIgFcy5nQEmE2T9dyh91XrqIZDkV+georRjA4387mTEjBnLnQw38cummoCOJiPRKhX4CFQNLWDDncj5y3nC++dTb/HDRu3qqkYhkJRV6CsqKI8y/dSKzJ1Ux70/r+dojb9AejQUdS0TkGCkVuplNM7PVZrbOzO7+gHGzzMzNrD59EbNDJBzif3/iAr523RieeH0rdzywjAOtupJURLLHCQvdzMLAPGA6MA6YbWbjehg3EPgy8Gq6Q2YLM+Pvrq3jh7MuZGnjbm7+t1fYqbsyikiWSGUPfRKwzt0b3b0dWADM7GHcd4AfAHnfcDfXV/Hz2y9ly57DfGLeS6zZeSDoSCIiKRX6KGBL0nxTYtlRZnYxUOXuT6cxW1abMqaC39x5BR0xZ9a9L7O0cXfQkUSkwKVS6NbDsqOneZhZCPgR8LUTbshsjpk1mFlDc3PuX4E5YdRgnvxvk6kYWMLf/OwvPP3mtqAjiUgBS6XQm4CqpPlKILm5BgITgOfNbCNwObCwpz+Muvt8d6939/qKiopTT51FKs8o4/G/ncyHqgZz18Ovc/8LjUFHEpEClUqhLwPqzKzGzIqBW4CFXSvdfZ+7l7t7tbtXA0uBGe7ekJHEWWhIWTEPfe4ybrhgJN/9/Sr+77Nrg44kIgXohIXu7lHgLmARsAp4xN1XmtlcM5uR6YC5orQozE9mX8LUcSOYv6SRw+3RoCOJSIFJ6Tx0d3/G3ce4+znu/r3EsnvcfWEPYz9SSHvnyUIh444P13CwLap7qYtIn9OVomk2qXooVUP78fjyrUFHEZECo0JPs1DIuOmSSl5a38LWvUeCjiMiBUSFngE3XVKJOzz5WlPQUUSkgKjQM6BqaBmX1w7lseVNujOjiPQZFXqGzJpYxcbdh1m+SU87EpG+oULPkOkTRlJWHOax5TrsIiJ9Q4WeIf1LIkyfcCZPv7mdI+2dQccRkQKgQs+gWRMrdU66iPQZFXoGXVYzlMoz+vG4znYRkT6gQs+grnPSX1zXwjadky4iGaZCz7Cj56S/ritHRSSzVOgZdvawMi6r0TnpIpJ5KvQ+MGtiJRtaDvHaZp2TLiKZo0LvA9MvOJN+RTonXUQyS4XeBwaURJh+wUiefkPnpItI5qjQ+8isiZUcaIuy+B2dky4imaFC7yOX1wxj1JB+OuwiIhmjQu8joZBx08T4Oenb9+mcdBFJPxV6H7rpklG4wxOv6Zx0EUk/FXofGj2sP5NqhvK4zkkXkQxIqdDNbJqZrTazdWZ2dw/rv2hmb5nZCjN70czGpT9qfpg1sZLGlkO8tnlv0FFEJM+csNDNLAzMA6YD44DZPRT2w+5+gbtfBPwA+Ne0J80TN+icdBHJkFT20CcB69y90d3bgQXAzOQB7r4/abY/oOMJvRhQEmH6hJE8/cY2Wjt0TrqIpE8qhT4K2JI035RYdgwz+5KZrSe+h/7l9MTLT13npOs+6SKSTqkUuvWw7Lg9cHef5+7nAP8T+GaPGzKbY2YNZtbQ3Nx8cknzyOW18XPSH9fZLiKSRqkUehNQlTRfCWz7gPELgBt7WuHu89293t3rKyoqUk+ZZ+L3SR/Fi2ub2bGvNeg4IpInUin0ZUCdmdWYWTFwC7AweYCZ1SXNfgxYm76I+emmiZXEHJ54XX8cFZH0OGGhu3sUuAtYBKwCHnH3lWY218xmJIbdZWYrzWwF8FXgtowlzhOjh/VnUrXuky4i6RNJZZC7PwM8023ZPUnTX0lzroIwa2Il//D4m7y+ZS+XnH1G0HFEJMfpStEA3XChzkkXkfRRoQeo65z03+mcdBFJAxV6wG6aWMmB1iiL39kZdBQRyXEq9IBdUTuMswaX6rCLiJw2FXrAQiHjU5dWsWRNM8s36SHSInLqVOhZ4PNX1XLm4FK++dTbRDtjQccRkRylQs8CA0oifOvj41i1fT8PvLwx6DgikqNU6Fni+vEjuea8Cn70xzV6RJ2InBIVepYwM+bOnEA05sz93TtBxxGRHKRCzyJVQ8v48rV1/OHtHfxp9a6g44hIjlGhZ5kvXFXLORX9uee3b+tiIxE5KSr0LFMcCfGdGyewZc8RfvLcuqDjiEgOUaFnocnnlPPJi0dx35L1rNt1MOg4IpIjVOhZ6h8/NpZ+RWH+11Nv6/a6IpISFXqWKh9Qwj9MO59XGnfz1Ao9qk5ETkyFnsX+y6SzuahqCN/7/Sr2He4IOo6IZDkVehYLhYzv3jiBPYfa+eHid4OOIyJZToWe5SaMGsxtk6v51aubWbFlb9BxRCSLqdBzwFevG8PwgSV848m3dPMuEemVCj0HDCwt4p6/Hs/Kbft5aOmmoOOISJZKqdDNbJqZrTazdWZ2dw/rv2pm75jZm2b2rJmNTn/UwnbDBSOZMqaCf1m8hp37W4OOIyJZ6ISFbmZhYB4wHRgHzDazcd2GvQ7Uu/uFwGPAD9IdtNCZGXNnjKe9M8bcp3XzLhE5Xip76JOAde7e6O7twAJgZvIAd/+Tux9OzC4FKtMbUwCqy/tz1zXn8vs3t7NkTXPQcUQky6RS6KOALUnzTYllvfkc8IfTCSW9u/PqWmrLdfMuETleKoVuPSzr8Vp0M/ssUA/8sJf1c8yswcwampu1h3kqSiJhvnPjBDbuPsy9z68POo6IZJFUCr0JqEqarwS2dR9kZh8FvgHMcPe2njbk7vPdvd7d6ysqKk4lrwBXnlvOzIvO4t7n17Nlz+ETv0BECkIqhb4MqDOzGjMrBm4BFiYPMLOLgfuIl7mezNAHvj59LI7z0xcag44iIlnihIXu7lHgLmARsAp4xN1XmtlcM5uRGPZDYADwqJmtMLOFvWxO0mTk4FJuvGgUjzRsYc+h9qDjiEgWiKQyyN2fAZ7ptuyepOmPpjmXpGDOlFoeXd7EQ69s4isfrQs6jogETFeK5rC6EQO59vzhPPjKRo6064wXkUKnQs9xc6bUsudQO4+91hR0FBEJmAo9x02qGcpFVUO4/4VGOmN6spFIIVOh5zgz484ptWzafZhFK3cEHUdEAqRCzwNTx4+kelgZ9/15vZ4/KlLAVOh5IBwyPn9VLW807ePVDXuCjiMiAVGh54lZEysZ1r+Y+Ut0oZFIoVKh54nSojC3Ta7muXd3sWbngaDjiEgAVOh55NbLR9OvKKy9dJECpULPI2f0L+ZT9ZX8dsVWduzTU41ECo0KPc98/qpaOmPOv7+0IegoItLHVOh5pmpoGTdccCYPv7qZA60dQccRkT6kQs9Dd045hwNtUX79l81BRxGRPqRCz0MXVA5m8jnD+PmLG2mPxoKOIyJ9RIWep+ZMqWXH/lYWvnHcw6VEJE+p0PPU1WMqOH/kQOYv0e0ARAqFCj1PmRlzptSyZudBnl+tB3KLFAIVeh77+IfO4qzBpdy3ZH3QUUSkD6jQ81hROMQdH65haeMe3tiyN+g4IpJhKvQ8d8uksxlYGtHtAEQKgAo9zw0oifDZy0fzh7e3s2n3oaDjiEgGpVToZjbNzFab2Tozu7uH9VPM7DUzi5rZrPTHlNPxXydXEwmFuP8F3Q5AJJ+dsNDNLAzMA6YD44DZZjau27DNwO3Aw+kOKKdv+KBSPnHxKB5dvoU9h9qDjiMiGZLKHvokYJ27N7p7O7AAmJk8wN03uvubgC5LzFJfmFJDa0eMX7yyMegoIpIhqRT6KGBL0nxTYpnkkHOHD+SjY4fz4MsbOdLeGXQcEcmAVArdelh2SpcemtkcM2sws4bmZl3s0tfuvPoc3jvcwdyn3yHaqX9MieSbVAq9CahKmq8ETukGIe4+393r3b2+oqLiVDYhp6F+9BnceXUtv/7LZr7wiwYOtkWDjiQiaZRKoS8D6sysxsyKgVuAhZmNJZlgZnx9+li+e+MElqxt4eZ/e4Xt+44EHUtE0uSEhe7uUeAuYBGwCnjE3Vea2VwzmwFgZpeaWRNwM3Cfma3MZGg5PZ+9fDQ/u62eLXsOc+O8l3h7676gI4lIGlhQd+Krr6/3hoaGQH62xK3avp87HljGviMd/L/ZF3Pt2BFBRxKREzCz5e5e39M6XSlawMaeOYinvnQltRX9+cIvGnjw5Y1BRxKR06BCL3AjBpXyyJ1X8Ffnj+BbC1fyT79bSWdM908XyUUqdKGsOMJ9t07kjitr+PeXNnLnQw0c0hkwIjlHhS4AhEPGPR8fxz/NGM9z7+7i0/NfYef+1qBjichJUKHLMW6bXM39t9XT2HyIG+e9xKrt+4OOJCIpUqHLcf7q/BE8+sUriLkz696XeX71rqAjiUgKVOjSo/FnDeapL13J6GH9+dyDDfzoj2t4T3dqFMlqKnTp1ZmD+/HoF69g2viR/PjZtVzx/Wf5xpNvsb75YNDRRKQHurBIUrJ6xwF+/uIGnlyxlfZojGvPH87nrqrhitphmPV0/zYRyYQPurBIhS4npflAG79cuolfLt3E7kPtjDtzEJ+/qoa/vvAsiiP6B59IpqnQJe1aOzr57Yqt3P/CBtbuOsjwgSXcNrmaz1x2NkPKioOOJ5K3VOiSMe7OkrUt3P9CIy+sbaG0KMSsiZXccWUNtRUDgo4nkndU6NInVu84wM9ebOSp17fR3hnjqrpyrh5TwZQxFdQNH6Bj7SJpoEKXPtV8oI2Hlm7i6Te30dh8CIARg0r48LkVXFVXzpXnllMxsCTglCK5SYUugdm69wgvrm1mydoWXlrXwt7DHUD8To9T6sr5cF05l1YPpbQoHHBSkdygQpes0BlzVm7bxwtrW3hxbQsNm/bQ0emUREJMqhnKVXXlTD6nnHOHD1DBi/RChS5Z6XB7lFc37OGFNS28uK6ZNTvfv2Bp1JB+1JT3f/+roj+15f0ZNaQfkbBOj5TC9UGFHunrMCJdyoojXHPecK45bzgAO/a1smzjHja0HGJDyyEaWw7x1IqtHGh9/1a+RWHj7KFl1JQPoLYiXvbVw/ozcnApwwYUM7Akoj++SsFSoUvWGDm4lI9/6Kxjlrk7ew61xwu+OV7yG1oOsqHlEEvWNtMejR0zvjgSorx/McMGlFA+IP592IBiKhLfh/UvoTyxbnBZESURHdqR/KFCl6xmZolSLqG+eugx6zpjzra9R9i4+xDNB9rYfbCdloNttBxsZ/ehNpoPtvHujgPsPthOe2esx+2XREIM6lfEoNIIA0uLuk1HGFQanx/Ur4iBpREGlBRRVhymX3GYsuIwZUUR+hWHKQqb/mUggUup0M1sGvBjIAzc7+7f77a+BPgFMBHYDXza3TemN6rIscIho2poGVVDyz5wnLuzvzXK7oNt7D7UTsuBNloOtrHvSAcHWqPsb+1g/5H4931HOmjac/jost5+EfSUpazo/aLvVxyJF35xmNKi+FdJJJT4ClNSlDQdCVFSFKL06PIwxZEQxeEQxRGjOBymKGIUhePLisIhiiMhisLvLwuF9MtEUih0MwsD84DrgCZgmZktdPd3koZ9DnjP3c81s1uAfwY+nYnAIifLzBjcr4jB/YqorTi517Z2dLK/NVH8Rzo42BblcHsnR9o7OdLRmZiOL+tafrgjvuxIRycH26I0H2ijPRqjLRqjLdpJW0d8OtVfFqkIhyxe8KEQkbARCYcoCsW/R8JGJGREQvFfApFwiEgo/sug63XhxPr498R8L8uPfpkR6j5tJNaHCIcgZO+P75oOGUnTXa9LLA/1PMaOmY//Nw3Z++NCSa/pGtu13pLHJa1/fxx586+rVPbQJwHr3L0RwMwWADOB5EKfCXw7Mf0Y8BMzMw/qFBqRNOnaux4+MP3bjsWc9s5YouA7jxZ+a2K+Pep0dMaOfrV3Ou3RpPlojI6kZe2J5Z0xp6PTiXbGiMbi24h2OtFYfHzX98Pt0aNjY+5EY05nLL6+szN5Pml5Ynw+Si5/M8M4tvgN4tOhY9d1Hxsfd/wvDgNITH/l2rrj/l6UDqkU+ihgS9J8E3BZb2PcPWpm+4BhQEs6Qorko1DIKA2FE+fcFwUd56TEYk6nx4s+1vU9xtFlnYn1se7THh8X86TXuRNzEttITCe91kmMT6zreq13vSYxfXQ77ni3Ze7v/5yj45O3fXS+63Xx1zjHjvOj2wKna9sc8/OcpGVJr+tah8Pgfpn5751Koff0b5Huv6JTGYOZzQHmAJx99tkp/GgRyUahkBHC0PVf2SWVKzSagKqk+UpgW29jzCwCDAb2dN+Qu89393p3r6+oOMmDmSIi8oFSKfRlQJ2Z1ZhZMXALsLDbmIXAbYnpWcBzOn4uItK3TnjIJXFM/C5gEfHTFn/u7ivNbC7Q4O4LgZ8BD5nZOuJ75rdkMrSIiBwvpfPQ3f0Z4Jluy+5Jmm4Fbk5vNBERORm6y5GISJ5QoYuI5AkVuohInlChi4jkicAecGFmzcCmU3x5OboKFfQ+JNN7Eaf3IS6f34fR7t7jhTyBFfrpMLOG3p7YUUj0PrxP70Wc3oe4Qn0fdMhFRCRPqNBFRPJErhb6/KADZAm9D+/TexGn9yGuIN+HnDyGLiIix8vVPXQREekm5wrdzKaZ2WozW2dmdwedJyhmttHM3jKzFWbWEHSevmJmPzezXWb2dtKyoWb2RzNbm/h+RpAZ+0ov78W3zWxr4nOxwsxuCDJjpplZlZn9ycxWmdlKM/tKYnlBfiZyqtCTnm86HRgHzDazccGmCtQ17n5RgZ2e9QAwrduyu4Fn3b0OeDYxXwge4Pj3AuBHic/FRYkb6+WzKPA1dx8LXA58KdEJBfmZyKlCJ+n5pu7eDnQ931QKhLsv4fiHp8wEHkxMPwjc2KehAtLLe1FQ3H27u7+WmD4ArCL+SMyC/EzkWqH39HzTUQFlCZoDi81seeLRfoVshLtvh/j/4MDwgPME7S4zezNxSKYgDjUAmFk1cDHwKgX6mci1Qk/p2aUF4kp3v4T44acvmdmUoANJVrgXOAe4CNgO/EuwcfqGmQ0AHgf+3t33B50nKLlW6Kk837QguPu2xPddwJPED0cVqp1mdiZA4vuugPMExt13ununu8eAn1IAnwszKyJe5r9y9ycSiwvyM5FrhZ7K803znpn1N7OBXdPAVODtD35VXkt+pu1twG8DzBKorhJL+AR5/rkwMyP+CMxV7v6vSasK8jORcxcWJU7D+j+8/3zT7wUcqc+ZWS3xvXKIP0bw4UJ5H8zs18BHiN9NbyfwLeAp4BHgbGAzcLO75/0fC3t5Lz5C/HCLAxuBO7uOJecjM/sw8ALwFhBLLP5H4sfRC+8zkWuFLiIiPcu1Qy4iItILFbqISJ5QoYuI5AkVuohInlChi4jkCRW6iEieUKGLiOQJFbqISJ74/4CiA4/z07EwAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train set:\n",
      "accuracy: 1.0\n",
      "Test Set:\n",
      "accuracy: 0.68\n",
      "took 49.13385081291199 seconds\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "layer_dims = [X_train.shape[0], 20, 7, 5, y_train.shape[0]]\n",
    "num_iterations = 2400\n",
    "alpha = .0075\n",
    "l2_lambdas = [0, 0.001, 0.01]\n",
    "# interval in the iterations for which the costs will be stored\n",
    "cost_interval = 200\n",
    "\n",
    "for l2_lambda in l2_lambdas:\n",
    "    start = time.time()\n",
    "    \n",
    "    print(f'lambda={l2_lambda}')\n",
    "    params, costs = fit(X_train, y_train, layer_dims, num_iterations, alpha,\n",
    "                        l2_lambda, cost_interval)\n",
    "    y_pred_train = predict(X_train, params)\n",
    "    y_pred_test = predict(X_test, params)\n",
    "\n",
    "    print('Train set:')\n",
    "    accuracy_score(y_train, y_pred_train)\n",
    "\n",
    "    print('Test Set:')\n",
    "    accuracy_score(y_test, y_pred_test)\n",
    "    \n",
    "    end = time.time()\n",
    "    print(f'took {end - start} seconds\\n\\n')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Performance on generated dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [],
   "source": [
    "m = 2000\n",
    "# data = make_classification(n_features=2, n_redundant=0, n_informative=2,\n",
    "#                            n_clusters_per_class=2, n_samples=m)\n",
    "# data = make_circles(n_samples=m, factor=.5, noise=.03)\n",
    "\n",
    "data = make_moons(n_samples=m, noise=.25)\n",
    "\n",
    "\n",
    "X, y = data[0].T, data[1].reshape(1, -1)\n",
    "\n",
    "idx = np.arange(X.shape[1])\n",
    "np.random.shuffle(idx)\n",
    "\n",
    "X_train = X[:, idx[0:int(m * .8)]]\n",
    "X_test = X[:, idx[int(m * .8):]]\n",
    "y_train = y[:, idx[0:int(m * .8)]]\n",
    "y_test = y[:, idx[int(m * .8):]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "lambda=0\n",
      "cost after iter 0: 0.7322637045312831\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAbLklEQVR4nO3de3TU9Z3/8ed7LskkmCFQk1QuCrp4wYoiKVWrrv0d22LtQi/aYrXXtWgr63a7v921uz1ul9/u2Z/d2vb0J7Wi69HtrlJru5ZVWtraWkWrJViKQooi3iIIUQmg5DYz798fM5MMYUKGkGQy3+/rcU5OvpdPZt7zPcNrPnzm+/l+zd0REZHKFyl3ASIiMjIU6CIiAaFAFxEJCAW6iEhAKNBFRAIiVq4nPvroo33GjBnlenoRkYq0fv3619y9odi+sgX6jBkzaGlpKdfTi4hUJDN7cbB9GnIREQkIBbqISEAo0EVEAkKBLiISEAp0EZGAUKCLiASEAl1EJCAqLtDXvfAGN/zsj+iyvyIiB6q4QN/YtoebH3qOPZ295S5FRGRcqbhAb6yrBmDn3u4yVyIiMr5UXKA3JRMA7NrXVeZKRETGlwoMdPXQRUSKqbhAb6xTD11EpJiKC/Saqih1iRi71EMXETlAxQU6ZL8YVQ9dRORAFRnoTcmExtBFRAYoKdDNbIGZbTGzrWZ2XZH93zKzDbmfZ8ysY+RL7aceuojIwYa8Y5GZRYHlwHuBNmCdma1y9835Nu7+VwXt/wKYOwq19sn30N0dMxvNpxIRqRil9NDnA1vdfZu79wArgUWHaH8ZcPdIFDeYhrpqelIZ9namRvNpREQqSimBPhV4uWC9LbftIGZ2HDAT+NUg+5eYWYuZtbS3tx9urX3yk4t2athFRKRPKYFebExjsCtjLQbudfd0sZ3uvsLdm929uaGh6E2rS5Kf/q9TF0VE+pUS6G3A9IL1acD2QdouZpSHW6Cgh75XPXQRkbxSAn0dMMvMZppZFdnQXjWwkZmdBEwCfjuyJR6sMTf9f9c+9dBFRPKGDHR3TwFLgTVAK3CPu28ys2VmtrCg6WXASh+DC5XXVsWoq46phy4iUmDI0xYB3H01sHrAtusHrH9t5MoaWkOymnb10EVE+lTkTFGAprqEeugiIgUqNtAbk9UaQxcRKVCxgZ6dLdqle4uKiORUbKA31lXTncqwt0uzRUVEoJIDPX8rOo2ji4gAlRzodToXXUSkUMUGumaLiogcqGIDXT10EZEDVWygT6iOcZRmi4qI9KnYQIfcnYt0xUUREaDSAz2pW9GJiORVdqDX6WbRIiJ5FR3oTbkeumaLiohUeKA31iXo6tVsURERqPRAz93ool3j6CIiFR7odfnJRRpHFxGp6EBv6rsVnXroIiIVHeiNSfXQRUTyKjrQj6qOMaEqqslFIiJUeKBD7kYXGnIREan8QG+oq6ZdPXQRkcoPdPXQRUSyKj7Q8xfo0mxREQm7kgLdzBaY2RYz22pm1w3S5mNmttnMNpnZXSNb5uCakgk6e9Ps69ZsUREJt9hQDcwsCiwH3gu0AevMbJW7by5oMwv4CvBud99tZo2jVfBA+dmiu/Z2k0zEx+ppRUTGnVJ66POBre6+zd17gJXAogFtPg8sd/fdAO6+a2TLHFx+tqhuFi0iYVdKoE8FXi5Yb8ttK3QicKKZPWpmj5vZgmIPZGZLzKzFzFra29uHV/EAfT103YpOREKulEC3ItsGfgMZA2YBFwCXAbeZWf1Bf+S+wt2b3b25oaHhcGstSjeLFhHJKiXQ24DpBevTgO1F2vzE3Xvd/XlgC9mAH3VHVceorYqqhy4ioVdKoK8DZpnZTDOrAhYDqwa0uQ94D4CZHU12CGbbSBZ6KE3JhHroIhJ6Qwa6u6eApcAaoBW4x903mdkyM1uYa7YGeN3MNgO/Bv7G3V8fraIHaqirVg9dREJvyNMWAdx9NbB6wLbrC5Yd+HLuZ8w1JRM81dZRjqcWERk3Kn6mKORmi+7TbFERCbdABHpTspr9PWne1GxREQmxQAR63+QijaOLSIgFI9Bzk4t0pouIhFkwAj3XQ29XD11EQiwQgd6kHrqISDAC/ajqGDVx3VtURMItEIFuZjQlq9mpIRcRCbFABDpkx9F1CV0RCbPgBHpS0/9FJNyCE+jqoYtIyAUm0JuS1byl2aIiEmKBCfT+e4uqly4i4RSYQG+qy9+5SOPoIhJOgQn0/nuLqocuIuEUoEDPXaBLPXQRCanABHpddYxEPKIeuoiEVmACPTtbNKExdBEJrcAEOuTvXKQeuoiEU7ACPZnQGLqIhFawAr1O0/9FJLwCFehNyQRvdqd4S7NFRSSEAhbo+XPR1UsXkfAJVKA39s0W1RejIhI+JQW6mS0wsy1mttXMriuy/zNm1m5mG3I/V458qUNTD11Ewiw2VAMziwLLgfcCbcA6M1vl7psHNP2Buy8dhRpL1lCXny2qHrqIhE8pPfT5wFZ33+buPcBKYNHoljU8yUR+tqh66CISPqUE+lTg5YL1tty2gT5qZhvN7F4zm17sgcxsiZm1mFlLe3v7MMo9NDOjsS6hMXQRCaVSAt2KbPMB6/8DzHD3OcAvgTuLPZC7r3D3ZndvbmhoOLxKS9SUrFagi0golRLobUBhj3sasL2wgbu/7u75cY5bgXkjU97ha6xLaMhFREKplEBfB8wys5lmVgUsBlYVNjCzYwpWFwKtI1fi4WlMVmv6v4iE0pBnubh7ysyWAmuAKHC7u28ys2VAi7uvAq41s4VACngD+Mwo1nxIjXX9s0UnVA/58kREAqOkxHP31cDqAduuL1j+CvCVkS1teArPRZ+pQBeREAnUTFHony2qc9FFJGwCF+j5HvpOfTEqIiETuEBXD11EwipwgZ6siVEd02xREQmfwAW6meVOXVQPXUTCJXCBDtBUp5tFi0j4BDLQG5O6WbSIhE8wA71ON4sWkfAJZqAnq9nXnWJ/j+4tKiLhEchAb+o7dVG9dBEJj0AGeqNuRSciIRTIQG9K6mbRIhI+gQz0xjr10EUkfAIZ6BNr4lTFIppcJCKhEshAz95btFo9dBEJlUAGOmTH0TWGLiJhEthAVw9dRMImsIGuHrqIhE1gA72hrpp9XSk6e9LlLkVEZEwENtDz56LrIl0iEhaBDXSdiy4iYRPYQNdsUREJm8AGel8PXRfoEpGQCGyg19fGqYpG2KkxdBEJiZIC3cwWmNkWM9tqZtcdot0lZuZm1jxyJQ5P/t6i7eqhi0hIDBnoZhYFlgMXAbOBy8xsdpF2dcC1wBMjXeRwNdZVq4cuIqFRSg99PrDV3be5ew+wElhUpN3/Ab4OjJsEbUrqVnQiEh6lBPpU4OWC9bbctj5mNheY7u73H+qBzGyJmbWYWUt7e/thF3u4GuuqdZaLiIRGKYFuRbZ5306zCPAt4K+HeiB3X+Huze7e3NDQUHqVw9SYTLC3K0VXr2aLikjwlRLobcD0gvVpwPaC9TrgHcBDZvYCcBawajx8MapTF0UkTEoJ9HXALDObaWZVwGJgVX6nu+9x96PdfYa7zwAeBxa6e8uoVHwYNP1fRMJkyEB39xSwFFgDtAL3uPsmM1tmZgtHu8Ajkb9Z9E710EUkBGKlNHL31cDqAduuH6TtBUde1shoqlMPXUTCI7AzRaFgtqh66CISAoEOdDOjoa5aPXQRCYVABzpkx9F1louIhEHgA72pLqEeuoiEQuADvTFZrTF0EQmFwAd6UzLBns5ezRYVkcALfKA35GaLtutWdCIScIEPdM0WFZGwCHyg56/nonF0EQm6wAd6Xw9dl9EVkYALfKBPqo0Tjxo7NYYuIgEX+EA3MxrrdOciEQm+wAc6oOn/IhIKoQj0pqRuRSciwReKQG+sS7BLY+giEnChCPSmZDUd+zVbVESCLRSB3pi70YVmi4pIkIUj0HO3otu8Y2+ZKxERGT2hCPS50ycxtb6GpXc9ye1rn8fdy12SiMiIC0WgT6yN88C15/KnJzay7P7NXPX99ezZ31vuskRERlQoAh2gvraKWz81j69efAq/+uMuPvCdR9jwcke5yxIRGTGhCXTIzhq98rzj+eHVZwNwyc2Pcdsj2zQEIyKBEKpAz5t77CRWX3se/+vkRv75gVY+/x/r6djfU+6yRESOSEmBbmYLzGyLmW01s+uK7L/azJ4ysw1mttbMZo98qSNrYm2cWz45j3/8s9n85pldXPydtTz50u5ylyUiMmxDBrqZRYHlwEXAbOCyIoF9l7uf5u5nAF8HvjnilY4CM+Oz757JvVefQyQCH/veb7n1YQ3BiEhlKqWHPh/Y6u7b3L0HWAksKmzg7oUneE8AKioRT59ez/1/cR4XntLEv6xu5co7W9j9loZgRKSylBLoU4GXC9bbctsOYGbXmNlzZHvo1xZ7IDNbYmYtZtbS3t4+nHpHzcSaODdfcSb/tPBUHnn2NS7+ziOsf/GNcpclIlKyUgLdimw7qAfu7svd/QTg74CvFnsgd1/h7s3u3tzQ0HB4lY4BM+PT58zgR184h1g0wsdueVxDMCJSMUoJ9DZgesH6NGD7IdqvBD50JEWV22nTJnL/tefyvtnZIZhr7nqSN7tT5S5LROSQSgn0dcAsM5tpZlXAYmBVYQMzm1WwejHw7MiVWB7JRJzvXn4mf/+Bk/nZ06+y6Ka1bN21r9xliYgMashAd/cUsBRYA7QC97j7JjNbZmYLc82WmtkmM9sAfBn49KhVPIbMjCXnn8B/Xvku9nT2suimR3lg445ylyUiUpSVa3y4ubnZW1payvLcw/Hqni6++F/refKlDq48dyZ/d9HJxKOhnJclImVkZuvdvbnYPiVSid4+McHKJWfzmXNmcNva57n8tid0n1IRGVcU6IehKhbhawtP5dsfP4ONbR188DtraXlBpzaKyPigQB+GD82dyn9/8d3UVkVZvOJxXWNdRMYFBfownXJMkp8sPZcLTspeY/3alRt4S6c2ikgZKdCPwMSaOCs+OY+/ef9JPLBxOx/+7qNsa3+z3GWJSEgp0I9QJGJc854/4c7Pzad9XzcLb3qUX2zeWe6yRCSEFOgj5LxZDdx/7Xkc3zCBq77fwt2/e6ncJYlIyCjQR9DU+hpWLjmL809s4Cs/foqbfvWsviwVkTGjQB9htVUxbv1UMx+eO5Vv/PwZ/ul/NpPJKNRFZPTFyl1AEMWjEW689HQmT6ji39c+zxtv9fCNS0+nKqbPTxEZPQr0URKJGF+9+BSOPqqaG372R3bv7+F7V8xjQrUOuYiMDnUZR5GZ8YULTuCGj57Go1tf4xO3PcEbuhOSiIwSBfoY+Pg7j+XmK+bRumMvl37vMV7p6Cx3SSISQAr0MfL+U9/O9z83n117u7nk5sd4dqeurS4iI0uBPobedfzb+MFVZ9Obdi695bc8+dLucpckIgGiQB9js6ck+fEXzmFiTZzLb32Ch7bsKndJIhIQCvQyOPZttdx79TnMPHoCV97Zwn2/f6XcJYlIACjQy6ShrpqVV51F84xJfOkHG7h97fPlLklEKpwCvYySiTh3fHY+7z+1iWX3b+Yba7boUgEiMmwK9DJLxKN89/J5LH7ndG769Vb+4b6nSetSASIyDJq2OA5EI8a/fuQ0Jk+o4rsPPUfH/h6+9fEzqI5Fy12aiFQQBfo4YWb87YKTmTyhin9+oJU9neu45ZPNHKVLBYhIiTTkMs5ced7x3Hjp6Ty+7Q0uv/VxXSpAREqmQB+HPjpvGrdcMY8/vrqPS3SpABEpUUmBbmYLzGyLmW01s+uK7P+ymW02s41m9qCZHTfypYbLhbOb+P6fv4v23KUCtu7SpQJE5NCGDHQziwLLgYuA2cBlZjZ7QLPfA83uPge4F/j6SBcaRvNnTu6/VMD3fsuGlzvKXZKIjGOl9NDnA1vdfZu79wArgUWFDdz91+6+P7f6ODBtZMsMr9lTkvzoC2dzVCLGJ259nLXPvlbukkRknCol0KcCLxest+W2DebPgZ8W22FmS8ysxcxa2tvbS68y5I572wR+dPU5HDu5ls/e8Tse2Lij3CWJyDhUSqBbkW1FZ76Y2RVAM/Bvxfa7+wp3b3b35oaGhtKrFBqTCX5w1dmcMb2epXc/yX8+/mK5SxKRcaaUQG8DphesTwO2D2xkZhcC/wAsdPfukSlPCk2sifMfn3sX7zmpka/e9zT/78FndakAEelTSqCvA2aZ2UwzqwIWA6sKG5jZXOAWsmGu68GOopqqKLd8ch4fmTuVG3/xDJ+6/Xe89Pr+of9QRAJvyEB39xSwFFgDtAL3uPsmM1tmZgtzzf4NOAr4oZltMLNVgzycjIB4NMI3Lj2dZYtO5ckXd/O+b/+GFQ8/RyqdKXdpIlJGVq7/sjc3N3tLS0tZnjtItnd0cv1PnuaXrbt4x9Qk//cjc3jH1InlLktERomZrXf35mL7NFO0wk2pr+HWTzWz/BNn8uqebhYtf5R//WkrnT3pcpcmImNMgR4AZsbFc47hwS//KZecOY1bfrON93/7YR7dqnPWRcJEgR4gE2vj3HDJHO7+/FlEI8bltz3B//7hH9itC3yJhIICPYDOPuFt/PQvz+OLF5zAfb9/hQu/+RtW/WG7TnEUCTgFekAl4lH+dsHJrFp6LtMm1XDt3b/nc3es05UbRQJMZ7mEQDrj3PHYC9z48y0AXDpvGhfPmULzcZOIRIpNBBaR8epQZ7ko0EOkbfd+bvjZFn6+6VW6UxmaktV84LRj+OCcKcydXq9wF6kACnQ5wFvdKX7ZupMHNu7goWfa6UllmDIxkQ3306dw+rSJmCncRcYjBboMal9XL79s3cn9f9jBw8+205t2ptbX8ME52Z77O6YmFe4i44gCXUqyp7OXX2zeyf0bt7P22ddIZZxjJ9dy8ZxjOH9WA7OPSTKxNl7uMkVCTYEuh61jfw9rNr3K/Rt38Nhzr5POZN8nU+trOOWYOk45Jtn3c9zkWo2/i4wRBbockd1v9fCHtg4279hL6459tO7Yy7b2N8llPLVVUU56ex2zC0L+5LfXMaE6Vt7CRQJIgS4jrqs3zTM7s+HeumNfLuz3sq8rBYBZtjef/5nS95PoW1fgixy+QwW6/kXJsCTiUeZMq2fOtPq+be7OKx2dfb3459rfZHtHJ088/wav7u3qG7bJq6+NM2ViNtyn1ieYUl/DMfU1TK6tor42zqQJVUyqjVMTj+qLWZESKNBlxJgZ0ybVMm1SLe+d3XTAvlQ6w6593Wzv6OSV3M/2jk62d3TRtns/T2x7nX3dqaKPWxWLMKk2zqRc0E+eUEV9bVXBtiomVEVJVEWpjUepqYpSWxUlEY9SWxWjJh4lEY/oQ0ECT4EuYyIWjfQNuxT9vyKwt6uXV/d0sfutHnbv76Vjf+Hv/uVndr7Zt29gr/9QanJhn/9dHYtQFYtQFc3+zq9Xx6J926qKtKnOtamOFyzHIrn16EH7838fi+pKGzK6FOgybiQTcZKJ0k+LdHf2dafoeKuX/b0p9vek6epJs78nTWdvms7c7/71VP96T5ruVIae3M/erlRuOU1Pun97TypDTzpDb/rIv2uKWPZuU4UfEFWxSHZbke2xiBGPRohFjVgkQjxqA5YjxCPZ39GIEY8a0UiEqEE0kluOQMQst259y/3byP2NEYlA1LLPUdguFrXc/uzvaO6Mpu5Ums6eDF2pNF29abp6M3T2Zpe7c+tdvenctgw96XTu7yO5WrO1xyL513Xgvngk+7oiETAMs+z/Ao3sdzT5bdlycvvJvt5I4evO1174uvOvN3Lwa8u/3mgke6wjEQ78natjPFKgS8Uys8P+EBiuTMbpSWf6PgS6U9kPhO7e/uXs9tx6b8Fybl9vur9Nfjn7YXHg9v09KTo6M6TSTm86QyrjByz3prP7UpmR+aAZbbGIkYhHqYpFyLj31Z79Pf7rLyb/AZkP/uwHS/5DI/vejFr/cv7DMmLZtl+68ET+7PQpI16XAl2kBJGIkYhkx+XHE3cnnfG+oM9kIJ3blvHs9kwmu5723HJ+f1/bDOkMfX+TzhT85P4mNWCfk/1iPP/9RCIeJRGLUlOVHW5KFGyPH2KoqbD+dKYg7HPbUukMGc+2c8AHLpN9HY6TP2Ev407GD3w9B7xu94Ne78DXnX+9qXT/cSjc13/MsvVlcsepb9mzdeVr6dvm2c5B/ShN0FOgi1Qwyw2JxKKMuw+bUhTWL0dO39KIiASEAl1EJCAU6CIiAaFAFxEJiJIC3cwWmNkWM9tqZtcV2X++mT1pZikzu2TkyxQRkaEMGehmFgWWAxcBs4HLzGz2gGYvAZ8B7hrpAkVEpDSlnLY4H9jq7tsAzGwlsAjYnG/g7i/k9mVGoUYRESlBKUMuU4GXC9bbctsOm5ktMbMWM2tpb28fzkOIiMggSumhF7towbDm67r7CmAFgJm1m9mLw3kc4GjgtWH+bZDoOPTTscjSccgK8nE4brAdpQR6GzC9YH0asP1IK3L3huH+rZm1DHaB9zDRceinY5Gl45AV1uNQypDLOmCWmc00sypgMbBqdMsSEZHDNWSgu3sKWAqsAVqBe9x9k5ktM7OFAGb2TjNrAy4FbjGzTaNZtIiIHKyki3O5+2pg9YBt1xcsryM7FDNWVozhc41nOg79dCyydByyQnkcynaTaBERGVma+i8iEhAKdBGRgKi4QB/qujJhYWYvmNlTZrbBzFrKXc9YMbPbzWyXmT1dsG2ymf3CzJ7N/Z5UzhrHyiDH4mtm9krufbHBzD5QzhpHm5lNN7Nfm1mrmW0ys7/MbQ/le6KiAr3E68qEyXvc/YyQnW97B7BgwLbrgAfdfRbwYG49DO7g4GMB8K3c++KM3AkNQZYC/trdTwHOAq7JZUIo3xMVFegUXFfG3XuA/HVlJCTc/WHgjQGbFwF35pbvBD40pkWVySDHIlTcfYe7P5lb3kf21OqphPQ9UWmBPmLXlQkAB35uZuvNbEm5iymzJnffAdl/4EBjmespt6VmtjE3JBOKoQYAM5sBzAWeIKTviUoL9BG7rkwAvNvdzyQ7/HSNmZ1f7oJkXLgZOAE4A9gB3FjecsaGmR0F/Aj4krvvLXc95VJpgT4q15WpRO6+Pfd7F/DfZIejwmqnmR0DkPu9q8z1lI2773T3tLtngFsJwfvCzOJkw/y/3P3Huc2hfE9UWqDrujKAmU0ws7r8MvA+4OlD/1WgrQI+nVv+NPCTMtZSVvkQy/kwAX9fmJkB/w60uvs3C3aF8j1RcTNFc6dhfRuIAre7+7+UuaQxZ2bHk+2VQ/byDXeF5TiY2d3ABWQvj7oT+EfgPuAe4Fiyd8+61N0D/2XhIMfiArLDLQ68AFyVH0sOIjM7F3gEeArI32Dn78mOo4fvPVFpgS4iIsVV2pCLiIgMQoEuIhIQCnQRkYBQoIuIBIQCXUQkIBToIiIBoUAXEQmI/w/0UEBcQICYZAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train set:\n",
      "accuracy: 0.946875\n",
      "Test Set:\n",
      "accuracy: 0.94\n",
      "took 4.786148548126221 seconds\n",
      "\n",
      "\n",
      "lambda=0.001\n",
      "cost after iter 0: 0.6313815503597857\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAbK0lEQVR4nO3de5hUd53n8fe3qvpaQBOqG0igLxDJBZJAtAMZYzAaNWTWCboac9EZdTNBfZJVV5/ZieuOs5OdnZ3VGXWefbIzopOJqyImO15whhFvMcQ4SWgyxAQICeHWDQG6IVybvlTVd/+o6qZoGrpoqjld53xez1NPncuvTn0tK58+/M6vfsfcHRERKX+xoAsQEZHSUKCLiISEAl1EJCQU6CIiIaFAFxEJiURQb1xfX+8tLS1Bvb2ISFlav359l7s3DLcvsEBvaWmhra0tqLcXESlLZrbzTPvU5SIiEhIKdBGRkFCgi4iEhAJdRCQkFOgiIiGhQBcRCQkFuohISJRdoK/bcZC//JeX0LS/IiKnKrtAf3H3Yf7uiVc5cLwv6FJERMaVsgv0llQSgJ0HjgdciYjI+FJ2gd6cqgVgR1d3wJWIiIwvZRfoMy+qJR4znaGLiAxRdoFemYgxY3IN2w/oDF1EpFDZBTrkul10hi4icqqyDPSWVJLtXcc1dFFEpEB5Bnp9kqM9aQ519wddiojIuFGegZ4f6bJd3S4iIoOKCnQzW2JmW8xsq5k9cIY2HzCzTWa20cxWlLbMUzVrLLqIyGlGvAWdmcWBh4B3Ah3AOjNb5e6bCtrMAT4H3ODur5vZ1LEqGKBxSg1mGosuIlKomDP0hcBWd9/m7n3ASmDpkDb3Ag+5++sA7r6/tGWeqioR55K6Gp2hi4gUKCbQZwDtBesd+W2FLgMuM7OnzOxpM1sy3IHMbJmZtZlZW2dn5+gqzptVn9RYdBGRAsUEug2zbeh4wQQwB7gJuAv4hplNPu1F7svdvdXdWxsaGs611lNoLLqIyKmKCfQOoLFgfSawZ5g2P3L3fnffDmwhF/BjpiWV5FB3P4e6NeuiiAgUF+jrgDlmNsvMKoE7gVVD2vwQeBuAmdWT64LZVspChxqYpGunul1ERIAiAt3d08D9wBpgM/Cou280swfN7LZ8szXAATPbBDwO/JG7HxiroiHXhw6wQ90uIiJAEcMWAdx9NbB6yLYvFCw78Jn844JonFKroYsiIgXK8peiANUVcS6eVK0LoyIieWUb6JD7xai6XEREcso60Fvqk+zQRVEREaDcAz1Vy8HjfRw+oVkXRUTKOtAHJunapbN0EZHyDvSW+vwNo9WPLiJS3oHePCU/Fr1LgS4iUtaBXlMZZ/qkal0YFRGhzAMdNEmXiMiAsg/0lpSGLoqIQBgCvT5J17FejvZo6KKIRFv5B7pmXRQRAUIQ6CdvGK1AF5FoC0Ggayy6iAiEINCTVQmmTqzSSBcRibyyD3TIj3TRvOgiEnGhCPTmVK26XEQk8kIR6C31SfYf7aW7Lx10KSIigQlHoGuki4hIOAJ9cKSLJukSkQgLV6DrDF1EIiwUgT6xuoL6CZUauigikRaKQIeBSboU6CISXaEJ9GaNRReRiAtNoLekatl7pIcTfZmgSxERCURoAr25Pn/D6IM6SxeRaApNoM/Kj0VXP7qIRFVoAr1JY9FFJOJCE+h1NRVMSVZqLLqIRFZoAh10w2gRibaiAt3MlpjZFjPbamYPDLP/I2bWaWYb8o8/LH2pI5uVSmo+FxGJrBED3cziwEPArcBc4C4zmztM0++5+4L84xslrrMozakkew6foKdfQxdFJHqKOUNfCGx1923u3gesBJaObVmj01Jfizu0a+iiiERQMYE+A2gvWO/IbxvqfWb2WzP7f2bWONyBzGyZmbWZWVtnZ+coyj275sGhiwp0EYmeYgLdhtnmQ9Z/DLS4+zXAz4FvDncgd1/u7q3u3trQ0HBulRZh1uC86LowKiLRU0ygdwCFZ9wzgT2FDdz9gLv35le/DrypNOWdm7raCibXVrBdY9FFJIKKCfR1wBwzm2VmlcCdwKrCBmZ2ccHqbcDm0pV4bpo10kVEIioxUgN3T5vZ/cAaIA487O4bzexBoM3dVwGfNLPbgDRwEPjIGNZ8Vi2pWtbvfD2otxcRCcyIgQ7g7quB1UO2faFg+XPA50pb2ui0pJL8+Pk99KYzVCXiQZcjInLBhOqXopAbuph1aD94IuhSREQuqNAFerNGuohIRIUu0Fs0Fl1EIip0gX5RbQWTqhM6QxeRyAldoJsZLfVJjUUXkcgJXaCDxqKLSDSFMtBbUrV0vN5NXzobdCkiIhdMSAM9SdZh9yENXRSR6AhnoNfr/qIiEj2hDPST0+gq0EUkOkIZ6KlkJROqErowKiKREspANzOaU7U6QxeRSAlloAO01GvooohES3gDPVVL+8Fu0hkNXRSRaAhtoDenkqSzrqGLIhIZoQ10TdIlIlET3kDPj0XXJF0iEhWhDfSGCVXUVsY1SZeIREZoAz03dFEjXUQkOkIb6JAb6aKx6CISFeEO9Pok7Qe7yWQ96FJERMZcuAM9VUt/xtmjoYsiEgGhDnRN0iUiURLqQNdYdBGJklAH+rRJVVRXxNipoYsiEgGhDnQzoyWVVJeLiERCqAMdyE+jqy4XEQm/0Ad6SyrJrgMauigi4Rf+QK9P0pfJsvdIT9CliIiMqdAHenNKN4wWkWgoKtDNbImZbTGzrWb2wFnavd/M3MxaS1fi+WnRWHQRiYgRA93M4sBDwK3AXOAuM5s7TLuJwCeBZ0pd5PmYPqmaqkRMk3SJSOgVc4a+ENjq7tvcvQ9YCSwdpt1/B74IjKvO6lgsf8NodbmISMgVE+gzgPaC9Y78tkFmdi3Q6O7/dLYDmdkyM2szs7bOzs5zLna0mjUWXUQioJhAt2G2DY4BNLMY8BXgsyMdyN2Xu3uru7c2NDQUX+V5aknVsvNAN1kNXRSRECsm0DuAxoL1mcCegvWJwFXAr8xsB3A9sGo8XRhtTiXpTWfZd3Rc9QaJiJRUMYG+DphjZrPMrBK4E1g1sNPdD7t7vbu3uHsL8DRwm7u3jUnFozCrPjfSZfNrRwKuRERk7IwY6O6eBu4H1gCbgUfdfaOZPWhmt411gaXwpuaLSCUr+da/7gy6FBGRMZMoppG7rwZWD9n2hTO0ven8yyqt6oo4H35zC1/+2cu8vO8ol02bGHRJIiIlF/pfig740PXNVFfE+PrabUGXIiIyJiIT6FOSlXygtZEfbtjNPs3rIiIhFJlAB7jnLbPIZJ1HfrMj6FJEREouUoHenEqy5KrpfPvpnRzrTQddjohISUUq0AHuvXE2R3vSrHx2V9CliIiUVOQC/dqmi1jYMoV/eGoH/Zls0OWIiJRM5AIdYNni2ew+dILVL7wWdCkiIiUTyUB/+xVTubQhyfK123DX/C4iEg6RDPRYzLj3xtls3HOE37x6IOhyRERKIpKBDvCea2dQP6GKr+mHRiISEpEN9OqKOB+9oYW1L3dq0i4RCYXIBjrABxc1UVsZ5+tP6ixdRMpfpAN9cm1uOoBVG/bw2uETQZcjInJeIh3okJsOIOvOI0/tCLoUEZHzEvlAb5xSy+9efTErntnF0Z7+oMsRERm1yAc6wMcWX8rR3jTf1XQAIlLGFOjA1TPr+J3ZKR7+9Q760poOQETKkwI9b9ni2ew90sM//XbPyI1FRMYhBXreTZc3MGfqBE0HICJlS4GeZ2bcu3g2L+09ypOvdAVdjojIOVOgF1i64BKmTqxiuaYDEJEypEAvUJWI89EbZvHrrV28uPtw0OWIiJwTBfoQdy9qIlkZ5xuaDkBEyowCfYi6mgruXNjEj3/7GrsPaToAESkfCvRh/Ie3zALg4V9vD7gSEZHiKdCHMWNyDe++5mJWPruLwyc0HYCIlAcF+hksWzyb430ZVmo6ABEpEwr0M5h3SR0LW6bwnWd2kc3qh0YiMv4p0M/i7kVN7DrYrfuOikhZUKCfxZKrpnNRbQUrnt0ZdCkiIiMqKtDNbImZbTGzrWb2wDD7P25mL5jZBjP7tZnNLX2pF151RZz3vXEmP924j/1He4IuR0TkrEYMdDOLAw8BtwJzgbuGCewV7n61uy8Avgh8ueSVBuSuRU2ks85jbR1BlyIiclbFnKEvBLa6+zZ37wNWAksLG7j7kYLVJBCaq4iXNkzg+tlTWLlOF0dFZHwrJtBnAO0F6x35bacws/vM7FVyZ+ifHO5AZrbMzNrMrK2zs3M09Qbi7kXNtB88wZNbNQujiIxfxQS6DbPttFNVd3/I3S8F/hj4r8MdyN2Xu3uru7c2NDScW6UBumXeNKYkK1nxjC6Oisj4VUygdwCNBeszgbPd1mcl8J7zKWq8qUrEuf1NM/n55v3sO6KLoyIyPhUT6OuAOWY2y8wqgTuBVYUNzGxOweq/A14pXYnjw10Lm8hknUfXtY/cWEQkACMGurungfuBNcBm4FF332hmD5rZbflm95vZRjPbAHwG+PCYVRyQlvokN7whxcp17WR0cVRExqFEMY3cfTWwesi2LxQsf6rEdY1Ldy9s5r4Vz7H25U7edsXUoMsRETmFfil6Dt45dxr1Eyr5zjOasEtExh8F+jmoTMS4vbWRX760j9cO6+YXIjK+KNDP0V3XNZF1+J4ujorIOKNAP0dNqVpunFPP93RxVETGGQX6KHxwUROvHe7hV1v2B12KiMggBfoo3HzlNBomVrFCF0dFZBxRoI9CRTzGHa2NPL5lP3sO6eKoiIwPCvRRuuO6RhxYqYujIjJOKNBHqXFKLYvnNPC9dbtIZ7JBlyMiokA/H3cvamLfkV5++ZIujopI8BTo5+HmK6YybVIVK57VxVERCZ4C/Twk8hdHn3i5k/aD3UGXIyIRp0A/T3csbMLQL0dFJHgK9PM0Y3INN10+le+1tdOvi6MiEiAFegncvbCJzqO9/GLzvqBLEZEIU6CXwE2XN3BxXbWm1RWRQCnQSyARj3HHdY08+UoXuw7o4qiIBEOBXiJ3XNdIzOC763SWLiLBUKCXyMV1Nbz9imk81tZOX1oXR0XkwlOgl9AHFzXRdayPn+viqIgEQIFeQosva2DG5Br+5uevcLi7P+hyRCRiFOglFI8Z//PfX832ruP8wcPPcKRHoS4iF44CvcQWX9bA//ngG9m45wgf/Yd1HOtNB12SiESEAn0MvGPuNP73Xdeyof0Q9zyyjhN9maBLEpEIUKCPkVuvvpgvf2A+63Yc5N7/20ZPv0JdRMaWAn0MLV0wgy++fz5PvdrFJ769nt60Ql1Exo4CfYy9/00z+Yv3Xs3jWzq5f8W/aQIvERkzCvQL4K6FTTy4dB4/27SPT6/coFvWiciYSARdQFT8we+00JfO8uf/vJmKuPHXH1hAPGZBlyUiIaJAv4D+8MbZ9KazfGnNFiriMf7X+64hplAXkRIpqsvFzJaY2RYz22pmDwyz/zNmtsnMfmtmvzCz5tKXGg73ve0NfOrmOTy2voM/+dGLuHvQJYlISIx4hm5mceAh4J1AB7DOzFa5+6aCZv8GtLp7t5l9AvgicMdYFBwGn37HHPoyWf72V69SEY/xp783FzOdqYvI+Smmy2UhsNXdtwGY2UpgKTAY6O7+eEH7p4EPlbLIsDEz/vMtl9Pbn+Xhp7ZTlYjxwK1XKNRF5LwUE+gzgMI7IHcAi87S/h7gX4bbYWbLgGUATU1NRZYYTmbGn7z7SvozWb62dhuxmPFH77pcfeoiMmrFBPpwCTNsx6+ZfQhoBd463H53Xw4sB2htbY1857GZ8We3zSOdzXW/PLfzdf7q9vk0TqkNujQRKUPFXBTtABoL1mcCe4Y2MrN3AJ8HbnP33tKUF36xmPEX772aL73/GjbtOcKSr67lu8/u0sVSETlnxQT6OmCOmc0ys0rgTmBVYQMzuxb4Grkw31/6MsPNzLi9tZGf/KfFzG+czOe+/wIffWQd+470BF2aiJSREQPd3dPA/cAaYDPwqLtvNLMHzey2fLMvAROAx8xsg5mtOsPh5CxmTK7h2/cs4sGl83h62wHe9ZW1/GjDbp2ti0hRLKiwaG1t9ba2tkDeuxxs7zrOZx/dwHO7DnHrVdP58/dcRWpCVdBliUjAzGy9u7cOt09zuYxTs+qTPPbxN/PHS67gF5v3c8tX1/LTjXuDLktExjEF+jgWjxmfuOlSVv3HG5g6sZpl31rPZx7dwOETurWdiJxOgV4Grpg+iR/edwOffPsb+NGGPSz56lqefKUz6LJEZJxRoJeJykSMz7zrcr7/iTdTWxnn9//+WT7/gxfoOqYRoiKSo0AvM/MbJ/PPn7yRe2+cxYpnd3HDX/6Sz//gBXZ0HQ+6NBEJmEa5lLFXO4/xjSe38Y/rd9OfzXLrVdP52OJLmd84OejSRGSMnG2UiwI9BPYf6eGR3+zgW0/v5GhPmutnT+Hjb72Ut17WoAm/REJGgR4Rx3rTrHx2F994cjt7j/RwxfSJfOyts3n3NZdQEVfvmkgYKNAjpi+dZdXze1i+9lVe3neMS+qquefG2dx5XSPJKt2kSqScKdAjKpt1fvXyfv7uiW08u/0gdTUV/P71zXygtZGmlGZ0FClHCnThuV2vs/yJbazZtBd3uPLiSdwybxpLrprO5dMmqq9dpEwo0GVQ+8Fu1mzcy09e3Mv6Xa/jDi2pWm6ZN51brprOgpmTdZMNkXFMgS7D2n+0h59t2seajfv4zdYu0lln6sSqXLjPm86i2VN0MVVknFGgy4gOn+jn8Zf285MX9/LEy52c6M9QV1PBzVdOZcm86dzwhnpdUBUZBxTock5O9GVY+0onazbu5eeb9nGkJ03M4LJpE5k/czLzGyczv7GOy6dNJKEzeJELSoEuo9afyfLMtoM8u+Mgz7cf4vmOQxzqzs32WF0R4+oZdYMhv6BxMjMvqtEFVpExpECXknF3dh3sZkP7IZ5vP8zzHYd4cfdhetNZAKYkK5k/s475jZOZd0kds+prmXlRLdUV8YArFwmHswW6OkXlnJgZzakkzakkSxfMAHJn8Vv2HuX5jkO5s/j2w/zq5VcYOFcwg4snVedfV0tzKklL/rk5Vau+eZES0Rm6jIljvWle2XeUXQe72dHVzc4Dx9l5MPfcdazvlLb1E6poSdXSlKqlaUotDROrSCWraJhYSSpZRWpCJROqEurKEUFn6BKACVUJrm26iGubLjpt37HedC7gD3Sz48BxduWf//XVA3z/ud3DHq8qEaN+Qi7cU8nK/HIV9RMqSU2oZHJNJZNqKqirqWBSTYK6mgqqEurmkWhRoMsFN6EqwbxL6ph3Sd1p+/rSWV7v7qPzaC8Hjvdx4FgvXcd6OXCsj65jfXQd66XzWC8v7T1K17Fe+jNn/hdmdUWMunzI1w2GfQWTqnPLE6sT1FYmSFbFqamIk6xKUFsZp7Yy9zywXpWI6V8HUhYU6DKuVCZiTJtUzbRJ1SO2dXeO9KQ5cKyXQyf6OXyinyP5x+FhHrsP9bD5taMcPtHPsd500TXFjMGQr6mMU52IU10Ro6oiTnVFLvCrK+JU558H1ytOrlcOPOLxweWKuOX2FWzLtTm5PxHLPesPihRDgS5ly8wGz7zPVTqT5XhfhhN9GY73penuzdDdl6a7L0P34LY03f0Zuntz6yfy+3rTGXr6s/T0Zzh8op/e/gy96dx67pGlJ52hlJen4jEjETMq4jESBUGfiBsVsdy2eH7bQNvcc+zU9Xy7k/tzz/GYEbOT22IxI26558Qp+yEeixE3iBW8JmYULBe8Pt8ud6xcm5OPk8c4bXmgzeBrcv9/x8wwctts8DUF60Pb5F9f+JrCdgPHDQsFukRSIh6jriY2qj8GxXB3+jNOTzpDb3+WvkyWvnTBI5P7I9Cf8VO2DSz3prOks046k2uTzmZJZ3xwuT+T25fOOv2Z3L50Nksm66SzPvh8oj+TX8+1yRTsyz3nt3tuPZs9uZzJOtlgxkxccAMhPxD8WO5fZsbJ0M9vJhYb+gdiyDon2xf+oTHIH9f41M1z+L35l5T8f4cCXWQMmBmVCaMyEYORe4/GLfdcqKezWbJZhoR97g9A1nPbs/ltg/ucwbY+0MZ98JgD7XzI8sljcMq2wuWhzyfb5t8PTm/LyXUveE023zgzsC/fjlNeD86pr3Xyz/lthe8zuI3Ta8AZsxMJBbqInJGZETeIxzRiqBxoIg4RkZBQoIuIhIQCXUQkJBToIiIhUVSgm9kSM9tiZlvN7IFh9i82s+fMLG1m7y99mSIiMpIRA93M4sBDwK3AXOAuM5s7pNku4CPAilIXKCIixSlm2OJCYKu7bwMws5XAUmDTQAN335Hflx2DGkVEpAjFdLnMANoL1jvy286ZmS0zszYza+vs7BzNIURE5AyKOUMfbqKDUf0g2N2XA8sBzKzTzHaO5jhAPdA1yteGiT6Hk/RZ5OhzyAnz59B8ph3FBHoH0FiwPhPYc74VuXvDaF9rZm1nmuA9SvQ5nKTPIkefQ05UP4diulzWAXPMbJaZVQJ3AqvGtiwRETlXIwa6u6eB+4E1wGbgUXffaGYPmtltAGZ2nZl1ALcDXzOzjWNZtIiInK6oybncfTWwesi2LxQsryPXFXOhLL+A7zWe6XM4SZ9Fjj6HnEh+DoHdJFpEREpLP/0XEQkJBbqISEiUXaCPNK9MVJjZDjN7wcw2mFlb0PVcKGb2sJntN7MXC7ZNMbOfmdkr+eeLgqzxQjnDZ/HfzGx3/nuxwcx+N8gax5qZNZrZ42a22cw2mtmn8tsj+Z0oq0Avcl6ZKHmbuy+I2HjbR4AlQ7Y9APzC3ecAv8ivR8EjnP5ZAHwl/71YkB/QEGZp4LPufiVwPXBfPhMi+Z0oq0CnYF4Zd+8DBuaVkYhw97XAwSGblwLfzC9/E3jPBS0qIGf4LCLF3V9z9+fyy0fJDa2eQUS/E+UW6CWbVyYEHPipma03s2VBFxOwae7+GuT+AwemBlxP0O43s9/mu2Qi0dUAYGYtwLXAM0T0O1FugV6yeWVC4AZ3fyO57qf7zGxx0AXJuPC3wKXAAuA14K+DLefCMLMJwD8Cn3b3I0HXE5RyC/QxmVemHLn7nvzzfuAH5LqjomqfmV0MkH/eH3A9gXH3fe6ecfcs8HUi8L0wswpyYf4dd/9+fnMkvxPlFuiaVwYws6SZTRxYBt4FvHj2V4XaKuDD+eUPAz8KsJZADYRY3nsJ+ffCzAz4e2Czu3+5YFckvxNl90vR/DCsrwJx4GF3/x8Bl3TBmdlscmflkJu+YUVUPgcz+y5wE7npUfcBfwr8EHgUaCJ396zb3T30FwvP8FncRK67xYEdwMcG+pLDyMzeAjwJvAAM3GDnv5DrR4/ed6LcAl1ERIZXbl0uIiJyBgp0EZGQUKCLiISEAl1EJCQU6CIiIaFAFxEJCQW6iEhI/H/j9R38+NvjDgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train set:\n",
      "accuracy: 0.944375\n",
      "Test Set:\n",
      "accuracy: 0.9375\n",
      "took 5.682788610458374 seconds\n",
      "\n",
      "\n",
      "lambda=0.01\n",
      "cost after iter 0: 1.0056345259838433\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAZK0lEQVR4nO3df3Dc9X3n8ed7d7Ura1cS0q78SzLIEENscGN7hEMDF2ACPUPv7MD0Guil7bW5uj/CXZtmOkN/pb30OnPXmTZpZ2h6XJqh7TThuMYQh7hNm9Q0uQ4By8Y2YOPEsQ2WZbBkGf+QrJ/7vj/2h1aybC225K/3+309ZjS73+/3s999syO/9OHz/Xw/a+6OiIjUvljQBYiIyNxQoIuIhIQCXUQkJBToIiIhoUAXEQmJRFBvnMvlvLOzM6i3FxGpSTt37ux397aZjgUW6J2dnXR3dwf19iIiNcnM3rzYMQ25iIiEhAJdRCQkFOgiIiGhQBcRCQkFuohISCjQRURCYtZAN7MvmdkJM3vtIsfNzP7MzA6a2V4zWzf3ZYqIyGyq6aE/BWy4xPEHgBXFn83AF668rIvbcWSA//kPb6Blf0VEppo10N39O8DAJZpsAv7aC74HXGdmS+aqwOn2HH2XL7zwQ86cH5+vtxARqUlzMYbeDhyt2O4p7psXuUwKgP7Bkfl6CxGRmjQXgW4z7JtxPMTMNptZt5l19/X1XdabZTNJAE6eG72s14uIhNVcBHoPsKxiuwPonamhuz/p7l3u3tXWNuPaMrPKpgs99AH10EVEppiLQN8K/ExxtssdwGl3Pz4H551RqYferx66iMgUs662aGZfAe4BcmbWA/weUAfg7n8BbAMeBA4CQ8DPzVexAC0NGnIREZnJrIHu7o/OctyBT85ZRbNIJmI0L6jjpIZcRESmqMk7RbOZpHroIiLT1GSg59Ip+s+phy4iUqkmAz2bSTIwqB66iEilmgz01nSSkwp0EZEpajLQs5kUp4ZGGZ/IB12KiMg1oyYDPZdJ4g6nhsaCLkVE5JpRk4FeultUUxdFRCbVZqBrPRcRkQvUZKDnyrf/q4cuIlJSk4E+uUCXeugiIiU1GejNC+qIx0xDLiIiFWoy0GMxo6UhqYuiIiIVajLQoTCOriV0RUQm1WygFxboUg9dRKSkdgM9ndLt/yIiFWo30DNJBjTkIiJSVruBnk5ydmSc4bGJoEsREbkm1G6gZzQXXUSkUu0Gelq3/4uIVKrdQC/20Ps1F11EBKjhQM9pgS4RkSlqNtBLPXTNRRcRKajZQE8n4yQTMV0UFREpqtlANzNyad3+LyJSUrOBDoVhFy3QJSJSUOOBntRFURGRotoO9HRKF0VFRIpqOtBzmST9g6O4e9CliIgErqYDPZtJMjqeZ3BU67mIiNR0oLemNRddRKSkpgM9W7xbVFMXRURqPNBz6qGLiJTVdKCXeuj65iIRkRoP9NbyErrqoYuI1HSg19fFaUwlNIYuIkKNBzpAayapBbpERKgy0M1sg5kdMLODZvb4DMevN7PtZvaKme01swfnvtSZZdNJreciIkIVgW5mceAJ4AFgFfComa2a1ux3gGfcfS3wCPDnc13oxWQzKa3nIiJCdT309cBBdz/k7qPA08CmaW0caCo+bwZ6567ES8tltISuiAhUF+jtwNGK7Z7ivkq/D3zczHqAbcB/melEZrbZzLrNrLuvr+8yyr1QNp1iYHCEfF7ruYhItFUT6DbDvunp+SjwlLt3AA8Cf2NmF5zb3Z909y5372pra3vv1c4gm0mSd3j3/NicnE9EpFZVE+g9wLKK7Q4uHFL5BPAMgLu/CNQDubkocDal7xYd0IVREYm4agJ9B7DCzJabWZLCRc+t09q8BXwEwMxWUgj0uRlTmUU2rfVcRESgikB393HgMeCbwH4Ks1leN7PPmtnGYrNPA79gZnuArwD/ya/SIuXl2/8V6CIScYlqGrn7NgoXOyv3fabi+T7gzrktrTrZ0gJdGnIRkYir+TtFWxrqMNOQi4hIzQd6Ih6jpSGpBbpEJPJqPtChePu/eugiEnGhCPTWtBboEhEJRaDnMin6dVFURCIuFIGezWjIRUQkHIGeTnH6/Bij4/mgSxERCUw4Ar14c9GpIfXSRSS6QhHouUzp9n+No4tIdIUi0CcX6FIPXUSiKxSB3prWei4iIqEI9FxxPRcNuYhIlIUi0JsWJEjEjJMachGRCAtFoJtZcS66eugiEl2hCHQozEXXGLqIRFl4Aj2TpF9DLiISYeEJ9HRS3ysqIpEWnkDPaMhFRKItRIGeZGh0gqHR8aBLEREJRGgCvTQXXb10EYmq0AR6aYEuzUUXkagKUaCXeui6MCoi0RSeQE+rhy4i0RaeQM9ogS4RibbQBHpDMsGCuriGXEQkskIT6FD8blENuYhIRIUs0FNaQldEIitUgZ5LJzWGLiKRFapALwy5qIcuItEUqkBvTacYGBzF3YMuRUTkqgtVoOcyScYmnDPDWs9FRKInVIE+ORddwy4iEj3hCvTSAl2auigiERSuQFcPXUQiLFSBnisu0NWvqYsiEkGhCvSWhkIPfUBDLiISQVUFupltMLMDZnbQzB6/SJufNLN9Zva6mX15bsusTjIRo6k+oSEXEYmkxGwNzCwOPAHcD/QAO8xsq7vvq2izAvhN4E53P2VmC+er4NnkMin61UMXkQiqpoe+Hjjo7ofcfRR4Gtg0rc0vAE+4+ykAdz8xt2VWL5tJqocuIpFUTaC3A0crtnuK+yrdDNxsZv9qZt8zsw0zncjMNptZt5l19/X1XV7Fs8imU1rPRUQiqZpAtxn2Tb+3PgGsAO4BHgW+aGbXXfAi9yfdvcvdu9ra2t5rrVXREroiElXVBHoPsKxiuwPonaHN19x9zN0PAwcoBPxVl82kODU0ykRe67mISLRUE+g7gBVmttzMksAjwNZpbZ4D7gUwsxyFIZhDc1lotbLpJO5waki9dBGJllkD3d3HgceAbwL7gWfc/XUz+6yZbSw2+yZw0sz2AduB33D3k/NV9KXou0VFJKpmnbYI4O7bgG3T9n2m4rkDv178CVR5PZdzI0BjsMWIiFxFobpTFApL6AKaiy4ikRO6QM9mKnvoIiLREbpAv25BHTHTGLqIRE/oAj0WM1rTmosuItETukCH0t2iGnIRkWgJZ6DrblERiaCQBrp66CISPeEM9HRSF0VFJHJCGei5TJKzI+MMj00EXYqIyFUTykAvzUXXV9GJSJSEMtBb0/puURGJnlAGevn2f10YFZEICWWgTy7QpR66iERHOAO9tITuoHroIhIdoQz0TCpBMhFTD11EIiWUgW5m5NJJ+hXoIhIhoQx0gNZMkgENuYhIhIQ20LPplNZzEZFICW+gZ3T7v4hES2gDPZdJ0X9uhMLXnYqIhF9oAz2bTjIynmdwVOu5iEg0hDfQ9d2iIhIx4Q30dOn2f42ji0g0hDfQM1qgS0SiJcSBriEXEYmW8AZ6urSei3roIhINoQ30+ro4mVRCS+iKSGSENtBBNxeJSLSEO9DTSS2hKyKREepAb02n1EMXkcgIdaDnMkldFBWRyAh1oGczSQYGR8nntZ6LiIRfuAM9nWIi75w+PxZ0KSIi8y7cga7vFhWRCAl1oOeKd4tqPRcRiYJQB3pr6W5RBbqIREBVgW5mG8zsgJkdNLPHL9HuJ8zMzaxr7kq8fJMLdGnIRUTCb9ZAN7M48ATwALAKeNTMVs3QrhH4r8BLc13k5Wpt0BK6IhId1fTQ1wMH3f2Qu48CTwObZmj3B8AfAcNzWN8VScRjtDTU6aKoiERCNYHeDhyt2O4p7iszs7XAMnd//lInMrPNZtZtZt19fX3vudjLkc3oblERiYZqAt1m2Fe+U8fMYsDngE/PdiJ3f9Ldu9y9q62trfoqr0A2rQW6RCQaqgn0HmBZxXYH0Fux3QjcBrxgZkeAO4Ct18qF0VwmRb+GXEQkAqoJ9B3ACjNbbmZJ4BFga+mgu59295y7d7p7J/A9YKO7d89Lxe9Razqpr6ETkUiYNdDdfRx4DPgmsB94xt1fN7PPmtnG+S7wSmUzSd4dGmNsIh90KSIi8ypRTSN33wZsm7bvMxdpe8+VlzV3St8tempwlIVN9QFXIyIyf0J9pyhALq256CISDaEP9FIPXXPRRSTsIhDoWs9FRKIh/IFeHnJRD11Ewi30gd5UX0ciZpq6KCKhF/pAj8WMVt0tKiIREPpAh+J6LrooKiIhF4lAz2WSmrYoIqEXiUDPppPqoYtI6EUj0LWErohEQCQCvTWdZGh0gvOjE0GXIiIybyIR6LnSzUUadhGREItEoGfThdv/T5xVoItIeEUi0FcsypCIGb/0Nzv5xt7juPvsLxIRqTGRCPQbsmm2/MqHWNiU4pNf3sXPPbWDowNDQZclIjKnIhHoAD/ScR3P/cqd/O6/W8WOwwPc/7l/4c9fOKgvvhCR0IhMoAMk4jE+cddyvvXpu7n75jb+6B8O8ON/9l26jwwEXZqIyBWLVKCXLGlewP/66S6++DNdDI5M8BN/8SK/uWUv7w5prrqI1K5IBnrJfasW8Y+f+jCbP3wjz3T38JE//heefaVHF01FpCZFOtAB0qkEv/XgSr7+2F0sa23gU/9nDx//y5c41Hcu6NJERN6TyAd6yaqlTWz55Q/x3z96G3t7TrPh89/l89/6PsNjurtURGqDBTW80NXV5d3d3YG892xOnB3mD57fz9f39NJUn+Chte187PbrWbW0KejSRCTizGynu3fNeEyBfnEvHx7gb196k79/7W1Gx/Osbm/mY7cvY+OapTTV1wVdnohEkAL9Cr07NMrXdvfylZff4o23z1JfF+PHVy/lkfXL6LqhBTMLukQRiQgF+hxxd149dpqndxxl6+5ezo2Mc2NbmkduX8bD6zrIZVJBlygiIadAnwdDo+N8Y+9xnuk+yo4jp0jEjPtWLuJj65fx4RVtxGPqtYvI3FOgz7ODJ87xTPdRvrqzh5ODoyxqSrFpTTsPrW1n5RJdSBWRuaNAv0pGx/N8e/87fHVXDy8c6GM877x/cSMPr2tn05p2FjXVB12iiNQ4BXoABgZHeX5vL1t2HWP30XcxgztvyvHQ2nY23LaYdCoRdIkiUoMU6AE71HeO5145xrO7j3F04DwL6uL821sX8dC6Du68KUsirvu7RKQ6CvRrhLuz881TbHnlGM/v6eXM8DhtjSk2fmApD61t59alTZoCKSKXpEC/Bo2MT7D9jRNs2XWM7QdOMDbh3LKokYfWtfPRNe0sbtZ4u4hcSIF+jTs1OMo3Xj3Oll097HpL4+0icnEK9BpyuH+QZ185xrOv9JTH2zfctpiH17XzoZtymt8uEnEK9Brk7nS/eYotu47x/N5ezg6Pl+e3P7yunfcv1vx2kShSoNe44bEJ/vmNE2ypmN++ckkTD69t599/YKnG20Ui5IoD3cw2AH8KxIEvuvv/mHb814H/DIwDfcDPu/ublzqnAv3ynDw3wvN7C+Pte3pOYwZ3LM+yac1SHrhtCc0NWgVSJMyuKNDNLA58H7gf6AF2AI+6+76KNvcCL7n7kJn9MnCPu3/sUudVoF+5H/adY+vuXr6+p5dD/YPUxY27b17IxjVLuW/lQhqSupgqEjaXCvRq/sWvBw66+6HiyZ4GNgHlQHf37RXtvwd8/PLLlWrd1JbhU/ffzK/dt4LXjp1h655jbN3Ty7f2v0NDMs6PrVrExjVL+Tcr2qjTzUsioVdNoLcDRyu2e4APXqL9J4C/n+mAmW0GNgNcf/31VZYoszEzVnc0s7qjmccfWMnLhwfYuqeXba8e57ndvbQ01PHA6iVs+sBSbu9sJaaZMiKhVE2gz/Svf8ZxGjP7ONAF3D3TcXd/EngSCkMuVdYo70E8ZvzoTVl+9KYs/23jrXzn+31s3dPLs7uO8eWX3mJJcz33r1rEvbcs5I4bsyxIxoMuWUTmSDWB3gMsq9juAHqnNzKz+4DfBu5295G5KU+uRDIR475Vi7hv1SKGRsf5p33v8PU9x/m/3T389YtvkkrEuOPGLPfe0sY9tyykM5cOumQRuQLVXBRNULgo+hHgGIWLoj/l7q9XtFkL/B2wwd1/UM0b66JocIbHJnj58AAvHOjjhQMnONQ/CMDyXJp7iuH+weWt1Nep9y5yrZmLaYsPAp+nMG3xS+7+h2b2WaDb3bea2beA1cDx4kvecveNlzqnAv3a8ebJQV440Mf2Ayd48YcnGRnPs6AuzoduypYDfllrQ9Bligi6sUjeg+GxCV48dJIX3jjB9gN9vDUwBMAN2QbWd7Zy+/JW1ne2ckO2QStDigRAgS6Xxd053F/ovb946CTdRwY4NTQGQFtjqhDwnS2sX57llsWNWmdG5Cq40nnoElFmxo1tGW5sy/Dzdy0nn3d+2HeOl48MsOPwADuOnOIbrxZG2RrrE3Td0FLuwa/uaCaV0Bi8yNWkQJeqxWLGikWNrFjUyH/84A0A9JwaYseRAV4+fIodRwbYfuAAAKlEjNXtzdy6tIlVS5u4dWkz71uY0YVWkXmkQJcr0tHSQEdLAw+t7QAK36W6o9iD3330Xf5uZw+DL04AkIgZ71uYYdWSQsivWtLEyiVNtKSTQf4niISGxtBlXuXzzlsDQ+w7foZ9vWfKj2+fGS63WdpcXw749y9p4sa2NJ3ZtHrzIjPQGLoEJhYzOnNpOnNpHly9pLz/5LkR9h8/y77jp9nXe4bXe8/wz2+cIF/Rv1jaXE9nLs3yip/OXJplLQ0kE1qbRmQ6BboEIptJcdeKFHetyJX3DY9N8IN3znH45CBH+gc5XPx5fu9xTp8fK7eLx4yOlgV0ZieDfklzPYub61ncVE82k9KMG4kkBbpcM+rr4uVFxqY7NTjK4ZODHO4b5MjJybDvPjLA4OjElLbxmLGwMcWipkLAL26uZ1FTPUuKj6Xg1zo2EjYKdKkJLekkLekk665vmbLf3ek7N8Lbp4d5+/Qw75wZ5u0zw7x9eoS3z5znByfO8v8O9nNuZPyCc9bXxWiqr6OxPkHTgjoa6+toqk8UHhckaCpuF44VthuSCZKJGKnyT5xkIkYyEdP/FUjgFOhS08yMhY31LGys50c6Lt7u3Mj4ZOCfLoT+u0OjnB0e58zwGGeHxzk9NErPwBBnhsc4c36c0Yn8e6olEbNy2Ccrwz4eIxE34jEjESs9xqZsx2OVx2MkYkYsZsRjkIjFiFnheazUxorHzYjHi48xI2ZGzAr/l2LlfYXPqfAaim2mHjMK+80mH614LqPwSOkYhTpi5TYXvn7yfS7SJjb13KX3KtVWeK+KWijVNPU1BuVaRYEuEZFJJXjfwgzvW5ip+jXDYxPlwD9zvhD6Q6PjjIznGRnPM1rxWHg+MXXfxOS+ibwznvfy4/mxieJ2nvGJwv6JvDPhzviEM57PM5GHvBf250uv98LzCXcCmqB2zSr+HZjyR6C0+Hc5+Et/GJj8g0Dl9rRjVmwwuf/Cc5Tfu/xek8eZ4X0AfvW+m9n4gaVz/hko0EUuor4uTn1dnLbGVNClzKgU7BN5rwj+4h8BL+wrbVc+L/wx8Cl/MADcJ9s6heGswr7C87yD4+V2F3usbF8+3yXaVr5fPj+5D2AiXzo2+d5T6irtm1Lv5Gug0IZpbaafk9Kx6e9VPk/pvS88Vjp/odXkeSgfn7oPh+sWzM93/yrQRWpULGbEMDRdX0o0mVdEJCQU6CIiIaFAFxEJCQW6iEhIKNBFREJCgS4iEhIKdBGRkFCgi4iERGBfcGFmfcCbl/nyHNA/h+XUKn0Ok/RZFOhzKAjz53CDu7fNdCCwQL8SZtZ9sW/siBJ9DpP0WRTocyiI6uegIRcRkZBQoIuIhEStBvqTQRdwjdDnMEmfRYE+h4JIfg41OYYuIiIXqtUeuoiITKNAFxEJiZoLdDPbYGYHzOygmT0edD1BMbMjZvaqme02s+6g67lazOxLZnbCzF6r2NdqZv9kZj8oPrZc6hxhcZHP4vfN7Fjx92K3mT0YZI3zzcyWmdl2M9tvZq+b2a8W90fyd6KmAt3M4sATwAPAKuBRM1sVbFWButfd10Rsvu1TwIZp+x4Hvu3uK4BvF7ej4Cku/CwAPlf8vVjj7tuuck1X2zjwaXdfCdwBfLKYCZH8naipQAfWAwfd/ZC7jwJPA5sCrkmuInf/DjAwbfcm4K+Kz/8K+OhVLSogF/ksIsXdj7v7ruLzs8B+oJ2I/k7UWqC3A0crtnuK+6LIgX80s51mtjnoYgK2yN2PQ+EfOLAw4HqC9piZ7S0OyURiqAHAzDqBtcBLRPR3otYC3WbYF9V5l3e6+zoKw0+fNLMPB12QXBO+ANwErAGOA38cbDlXh5llgK8Cv+buZ4KuJyi1Fug9wLKK7Q6gN6BaAuXuvcXHE8CzFIajouodM1sCUHw8EXA9gXH3d9x9wt3zwP8mAr8XZlZHIcz/1t23FHdH8nei1gJ9B7DCzJabWRJ4BNgacE1XnZmlzayx9Bz4MeC1S78q1LYCP1t8/rPA1wKsJVClECt6iJD/XpiZAX8J7Hf3P6k4FMnfiZq7U7Q4DevzQBz4krv/YcAlXXVmdiOFXjlAAvhyVD4HM/sKcA+F5VHfAX4PeA54BrgeeAv4D+4e+ouFF/ks7qEw3OLAEeAXS2PJYWRmdwHfBV4F8sXdv0VhHD16vxO1FugiIjKzWhtyERGRi1Cgi4iEhAJdRCQkFOgiIiGhQBcRCQkFuohISCjQRURC4v8DmsO1hFeHuiwAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train set:\n",
      "accuracy: 0.946875\n",
      "Test Set:\n",
      "accuracy: 0.945\n",
      "took 6.017758369445801 seconds\n",
      "\n",
      "\n",
      "lambda=0.1\n",
      "cost after iter 0: 0.8538221970264084\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAcnklEQVR4nO3deXRc5Znn8e9TpdVWyTZ2lbElryDAhhgMagcOCZgtY2Cw01l6bKY7y0na9CQm6Uk6M2R6hs6hT870kqXTM3QaQ6eTkMQOWREZJyRhNQEHC7AJtjEY4UW2seR9laWqeuaPKslluSSVbVmluvf3OUen6t776uqhKH718ta972vujoiIlL5IsQsQEZHBoUAXEQkIBbqISEAo0EVEAkKBLiISEGXF+sPjxo3zqVOnFuvPi4iUpJdeemm3u8fzHStaoE+dOpXm5uZi/XkRkZJkZlv6OqYhFxGRgFCgi4gEhAJdRCQgFOgiIgGhQBcRCQgFuohIQCjQRUQCouQCvXnzXv7+V6+jaX9FRE5WcoG+tvUA33z6LQ4c6yp2KSIiw0rJBXoiVglA26HjRa5ERGR4KblAj2cDvV2BLiJykpIL9BM99I4iVyIiMryUXKCrhy4ikl/JBXpNZRnV5VHaDirQRURylVygmxnxWCXthxXoIiK5Si7QITOOrh66iMjJSjLQ47FKfSkqItJLSQZ6IlapL0VFRHopzUCvreJgR5KOrlSxSxERGTZKMtDjNbp0UUSkt4IC3czmmdlGM9tkZvfkOT7ZzJ4ys1fM7FUzu23wSz0hXqvb/0VEehsw0M0sCtwP3ArMBBaZ2cxezf4n8Ii7zwYWAv8y2IXmUg9dRORUhfTQ5wCb3L3F3TuB5cCCXm0cqM0+HwXsGLwST5Wo7Q50XekiItKtkECvA7blbLdm9+X6EvCnZtYKrADuznciM1tsZs1m1tze3n4G5WaMHVlJxNRDFxHJVUigW559vVeXWAR8293rgduAh83slHO7+1J3b3T3xng8fvrVZkUjxtiaSo2hi4jkKCTQW4FJOdv1nDqk8gngEQB3fwGoAsYNRoF9iSvQRUROUkigrwYazGyamVWQ+dKzqVebrcBNAGY2g0ygn/mYSgEStbq5SEQk14CB7u5JYAnwOLCBzNUs68zsPjObn232eeDPzWwtsAz4mJ/jRT8Tuv1fROQkZYU0cvcVZL7szN13b87z9cC1g1ta/+KxSnYf7iSddiKRfMP8IiLhUpJ3igIkYlWk0s7eo53FLkVEZFgo2UDXykUiIicr2UA/sbaoAl1EBEo40NVDFxE5WckHuq50ERHJKNlAH1FRRk1lmZaiExHJKtlAh+zKRVosWkQEKPFAj8cqaVcPXUQECEKgq4cuIgKUeKAnYlW0HdSXoiIiUOKBHo9VcqQzxZHjyWKXIiJSdCUd6Aldiy4i0qOkA73n5iKNo4uIlHagd68tqmvRRURKPNDjNVosWkSkW0kH+pgRFZRFTBN0iYhQ4oEeiRjjtLaoiAhQ4oEOWltURKRbQYFuZvPMbKOZbTKze/Ic/7qZrcn+vGFm+we/1Pwya4sq0EVEBlxT1MyiwP3ALUArsNrMmrLriALg7v81p/3dwOxzUGte8Vgla7YdGKo/JyIybBXSQ58DbHL3FnfvBJYDC/ppvwhYNhjFFSIeq2LPkeMkU+mh+pMiIsNSIYFeB2zL2W7N7juFmU0BpgFP9nF8sZk1m1lze3v76daaVzxWiTvsPaLFokUk3AoJdMuzz/touxD4sbun8h1096Xu3ujujfF4vNAa+6W1RUVEMgoJ9FZgUs52PbCjj7YLGcLhFtDaoiIi3QoJ9NVAg5lNM7MKMqHd1LuRmV0MjAFeGNwS+5fQ2qIiIkABge7uSWAJ8DiwAXjE3deZ2X1mNj+n6SJgubv3NRxzToyr0XwuIiJQwGWLAO6+AljRa9+9vba/NHhlFa6qPMqo6nLNuCgioVfyd4pC9uYi9dBFJOQCEehaW1REJCCBnrn9X1+Kiki4BSLQ47HMBF1D/H2siMiwEohAT8Sq6OhKc0iLRYtIiAUi0HVzkYhIQAK95+YiXekiIiEWiECP625REZFgBHoiVgVoyEVEwi0QgV5bXUZFWUSBLiKhFohANzPiNVpbVETCLRCBDpnFojUnuoiEWWACXT10EQm7wAR6poeuq1xEJLwCE+jxmir2He2iM6nFokUknAIT6InazLXouzXrooiEVGACPV6j2/9FJNwCE+jdPXRd6SIiYVVQoJvZPDPbaGabzOyePtr8iZmtN7N1ZvaDwS1zYLr9X0TCbsA1Rc0sCtwP3AK0AqvNrMnd1+e0aQC+CFzr7vvMLHGuCu7LuJpKzDTkIiLhVUgPfQ6wyd1b3L0TWA4s6NXmz4H73X0fgLu3DW6ZAyuPRjhvRIWGXEQktAoJ9DpgW852a3ZfrouAi8zsd2a2yszm5TuRmS02s2Yza25vbz+zivvRvXKRiEgYFRLolmdf77XeyoAGYC6wCHjIzEaf8kvuS9290d0b4/H46dY6oHhMt/+LSHgVEuitwKSc7XpgR542j7p7l7u/DWwkE/BDKh6rZLcCXURCqpBAXw00mNk0M6sAFgJNvdr8HLgBwMzGkRmCaRnMQguRiFVpsWgRCa0BA93dk8AS4HFgA/CIu68zs/vMbH622ePAHjNbDzwFfMHd95yrovsSj1XSmUpz4FjXUP9pEZGiG/CyRQB3XwGs6LXv3pznDnwu+1M0PWuLHjrO6BEVxSxFRGTIBeZOUci5uUiLRYtICAUq0Lt76O2HdbeoiIRPsAK9NrNYtHroIhJGgQr0kRVRqsujurlIREIpUIFuZlpbVERCK1CBDlpbVETCK3CBrrVFRSSsAhfo6qGLSFgFLtATtVUc7EjS0ZUqdikiIkMqcIGutUVFJKyCF+haW1REQipwgd5zt6i+GBWRkAlcoMdjGnIRkXAKXKCPHVlJxDTkIiLhE7hAj0aMsbp0UURCKHCBDplxdPXQRSRsAhno8Zh66CISPoEM9EwPXVe5iEi4FBToZjbPzDaa2SYzuyfP8Y+ZWbuZrcn+fHLwSy1cPFbJ7sOdpNNaLFpEwmPANUXNLArcD9wCtAKrzazJ3df3avpDd19yDmo8bYlYFam0s/doJ+Oyd46KiARdIT30OcAmd29x905gObDg3JZ1drS2qIiEUSGBXgdsy9luze7r7YNm9qqZ/djMJuU7kZktNrNmM2tub28/g3ILc2JtUQW6iIRHIYFuefb1Hpx+DJjq7rOA3wLfyXcid1/q7o3u3hiPx0+v0tOQiHWvLaovRkUkPAoJ9FYgt8ddD+zIbeDue9y9uzv8IHDV4JR3ZuLqoYtICBUS6KuBBjObZmYVwEKgKbeBmU3I2ZwPbBi8Ek9fdUWUWGWZxtBFJFQGvMrF3ZNmtgR4HIgC33L3dWZ2H9Ds7k3AZ8xsPpAE9gIfO4c1FyQeq1QPXURCZcBAB3D3FcCKXvvuzXn+ReCLg1va2YnHKmlXD11EQiSQd4qCeugiEj6BDfRErEpXuYhIqAQ20OOxSo50pjhyPFnsUkREhkRgAz2hlYtEJGSCG+haLFpEQiawga61RUUkbAIb6D23/2tedBEJicAG+ujqcsoiph66iIRGYAM9EjHiWltUREIksIEOWltURMIl0IGeUA9dREIk0IGe6aHrS1ERCYeAB3oVe450kkyli12KiMg5F+hAT8QqcYc9RzqLXYqIyDkX6EDXzUUiEiaBDvTu+Vx0c5GIhEGgA109dBEJk1AEutYWFZEwKCjQzWyemW00s01mdk8/7T5kZm5mjYNX4pmrLIsyqrpcKxeJSCgMGOhmFgXuB24FZgKLzGxmnnYx4DPA7we7yLORiFWqhy4ioVBID30OsMndW9y9E1gOLMjT7m+BfwCG1TeQmflchlVJIiLnRCGBXgdsy9luze7rYWazgUnu/ov+TmRmi82s2cya29vbT7vYM5HQYtEiEhKFBLrl2ec9B80iwNeBzw90Indf6u6N7t4Yj8cLr/IsxLNDLu4+cGMRkRJWSKC3ApNytuuBHTnbMeAy4Gkz2wxcDTQNly9GE7EqjifTHNJi0SIScIUE+mqgwcymmVkFsBBo6j7o7gfcfZy7T3X3qcAqYL67N5+Tik9Tz9qi+mJURAJuwEB39ySwBHgc2AA84u7rzOw+M5t/rgs8W/Ea3VwkIuFQVkgjd18BrOi1794+2s49+7IGT08PXVe6iEjABfpOUYB4TWaxaPXQRSToAh/otdVlVJRFFOgiEniBD3QzI16jtUVFJPgCH+iQGUfX2qIiEnShCPR4jW7/F5HgC0WgJ2o15CIiwReOQI9Vse9oF51JLRYtIsEVikDvXuhitybpEpEAC0Wgd68t+oftB4pciYjIuROKQL904ihqq8q46+GX+OA3n+fRNds1/CIigWPFmla2sbHRm5uHbv6uA0e7+NFL2/jeqi1s3nOUcTWVLJoziTvfPZkJo6qHrA4RkbNhZi+5e97ZbEMT6N3SaefZN9t5+IUtPLmxjYgZt8wYz0eumcI1F4zFLN/07yIiw0N/gV7Q5FxBEokYcy9OMPfiBNv2HuV7v9/CD1dv41fr3uHCRA0fuWYKfzy7jlhVebFLFRE5LaHroefT0ZXisbU7eHjVFl5tPcDIiigfuLKej1wzhYbxsWKXJyLSQ0Mup2HNtv1894XN/OLVnXQm01xWV8stM87n5pkJZk6o1ZCMiBSVAv0M7Dl8nJ+83MqvXnuHV7btxx3qRldz84wEt8w8n3dPP4/yaCguEhKRYUSBfpbaDnXw1Ott/Gb9Lla+uZvjyTSxqjLmXpzglpnjuf6iOKOqNeYuIueeAn0QHetMsfLNdn67YRdPbGhjz5FOyiLG1dPHcvOMBDfPHE/9mBHFLlNEAuqsA93M5gHfAKLAQ+7+d72O/wXwaSAFHAYWu/v6/s5ZqoGeK5V21mzbx6/X7+K363fxVvsRAC45P8bNM8Zz44wEl9ePJhrRuLuIDI6zCnQziwJvALcArcBqYFFuYJtZrbsfzD6fD3zK3ef1d94gBHpvLe2H+c36XTzxehsvbdlHKu2MHVnB3IsT3DQjwXsbxulySBE5K2d7HfocYJO7t2RPthxYAPQEeneYZ40EijOOU2TT4zXcdX0Nd11/AfuPdvLMG+08+Xobv92wi5+83Ep51Jgz7TxuvGQ8N12SYOq4kcUuWUQCpJAe+oeAee7+yez2nwHvdvclvdp9GvgcUAHc6O5v5jnXYmAxwOTJk6/asmXLoPxDDHfJVJqXt+7nidd38eSGNt5sOwzA9PjIzNDMJQmumjJGV82IyIDOdsjlw8B/6BXoc9z97j7a35lt/9H+zhvEIZdCbd1zNBPur7exqmUPXSln9Ihybr1sAguumMicqecR0bi7iORxtkMurcCknO16YEc/7ZcD3yy8vPCZPHYEH792Gh+/dhqHjydZ+UY7v1r3Dj9/ZTvLXtzK+bVVzL9iIvMvn8ilE3Uzk4gUppBAXw00mNk0YDuwELgzt4GZNeQMsdwOnDLcIvnVVJZx67smcOu7JnC0M8lv1u/isbU7+NZzb7P02Ramx0ey4PI65l8xkWkacxeRfhR62eJtwD+RuWzxW+7+ZTO7D2h29yYz+wZwM9AF7AOWuPu6/s4Z5iGXQuw70skvX3uHR9ds58XNe3GHWfWjmH/5RO64fCLja6uKXaKIFIFuLCpxOw8c4xdrd/Lo2u28tv0gZnDN9LEsuGIit8+aSE1l6CbNFAktBXqAbGo7TNPaHTSt2c7mPUcZURHljlkTWThnEldMGq3xdpGAU6AHkLvzyrb9LH9xK4+t3cmxrhSXnB9j0ZzJvH92neaWEQkoBXrAHeroomntDpa9uJXXth+ksizC7bMmsGjOZBqnjFGvXSRAFOgh8tr2Ayx7cSuPrtnB4eNJLkzUsPCPJvGBK+s5b2RFscsTkbOkQA+ho51JfrF2J8tWb+WVrfupiEZ436XjuXPOZK2dKlLCFOght/GdQyx7cSs/e2U7B451cfH4GIuvm84dl0+kokzTDYiUEgW6AJm1U3/x6k4efLaFjbsOMWFUFZ94zzQWzpmsSx9FSoQCXU7i7jz9RjsPPPMWq1r2Eqsq48+unsLHrp1KIqYblkSGMwW69Gnttv0sfbaFX762k7JIhA9eVccn3zudC+I1xS5NRPJQoMuANu8+wkPPtfCj5lY6U2neN3M8d11/AVdOHlPs0kQkhwJdCrb78HG++/xmvvPCFg4c62LO1PO46/rp3HBxQlP6igwDCnQ5bUeOJ3mkeRsPrXyb7fuPcenEWv7Xf5zJ1dPHFrs0kVDrL9B1zZrkNbKyjI9fO42nvzCXr/3J5ew70snCpav4L997ia17jha7PBHJQ4Eu/SqPRvjAlfU8+Vdz+fwtF/HMG+3c/LVn+N+/3MChjq5ilyciORToUpCq8ih339TAU381l/lXTOSBZ1q44StPs+zFraTSoVwTXGTYUaDLaRlfW8VXPnw5TUuuZdq4kXzxp3/g9n9eyfNv7S52aSKhp0CXMzKrfjSP3HUN9995JYc6ktz54O9Z/N1mNu8+UuzSREJLgS5nzMy4fdYEnvj89fy3eRfzu027ueXrz/Dl/7eeA8c0vi4y1AoKdDObZ2YbzWyTmd2T5/jnzGy9mb1qZk+Y2ZTBL1WGq6ryKJ+aeyFPfWEuH5hdz0PPvc0NX3ma7/9+C2mNr4sMmQED3cyiwP3ArcBMYJGZzezV7BWg0d1nAT8G/mGwC5XhLxGr4u8/NIvHlryHCxM1/PXPXuPDD7zAG7sOFbs0kVAopIc+B9jk7i3u3gksBxbkNnD3p9y9++LkVUD94JYppeSyulH8cPHVfPXDl9PSfpjb/3klX/31Rjq6UsUuTSTQCgn0OmBbznZrdl9fPgH88myKktJnZnzwqnp++7nruWPWRP7Pk5u47RsreeGtPcUuTSSwCgn0fBN45B0YNbM/BRqBf+zj+GIzazaz5vb29sKrlJI1tqaSr/2nK3j4E3NIpp1FD67iv//4VfYf7Sx2aSKBU0igtwKTcrbrgR29G5nZzcBfA/Pd/Xi+E7n7UndvdPfGeDx+JvVKiXpvQ5zH//I6/uL6C/jxy63c/LVnaFq7g2LNJSQSRIUE+mqgwcymmVkFsBBoym1gZrOBB8iEedvglylBUF0R5Z5bL+GxJe+hbnQ1n1n2Ch//9mq27dXcMCKDYcBAd/cksAR4HNgAPOLu68zsPjObn232j0AN8CMzW2NmTX2cToSZE2v56aeu5W/umMmLb+/lfV9/lodWtpBMpYtdmkhJ0/S5UlTb9x/j3p+/xhOvt3FZXS1/94FZXFY3qthliQxbmj5Xhq260dU89NFG/uU/X8mug8e54/8+x93LXmHjO7p2XeR0aal3KToz47Z3TeDaC8fxr8+8xXef38xja3dw62Xns+TGC7l0onrsIoXQkIsMO/uOdPLvv3ubf39+M4c6ktw8YzyfuelCZtWPLnZpIkWnJeikJB041sV3nt/Mvz33NgeOdTH34jh339jAVVO0cLWElwJdStqhji4eXrWFh1a+zd4jnbznwnHcfeOFvFvrm0oIKdAlEI52Jvn+qq088GwLuw8f593TzuOzNzVwzQVjMct3Q7NI8CjQJVA6ulIse3Er//rMW+w6eJyrpozh/bPruL4hzuSxI4pdnsg5pUCXQOroSvGjl1p5aGULW/Zk7jadNm4k1zWM47qL4lw9fSwjK3UhlwSLAl0Czd1p2X2EZ99o59k32nmhZQ8dXWkqohEap47huoviXNcQZ8aEmIZmpOQp0CVUOrpSNG/ex7NvZgL+9exNSolYJe9tiHPdReN4b0Oc80ZWFLlSkdOnQJdQe+dAR0+4P7dpN/uPdmEG9WOqqRtdTd3oEdSNqaZ+dHXmcUw1E0ZVU1GmG6ll+FGgi2Sl0s6rrftZ+eZuNrUdZvv+Y2zfd4xdhzrI/U/BLNOjrxtdTd2YEdSNzgT9+NoqYlVl1FaVZx6ry6mpLCMa0VCODI3+Al3fGEmoRCPG7MljmD355JuTOpNp3jnQQeu+o7RmQ7477Ndu288v/7CTZD8LXscqy3oCPl/gV5VHqSqPZB7LolR2Py+PUlWW8zynTUVZhPKoEY2Yxv6lIAp0EaCiLMLksSP6vOwxlXbaDnXQfug4hzqSHOro4uCxJAc7ujiYs32oo4uDHV28c7CDN9q6sm2TpPr5MBiIGZRHI5RHjPKyCOXRCBXRTNiXRzPb5WWZ42VRoywSIRoxyiKZD4OyqBGNRE5sn/QYIRqBSMSIWmZ/xDI/vfebGVHLfChGsu2iZpiRbX/y84iR/R0jEsk8z5w706a7bfc+yzl20vEIGCfanPi9zH6zTJ3Gif30PjeGRbJ/l5P//knnK/EPTgW6SAGiEWPCqMzY+pnoSqXp6ErR0ZV5PJ488bzn8aR9KZJppyuZpiuVpjPldKXSOT8ntjuTnn1Mk3LnWPZ3U+k0yZSTSmd+kj2P6RPbqeyjO+nsY9gXkeoJejKPmQ+HEx8qJ/ad+mGQ+6FyyodPzvZnb2rgjssnDnrtCnSRIdDdk45VFbuSgbk7ac/8X0navecxnYZU7na2XTqd8zz7wdD9PJXOfECkvfvDonubnnN4z3Oy25m/dfJxcDJtvNfv5G47mbo98w+S83dObddz7nTfv9fdDuekWtLZT73e9eetjVOPj6ouPyf/7hToInKS3KEVKS26LktEJCAKCnQzm2dmG81sk5ndk+f4dWb2spklzexDg1+miIgMZMBAN7MocD9wKzATWGRmM3s12wp8DPjBYBcoIiKFKWQMfQ6wyd1bAMxsObAAWN/dwN03Z49p2XYRkSIpZMilDtiWs92a3SciIsNIIYGe76vuM7pS1cwWm1mzmTW3t7efySlERKQPhQR6KzApZ7se2HEmf8zdl7p7o7s3xuPxMzmFiIj0oZBAXw00mNk0M6sAFgJN57YsERE5XQXNtmhmtwH/BESBb7n7l83sPqDZ3ZvM7I+AnwFjgA7gHXe/dIBztgNbzrDuccDuM/zdINHrcIJeiwy9DhlBfh2muHveIY6iTZ97Nsysua/pI8NEr8MJei0y9DpkhPV10J2iIiIBoUAXEQmIUg30pcUuYJjQ63CCXosMvQ4ZoXwdSnIMXURETlWqPXQREelFgS4iEhAlF+gDTeUbFma22cz+YGZrzKy52PUMFTP7lpm1mdlrOfvOM7PfmNmb2ccx/Z0jKPp4Lb5kZtuz74s12XtIAsvMJpnZU2a2wczWmdlns/tD+Z4oqUAvcCrfMLnB3a8I2fW23wbm9dp3D/CEuzcAT2S3w+DbnPpaAHw9+764wt1XDHFNQy0JfN7dZwBXA5/OZkIo3xMlFejkTOXr7p1A91S+EhLu/iywt9fuBcB3ss+/A7x/SIsqkj5ei1Bx953u/nL2+SFgA5nZYEP5nii1QNdUvic48Gsze8nMFhe7mCIb7+47IfMfOJAocj3FtsTMXs0OyYRiqAHAzKYCs4HfE9L3RKkF+qBN5RsA17r7lWSGnz5tZtcVuyAZFr4JXABcAewEvlrccoaGmdUAPwH+0t0PFrueYim1QB+0qXxLnbvvyD62kZkYbU5xKyqqXWY2ASD72FbkeorG3Xe5e8rd08CDhOB9YWblZML8++7+0+zuUL4nSi3QNZUvYGYjzSzW/Rx4H/Ba/78VaE3AR7PPPwo8WsRaiqo7xLL+mIC/L8zMgH8DNrj713IOhfI9UXJ3iuabyrfIJQ05M5tOplcOmXVhfxCW18HMlgFzyUyPugv4G+DnwCPAZDILln/Y3QP/ZWEfr8VcMsMtDmwG7uoeSw4iM3sPsBL4A9C9pvH/IDOOHr73RKkFuoiI5FdqQy4iItIHBbqISEAo0EVEAkKBLiISEAp0EZGAUKCLiASEAl1EJCD+P2bMv2B17fA8AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train set:\n",
      "accuracy: 0.94875\n",
      "Test Set:\n",
      "accuracy: 0.9375\n",
      "took 6.237306356430054 seconds\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "layer_dims = [X_train.shape[0], 20, 7, 5, y_train.shape[0]]\n",
    "num_iterations = 2400\n",
    "alpha = .075\n",
    "l2_lambdas = [0, 0.001, 0.01, .1]\n",
    "# interval in the iterations for which the costs will be stored\n",
    "cost_interval = 20000000\n",
    "np.random.seed(1)\n",
    "for l2_lambda in l2_lambdas:\n",
    "    start = time.time()\n",
    "    \n",
    "    print(f'lambda={l2_lambda}')\n",
    "    params, costs = fit(X_train, y_train, layer_dims, num_iterations, alpha,\n",
    "                        l2_lambda, cost_interval)\n",
    "    y_pred_train = predict(X_train, params)\n",
    "    y_pred_test = predict(X_test, params)\n",
    "\n",
    "    print('Train set:')\n",
    "    accuracy_score(y_train, y_pred_train)\n",
    "\n",
    "    print('Test Set:')\n",
    "    accuracy_score(y_test, y_pred_test)\n",
    "    \n",
    "    end = time.time()\n",
    "    print(f'took {end - start} seconds\\n\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.collections.PathCollection at 0x1f928b05688>"
      ]
     },
     "execution_count": 131,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAEGCAYAAABsLkJ6AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nOy9Z5hcV5Wo/e6TqqpzUivnnIMlS3KSLedsbAO2sbFJnjEwBoYMc4HhfpjLwAAe8Ax4MBiDIziAc8ZRtiRLsnKOLbXU3ercFU7a34/TqbpOdVBqhf0+jx9Q1Qm7TlWvtfaKQkqJQqFQKE49tP5egEKhUCj6B6UAFAqF4hRFKQCFQqE4RVEKQKFQKE5RlAJQKBSKUxSjvxfQFyKxEpmTP7S/l6FQKBT9QizPAiA3ClHdBsC0W5Atze3HpBrjGedtaGqpkVIO6Pr6CaUAcvKHcu7HnujvZSgUCsUxZdaCUQDMmSwAmFq0i8F7lgJgL3u3/bhtL6wMPf+0197ZFfa6cgEpFArFcczhCv/uOKF2AAqFQnEq0Vn4Ty0KjPiuwv9QBH8bSgEoFArFcUab4IdM4X+4Vn9nlAJQKBSK44ij6fLpilIACoVCcZzQG+F/JAR/G0oBKBQKxXHA0fb3h6EUgEKhUPQjYVY/HHl/fxhKASgUCkU/cSz9/WEoBaBQKBT9QH+4fLqiFIBCoVAcQ45VimdvUApAoVAojhH97fLpimoFoVAoFMeA4034g9oBKBQKxVHnePD3h6EUgEKhUBwl+jPFszcoBaBQKBRHgePR5dMVpQAUCoXiCHOsWzocKkoBKBQKxRHieErx7A1KASgUCsUR4ERw+XRFpYEqFArFYXIiCn9QOwCFQqE4LI7XFM/eoBSAQqFQHAInmr8/DKUAFAqFoo+cqC6frigFoFAoFH3gREnx7A1KASgUCkUvOBlcPl1RCkChUCh64GRx+XRFpYEqFApFN5yswh/UDkChUCiyciKnePaGflMAQojhwAPAIMAH7pVS3t1f61EoFIo2TkZ/fxj9uQNwga9KKVcIIfKBD4QQL0sp1/fjmhQKxSnOyezy6Uq/KQApZSVQ2fr/m4QQG4ChgFIACoWiXziZUjx7w3ERAxBCjAJmA++HvHc7cDtALG/IMV2XQqE4dTjZ/f1h9LsCEELkAY8DX5ZSNnZ9X0p5L3AvQHH5dHmMl6dQKE5yjvepXUeTflUAQgiTQPg/KKV8oj/XolAoTj1OJX9/GP2ZBSSA+4ANUsqf99c6FArFqcmp6PLpSn/uAM4EbgHWCCFWtb72HSnlc/24JoVCcZJzqqR49ob+zAJ6GxD9dX+FQnHqcaq7fLqiWkEoFIpTAiX8M+n3LCCFQqE42ih/fzhKASgUipOWUznFszcoBaBQKE5KlMunZ1QMQKFQnHQo4d871A5AoVCcNPQ2xROU8AelABQKxUlCb61+UMK/DeUCUigUJzxK+B8aagegUChOaHqT4tmGEv7pKAWgUChOSLrz94MS/r1BKQCFQnHC0Z3LB5Tw7y0qBqBQKE4olPA/cqgdgEKhOCHoyeUDSvj3FaUAFArFcU93LR3aUMK/7ygXkEKhOK5Rwv/ooRSAQqE4bskm/DujhP+ho1xACkUf8KoqcDevQvo+5rjpaINHEUw3VRxJuvr7IV34q1TPI4NSAApFL0ktexVn5ZvguoDE3fgBxsQ5RBdd3d9LO6lQwv/YoVxACkUv8BtqcVa8Aa4DyOBF18HdtALvwJ5+XdvJhBL+xxalABSKXuDu3pTlDQd3x/pju5iTlDDh3xkl/I88SgEoFL1A6AaE+fo1HQzz2C/oJCOb8FftHY4uSgEoFL3AGD2l3fOThhCY42ce8/WcTByq8FccPkoBKE4qpOciff+IX1fEcolc+LHA2jctMCzQDayzr0QrLD3i9ztV6K3wD0NZ/4ePygJSnBR4NZWk/vEkfvVeEBr6uOlEz7kKYUWP2D3MMVMxbv12EA/wfYwR4xGxPACklHg71uOseQ9pJzHGTcecNh9hRo7Y/U82+iL8levn6KAUgOKEx29pJPHkveCkghekh7dlNfGqvUQXXY02eCRC04/IvUQkGurysZe8gLP2vdYsIbBrq3A2rSDn+i8gVIwgAyX8jw+UAlAcFfzmBpwNy5CNdehDxmCMn3HUBKGzbil4bvqL0kfWV5N49o9gmMQu+yT6oBFH5f5+SyPOmiXpa/AcZGMd7uYPMafMPSr3PVHpLPw7o4T/sUfFAE5RZKKF1Af/IPHiw9gr30Qm40fs2l7lTuIP/QLngzdwN60k9fbTxB/9L2QqccTu0Rn/4H7wvfA3XQeScRLP/AHp2Efn/pW7gmygkHu7uzYelXueqHQV/qq/T/+iFMApiF9XTctDP8dZ/hretjXYy16l5cGf4zccPOxrSylJvvwYuHaHUHZsZGMtiRcfPqKKpg2tfBjoPWxmfR9351ESxqYVroCEQOQVHJ17noAo4X/8oRTAKUjqzb9BKtnhsnAdsBOk3nr6sK8tm+qQyZaQNyR+xVZaHvgJbsW2w75PZ6yppwdCuDtcB9lcd0TvC4H7J/na45kuKAiyhKbOP+L3PBFRwv/4RCmAUwwpJd6+HWQktUuJdyQEs2GC7CYN03VIvvgQ0svisjkERDSHnOs/Dz2kY/rx5tDX7Q3LaXngP2j+zf8h/sjduLu39Pre9pIXIGxXIwSRxdejlQzs9bVOVpTwP35RCuBUJFtGjH74mTJaTj5a6eDwqtk2fB//wO7Duo/0fWQy3p7zrxWUYAwb1/1Jicydib36Xey3nkY214Pv4dceIPnCn9t3KTLRTPL1J2i+7//S/Ie7SC15Adma6QMEbqUwhSdBOincbWuRbdlJCiX8jzNUFtAphhACY8Is3E0r0/3WuoExcU6frhXsJrbjbFiJbKpFxPLwWxqQVXsJL5s9PKQMrumsWYK97BVwnMDNctoizNmLMEZPxt34QRZ3jIk2MD0LSPo+9rJX21M323Edkn+/D2K54PtBemmronFWv4u3fzexaz6HEAJhGMjQ2LLEfutpEBpISfTiGzFGTjwCT+HEoqfmbm0o4d8/KAVwChI583L82ir8g5WBpS4l2oChRBZe0qfrpN58Gnfj8nCB2x2aliGMAaTvYa98E7e1mCoIrvrg2IGVLSVY0Y5/A/ge9vLXQTcwZ5yJPmIC3o4NdFVAIhrDnDg7/YZ2MrhWNkJ2DHgufvU+/AN70AeNwJg8D2fVW+HPoJNiSb74ELmf/BYiGst+v5MM1dnz+EcpgFMQYUWIXftP+NV78euq0UrK0QcM7dM1vOp9uJuyWNvpdwMtUDJtmTrRS25CtLqb/NoD2GveQzbW4idakHVV6cHprtjJzNdcB/v9VzDGzyR6yU2429djL3sF2VgHQsMYOw1rwUUIq6MqVzo2qaWvZE8f7Q7p49fsQx80AmvuefhVFXiVOwNl6nlZrilwd6zHnHxa3+93AqKE/4mBUgCnKEII9PJh6OXDDul8b9fGcAGdgUQbPAZj2FhEJIYxbjoimgME/vPkSw8HAl8epsvItYn/6aeYp52LPngUOVd/tr1NQ8aKpCTx9B+CthGHgqYjCkqAoEto7MpP4dVU4tdU4u7ejLd1ddhNke7RqUM43lDC/8RBKYCTBL+pHvuD1/EqtiJy8rFmnYMxZsrRu6Fhtvq3e2i8ppsYIyZgzT477WXp+6Ref7yXSqSXeC7O0ldwWl1H5pR5WGddjhDpuQ5+VQV+zb6+u64AhIaI5aJ3CTjrZYPRywajFZaS2Lkh5HNJjBEnfwxA9fQ/sVAK4CTAb2kk/tivA/eI9JGNdSRfeRRr7mKsOYt6PF/6Hs6G5Thr3wffw5gyF2vawqAHfhaMcTOw338ZvO4UgADLCm2FIBsOIp0jKPw70+rXdzYsRxSVYU1fmPa2X3ug99fSDYQVRaaCVE996Fgii69DaOlKRfo+XuUOZDKBPnJy6w7JBgQYBubMs9AKSw7rYx3vqLbOJx79qgCEEL8HrgCqpJTT+nMtJzLOijeDTJXO1rjrYC9/DXP6QkQ3RVJSSpLPPoC3ZyttgVPnnedx1y8n54YvZVjPbWh5hUQWX0fq1b+m+7zb0j81A33URCJnXAZWFHvt+zjLX0PGmxGFpVinLep595ANTQMjAnYPrSVcB+fDd9IUgLd/N87GFa1zfXsgEsWceRbWaecGGUdCtD9Lv7YK+8O38euqEEUD8HZu7NhR+B76+FkIgriHOXH2UetDdLxwOMJfWf/9R3/vAO4Hfg080M/rOKFx924LDzxqGn5tFfrA7H5+v6oiTfi3Ieuqcda+n2E9d8YcPxNjxEScXRuR1fsQ0RjawOHoQ8ciWhWBTDSTeusZ3A3L2gWkbKgh9ebfEQUlyLrqjHt3ixCYsxcFu5VeIJsbiP/lHkReEV7NXmiq7/EcrXwosevuQDbWk3r7GVru/UEgyCefhjX/IrwDu0k+88cg4IuE/Zk1Dd7WD4ldfiv60DG9/2wnKKqn/4lLvyoAKeWbQohR/bmGkwEtrwgvzK3heYjc/G7PdbauIZsAdtYt7VYBQNAe2ZowCybMan9Nug5eQw32h0vwtqzKniIpBBQUQ2Ntt/fojDZoJM6H77S6V3qB7wXB3t4GfA2TyJlXIOPNxP9yDzjJIEDtuTgfvhN0HtWMnuMHroO97n1iJ7kCUG2dT2z6ewfQI0KI24HbAWJ5Q/p5Nccn1pxzSOzbnh541HT0IaPQ8gq7PberLzuN1uvJZDwIfkZ6Hq5ir3gDe/lrQf5+DymWsqmOnM/8H+J//Akkwts0pGGYaKWD8asqwt/vTVA6DE1H5OSjlQ1CHz2F1BtPZY8TuA7Qy9hFMt1F5cebcLd8iEy0oA8bm7ZTOhFRwv/E57hXAFLKe4F7AYrLpx/58tKTAH3IaCLnXE3q7WcCAej76MPGEr3g4z2ea05bgLPyzdD3tCGjiD/6X/h11cG/Bw0nev7H0PKLQo93Nq8KhH8vM3u0wlI0TSd29WdIPPU7CGsi14rIKyRyztV4+3YeWvZON4j8InJu+ldkfXVg9R+JzCTDxBg3o/2fbsVWks/9qWM3sXoJ+pBRRC+7BewU3t7tYJjow8Z2G3w/XlA9/U8Ojv9fmqJXmJPmYIyfiWw8iIjmZM2B74qWX4QxcQ7uphXpb1gxvG3rOqZsAX7lbhJP3kvOzV9FaHrQ0E3T2q1YZ8UbvReehom14GIA9JKBRC+9meRT/5vVgpctjaSWPB/41A0z/D6HGFSWzfUkHvsVWumgI6NcDAutpBxj4qzWZXkkX3w4fc2ujbdvB8nXHsfbtrajP5MQxK647bgOGqvmbicPSgGcRAhdRxSX9/m8yOLrEMPG4ix9GZIJtMEj0cqH43bdGUgfmUpgf/AG7uZVyIYaMCOYMxZizbsAP97Uu3UWlmLNXoRfV41dewBj9BT8A3ta21JkOUlKZF01blN9MJD9iNYPePgH9+PXVh1eQVphKXpxOcaYqcEEtFZL3j9Q0d5LKA3Xwdv8ISDTFE/imfvJ/dR3DmknID0Xf/9u0A208mHdu/gOASX8Ty76Ow30YeBcoEwIUQF8X0p5X3+u6WTG3bkRe9kr+E316GVDsBZcFFQCuw7emiVBW2PPw6/chb9vB3ghQtZzcT54rUOgOakgKGun0AeNxNu5IfsChMBceBnCNEm99XeClplgv/8y+qjJQTfSnlozuM7hCWmtm3v0dQehacFz0A1EJErk7Ctxt60NOoB6LubE2cEYzG6FcMhnkRJv9xaM0ZP7tBx3x3qSr/wl+HytCkUbMproedei9dAquzco4X/y0d9ZQDf25/1PJZyNK4JBMK2Ws1exlcRTu4hd/VncbWuDsYqd8tiB9kZxafg+GULLdYKMocXXda8ApMR599nQ63o7N2RvU92VQ3XTxHKJXnM7yUfu7r2wN8zgeXS14DUdbchohGGhDx6JNAySLzzY3tbC27sNZ80Scq67A23A0KAPUq/bQss+t5D2G+tIvvxoxs7I37eD+F/uIefGL6PlHvp0sqMp/IUpwQfpnbgB8RMV5QI6BZDSJ/Xu86Ftj+33XsTv3IAt/cR0i7nNJRF2rO/h7d+d3T/f9boZ5/sYk07D3b0Zmg5hcpduBPfOL4KayvBj7BTy4H704ePxdm/q+ZrRXIxx04MW010VgO8hmxvIuekrSMem5Q93dfHxO8iGWpz1y7Bmnok5bT7O8td691lag/h9wdn4QfadjWPjrFlCpDXmcrgcKeFvDXQZ8bkmcicEz813QehgV+lUPppHw/JI1nMVRwY1EOZUIJkI76JJ0NVTZnOpCIExaQ4ilofIK8ScfU5GD5y0a21Y3nsrvivSx2+oIef6zxM579ruB8pkrFNDHzs96NGfaAnSQUMX6JJ65VG0gcMg0lNb5mCerzn1dCB8LbK+GnvjB61N5UKeoefgbluDu2sTzoblWW4j0gLAGCbW6eej5XRfvwEg7WT7cBqZaAmPMwBIP1DOh8jR6OmvxXwm/HsduRMdhB4Ifj0SlFhEh3iM/HwDhaeH/2YVRw61AzgJkL6Hu3UN7uZVYJiYk+eij5jQkWNuRVr91ZkWopZXgDZkNO765V3eF4jicoRhog0djTFyEsa46aSWvwa7sgxXFwJj/Mwgo6gXdQBd8St3EX/o5+gjxvfpPHQdY8ho7HeeRcabuo8R+D7OyjeJXn4bqTeeRDYczHK8RNZUknz1r8EMgiyFZ/ZbTxM5/6NZdz1+1d4O11AYhoU54wz8hhqEGcWcfFqPGUDegQpSrz+OX18NCPRRkzDGTg2ee+g6BNohJAfA0evsWbwwhTBlVl2tRWDIjc00LO259kRx6CgFcIIjpU/yuQfw9u1qF1Le7i2YU+YROetyIGhZbE5bgLP2vXQBYZhYc89HHzYOb882ZLwxaKRmmIBANhzEqQsyY7ydG7E/eB3ZXSsFKdGKysi54Us4G5YHwdD6mvBj9ZBqWs8F38XfuyNcKGt6YIx3nScsJdrgkXhvPtW7ALHn4m5ZRc5Hv0jq3edx12VvKyEPViJyC7ppViFwtq3Nfq8elKAwTax5ixGtuwApfex1S3HXvod0nWCWwexF7UV4flM9ib//Lm2QjbdzI7KpDjFgGHL/zsxnYBhYM8/sdh1h9Fb4O+teJXr2JoyhNUhfo+ZlA2HmIZ3su7jIEBe9B9luDWiLN6nYwNFCuYBOcLzdW/Aqd6VbqK6Ns+59/IaD7S9ZCy7GnDo/EO66CZEY1sJLW/vzx8i54U4i512LOetsrIWXtAraTn36XSewlrsLwEof78AenNXvYoyZSs51d0BIPYI5/yJybvhy+N+1BJlo6og3dEYItMFjgjTQNgwTc9bZQcVzb5ODpESmEkg7iZZXgCgbnN1t1HqsyDYwR/rIbDGHntB0Ihd8vF34A6Re/Sv2O8/iH9wfKOAP3yH+13vaXT3OuvczlYrv4ddVE1l4MdbZV3Z65gKKyohd+Sm0orI+La3XbZ1XvUHOZe9hjDiAMD20iEPZRQnGfL37nkuJnQZeT738GgVK+B9d1A7gBMfdtSl8rKEQeBVb29P/hKYROfMyrPkXBgItlpsmeIRuYI6bAeNmBNOtwv7werSuBd7W1XiIIPg573xyb/k6zvrluFtXI2K5QeppqzsiEpOk4pn3MU0fOWYa7tZ1wY5F00EIrLOvwpw0h+FmDdXvvYtmRRi06FwKJwXpkqvfnEDj5k09r9OwILeQ+AM/6WVKqcAYPg6ndn/G7kPkFyMPtaup75N642/oN30FIQR+fQ3utrXpStZzkS2NuJs/xJwyN6jK7roDgkCBNTdgTVuANW0B0nODXlBW3wOpfWnxYI7dizDcNP2pWZAzziE2yiGx0wy9R/37UQZd34IW8UN1r5Sw//HcPq9d0TeUAjjBEdGc8Nx2oYGVGegUhhnkpneHYZHdnBbh76W5dGSQYbQ0GNNozViINSOzqdy5H2nktb/m46Q6JIBp+ZxzdSPrR1xI/Y4FuDvWB1b+hFlohaWtwmkMZfNOz7jehM/9Ex/+8Ps4zc1ZA6JC18kfM5rGtUt6X0/ge4i84lDBq4+Zin+wEj+bq6tbJDLeiL9/N/rgkXgH9gSxmq63cR3cvdswp8xFHzQCb/eWzBoN3wsqmVsRuhG+i+qBvvb3iZ7ZiDBDnrUP0eFuVgUgHcGW75Uw5JYmihekQHTE/aWE+vcsDr6a0+f1K/qGUgAnOObEOeG9fAQYoyYd0jW1ssGIWC6y687CMMGMBBlFbcLeMNHKBgfVpxlr0PB2b0ELGQgDcPZthVRsaWbrmhiaFvzhj5yY5OZ/reTzdx1AHz6pPSCarfeM09hIxbNPU7tqJWZ+PmNu/iRb//gHvJbwvkLCNInv25s9YwbSFaphYkyYhf3+i6GHuivfJHLJJ0jt2RruHuts3obuFASypTH4f7kFhO68NL29/5I5eR72qrcg2ck9p5voI8ajFQ/I/pl6waG0dfbr85CuhjC6fDYBqf3dixe3UWP3PYXse9BjwGUtFMywcRs1qp7NoelDFfw9FigFcIKjFZYQOf+jpF57vHX4OqDrxC77ZLeDYNqQUuLv341XuRMRy8UYOw1hRYldfiuJv93XqgRkMGJx2gLM087FWf0u7tY1gUvntHPx9mwNWjl0tagFYGT/if3lvU9x12PfZOc6jT1bowwfl2TkhCT//d2hJJ5/mJyb/hUtrzC78G9uZuW/fRunuQnpuiQqoXnXTvxuhr34ySQ9OWyGXHwJle8sQVgRzOkLMSbNwV2/LMsFPbSBwzCnzMNZ8x4ZuyMhiH78X3DXvh9co6uS8D20gcMB0IeORkRzgtnBnZ+lprWmoxLEaz76RVJLXsDbtQlhmBjT5mPN7nnyW3f0JPy70pb142wdhjVtB9LvcOX4TiD841u6Fy9mmcfwTzeSPy3YzTSustjz+3zc+kNMJVb0GaUATgLMcdMxRk7Eq9yF0HW0wSPT/PvZkL5H8vkHg06Urg26QerNp4le9SmMwaPI+eQ38fZuRyZb0AcHraWdzStxVr0dnN9Uh73kBawzLsVZ/S7IzOwcffh4Uu+9GAxwsZOI/GK0IaMxx05jqz+B7971OW5a/ADzFjdSVWHx48+P5P2XC0HzcTauYP4XP5V1/ZUvv4TT0ozsJPD9VKpvNQQhHHjjdWZ+8zvkjR7Dqvd29ni8sKKY0xfihAl46eMsf53Ioqvxdm0MrH2v0+5i4ux2614Ijdg1nyX54sNBZbYQiEiMyPkfRcsvbr+klldI7MKeO732lmwKtjOdrX8R3Ys5eScyaeHuKSf+wnzkxKXkjnOQEhqWR6j4fT6ddzNaRFJ2cZzihUl8W3DwtSiDP9qCXiARrT/Vgpk2439Qx4avloKqCj4mKAVwkiBMC6OP+fPO+mV4e7Z0uDtahVfyyXuJXHoz5ugpGMM7Cr+86r2k/vFUWiqpf3B/kCsf4k/Xx0wj9c6zQVfRVp+1bKrD21SHt20NWslANo+fyXfumxDq0y4rFLTs3k39ujXoOTmUzT0dIzeXOZMFru2wbsW7oXOFNctC81N4nkT6bTGL3gsULx5n/S9+xrxf/rpdOK5+fQqNG9ZnHCuKy9F0A7euOnAdZSgAibdlNSnHJnbd53E+fBt329rW3cUZGBNnp689v5ic6z+P39wAroMoLMk6lrP9Fr4XzGs4BMWXrcUDhFn/PsbQx9Bim2C4BF+DeRuIvzyPrT8sRugy+Bn46esQhmT892uJDPLQWmPSsRHNoKXXDQoDjFyfwlk2DR+oKuBjgVIApyjegQrst57OGghNvfgQ3oTZiKJSzAmzA+t/9ZJQAUerDzvjHtvXBX7vLBPB/IP78QeOICyoLCyLRGUlH/7we4F7QdfZ+dADXPWLb1C1IcqTX7wLuyU8j1ATku/et5fmWpdx05K8+kQRf7tvQFqwGQJFIaUMVSJuMknzzh3kjwlaMkz9ytf44DvfwK7pFOyN5hK75vbgWkVl3eT8S7xdm0i+8hiR0y/ssSWDlH4Q3NeNboW6s30d9jvPIZvqgrTeOYswZ519xIfMtFn/WuFKtNytCK3ViaYHn9eY/z48XNreyycy2KVkUQKjQNK4wkKLSqyBfrvwhyBTKAxhBeeDUgDHAqUATkGk75N47o89Vsy6Gz8A3cBZ/jrRSz6B39LQt06cnhsEjrPVDnguXsWWoHp46+r2nYVmRYgMKKNxy2Z8OwhES9fFB5752s/QLYtUY3iQVzN0BowbwoLzV6CJQECVDarm+T+V4aQ6dgKaBgWlOgXDhrF72ZaM6wgEfifFoEejzPvPu2ncvImmbVvYH89BHzqmXdhqxQPQh47Bq8gyn1n6+Hu2kKjciTn7HCLzzg/iL3XV+E216IVliMIS7GWvtY+8FLkFWGdejjl2Wsbl3N2bSb3yWMduLJXAXvYq0nOJzF0c/ry70DfrH/SiFQgtU1ka+T7RYR7JCoOiBQlG3N4EetDWoWh+Ej8p0KOZvxspM7110hEk9iixdKxQT/oUQ3ou3v5d0E2gNI02t9DLj2LOXoS/b1fQtas3mGZ4znonhNCInHct+vDxRPeuQToOA848m9oPV5LYmznH13M9PDue7WIMmj6eT/3XDJLx59CETyQmKSjx+OkTW/npnSPYszWwLCfPjfOln2zAKtjBFxYPpak2fZ267pA/dlyXywsKJ06icOIkhkFGfCB6ySdIvvJYsPPJhuvgrHgDrbic1Jt/T5+CFokG30vrM5fNDaRe/QvCiqa54iBooR3W3M9Z+WZQOawfmUBqZ9+/bGmEsBZKUiB0ibAkwz/XlGbp69HABSTdwMWTfl4QNmp73XfArtFoWt1z8oLiyKAUwCmC1+qrlzX7SEu67i2+HyiOrsJf19HHzsDbtibE0heIYWOQe3eE96gxTIzJcxFCMO+Wq4Cr2t+qW7Ui83hAQ+JmUV5WzGDQQIO7Fj2G9IPisNJBDl/40V7OvLSR/3llM421OoYpyS0I3Bi2XcOnvi259/vlJOM6puWj6fCt/95J3twnqbdn8caacYTFEGYtGJWmBIRhYk6Z170CAJCQeuXRzFTUVEjzM9fBXvZKhgLwG2vDr+0HQ+Qs48kAACAASURBVHv8ZAv28tfwq/ehFZdjzT0vmP3Qae2d6U3mj7NtCFphc2bev5C4zYLc8U6r/z/d2teMzAxY6YPTIGhaa1E0zwYf6t6LUPlwHkgVAD5WKAVwCuDHm0j89Z5O1rjs+1AV18EP6/UvdKKLryVeVYGsr05/z04i92zDGD8jqHB1nVbFIwLFMXgU5rQFoVko8z9+Di9vWo+bSO+Lr2kC6Yev3Y47rHhxbav8CYTIwf0WP/niSH74xx3MPLOZwtJ0S9+yPC67qYoFF9RSWOqCBMcWRHMkkv+LLy3mnlvMB7W/otGZzIoN6ffurASklKTeeCrbE+xA+n0aPtO5pUcbWvGA8NoL3cBvqCX59H0dswkaDpKo2Er0slswuunm2pWMNs/acIyRB9BLG8EInqMQgQU/6T9q2fdwXtZYe8sWIwgCWyA0SapKZ8cvCrEPGOz5Ta+XpDjCqF5ApwCppa9md8W0ZZhoRvetnLMJLOnj7dqMzGqRevi1VeTd/u/kfO4HRC74GNbCS4hd9RmiV9zG7DODIKsubCaXv8i84X/mvBnrGLf4dEadMRMzFgmK2iwDM2ryuV/eRDSWuU5Nk9z6jb1Ykcx1phIaD/xsYLebntKBLkbrSIFYrgy6NAsPQ0sQ0/cxv+xTaCSZM1lk9MZpU2Ay0RJ0I+2RvilfvXQQfnMDXuUuZDJwf0XmX9TatK8Thok1dzGpd5/NnJzmOoHLiUOz/gHwNRIvz8PZWwad/PeaEbh6Bl4Rxw+ZY+Ml4cCTuaz7Qhlb/r2Yjd8uYdO3SrEPKPuzv1HfwCmAtyczyNmGKB+GXjYIY+QkvOq9wWD3vkzc8j1k2OjIzofU7EN6LpppoY2f2f56myAqzdnOrXNvxdBSGJqDROdgaj5T/+erbPtgDuve2kROYQ5XzPEpKW3kwLXjeOqxzThOIOCEJrnnxc1YUZ9HfzUobAns2xmeVeK6Al3vXiALAQYOc0sfZenBW4FAaHbeDcxaMIqVb23uWbYLDX3q6Xhr3+vhwFZ0A+m5xB/8z/Z2G+a0BVhnXEr00luw330Ov64akZOHOXcx+siJ2O8+F3opWV8T9AjqBdmHvAiMsobQ/j1Ggc+OuwsYeUcTQg80hNAlNS/GaFoTPP/kIQZ4jUKP/Ok2fkrQ+GEEaYdocyEpPM2m+KwE+ILaN6M0rrJQDeWyoxTAKYAwzKxyyRw7DWvWWQBo5UODIq++KAAJxqjJOEVl+LUHsixAy95tE7h+xleImXVoomOVA6JLqLOfY9zcqzh7YFXa8ddOqWf7OIs1W1PomuD0CxsYMspGNzuKiroucsyUjpRR34NkQmBFQdc0hOh5boH0HbYuXUHTngIip1+KlZPZqkCYFvqoSXg7N2ZPCdX1nsMvQoDQ0AYMQVhRvH07gu+k9Xtx1r2PKCrFmjof4+N3pp0af/K32a9rWsw6I/uksW6t/05IN8tOUUBih8W6L5SRP9NGz/WJbzV6bAnREwMua2HwR1uCNFMJiCa2/6yQlo2dg8WSkXc0UnBaqr3NdP7MFHXvRKn4/aGPwjzZUS6gUwBz+oKs7xmjOgaPazn5xK7+DKK4vNeTvbTRkxGGibX4+nAhr+kY42cgugxGb7P+C6N7KY7tSRP+AJpIkW88n2GJ2svexTQE376tmJ/eWcYd1xXwmdsk0Vwf05LcdOcBorF04RuJSW79xn4AfEfjwIflUF+ADgitd0NrnJTPQ/9vDw99/yl+f9nn2b9ua7vrRPo+nm0zc/5Iouddh9btQBeBt3dnj/czJs4mdvVn8fZtz1TIroOz8q2Mc/zmBvyqzMyptvua0xZk1Ah01+oZMge9AFS9JDNyAdrCGuVXtWAU+AhDMvhjLUz6SR3T/7eGQdc3g+hj3AmIjXYYfH0LmgV6TKLnSPSYZMxXG4JZwq3kjHPThD8EbqmSs5JEhx/iDOlTALUDOAUwJ8/FXv46hPin43/9NfrAEZgzz8QYMQG9fBi5N34ZmWgm8dIj+Hu3d3ttf89WWh76BbKhtUDKsAL/c2sPIK18GJGzr0o7p7MPWhdu1sIlw6/FMRrBLQgVRMkVaxgIpNYm8OYEf/Af/UI1+SUej/xXOXVVJsNHJrnjR3sZPzmBdHSE8Bk07SCYXlZLvM113vZ+Iq7x/sv5bF4VAYK6hGe++p98+plf4bz1N5b/+Vm8VIpIcTGjb7qFims+R/zx3+AfCAnSujYivxBZuz/85q0LcLevw5p9TtamdTIVUgRnJ8O7iQJYFtb8C7PesrfWv1bSSPnieFrnTiCI60eh7MIEpecmEBrt6aB6TDLg0jjCkFQ+0vOoy86ULEpkpo+2kj/DprG1YrhgRiq8uEyD/On2IbueTnbUUzkFELqBMK1wN5CdwtuzBa9yJ9a88wOhA4hYHuaUeaQOVGQdhwiAa6dn/7g2RGJEzr4SrXQQemm6T76z8K/9cCXrn3yI6x/2ae2H1o6UIPQWrFG/wNk2FJgMiNBRg/XvRxhyYzPSCkYMXnpTLRd/vBavUWP9l0vx7s1lU3mECT+sa7UQs1v9UoK7ayDewXzMkVXsrvR4/L4SXvlLcdpxqeY493/kyzQfqG2XgqmDB9n82//GuvhmzDnnkHr+z5k3ECLo+qlp3XckdWziT/8+S7aWQB8yOvPVorIgTtC1i6umY045ndld3D+HYv2LKcvQO9UCdFWimgFSz3xdj8KAixLsf7z7SWFd0SKEuvW0mGT4pxvZl5NH3VtRvLgW1Bp0UQLSAy+hYgDZ6NYFJIQoEEJkOA2FEDOO3pIURwOZZSh8O639+zsfZ4ydHsznNfpYmOO54DropYOQiWacratxd21ixtyOPPTShjVs/vXd1G7fy113jCDerJFs/UNtqxAVmocwfMwx+6iLL8k6Z9ZPamz592LiOwx8JyhViG8z2PtgLqO/Vs/EH9cy/NNNPcYCpQ+NK03WfVfirB9L/PmF3P3Pk3n5sRJkl9x0L2XTvD9znrBv25gb3wpmLoT145cSWVeNOXdxD1lXQCJLwZthEll4ScbLQtOJLLomyA5qk8C6EXRtbVXsYUwr2kye8SKl1n+il7wBepZMJt0jNrJnd0p3MQ4jv2/DcxqWRvBCfrpCgFkkGXZbEwMujVP3XiRrAL5hqWorkY2sOwAhxMeAXwJVQggTuE1K2dYT935gztFfnuJwkI6NdFKIWB7GiAnB0Pju8v91He9ARdC2oOEgeukgIhfdiDywB2f9MtwtH/Zu0Lvr4NdWYa96K6hY1XR0XWPpP/7K1K9/i0WXj+PhWx7BTQWW6sYVudy2YDKX3VzNLV+vomsRqzA9BlwSp/bNsDLUgFSlwZbvlaDnBAKmYE6K4bc3obfqLllOzxEvAbmTHGY8UIUbfxF30zjOnTOY7RUOqe4TndLwa7bxpVur+NmLMjP7VghEXiGRuYsxxs0g8fAvw1NsLQvC3DyANfe8rCMezbHT0ApLcVa/GxSLRWLIxjp49c/sTZ7N4PMvQLOsduvfFPUMif0zuqhHE0lkmYFe+jbxl2YDRWnX3r9tGeMOx5jWwGnoW9ixcZVF0xqL/Gk2eiyzfYQehUHXtlD9Ug47f1XIyC80pimCHb8sxGtRoc5sdOcC+g5wmpSyUghxOvAnIcR3pJRPoPKqjmuknSL5jyfwtq8PBE4sF+v0C1vHR6ay1wTYKZLP3N/67QbFWqJoAJGFFweTr+qq8Q9W9pwlpBtgRbCXvtKevdKWKbrx5z/hrIv/h/rd6XN0G2oNXnuyhBvurEbPyVRSWshrYXhxDRHxg5YEnX7dQg/XfZ0FihBgtA6hsvLBPG0ri0sOEp2Zw95KyatPFtFw0MJ1uluLZPzUBhafsZK/jJ/Izo3RdJ2pdwxo14vKMKctwFm/NP2ZGib6iIl4Wz4k1KztIUCvlw1GX3wdief+FKQAuw4tB2HX/kpqli9lxne/B+gIXM4q/yKG2N/xDLRgHbEz19Dyt7Po/Kc+4JJwhdRVKIf1+AHwbdH3Ns9SsPPuQsqviDP44y2h1xUmmIU+jSsjrL2jjLxJNlIKWjaaSFeJqu7oTjXqUspKACnlUuA84LtCiDvpayWL4piSfPHBQPj7XjBTtrmB1Jt/I3rxjZhzFqGVDcm+T5d+4Jv2PXBsZPVeks/cT+qVx/Br9qENGoHIL4JoDtnsABHNwW+qD+035NkOy373JGVDMlPzqios4s2Zws13oXFFxzY+Z5zD8NsbGfWVeorPTEKXPP5htzaF+o2FSFcC2QRV5+PzR9dx8af38qlvVfLAexv4xjfjWFb2P5tITHLbN4MA748e3MaEWQk0UwPTAitK5Nxr01oyWGdeijllXuC20fRW5RnD37+LbH9mTjd1HW14ByrwKramt+62bVp27aK0fjUgOa30i8T0FeFCNSeJiHVUdW17YSVmqR/+vGTQx8eLC3w7e82gHpVM/PFBiub34I5EkjvRpvzqFsoujGMU+kHcoJsppW5T8J1IR9C0JkLzWksJ/17Q3Q6gSQgxVkq5DaB1J3Au8BQw9VgsTtF3/MZavH07M101nouz4QNiF3wM5l2AvX4Z9jvPtkY9e/Bv+D5tJZ7+gT1EL7sFfegY4s88gNyzOeNwUTqoNXCc+RfrJlOsfPDveK6Prgs8r+MYyxCsemgii+5Yh2YEVruXAq9F48BTwYDwAZfEGfSxZjQzyDrNn2ZTer7B1h8Vt1uXRafbWQW7EMGjqdlvUDrQ7XFsblvnCqFJNOCc27ey8h/zeHWZjZ1Kl3QDhjp889e7mHxa4LsvKXe5++nNvL7iCu5+4Bq04vKMJm1C04mcfSXWwktIPPtAkDkUb+zWwpJ7ttD8hx+hFQ1AHzIac8rctIExAN7+naFBZt9O8cGfnuG081IMiCzNrgCFRHrpiq5ptUVsmJuRbSMd2PRvxZSck6T03AR6bpZdgQ6xER7D/6kRo9Cn5qXMmb+5E2xGf7UevW0evIQhNzVT/WIM3wG9izu/XaErk/SQ6O7nfwegCSGmSCnXA0gpm4QQlwA3HJPVKfqMbKoHPXwwiew0uNyaMg9zwixSbz2Nu2F572/gOiRfeyLoYplFcfh7tzPmppvZWbElmNDVBScZrM0wBEXFEerrUhTkCq4/P4/ipZvZvLeEARfFsQZ6NK83cZt0htzUhNsiKDs/idapA4IehdhIl6L5KerfDZLAhdG9NNB0KBvkYrdo6AV9C0riCz79T0laWqIsWWdjmhqu43PBrWP44vefR9fSA7eerxMrLUEvG9z9ZQ8eCIR/T8q4jUQLfqIFv3IXzqq3iF5+K8awjnwNkZPfmmmU6e7bt2ojeSkQhFvi0gevpgjsQNK3Bd+rX8ih9NwECNn+HXhJOPBULjljXMouSqQJaClpn8WT5rePwOCPtnDw1Vj7DAEAs9RjzLfq04W8CDJ7BlycCHYXVvq1hADfERTMStGwXM0R7itZFYCU8kMAIcRaIcSfgP8Aoq3/Oxf40zFZoaJPaCUDw338QsOvr6H53u+jlQ4mcsal6INHopcPC4K7vRU8AM313b6tWxbRAQMoGDeelu1bcBIhDWIA15XENIePXZvHonMk+1/fiCs0UvsMKu4vQJiS8d+rIzLERY8Gsiys1kyPQtG8ZLsCaF5nkT/D7q74GKFB4xILc1EyrXioN5gG3HlDEbe1+DQMmcqgIXk0TZ7Hmvfe4LffG8aODVHyijyu/Vw1132hkVX7rmX2guHoB//K+CGbqGkcwNvrFpGwOyzgoNNqH5URABI8l+SLD5P76e+0Tw8zRk0m/KkHNG49gBxqIkj/3tsmeml5cXKueAdn44iggEsKvCaNTd8pZsgnWsibYuM2aOx/IpfGD6JM/kVNhnUu2gp3w3YZmsQs8bGrO3ZEpYsTaXGbrh8zudcgb2KmW1GzJEbhoTw7RW/C4/OB4cC7wDJgH3Dm0VyU4tARsVzMqfMzG4VJPygUch38A7tJPP17vP27McbNCCzFI4h0HXKGDePm332LC39wB8PmTkE3wu9RedDngRca+OcfNFBwZxOjv1qPFgv+mEvPS7QLfwgs9zBhIj3SMj0qHsgL/NHdSUAgvt1k7x/zsWs0pA/x5kAAdv4vA03i7gsycApyNcZOKCY3z6Tl1df5PzePYPv6GFIKmuoMHvnVQP7tK5dTnxjKZ06/gS9f8x9cf/aj3Hbh//Kbf7mNkeU7Oi6bW0BG+hP0vm13Ko5f21GPIUyLoRdeFHqoEIItG6eQ9c9fSLTcFHpxM+bs9Qz7dJAWqkV8Rn2pkcLTUugxSWSQx8Ar42hRH7O4bwJYGOA2pn+2yEAvSysPgjYTO40s30nbFDFFX+nNX74DJAhGQUSBHVL2oZet4phjnXkZ1pmXB4VB0Zxws9l1sN9/CRGJErvqM9BWnIQA00QMG4c2YXZ6TnmbQ7wbNNOkeOZsomUD0AydcYvn8437bkA3s2euJOM6zQ06P/vacPKn2Yz6l2DEZPHC3lnn0oWa1zpSRO0DBhu+Wsr+p3Kxa7MIcgLDtvbNGOu/VMaHt5Sz/f8r5tmflPLUPaU8+eMydj0XC9wOXut/Luy+LwcRtYnM20DswmXg3A96Ew8/sBEnlb7zSiU01j27i9PK76UsdxuxSOByiVopcqItfPman7Yfq4+enKm0oU9tu/1OLj6AwRdc2F6R3RldF0w663Sqkt9HelGkF0F6FtLRwRNpQritnYJZ5jHkpmZyRgcKue2/2CiXoTc3k9zXt5pSt17D7zKis3mDiZdNaQtoWhfyfAh+liVn9aDtFaH0RgEsI1AA84CzgBuFEH89qqtSHBZCCKypp5N707+S8/E7s1r43sEgW0Wmkq05563DPBwHuXcb/p4tmHPPh0hOe4OybqNtpsXgCy9m4ue/mPZyJMfixu9fTSSiZ91sSCnYuiZGIqmRN9nGLPbwUuHKRsrA99yWdbL3kTwS29OFgxcXxIa7mIXZl5sz1gEkOWMdBl7TQtEkh1G7BKOX6IxZp9HyQg5eqiOAKX0YdmszuVe+gzl+D8agWqwpO7FG/JJdO6tDZbVuCAZ6L2LqXeYaCBhUvI/ivINIJ4X9wT9Ab80EElqgDMw+FDAJgYh2aMtZC0YRHVDO6I/fiGaaaIaOZujoEZMr/uUCBo4eQPEuB3vLN3Aqbib+0mzc/cWh8RPpCnJGORSfldluQTOh+Mwk+x7KzdhxZcuykj40rGq9kJCUX9nC1HuqGXpLc+A28tKvIT2o+EM+0SHZ7U4toqLAh0Jv1PZnpJRtUcL9wNVCiFuO4poURxARzcnqRtAKSpCeR/LFhzJjAFJCohnn/Rc7/i17KALzfUpmz0ELsTrPuG4us4pref7v23n9hZ04YWGKVh0jXYFR6NO4IkLeJCct6Ct9sKsFe35XgBaDlg0mXjxTq5RdFKfwtFRWl4IQYJV7jLijkcK5KTQzsPCH3NjMjl8U0rQmwpCbmjFyOloRaFabUOsQREKXSOEyZlotVZX5GRXDeElGjs4yK0GA70oST/w2sN7bAve6gVY+DHLy8beuDj+3K1KGKoyhF19Kyaw5RHcvRUq45OqhDBzVuYhMRyZG4NdUIJtzkL5AaF2EqQCnTkfLElwXBjSvtah6LoeBV8fbN5zZvFe+AzUvBPGPYbc2UXx2+k7Pd8FtDG7cstlk35/zsKsNCk9PtgeVu370RMWRGYF5qtHjDqCT8O/8mgoAnyAI3cCcsTB8eMjpFwSZJ925GbI6w0PwXNb+5C68kMwfgLHji/jiV+dw2Vm5mF10hKb7TJ4bJyfPB00y4IoWhtzQDK25+9IPrH63UbD9Z8U0r4/Q+EEkVPgDDLgwkTabtiuBkNEonBt0kBR60HdGi8CoOxsRhqRgpp2hQEJz5jW46V/3Y3UZfB6JeVxxSw35eXF8mb5O34fdOwqo+seKoGK3c9aW5+JXVWCUDe5TG47kE7/Ft1MZA1/OPHcQp33yKubeelW78A/r+WNvGtE60rED6QZzeuPbDJxGka01EeVXtZA7wek28A7Bd7nzF4WkKg30PJ+SkCC8END4QZR1dwxg5y+KsKuDH0vjighePPznuOe3quXzoaBqpE8BrPkXYc48KyhGEhoit4DIeddijJjQ+yBjBlncM65L7aqVoY3GIBA2H70gl5GDDSzdRzd8YrkexQNcvnH3brwkNK60KJxtBwLZ6FiiU6+x7l/KSIX4m40Cn+GfbWTab6uZek91t1khbemJRrGfNcaQO8nB70Ni1PjpCX704HbGTk0gNElBscuNd1bxue9Xomk+AontRXFdjXizRmOtwV2fGoCz8s3M5m2ti5SGgTZwWCclIMAwEcPHZym19Ui+/GjWNfbU8VM25ZL4x2z8eAQvFVjqLVtMtv24CBDorZPSuiIEDP5onNyJTve2hAcNS8324TCRQV5osZbQITYm8+FLV7DlByWk9mrtcRm3CbbeVUiyIjw+oOge1Q30JED6Hu6W1UE6p2FhTpmHMWJ8+/tCaETmX4g17/zA1WNa7S2YtYHDe937v9MFu90VuM0dzcTahE5nizNqafz4C6W89Mh6Dng6Y85t4ZxbapEtGrseKWTQ9c2ZVqEGVkmQbeIc7FJMZUkm/LAWo9hvTyOUXnYftBAgBeSND5fwwpSUX9FCyyaTgpl22k6izQvWeWcgXQ2vLo9pc5v575czC+MAXJnL+qrbWf6Hx6jao/HuC4U4KQ3Ioqg0HT2vGP3My7Hffgb/4AFEXgGR+Rfh7duRtRpYVmzBdxw0MxCIfe346VWWsfr2AqwyHz8lcBs72YjdpH70MPMnqDdsEuz9U4elblfpoTEH38s+OSxVabDxm2WYJUHGkF3dmrigOCSUAjjBkb5P8pn78fZ3FBF5uzdhTl+Y0TFSaBpYkS6v6cQu/QSJZ/+YOUc2KwLyi6CpLvTd4pmze76CEIwrthkHsFZjyzc7/NJDbmoOPUf6oJmZ6ytemETP9zN6/3SHZgRuGM+mvWFc+7kGFEx38JKtdVQpgoIlTZKq1PHiGrnjHHxPoFsCd/dAku9NJnrGOoyRVUCmpawJm/ffHMHTfxiK09JTKwQRTHETgtQTvwnqOqSPdG1Srz+OtfDSrGdKz2PPs08z8pprSTXUsfqvy/Edj/MuKIWigT3ct+0iIi0/HyERJtQtiVB6XqrHTWOY4pUO7H8iF6eu47puo0b90giF81LpBWQuVD2dS3c4tcrnfyToVwXQWlV8N6ADv5NS/r/+XM+JiLdrE97+PelBXNfBWf0uxtTTEVKC5yGKB2QdvKIPGU3uJ7+Js3UNXsVW/NqqwAXRcDC0sYvIySNy/vUk//a7DIVROm8+0bLwTpWdydbaGQJBM/DKeEbGiRfXSO3P/MPPGeuEunJ66vWDCCzN2FAX9EBpiE5Vq3o0aEVR9VKM5E6TVJVOYrsBCCKDXKyBHgPHzkTGgxTU5Fuz0NbXk3PJCiRO+3V8GaHFvZBBMY9s1qoWjYLvI6VEFJQRueiG4Pl2/l49F5n0cSt3BorcDo+17Pn736hZ8i6Jyn3tr737a4MzrjuNr362ow9UZ+t/xz6HHXsdROU25nykhcLpNvZBHWH5FM0NYiGpAxpeE+j5h+A5DAneAuy5twC3sZnS8xNoZtBxpGmD2ZfsV8Vh0G8KQAihA/cAFwIVwDIhxN/b2k4oeoe7c0PWgS2Jx38TdP8ERCRG9KIb0QePDD1WRGJYU0+HqacDwc6i5f67INmlJ70Q6CMmYAwZTfQjt2P/46kgg8WKMPqajzD0ssuzrjVswEgY1c/lUDQ/hVXmo0dle4Ox3f9dQJgUSe7T8VKZfWJ6QrqCA0/m4tRqlCwK+th0HSiiRyB/isP+TpOsIkNcCmbZwQCSgmaE6eE35AIC/2ARLX+fT+z8arScneBF8eoWUpt3B1PO8tCFT1fHkxG1mPy1bxIpLkboOpGSUpI1NSz/S0j3Tenj795MZPH1pF54MPyDuU6a8AdwbZdlzyxn55XFjJvqI1PlIHwcR/CTP9axYYdNQanLL56ppqDYJRKTSN9Ja+MQHeLjJaH+PYuC1hgNsnvXTzsCGldlfkHSE9S8FKN0cQIEaFEonO1QOLOWij/ncvCl7ncCisOjP3cApwNbpZTbAYQQjwBXA0oB9AHRVujV1VL3XEh0uFKk65B45g/k3vw1RCyv5+tqGtZZV2C/8hfScv+lBCswt41BIzFu+FL7W0O7ZJ+00dnn3Bv8pMbm75ZQND9F3hQbu0aj9o1Y1m1/3VsxBl0bDyaCdekTkzUX3W2tRJWS0vMT7SUQoevpVI8w+IYmBlycAK3NzbQiCBYncki8PgfZlItsyiX+VC4wCmveGcF5dUEy3Rd+eyv33P7HYA2+xPUk8267hnlXTwJgxQaJwOPcyX9glWbjhuRpCCsajPfU9N7NZwA0TXLZJyoZPf9D9KgHrk7OVREe+tZU1u+wsR34yD8doKjUac8mbRPsTXU6L/2liCs/WYthSqQnWPOZctB8xny9gdwJHTswLwmpAzqRgV77Dk46sP+pXJya8O9vzNcbgslfnb8nHYZ+ooWG96O4Dcrdc7ToTwUwFNjT6d8VBG0n0hBC3A7cDhDLG3JsVnYCYU6ei7P6XfB6UZzteTibVmHNOqtX1xZShjaWc9e+hz/7HLScnhVJnxGSkkUJyi5MokUk9e9HqH4hBz+R3cz0WjR2/rKAsd9uCHUz+F6HrGy3VjXQcnxGfqExGDTiBq9JP92i9ZJw8JXAxZM7wabswkSGa0q3QBpxci5cRsuTi4IIcyttu542RbBo6B7OeGYab2wZzJ4twXNNFA3ESSQxY1HmTBYUOXczb/gjvHHWYFa+lYfrdCxIi0QY85Gr8e0UO3QN2UsF4PuCmkqTSKz1eNND0xIMW7wN+6lgHueCixpDa88MS/LyoyW8/UwxP3tyK9FhrdfwNbb/tIiiBSmKz0ji24La16M0HlvdpQAAIABJREFUrbHInWRTfnkCs9Qjvt3MOpXLKPKIDAqfzyw0KJhlU/tG9kFAisOjPxVAls7iXV6Q8l7gXoDi8unKM9gFraiMyOLrSL3yWM8BXN/Dq9rT/TGdcHdsCB/+oun4+3agjZve7fm9HTTemeGfa6JofkduePllcYrmp9j07ZJuZ8lKX+AlBEZu+jMQAtDAbRZoEdkRHBZg5HXqctGWPSTBSwQHCF1S91aU+vcD4VV0RjJ88DiBsPKFzYE9Sxk4LMOOwV72DrHLmtGLl2JJnYvHOLwn8vn5V8fiea/zD+CKu7/N0JmjmT/iIQwtyTd+tZvvfXI029dHMQxIpTQGnHMuAxedS6qmhp2PPhyyaQl3tkdiHlPntaSvWZfMv6SeoNUXtDSGW9q6Liko8bjxzgNIGfTdmfCjg/g2NHwQ5eArsfZGfABokoFXJcidaKNHITrUo+SMJLv/N5/6JenCPNvA9/ZPo3r6H1X6sw6ggrZfXsAwgkZzij5ijp9Jb1PhvB0b8JsbenWsiGWvIibSN6usN/5/a6Cb0f9Hs8As9ig+o/vMmdQ+IzRDCIKPYOTJDGETmkqfgKrnc6i4P4+N3yil4v6OuEOPgU8BZoHPthdWZgS5zYm70QveQ2guQk9hmD7zFjfyme/uJNmSItmS4pkv/5gpeWvaK40Lij1++fRW/uu5LXz7N7u49+0Kxt5yK0IIogMGMOIj12FE2ip3JUJIxkxNMOus/7+9O4+TrKwOPv4799ba1fv09OwLzAIDCKiAGFREIUGCEjVG4hojWYxr1LyuyRuVJK+vBpeYN0hejeaD+4KiIIoEBcI2iMMyzADDTM8wW09Pd1cvtd6698kft6q6q6uq1+quru7z/Xz6o11dy3N76HvufZ7znDNKKDJ2R2gHPGKtHpe+rjxry7YEO38WuOnfV5JKlB5kzoEDeyJ84j96OOeiBIGAvy4S3eTSvN1lzWsTbL92ECv/eeE1ObZ+fJCW52THivgF/A12G64ZKSvZ4Jy0cOJVTkOevydEzZ96BoCdwDYROUVEQvg9Bm6u43gaW3CafyjG4OzZOfXzgOCZF1RsbC6BIPa6U6u+rtomsPG2XF6eKhrbkiupD19gR/zGL5PJDVsM3BOZtALo9DJXhORTQQbviZamQgKD90XwJhmG2IZMv0X7hWlip2V55raHiz8L7TiIBEun6cJRw6WvGySQfzw9muHz1/wIzyv9t9x8WobzLxkh27St5PH1V76Kz958mKvf3cvr33WCf/nZ0/zb7U9z7Y37+aN3nmDF6iwt7TkueXWcL932FLEJDdmNKzg9q2gOuoQsj//6YTu3faOTbFpIDFukEhY9T0boPRIkGPFK6jgVfpdWCEIrXFa8NEV0k8P2aweJbctVXBg2ntB8dobOl6RYeUWS6CYH8Fs+euMykAs7v3v+X0vVnd6qNuo2BWSMyYnIu4Cf46eBftUYs7te42l0wTPOx3nsvqn79XouZqhKbZoJ7K61hF70SrL3/MQvKGdAQmEiV/6Jv6dgnInlByYKnf87U94FVLsS9BzI9E29EHj4qy1kT9h0vyqJHa28a3UqxoPRJ0IgfimIlrOzOEPC4D1REnuDDPw6WpItVPgMNw3ZfputH44Xg5gTt3jmHx7CGbQ55/WVI4dtG8JRU8z23Pfgfn76nVdw1R//GEvGolnOi3A0976y4Lp9zSg7nlN6dxQMGd70/l7e9P6+krpF/g7o/Gq3a+Mlw5jHTucDF+zmt71RDg0HOfD1Vh683+JOr4mjR0Ls3x3lK3fvrVRUtMgKQedL0n5mUMhUzQoSy7D5HcP+OksgXxRuZ5hD17fyxHu66Pq9BLHtOdJHbXq/HyM3Mr3F32CHS2iVS+a4TS6uC8YzUdd9AMaYW4Fb6zmGpSL0gsvwBvv8PrC27W8ecl3KllUCQey1p0z/fc84j+C25+AeO4SEwlir1hebjkyXyV/aTQwCWy5/bslUyeieILkRQUKmZHOycWHgv6Yx5WSEEz+JMfJoiG3/exCp0D7QeGMbn72cfwL3r+r97iX7P9uOAbZ8OO7vL4j6AWj1VUl6vtjGkf9sYeDXEdrOzxDdmCPU5ZJLWGSOW3S+JJPfjOYfrxV22fTuYfZ9soPh3Tat57hlJ8e+oyES43bbOo7Hf33lBFdefBW51Q9gWydxvFMZzL6NuHNO2SH3ZV5Id+TXWBNO9FnvVALWCSyTQMRgPBuTssk+thUJZ3HjLbhHunj61kfoGQrheMLZ3Wm2d2YIBFze/KZe7tnZiojhaE+YtadkJ20bEezyCHV7k6aEWqHylNG28zJ+N7f7Ixz/XkvlF1YhtmHjX/rF/DxHsIKG+INhDt3QOvPm88uU7gReIsQOEP39t+DFT+INnMBqX0HmwV/iHnpqbDORZSPRZgLbyk8kk753MFxSWmK6dh1fzRPXX8+9P/gNuYzD6Weu4B3vO5e1/VVu9Iyw79oONr93iOiGnF8ALmlx6PpWslVSCCtJHQwyuidEbEe2uDfAuOBmhNG9Nk2bXNykRd/Po4w8GqLlTAc3JQzvCmMcofPiFE1bx1IbC9VIN/7VMLvf0UXqYJDUwdLaM2dd31fWzUosf5NaoM3l6LeaaT59ECvor0V4LmQzwhc/vJ6J6zfZjIs3ciaMnEkOOL7B35txZvtBdsdL93E8MfRhOsMPY5k0ASuDZ4JAgJy3hqB1BJGxCwBvpAln3/piltJo0uPzO7sYTNu4RgiGPa56Wx9v+5h/h7jpxWne8NcnSI1aU95N2U2marFYY/ItpaV8r4Ydgc6XpojfP7bwE9mQY+3VozRtcXDiFr0/jhG/r3yn3+rXJWh9vl+i2gr5x9l+fobsyQTHvzcPGWpLkAaAJcZq78Jq93fiRn73apxH78PZ/QDkHAJbziJ03suQ6a4XzNLDewzP2yHc8sHrOLbrCZyMPy215/F+PvSeX/P593Wwoq3yCd3pt3n67zoJdrpYIUOm1y5Jq5yuA9e10X1lku4rk1hRAxbYUUPbc3N+A5k7Igz8KkJko4vd6tFytkNsm8PJO6J0vKhyIxoR/4SeeMr//YVW+c1QWs7Mlm0gK77G8q98M0cCPPmRTrpfmSC21SHTb3Htpzby2AOlJ6pQ2HDR9tK5+sI+imMbLqiYWXU89WWagz8jbD1J1juFlPs8Vkc+hiVj005iudgrRrDX9eEe7gbgS1/u4WQyipv//ba1eLztY71leylird60duZWbeeI38ylZUeOSpstxncTi6zLse3vB4tTSYEWlw3XDBNsd+n7WemmsK6Xp8oCihWGrstSGgCmSQPAEiaWTejcF007738udt3fU7IOcM8dhzm6ay+5zIQ9BI7HbfcleePlk9/uz7XWi3GFod+GWHVVouzqVYKw+g8SdF+RwG6iuNvVuNB1aYrsYLVJ7LG0xECrx/ZPDmI3VZ/zLrBbXKJNHqlDAQ5/1S+Gtv5tw/zxu3t56tEorivkshaRJpd1m7Nc8fL2sv0D4AeCY/m7gfE82hh2ri5+3xyoPKsqQZfAmpPFAPDIiUjx5A/whvcdr3oM4+v/zXRtRQRaz3ZKNtSNF17lEtnokD4UZPVrE/5d0rjfqR2B1a9NcvL2pnFpoQYrUjkq2VUeV+U0AKiaGR8EUseOYgVsJnYmdxyPA0dmUGd5DtovSCNVqgT70walj0m+HlB4lVdxB7GXFpIH/D+ZrsuSky54wlgrhW1/F/f3CThw6PpWhnZGiG7O8fytaf7/r5/ktm910nc0yPNeMspFlyXI3duG2+uveUwMBOPvBgom7rS2WnqR1fipFePH4wom7R/0M7f9FsPqkp93rXEmPbkXgmR2WAi2mZIyEVMFBQlA7/ebWHt1eUDG8q/aD38lSNMWp0ohP0NwhUu2t3DKEpL7A8S2lic9JJ7R09p0aY6Vqqld9/cA0LR2Pbls+R9nMGix/YLpL0LPRdt52VlVCh5fEM4YvyBcLiHs/+e24nRU05Zc1U1hhdeBv+BsBf3AYkdg83uHaTk3Q+pAAC8H3esd3vI3vXzgc4e55NVxAiEHb7ip7P2yO+8tWUBf8+yDxa+JvNHtVPzTNoLzzLrit2esSGONWyO466ft05rqKZRqnumdwJo/TFRM07Vsv9Q3+CWiKxGb0tLUwOGvteCm87u48f/XTQtHvj6zxeTlTAOAqrld9/cQXbOGtjPPQoLjLsEFgiGLK64a20NQaT9ALYTX5AivqlxiYEYMpHoC7H5nF6kDY8eSOW4XTzxlL8k3nKlUP18ENv3FEAP/HcZMuBHyMhB/MMy+m/ZUrZY6MRBUHkAQ59Bb/TRPx/K/sjapu87FJKPF937ltmFaQi6RJn/19sFftjDQa08aBMT2p3NmGlhF/EBYqUubm4Gh/IavoYdDZZ9vPL9C7MRyIKkDQZ78aCf9v4qQ2Beg/1cRnvxoB6kebQ4zXXqvpObNjne9l57vf5eTd9+Jm8mw/QWn8q5rNtO5Ikp2//x+dtMWJ5+PP7f5YMn3Gzn1g3HCa13Sh2ycYYuOCzNlUyzF1wiTniDtFtj6sSFyIxaZPiG6zsVNCSd/GeX4D8cWOgsn6kpBcuLUUFlQCOSwXtKMvXoAPAssg726H/fIWKnu1rDH37ygjycyIZKneZxz8Sidq/xgMP4kXK2pzmwYx/8XKdw9uRl/4X/w7ihgWPmKVPl7G8hV2SOS7Q1w+D+0HeRsaQBQ86KwHnDqG97EH37qzYCfxjjTyqCzNZO00cl4WWg6JVdMBQ22eyVz3zBJxdFJOpJJ0J/2sJtgzwdWlO06Hm86gWCiyIW7sVcPILYB2z+ph7Y9S++9/cDYFFPQhvPWpWjdlmbtxcnydY+cv3ZhRWZ/0h/PeH5D+KYtDnbMEN8Z5uTtUbyMEFqVw45V6D9hQ9sLshz77tw/X5XSAKCWpMSTQZxBCyvkTtodrGL3qvxjbho/d33cXH+lRd/JToxTNaURG1a8PMWxb0+dtjhZICgRyBHY2Ouf/Md/VtCj+4oUA3eOBYAVlyZZ98ZR/6q8wsyJFYDssIXYXtVU15mwwtD7kyheuvx4jSNVF9XN5JVA1CzpGoBamoyw7x/aSTwVxMuVTmkUsnM8p3qvgJE9QZ7999aKJ8WKHzdx3jq/DjDVVbMVhMjaKcp3TFCp2Nx4EqjeecyOeTSfkWXLhwc54wsnWf+WUayQH+SqjdXLyFxn0oqMA6GVld/MGbBJPRsoa3HgpuHkL7Qk9HzQOwC1oI5tuGDBpoFycZt913ZgN3s078jSdWmK8GqX1LM2J29vwm7y2HDNSPmmLwNHv9FMqsdmw9sFu2nqs1+1evZTMQY8b+z9JWjoeFGKtudmceIW/XdESR0MEl6TI9jukToYKBZIq3ZHYNIhnDiEVkz4LM9v63jqB+PFxdipsn6MgVCX68/dm+pTX8bFb5IzVb9gr2KX0aKeL7ax9eODBJr9J4kNQw+F6f+VBoD5oAFA1cV0isPVijtqMbQzwtDOCWd6y7DipelizZ/iicmCrR8fxEtZxH8Tov38TEmQKJzErMBYk5myzWbTnC8XgaZN/h2AFTZs+/sBQt0udsR/784XpXHiFsF2Dy/n17s5/qMYJ35cvlg8XsuxFk557xAE8s1wHD/LKLLWK8nEmTJ/P79eYQIVnmv8NQI3KQzcHcFuMnS+OO03qHP9jFl73B2U8fxUzsyR6nNyzkmbPX+9guYzHIKdLsl9QTLH9DQ1X/Q3q+ZdoTTEouMJB77Qyo7PDPh16gsljgNAAOyIR8cFGdwsxZaFhYJyA3dGCK1ykQA0nVq5Kf102fmL2xWXJAmtcovlDSwbsPGLrAnFWvqrXpUgfTjA8G+qN0EeeSTM05/oYOUVScJrXBJ7gwzeF2bb38ZnP9AJxIL4zjAHv9hWfOzI11uwIgY3Cae8b5jmM7N+5U/HLwfd8/k2xk9PBdpd2p6fAYHhh8P+DnAjjO7WPgALQQOAqruJVUHnS3SzQ9OpObJ9FiOPh8DAadcOEmirXjpaAn5x1fE7XiUAbedn2f2uFQTaPM74XH/Z66Za/B3/vNRhf0qn7QWZio3tJ76PHYHuVyRLAkCg1aP7lQlaz82SG7bou7WJod+EOfRvYydnK+xVXfWb7njHc9OQ2FO6SGJygjvqv9GB69po2pIjdppDbsgi8VSQULdLaKVLts+m86VJ1r91tDgNte6Noxy5sZn+O8o3wqn5oQFAzZuJ9YF2xzfNqk3kXEnAcMr7h4idnvUXM42QGxEG7wsXr66rvtauvJxqRTzCa10yRwIc+0GMNa9JIAH/+W7a3ygW7vaQkN+GcrLP6P2RnxHjJqafkxHZkGPLRwcZ3RNk8L4I2z4ex272/EXrtS7RzUP0/jjGiZvHpoq8jEX8vgjtF5a2tpzO7l/j5qe98q/zHH9qbeDuyW59hOQzQZL7A6z/0xE2XDNcLNuc2BckttUp20297k2jjDwanjQtVtWOBgC15HX/fpLY6dlxV9cGK+z3rZ3OomW11E+T9V/c99MmxPLrA0kQhh4KceQ/W8AVOl6cZt2bKyw0F3iQ2OufBU/eHqV5R3bK6SRj/PLLLWf6FUy7r0z6VUfHXYzbEVjzugTh7hwnbokV59Gf/WoLsR1ZQl1eyV1Ntc8BPwWz/84ImeM2XZelsSKGoYfCHL8phpeeOmitvDxJR76fcqFsc+w0p3KekkDbBWn6bolV+qmqMQ0Aasnr+t1k2dTKdDJ0vCy4Kb+ZfMkisAuZXrt4lbr+T0boePFYCenOizK0nuXQe2uU6IYcbtK/6p2YJeNlIf5ApFjhcuSRMCdujbL61eWBqTBFU5yqGdeSseqGMws6X5yh/YUZ9n+mncTeECbnb0CbznSPiB8An/5UO6kDhSA18xPzystTZUHNsqvceQiT7ttQtaX7ANSS5jdkmVkSuzGQPWlx4mdN7P2bTuIP+r2A3bQfEJwhiwOf8+fWQ905Oi+u0Mi+y2P9WxKsuCRDqNNvllL48rL+FMro3iCH/6N0Q1T8/mjFgmmFzmWVUijHl2ou+1nAvxvY8PYRwBDbNsM6Poay5jczNZ002iLXbxOpFobeAaglbfVrRys+Pln5hp4vtDC0cyzv/Nkvt3Li5iZi2xycQYuR3SHw/Bc3n+5U7IQ18b0LV+omv0fLeH55Zi9X+kQ3UX03bOaETWRdlbZbUwh1u9hN/tSXlyvd3VxQlufvwvAjY8c6WyO7g7Sdl62YKmvGFc0zOej9SUzTPheQ/qbVkhbdOLOKoF4W0kfKr3gzxwIVT0zOkDXWaH0KhQyiguYzsnRdluTkbWPTKrm4TeLpILHTnJIOW54L4e7qJ3/j+f0KrGiVHgUGPEdIPB1EKgy1sHPZAHj56a+ExbNfmXtp5aPfaqbtvIGqP++9OYrxhKGdkWKpabUwdApI1d18poCmq206qrIj1QrB5vcM0XzG9IrPjDweqjo1MxU7DF2XpMse7/liG6n9Abx8HwLP8cdrTVKuweTgsXd0ceLWqF/DaBwv65eZNo7gpS2e/VoLnlNe8bOwoc0YOPKNZp54/wpy8blPyGd7A1VLZ2Og/84men/YrCf/OtDfuFrSjv8gRmxbvGT3q5uGgbsjdFyYwYoarMDY9IcIRDe4nPrBOE9f20Fqf/ndgAQMnRen6Hih33EsO2hhRdziZrGZCK9xOe3T/fTd0sTAXRFAcEctnv5EZ7H8Q9flSdrPqx6QPAe/qborHPt2M4GYoeOidL7uMiQP2nhpOO3T/TgnLfpua/JTOquN1UCgxYBbu817yf0BYttz5dVGs9VLPav5p795taQlngpx4AttpI/5jU5yCaH35hhHvt7Ckx/ppP/OiJ/qWaVvcBnbsPXjg6x9wyjNO3LEtuaIneJOmrpZKD5X6XGxIbreZd1bR1jz+tL1isyxAKN7QpisVLzDKHQry/TaHL0xv5hsBKdwQrX8O5PYVpfOl2aIrndpPddh8/uHJl0HtoIQbJvFLc0kjtzYUlbR083A8R/G8n0bVD3oHYBa8kYeCbP3kTCIKbZ0BHAG/aJwnS/KYEcnlE62ILK+dN5CgoaNfzFE05Zc9b0BlRaXPTh4Qwsb/mQUsQxWuPx5dsRPlzzxk1ix2FvBwF0RWp+XKQsyxoGef2llZFe4eFyx7VlWXpGs2O+4+FmhyTd/uSl/aquWUvuD7PvHDta+YZToJn9n8PGbYgxOupFMzTcNAGr5MOVXmk6/VVY3H/wr59ShcX8eYtjy0UFiVU7+xadNzN93YfiRIPH/juD02bSck2HFxWmC7RU+MydE1udIPFV68h15LET/nVG6XpbyM4jymUQH/rmd0SdKn9v5kvS0S1hX4mUh2RNkeNf0A0Cww6XtggxW0DD02zCZI5VPK8l9QfZ9smP2g1M1pwFALWte2qLvF1G6Li3drGQc6L1pLDun5aws0Q25aW9SMp6/MDt4b4TB+8Oc+aX+fME5P9Wm0g5jCRiyA5U+QDh6Ywv9d0RpOTuLlxLiO8t75BbeY1plqPOlHcSmuC/AiQu9P4wxcFe0YrCspP13Umz8sxH/GwtWvyZB38+jHPuONmZvBBoAVF0sVCno6Tj27WZywxbdv58k0GxI9gQ4emNzyQaopq05rGnOVngO9N0W5fj3mrGbPXZc119WTnriJLznwOieEM4krSyrpaKOF78/Qtv55dNFEz/Ly/i7kwsBzcvAyKNh+u+cfiE2u9lj45+NlE43BWDl76UYeihC8hltzr7YaQBQdbUQVUB9hpbnZFlxSRoJGeL5K3M8ASP03RKbtP6MM2DhZSifh8/P5JRsoMr5WTRnfKEfCXkluf/gX/m7Gb8FohUyxVLIh26Y+1Xz8K4Qww+HaX1ext98VljLNfmWiwFD6nCAyNpcSWaUFYaOF6Y58dOmaW/Eaj03U3kTXMB/Lw0Ai58GALVg6lEJtGDNG0bpevnYNE/zjiwdLw6y//+2T2u6I/5AmLVvHMV4Y1Ms/uYrSB4I0HxaDsRfN7BbPDoumnou/vj3mxh6KIKbEmLbHbZ8KE5opUfimQDHvz/bvHjh4L+2Ejvdoe35GdyUMPjfEZxBm8jaHE7cYvWrE8ROLU/MN8bf2VyTnbia2NMQNACoJS+00mXlZamSqQo7ArHtDq3nZBneNXXtGS9tse9THWx+9xChlf5lb6bX5uC/tJE+EgDLn05pvyDD+reNTGshdvTJEM6gTcdFKTa8faR4Rd72vCwtZw2w75Mds6zDIyT2hopVRgtSPf57OYMWnlOhCbwn5IbLFxBaz82w6jUJQitdUgcCHPtuM6meIMO7wog9UvZ8LweD92l2TyPQAKAW1EL1Ax6v+cxsxTx6OwKtz81MKwAApJ8NsPd/rSC4wgXjNzEv8vxc/aZt2bKU0oncNAz/Nkz6UBDEsO5No6VtGi1/1++aP0qw/zPt0xrbTAzcFaX7VcmSxwqL1hOzfzouSrH+7SPFaqqB5zjETh9kX36T3KF/b2HjNSP+Fb8FuH4D9+Q+nf5pBBoA1JLnJqVivR7PgdzIzPdCOv3VF2qzx23cDBU7exU+88jXW4qNVAItBitSHjDE8iuZzgdnwKbnc21seuewvz4hhtyQxYF/bi/dlCWGtW8cLTmWQt+Bta8f5Zl/6iB+b5TE3hDtF2SQoGH44bB/R6Qagv5LqQW30BlAw78NY7zyqQo8puhoNXMD90RZ/ZoEpkrdHpMTP80yzw9Old/LGZy/jfojj4V5/K+6iG7KYXJC+tlx+aB5dsxULOUsFkRPGVtDcAZs+m7TNo6NSEtBqHlTaAdZrSH8QmUAGUfY/3/ayQ0LbjL/lYaD17eS7a3tNZA7avH0te0Vs2OMB8lnSj/P5IT+X5X3AHDT0Pujee6K5QmpA8H8YnP5v5GXkorHAX5WlGp8egegFkQ9M4AAkvuDPP7OLmLbHaygYfTJULGlY62lD4Xo+Vwbm98zhAT9K2bP9QPR0W82lz3/yDeaEcvQeXHaX6vwhGPfixF/oL4LqcYVTv4yStdlqZJpIDft1/BRjU8DgFow9VgALuFJWWbMfBneFWbfP3aw6g8SRNa4JPcHOP6jWOUyCa5w+GutHP1WC4EWD2fQWjQF0o59pxkR6Lo0BYCXE459N8bQg5rlsxRoAFBqniT3BTnw2eln8XgZIZtZZA1xPeHoN1s49r1m7Jjnp4nOsUOYWjzqMpEnIq8Tkd0i4onIefUYg6qPxVQCQk2fccRvDqMn/yWlXncAjwOvAb5cp89XC6TeC8ALzW72WPmKJK3nZsjFLU78LMZojUsrK1UrdQkAxpg9ADLT9kmqYRQygJYTu9njtH8aINDs5Xcdu8ROj3P0O830/0LTJNXis+hzuUTkz0XkIRF5KJOq3lhaLV71zgBaKCsvT447+fvsCKy9etQvBa3UIjNvAUBEfikij1f4umom72OMucEYc54x5rxwtHO+hqvmWd0zgBZA6znZsk5cALhCZGO1ruhK1c+8TQEZYy6dr/dWajFy4lblRi+2ITes051q8Vn0U0CqcU1cAC5kAC3VBeC+W5vwJjQ+93J+ieha7zhWqhbqlQb6ahE5DLwQuEVEfl6Pcaj5sRwXgMHv6HX0m824ab/Gj5eB1IEAB66rfUVPpWqhXllANwE31eOz1cJaLgvABf13NDFwd5Tohhy5YYts3yLb2KXUOHpfqubdclgAHs9kRdshqoagawBKKbVMaQBQC0JLQCi1+GgAUPNiuZWAUKoRaQBQNTU+A2i5LQAr1Wg0AKh5tdwWgJVqJBoAlFJqmdIAoOadLgArtThpAFA1pwvASjUGDQCqZnQBWKnGogFAzRtdAFZqcdMAoJRSy5QGADWvdAFYqcVLA4CqKV0AVqpxaABQNbFcewAo1cg0AKia0wwgpRqDBgA1LzQDSKnFTwOAmje6AKzU4qYBQNWMLgAr1Vg0AKg50wVgpRqTBgBVU7oArFQ9MLvBAAAE+0lEQVTj0ACgak4XgJVqDBoAlFJqmdIAoOZFIQNIF4CVWrw0AKiaqJYBpJRavDQAqDnRHgBKNS4NAKqmdAFYqcahAUAppZYpDQCq5rQEhFKNQQOAmjMtAaFUY9IAoGZNF4CVamwaAFTN6AKwUo1FA4BSSi1TGgBUTekCsFKNQwOAmhNdAFaqcWkAULOiC8BKNT4NAKomdAFYqcZTlwAgIp8Rkb0i8qiI3CQi7fUYh1JKLWf1ugO4HTjLGHM28BTwkTqNQ9WQLgAr1VjqEgCMMb8wxuTy394PrK/HONTc6AKwUo1tMawB/Cnws2o/FJE/F5GHROShTGpgAYellFJLW2C+3lhEfgmsrvCjjxljfpx/zseAHPCNau9jjLkBuAGgo/s5Zh6GqmZIM4CUWhrEmPqcU0XkrcBfAi83xiSn+Zo+YDGccbqAk/UeRI3pMTUGPabGsNiOaZMxZuXEB+sSAETkcuA64GJjTN+CD2COROQhY8x59R5HLekxNQY9psbQKMdUrzWALwEtwO0isktErq/TOJRSatmatzWAyRhjttbjc5VSSo1ZDFlAjeiGeg9gHugxNQY9psbQEMdUt0VgpZRS9aV3AEoptUxpAFBKqWVKA8AsLMVidiLyOhHZLSKeiCz69LXJiMjlIvKkiOwTkQ/Xezy1ICJfFZETIvJ4vcdSCyKyQUTuFJE9+f/u3lvvMc2ViERE5EEReSR/TJ+o95imogFgdpZiMbvHgdcAd9V7IHMhIjbwr8ArgDOAPxaRM+o7qpr4GnB5vQdRQzngA8aYHcCFwDuXwL9TBniZMeYc4FzgchG5sM5jmpQGgFlYisXsjDF7jDFP1nscNXABsM8Ys98YkwW+DVxV5zHNmTHmLmDJFMMyxhwzxjyc//8jwB5gXX1HNTfGN5r/Npj/WtRZNhoA5m7SYnZqwa0Dnh33/WEa/MSy1InIZuC5wAP1HcnciYgtIruAE8DtxphFfUx12QjWCGpVzG4xmc4xLQGValQv6quw5UxEmoEfAO8zxgzXezxzZYxxgXPz64I3ichZxphFu26jAaAKY8ylk/08X8zuSvxidg1xgpnqmJaIw8CGcd+vB47WaSxqEiISxD/5f8MY88N6j6eWjDFxEfkV/rrNog0AOgU0C/lidh8CXjXdSqZqwewEtonIKSISAq4Gbq7zmNQEIiLAV4A9xpjr6j2eWhCRlYWMQBGJApcCe+s7qslpAJidJVfMTkReLSKHgRcCt4jIz+s9ptnIL86/C/g5/sLid40xu+s7qrkTkW8B9wGnichhEXl7vcc0RxcBbwZelv8b2iUiV9R7UHO0BrhTRB7FvxC53Rjz0zqPaVJaCkIppZYpvQNQSqllSgOAUkotUxoAlFJqmdIAoJRSy5QGAKWUWqY0AChVIyJym4jERWRRp/4pVaABQKna+Qx+brtSDUEDgFIzJCLn53tBREQklq/9fpYx5g5gpN7jU2q6tBaQUjNkjNkpIjcD1wJR4MbFXPBLqWo0ACg1O5/E3+6fBt5T57EoNSs6BaTU7HQCzfg1oSJ1HotSs6IBQKnZuQH4W/xeEJ+u81iUmhWdAlJqhkTkLUDOGPPNfA/ie0XkZcAngNOB5nxl1bcbYxqyqqpaHrQaqFJKLVM6BaSUUsuUBgCllFqmNAAopdQypQFAKaWWKQ0ASim1TGkAUEqpZUoDgFJKLVP/AyK7AUNX9XQUAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Set min and max values for meshgrid\n",
    "x_min, x_max = X[0, :].min() - 1, X[0, :].max() + 1\n",
    "y_min, y_max = X[1, :].min() - 1, X[1, :].max() + 1\n",
    "\n",
    "# Generate a grid of points with distance h between them\n",
    "h = 0.01\n",
    "xx, yy = np.meshgrid(np.arange(x_min, x_max, h), np.arange(y_min, y_max, h))\n",
    "\n",
    "# Predict the function value for the whole grid using previous params\n",
    "L = len(params) // 2\n",
    "X_mesh = np.c_[xx.ravel(), yy.ravel()].T\n",
    "Z = forward_prop(X_mesh, params)['A' + str(L)]\n",
    "Z = Z.reshape(xx.shape)\n",
    "\n",
    "# Plot the contour and training examples\n",
    "plt.contourf(xx, yy, Z, cmap=plt.cm.coolwarm)\n",
    "plt.ylabel('x2')\n",
    "plt.xlabel('x1')\n",
    "plt.scatter(X_test[0, :], X_test[1, :], c=np.squeeze(y_test))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "stenv",
   "language": "python",
   "name": "stenv"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}