CH08 Regression.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# CH8 Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## MLiA"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### (LR) Linear Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def loadDataSet(fileName):\n",
    "    numFeat = len(open(fileName).readline().split(\"\\t\"))\n",
    "    dataMat = []\n",
    "    labelMat = []\n",
    "    fr = open(fileName)\n",
    "    for line in fr.readlines():\n",
    "        lineArr = []\n",
    "        curLine = line.strip().split(\"\\t\")\n",
    "        for i in range(numFeat-1):\n",
    "            lineArr.append(float(curLine[i]))\n",
    "        dataMat.append(lineArr)\n",
    "        labelMat.append(float(curLine[-1]))\n",
    "    return dataMat, labelMat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def standRegres(xArr, yArr):\n",
    "    xMat = np.mat(xArr)\n",
    "    yMat = np.mat(yArr).T\n",
    "    xTx = xMat.T*xMat\n",
    "    if np.linalg.det(xTx) == 0:\n",
    "        print(\"This matrix is singular, cannot do inverse\")\n",
    "        return\n",
    "    ws = xTx.I*(xMat.T*yMat)\n",
    "    return ws"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[1.0, 0.067732], [1.0, 0.42781], [1.0, 0.995731], [1.0, 0.738336], [1.0, 0.981083], [1.0, 0.526171], [1.0, 0.378887], [1.0, 0.033859], [1.0, 0.132791], [1.0, 0.138306], [1.0, 0.247809], [1.0, 0.64827], [1.0, 0.731209], [1.0, 0.236833], [1.0, 0.969788], [1.0, 0.607492], [1.0, 0.358622], [1.0, 0.147846], [1.0, 0.63782], [1.0, 0.230372], [1.0, 0.070237], [1.0, 0.067154], [1.0, 0.925577], [1.0, 0.717733], [1.0, 0.015371], [1.0, 0.33507], [1.0, 0.040486], [1.0, 0.212575], [1.0, 0.617218], [1.0, 0.541196], [1.0, 0.045353], [1.0, 0.126762], [1.0, 0.556486], [1.0, 0.901144], [1.0, 0.958476], [1.0, 0.274561], [1.0, 0.394396], [1.0, 0.87248], [1.0, 0.409932], [1.0, 0.908969], [1.0, 0.166819], [1.0, 0.665016], [1.0, 0.263727], [1.0, 0.231214], [1.0, 0.552928], [1.0, 0.047744], [1.0, 0.365746], [1.0, 0.495002], [1.0, 0.493466], [1.0, 0.792101], [1.0, 0.76966], [1.0, 0.251821], [1.0, 0.181951], [1.0, 0.808177], [1.0, 0.334116], [1.0, 0.33863], [1.0, 0.452584], [1.0, 0.69477], [1.0, 0.590902], [1.0, 0.307928], [1.0, 0.148364], [1.0, 0.70218], [1.0, 0.721544], [1.0, 0.666886], [1.0, 0.124931], [1.0, 0.618286], [1.0, 0.381086], [1.0, 0.385643], [1.0, 0.777175], [1.0, 0.116089], [1.0, 0.115487], [1.0, 0.66351], [1.0, 0.254884], [1.0, 0.993888], [1.0, 0.295434], [1.0, 0.952523], [1.0, 0.307047], [1.0, 0.277261], [1.0, 0.279101], [1.0, 0.175724], [1.0, 0.156383], [1.0, 0.733165], [1.0, 0.848142], [1.0, 0.771184], [1.0, 0.429492], [1.0, 0.162176], [1.0, 0.917064], [1.0, 0.315044], [1.0, 0.201473], [1.0, 0.297038], [1.0, 0.336647], [1.0, 0.666109], [1.0, 0.583888], [1.0, 0.085031], [1.0, 0.687006], [1.0, 0.949655], [1.0, 0.189912], [1.0, 0.844027], [1.0, 0.333288], [1.0, 0.427035], [1.0, 0.466369], [1.0, 0.550659], [1.0, 0.278213], [1.0, 0.918769], [1.0, 0.886555], [1.0, 0.569488], [1.0, 0.066379], [1.0, 0.335751], [1.0, 0.426863], [1.0, 0.395746], [1.0, 0.694221], [1.0, 0.27276], [1.0, 0.503495], [1.0, 0.067119], [1.0, 0.038326], [1.0, 0.599122], [1.0, 0.947054], [1.0, 0.671279], [1.0, 0.434811], [1.0, 0.509381], [1.0, 0.749442], [1.0, 0.058014], [1.0, 0.482978], [1.0, 0.466776], [1.0, 0.357767], [1.0, 0.949123], [1.0, 0.41732], [1.0, 0.920461], [1.0, 0.156433], [1.0, 0.656662], [1.0, 0.616418], [1.0, 0.853428], [1.0, 0.133295], [1.0, 0.693007], [1.0, 0.178449], [1.0, 0.199526], [1.0, 0.073224], [1.0, 0.286515], [1.0, 0.182026], [1.0, 0.621523], [1.0, 0.344584], [1.0, 0.398556], [1.0, 0.480369], [1.0, 0.15335], [1.0, 0.171846], [1.0, 0.867082], [1.0, 0.223855], [1.0, 0.528301], [1.0, 0.890192], [1.0, 0.106352], [1.0, 0.917886], [1.0, 0.014855], [1.0, 0.567682], [1.0, 0.068854], [1.0, 0.603535], [1.0, 0.53205], [1.0, 0.651362], [1.0, 0.901225], [1.0, 0.204337], [1.0, 0.696081], [1.0, 0.963924], [1.0, 0.98139], [1.0, 0.987911], [1.0, 0.990947], [1.0, 0.736021], [1.0, 0.253574], [1.0, 0.674722], [1.0, 0.939368], [1.0, 0.235419], [1.0, 0.110521], [1.0, 0.218023], [1.0, 0.869778], [1.0, 0.19683], [1.0, 0.958178], [1.0, 0.972673], [1.0, 0.745797], [1.0, 0.445674], [1.0, 0.470557], [1.0, 0.549236], [1.0, 0.335691], [1.0, 0.884739], [1.0, 0.918916], [1.0, 0.441815], [1.0, 0.116598], [1.0, 0.359274], [1.0, 0.814811], [1.0, 0.387125], [1.0, 0.982243], [1.0, 0.78088], [1.0, 0.652565], [1.0, 0.87003], [1.0, 0.604755], [1.0, 0.255212], [1.0, 0.730546], [1.0, 0.493829], [1.0, 0.257017], [1.0, 0.833735], [1.0, 0.070095], [1.0, 0.52707], [1.0, 0.116163]] [3.176513, 3.816464, 4.550095, 4.256571, 4.560815, 3.929515, 3.52617, 3.156393, 3.110301, 3.149813, 3.476346, 4.119688, 4.282233, 3.486582, 4.655492, 3.965162, 3.5149, 3.125947, 4.094115, 3.476039, 3.21061, 3.190612, 4.631504, 4.29589, 3.085028, 3.44808, 3.16744, 3.364266, 3.993482, 3.891471, 3.143259, 3.114204, 3.851484, 4.621899, 4.580768, 3.620992, 3.580501, 4.618706, 3.676867, 4.641845, 3.175939, 4.26498, 3.558448, 3.436632, 3.831052, 3.182853, 3.498906, 3.946833, 3.900583, 4.238522, 4.23308, 3.521557, 3.203344, 4.278105, 3.555705, 3.502661, 3.859776, 4.275956, 3.916191, 3.587961, 3.183004, 4.225236, 4.231083, 4.240544, 3.222372, 4.021445, 3.567479, 3.56258, 4.262059, 3.208813, 3.169825, 4.193949, 3.491678, 4.533306, 3.550108, 4.636427, 3.557078, 3.552874, 3.494159, 3.206828, 3.195266, 4.221292, 4.413372, 4.184347, 3.742878, 3.201878, 4.648964, 3.510117, 3.274434, 3.579622, 3.489244, 4.237386, 3.913749, 3.22899, 4.286286, 4.628614, 3.239536, 4.457997, 3.513384, 3.729674, 3.834274, 3.811155, 3.598316, 4.692514, 4.604859, 3.864912, 3.184236, 3.500796, 3.743365, 3.622905, 4.310796, 3.583357, 3.901852, 3.233521, 3.105266, 3.865544, 4.628625, 4.231213, 3.791149, 3.968271, 4.25391, 3.19471, 3.996503, 3.904358, 3.503976, 4.557545, 3.699876, 4.613614, 3.140401, 4.206717, 3.969524, 4.476096, 3.136528, 4.279071, 3.200603, 3.299012, 3.209873, 3.632942, 3.248361, 3.995783, 3.563262, 3.649712, 3.951845, 3.145031, 3.181577, 4.637087, 3.404964, 3.873188, 4.633648, 3.154768, 4.623637, 3.078132, 3.913596, 3.221817, 3.938071, 3.880822, 4.176436, 4.648161, 3.332312, 4.240614, 4.532224, 4.557105, 4.610072, 4.636569, 4.229813, 3.50086, 4.245514, 4.605182, 3.45434, 3.180775, 3.38082, 4.56502, 3.279973, 4.554241, 4.63352, 4.281037, 3.844426, 3.891601, 3.849728, 3.492215, 4.592374, 4.632025, 3.75675, 3.133555, 3.567919, 4.363382, 3.560165, 4.564305, 4.215055, 4.174999, 4.58664, 3.960008, 3.529963, 4.213412, 3.908685, 3.585821, 4.374394, 3.213817, 3.952681, 3.129283]\n"
     ]
    }
   ],
   "source": [
    "xArr, yArr = loadDataSet(\"./Data/CH8/ex0.txt\")\n",
    "print(xArr,yArr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "matrix([[3.00774324],\n",
       "        [1.69532264]])"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ws = standRegres(xArr,yArr)\n",
    "ws"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "xMat = np.mat(xArr)\n",
    "yMat = np.mat(yArr)\n",
    "yHat = xMat*ws"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXwAAAD8CAYAAAB0IB+mAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3Xl8U1Xex/HPSdpSkE2gFkRAEKhW\nHAFRVBZFqDjqwDyuCOKCo4IwqDPyPC4jjrgv48KIICqKKKg4zoArU9zAhWJZRCyW1QoIpYAgaKFt\ncp4/bhKSNKWhpGu+79eLl0nuzc0vtf3m5NxzzzHWWkREpO5zVXcBIiJSNRT4IiJxQoEvIhInFPgi\nInFCgS8iEicU+CIicSKqwDfG/GCM+dYYs9wYkx1huzHGTDTGrDXGrDDGdI99qSIicjgSDmHfftba\n7WVs+z3QyfevJzDZ918REakhYtWlMxh4xToWAU2NMa1idGwREYmBaFv4FvivMcYCz1lrp4Ztbw1s\nDLq/yffYluCdjDE3ADcAHHHEEaccf/zxFSpaRCReLVmyZLu1NqUiz4028HtZa38yxhwFZBpjvrfW\nLgjabiI8p9ScDb4PiqkAPXr0sNnZpU4HiIjIQRhj8ir63Ki6dKy1P/n+uw34N3Ba2C6bgDZB948B\nfqpoUSIiEnvlBr4x5ghjTCP/beBcYGXYbnOBq3yjdU4HdltrtyAiIjVGNF06qcC/jTH+/Wdaaz80\nxowEsNZOAd4HzgfWAr8B11ZOuSIiUlHlBr61dj1wcoTHpwTdtsDo2JYmIiKxpCttRUTihAJfRCRO\nKPBFROKEAl9EJE4o8EVE4oQCX0QkTijwRUTihAJfRCROKPBFROKEAl9EJE4o8EVE4oQCX0QkTijw\nRUTihAJfRCROKPBFROKEAl9EJE4o8EVE4oQCX0QkTijwRUTihAJfRGq0zJx8xs9ZSWZO/uEf7Jdf\nDv8YtZgCX0RqrMycfMbOWsYrX+Uxdtayiof+5s1w6aXQuzcUF8e2yFpEgS8iNdbCNQUUFnsAKCz2\nsHBNwaEdoKQEnn4ajj8e3n0XhgwBayuh0tohoboLEBEpS59OKczO3kRhsYf6iW76dEop9zmZOfks\nXFPABYUb6fnoXbBsGfz+9/DMM9ChQxVUXXMp8EWkRvEHtj/cT+/QDIChPduRkZ5a7nPvemkhYz56\nmVOXvc++o1JJfustuOgiMKbSa6/pFPgiUmWCwzw4vP2PN0pOZNrnGygs9vDqojxcxlDitbiA7Xv3\nA5QK/cAxO7Zg14vTeW/qwzT77Rde6jGI7GvHMvnic8p83XhjbDX1Z/Xo0cNmZ2dXy2uLSNXzn4D1\nd89MvKIbGempIY+7DXgOEklJbheThnUPhPZNry3l/W+30H7nZh7MnMwZPyxnRavO3DFwNN+lHkeS\n28X1fTsEPkSCX7e2MsYssdb2qMhz1cIXkSoRfgL28Xnfl3r8YGEPUOTxsnBNARnpqTw2L5ePluVx\ny6LZjFo0m/0J9Xjnxr/x71PP57s1OwP7/2fZplInfmtz4B8OjdIRkSrRp1MK9RPdgfu5+XsZO2sZ\n2/cW4XY5/ev+/5bF+I6TmZPPutfe5sNpo7nli1l8kNaL/n+aQvLYMVxxRofA6yS5XeT/sj/w/CS3\nK6oTv3WVAl9EqkRGeioTr+hG6ybJgccKiz18uHILHl8//cATWwbC2m2gbbMGIcfo0roxSQXb2H/Z\nEKbMuBOLYdjl93PLH8Zx6hnpZKSnBl7nqjPa0atjc0q8B7429OrYPG5b96AuHRGpYgV7iwK3DeDP\nYy9QWFTCxCu6hYzSGf3aUoo8XpKN5dH8z2l//oOY/ft5ovcwnut5MfUbNeD8jim0aJhEZk5+IPT9\n5wcWrd8Z6L8f2rNd1b/hGkSBLyJVZuGaAoo83sD9I49IYuevRSH7+MPab9Kw7qz94FOGvvQQTb77\nhh1nnsWwrsP5vpHzbWDY6ccGTsrOzt5U6qTsoQzrrOsU+CJSZRolJ4bcP71Dc+bn5FPk8ZLkdpVu\nge/eTcaUB8iYNAmOOgpmzaL55Zfz11XbAsM45+dsjXhSNjMnn1GvLqHEa0lwGdKPbhL3QzMV+CJS\nZfbsC53HpkXDJCYN6146iK2F2bPhlltg61a46Sa4/35o2hQ4MBbfP5wzmP9D5emP1gT670u8lmc/\nWYuFiN8C4oVO2opIlQkeqeOfKiEjPZUJg7scCOB165ypEC6/HFq1gqwsZ1oEX9j7BQ/nDDY/ZyuZ\nOfn8vHd/yOP+U7cVmpOnjlDgi0iVCR5BU6qVvX+/04rv0gW+/BImToTFi+HUUyMeK3yYp59/uOfJ\nbY8ss47wrqV4EXWXjjHGDWQDm621F4ZtuwZ4DNjse+gZa+0LsSpSROqO8JOyAHz8sdNtk5sLl10G\nTz4JRx9d7nH8I3r8ffm5+XsBpxXfomESo/t1ZH7OVpISXHy7+cBc+OFdS/HiUPrwbwZWAY3L2P6G\ntXbM4ZckInEjPx9uuw1efdWZyfLDD2HgwKifHvzh0bVN05CpG/zdReMGppWa1iFeL76KKvCNMccA\nFwAPAH+p1IpEpO7zeuH55+H22+HXX+Huu+GOO6B+/QofMrjFHz4S52Db4klUk6cZY94CHgIaAbeV\n0aXzEFAArAZutdZujHCcG4AbANq2bXtKXl7e4dYvIjVcqZkqv/kGRo6ERYugXz949llngRKJSqVO\nnmaMuRDYZq1dYow5u4zd3gFmWWv3G2NGAtOBc8J3stZOBaaCM1tmRQoWkdojuCvlvS9W03nbf2n3\n6gvQvLnTjTN0qOapr0LRdOn0AgYZY84HkoHGxphXrbVX+new1u4I2v954JHYlikitU1mTj6Pz/ue\nwqISBq7+ins+mkrLvTvgxhvhwQfhyLJH0UjlKDfwrbV3AHcA+Fr4twWHve/xVtbaLb67g3BO7opI\nnPK37JsXbObF+c/Rf93XrErtwKbnpnPaFedXd3lxq8JX2hpjJgDZ1tq5wFhjzCCgBNgJXBOb8kSk\nNvoy5yeuWfg6Y794Ha8xTBk8mo4T7mDA71pXd2lxTSteiUhsLVjA3hHX03Ddaj7ofCYPZNzA4D+c\nzriBadVdWZ1wOCdtdaWtiMTG9u0wYgScdRYNPUX8694pjLnoTjY1bMG0zzeQmZNf3RXGPQW+iBwe\nrxdefBHS0mDGDGc8/Xff8c3JvQJLFsbz/DU1iQJfRCru22+hb1/405/gxBNh+XJnBE6DBhEnSpPq\npemRReTQ/forTJgATzwBTZrASy/B1VeTuWobC+esDFxkpatbaxYFvogcmrlz4c9/hh9/ZPPFQ3ll\n8Ch6nNIJVm0LXGQVPOe8gr7mUOCLSHR+/BHGjoU5c6BLFxbPmMvVuYkUfrebV1Yv4/QOzSKuPCU1\nh/rwReTgiovh8cfhhBMgMxMefRSWLuXdRu1DAh5Qn30Npxa+iJTtiy+cic5WroRBg5xFSdo56872\n6ZTC7OxNgSmHh/Zsx9Ce7dRnX4Mp8EXqiFKzUh6OHTucqYtfeAHatIH//AcGDw7ZpayTsgr6mkuB\nL1IHBM9KeViLdFsLr7ziLEry888wbhyMHw8NG0bcXSdlaxf14YvUcoFZKcNOmAZvHz9nZflXuq5a\n5cxPf8010LkzLF3q9NeXEfZS+6iFL1KLBbfs/RJcJnDCNLzlP6J3e/bsKw5sX7imgLOOOYL+/3re\nOTHbsKGzEtWIEeAquz0Y0+4jqTIKfJFaJDxoF64pCAl7gBKvZfnGXaW2FxZ7mPLpWjwWXl/sLEh3\n5uos0jKnwO58p2X/6KOQcvDRNTHrPpIqpy4dkVrCH7SvfJXH2FnLyMzJD5m+INj8nK0AIdvdLhOY\n26bZz9t46q37efmteylMSOLh/5vsXC0bFPZldQWFf4hojpzaQ4EvUkuEB+3f564EYOIV3TipdeOQ\nfQektwQOjKS56ox2jDzrOBq6YcTXc5j/4ijOWZ/No32v4vxrJzLN3S4k2CN9uPhpjpzaS4EvUkv0\n6ZRCguvA+q+bd+1j1KtLABjbv3NgW4LL0LVN08B+Gemp9OmUQpMVS/jojXGM//h5vj4mnYzrnuXZ\nMy6j2J1IkccbaKlHOgk8Mysv0NoP/hBRd07toj58kVrEG7ZgUYnXMjMrjzbNGlDitYHHHp/3PeCE\n/SeLVrN9zF/405L32daoGaP+eAcfdD4zZPFwf0s90kngJLeLL9buoMhToDlyajkFvkgt4G91e8tY\noC74qleA3Py9jJ25lDcbb+DUB++m/i+7mNZjEE/2Hsav9RoEnteldRO6t20aOAk8fs7KkLBv3SQZ\njPNtAjRHTm2nwBep4SK1uv2S3C6G9mwX6GZ5fN735ObvpcOOTdyX+Swn5a1g90nduLTnvSxt3j7k\nuS4DN/fvFBLewR8cSW4XBXuLKPJ4A9vVZ1+7KfBFarhIQy+dsfYtAmEPTveNa/8+vr/5Tv705Zvs\nT6jHqvGPcML4vzIqdzt/n7sy0FIHSD+6SamWekZ6KiN6t2d+zlYSE9ys3Lw7sC0ttSG3DTxerfta\nTIEvUsNt31tU6rESr6VNswah4fvf/9L/ppvov24d35x1AbsmPMxZfbsAB+a3Gf3aUoo8XpLcLm7u\n36nUcTNz8pn2+YZACz/J7aLI46V+olthXwco8EVqsMfm5fL+t1sC910GvDasa2XLFrj1VnjjDWdK\nhPnzObl//1LHykhPZdKw7ge9Qjb420SRx0u/tBTaNGugK2rrCAW+SA3mv4DKr1WTZPqf4AyzzEhr\nAc88A3fdBfv3O0sO/u//Qr16ZR6vvNE1kaY8VtDXHQp8kRpsQHpLcvPXBu7/sdsxjBuYBtnZ0PMC\nWLIEzj0XJk2Cjh0P+/W0Dm3dpsAXqcHGDUwDnJb+gPSWjDu9pbOe7KRJkJoKr78Ol10WMqb+cGmM\nfd2lwBepwTJz8tmzr5jbzk0jY+VncPwtkJ8PY8bAffdBkybVXaLUIgp8kRrKP/7+qG0bOXf+FFi/\nFE45Bd55B3r0qO7ypBZS4IvUQJk5+Tz93gqu//RVRn/1Jvvdibx7/R1cOPk+cJeeHVMkGgp8kRom\nMyefmQ9O4+kPJnHczs3MPaEvj597I3ff0F9hL4dFgS9Sk+Tnc9RN1/HSZ+/xQ9NWDL9sAttO78vd\nuuhJYkCBL1ITeL0wdSrcfjsn/VbIpD5DmXjqxbgaNGCiwl5iRIEvUt2WL4eRIyErC845B9ezz9LZ\n05TLNRZeYkyBL1Jd9uyBe+6Bp5+GFi3g1Vdh6FAwhgxQ0EvMKfBFqpq18PbbcPPN8NNPbLzsKmb8\n4XpO7daRjBheQCUSLuolDo0xbmPMMmPMuxG21TPGvGGMWWuMyTLGHBvLIkXqjA0b4MIL4ZJLICWF\nrJnvcW6nIUz9dleptWNFYu1Q1rS9GVhVxrbrgJ+ttR2BJ4FHDrcwkTqlqAgeegjS02HBAnjySfj6\na96r3yZk7Vj/urIilSGqwDfGHANcALxQxi6Dgem+228B/Y3Rd1OpHTJz8gMLdFeKzz6Drl3hzjvh\nggtg1Sq45RZISKBPpxSS3M6foQtolJxYOTWIEH0L/yngfwFvGdtbAxsBrLUlwG6gefhOxpgbjDHZ\nxpjsggK1ZKT6+acveOWrvNh3qRQUwDXXwNlnQ2EhvPcevPUWHHNMyG7+hcm9wPML1qtbRypNuYFv\njLkQ2GatXXKw3SI8Vmq5ZWvtVGttD2ttj5QUrYsp1S94wY/wLpUKt/y9XnjhBUhLg5kznZb9d9/B\n+eeXOvbEj1ZTErQyeZHHq24dqTTRjNLpBQwyxpwPJAONjTGvWmuvDNpnE9AG2GSMSQCaADtjXq1I\njIUv+OFfRSp44fDZ2ZuYeEU3gPLniV+xwhlT/9VX0LcvTJ7s9NsHyczJDyw1GC7J7dIi4VJpyg18\na+0dwB0AxpizgdvCwh5gLnA18BVwCfCxtbZUC1+kJjq9QzPAWdTb37oOb/nPzMpj0fqdIR8AIaG/\ndy/ce69zMvbII+Hll+GqqyLOUz8zKy9i2LdumszfB3XR+HupNBUeh2+MmQBkW2vnAi8CM4wxa3Fa\n9kNiVJ9IpQluxSe5XXyxdgdFHi+zszcxond76ie6Ay1/oFTXTyCY58xxFiXZuBGuv94ZjdO81Cms\ngIIIi5LXT3Qr7KXSHVLgW2s/BT713R4f9Pg+4NJYFiZS2cIX7PYrLPawZ19xyFJ/yzfu4pPcA33r\njZITIS8Pxo6FuXPhpJNg1izo1eugr5mZk8/qrXsC990G+nZO0dqxUiV0pa3EreD++yS3C6+1lHht\noB89eKm/4BOpCZ4STp75HLz1nPPAY485V80mOkMqM3PyQ/r6M3PymZmVF3h+8IdL384pvHTtaVXw\nbkUU+BKHggPZ34pvlJzIc5+tA8Dj9QYC2h/4/g+HEzes4MH/TqZzwQ/wxz868+C0bRty7OCTvSN6\nt+f5BesDIZ/gMiS5XRR5vNRPdDO0Z7uqffMS1xT4EleCA/m1RXmMPLsjEwZ34dqXFgeGR3osfJJb\nwKL1OwMnZzNSE5i/agat/zWTwlatWf7P6bzdpjt99tYjI+j44Sd75+dsDWnRl3gt/dJa0KZZA82E\nKVVOgS9xJTiQPRamfLaOrm2aRty3sNjDwtXbyFj8Adx2G6127+b93w/nxXOG8+2WEoo25ZUasRM+\nzHNAeks2bD/Qwk9yu9RfL9VGgS9xpU+nFF5blIfHN2jY47UsXFPA0J7tAqN0/Lrs2sStD9wP2Yv4\nudtpDD/lalY2bwfbD4yyCR+xk5Geyoje7Zmfs5UB6S0ZNzCNrm2aBrqIgod+KvSlqinwpc4LP4k6\n8uyOTPlsHR6vDVxslZGeyqRh3Vm4poAjKeGUGc/Qe850XI0bwwsv8FSzU1mZtbHUsZPcLjbu/I3H\n5uWyZ18xjZITmfb5BgqLPfy4cwNd2zQNnPyNdDGXQl+qkgJf6rTH5uUy5dO1eCyBkPW3usOvms1I\nTyVjQzaMGQM//ADXXguPPgotWtA7J59Z2ZtDTr6e0KoRuVv38kluQWDIptsQ+PYQ3vqPNI2DAl+q\n0qFMjyxSq2Tm5Dst+bAADm/xA7BpE1x8sTNXfYMGzgyX06Y5K1FB4BtAv7QU+qWlMPnKU+jW9shS\nV8x6LLhdztW1wVM1gNOd5L+IK3ybSFVQC1/qjPAgX7imAE/QxGRu41wwFdyt8s9LT2LAR2/C+PHg\n8ThXyf7lL5CUVOr4wePy/V5fvDEk9JPcLq7v24E9+4pLjcLJSE8NuZhLrXupagp8qRMi9Y8Hj5hx\nuwwjzzqOPfuKA90qx//wHScOvgV+yHVmsnzmGWjfPurXzEhPpVfH5iFX4Pbq2JxxA9MO+hwFvVQX\nBb7UCZH6xycM7lKqRZ2Zk8+8hasY+9E0rlg+j6LUlvCvf8H//E/Eic7Kk350k5DATz+6Sczek0is\nKfClTgifJmHjzt/IzMkPbVFbS8bSTPq+fCsJP//MxqtvoN0/H4NGjSr8unv2FR/0vkhNopO2Uif4\n+8f7pTknQj/JLQhdwer776F/fxg+nHodj8O9JJt2L085rLAHnYiV2kUtfKkz/CdqizxOF0thsYeJ\n73zDcRM/pMO0Z+CII2DKFGcKY1ds2jo6ESu1iQJf6pTgrp2+65dwX+Zk2u3ayk+DLuHoqc9AauwD\nWSdipbZQ4Eud4Z+G+LR6hVw152n6f/sZ65odwxVDHqDTkEFMqISwF6lNFPhSJ2Tm5DP6lcUMzX6X\nvy6cQZKnhCf7DmfyqRfhrp/MCPWtiyjwpW746vUP+NfL93NS/jo+a9+duzNGcdzpv2OIpiEWCVDg\nS+22ezc/jrqVv73+MgVHHMlNg2/n/bReuIzhbk1DLBJCgS+1k7Xwxhvs//PNtN6xnendL+QffYaz\nt14DXAZGnd1RYS8SRoEvtc+aNTB6NGRm8kObNP561Z2sbNkRgLTUhtw28HiFvUgECnypPfbtg0ce\ngYceojgxiYfPu4mXThqI13XgwieFvUjZFPhSO8yfDzfd5LTuhwzhyXNv4MXc3wKb1bIXKZ+mVpCa\nbetWGDYMMjLAWpY8/wbjh9yFOfrokCkNFPYi5VMLX2Ii4qIih8PjgalT4Y47oLAQ7rmHjwZfy5i3\nV1G4No/6iW5G9G4fcd55EYlMgS8V5g/54HVcY7JW67JlMHIkLF7sTHj27LPQuTOfzVkZMgXynn3F\nTBjcJUbvRqTuU+BLhQQvOHKwdVwPyS+/OCtP/fOfztKCr70GV1wRmKe+UXJiyO7h90Xk4BT4UiHB\nC454LLgMeG0Fpwi21lmE5OabYcsWGDUKHngAmjYN2U1zz4scHp20lahk5uQzfs7KwPzyfTqlkOQ+\n8OvjMoZ+aSkh3Tnhz4lo/XpnecFLL3Vmsly0CCZNKhX2/tfU3PMiFacWvpQr0nqxACmNkti8ax8A\nJV5Lm2YNABg/Z2X5/fr798Pjj8P990NiIjz1lHMxVcLBfyVP79AMgKGaNkHkkCnwpVzh68XOzMpj\n0fqdgcfAaXE3Sk480K/vMni8NvCckH79Tz91um2+/56VZ57LzvsfoW+/rhFH+vinPC7YW8TqrXso\n8nipn+hmaM92VfozEKkLFPhSrvD1Yldv3RMS9q2bJvP3QV1C+/V9YQ9B3S/btsG4cfDKK2xrcTR3\nXn4v8489hfofb2VEUW6pbwQAo19bSpHHG1LPYZ0YFoljCnwpl38Zv5lZeXyxdgebd+8L2V6wpwiI\nPGrG7TKMOLMdGZ/Pgdtvx7tnL1N7DeGpnpewLzEZcAJ8fs7WkG8RC9c4yxSGhz1Aktul/nuRClDg\nS9R+2lUYMYCLPN5AQIfrvHUdl996J+R+A2efzTMX38oTm9wh+9RPdDMgvSU/7nRa+MEnZF9fvLHU\na/bq2Fyte5EKUOBLuR6bl8uUz9aFdNP4R+j4+9T9Ae3v+mlQVMgtn89kRPYcvEceCa+8AldeyQmr\ntlHf18+f5HbRq2PzwAnYrm2alurDv75vB2Zl5fFzYTHWN+xT/fciFVNu4BtjkoEFQD3f/m9Za+8J\n2+ca4DFgs++hZ6y1L8S2VKkOmTn5TPl0beDCKjgwURlQKqAnDulKwatvcOGMh2m8fSvf/P4y3h86\nlh7dO5JhDBnpqYzo3Z75OVsZkN6ScQPTAsddvnEXWet30Cg5kYz0VDJz8gP9+uEfDiJy6KJp4e8H\nzrHW7jXGJAKfG2M+sNYuCtvvDWvtmNiXKNVp4ZqCkLB3GUpNVObvzsk4Yh8Zd/wZ3nkHTjqJxU89\nz9XfJ1C4chev5C4LnIh9fsF6ijxe1mxbC8C4gWk8Ni+XSZ8493Pz17Jh+6+0aJgU6Ncv8nhp06yB\nwl7kMJQb+NZaC+z13U30/bNlP0PqguB5cpLcroh99/7x+cX79tNk4hOc89XruF0uZ3z92LG8+34u\nhcV5wIETsRt3/hY4ltfCs5+spWubpszP2Rpy7Pe/3cLofh2pn+gu1a8vIhUTVR++McYNLAE6ApOs\ntVkRdrvYGNMXWA3caq3dGOE4NwA3ALRt27bCRUvl8Y97/2LtDoo8XpLcLpISDEW+UZheCzOz8shI\nT2XhmgK6rP+G+//7LGnbfySn5zmkv/kS+P7fBg/n9Af2xI9Wh7yexTnegPSW5OavDdm2Z18xE6/o\nFttZOEXiWFSBb631AF2NMU2BfxtjulhrVwbt8g4wy1q73xgzEpgOnBPhOFOBqQA9evTQt4QaJviK\nWr8ijzcQ9iG2b+fGVx6k9duz2NT4KEZddg8X3TOK9LYHQtk/nDM4sGdm5UV87XED09iw/Vfe/3YL\ncGDsfkZ6qoJeJEYOaZSOtXaXMeZT4DxgZdDjO4J2ex54JCbVSZUKvnCqLPVccNtPX8Lx59N69242\nXDeGVzOGc9FJkU+mhgf20J7tWLD6wHmBBJcJjLp5dlj32M+rLyIB0YzSSQGKfWFfHxhAWKAbY1pZ\na7f47g4CVsW8Uql0fTqlMDPrR0p8wy+NAbcxlHgtSW4Xl9bfzW3/eZojlyyC3r1h8mTad+nC3Yf4\nOm6XC4/Hiwu48azjQoJdLXqRyhNNC78VMN3Xj+8C3rTWvmuMmQBkW2vnAmONMYOAEmAncE1lFSyV\nyxs01t5a6NO5BR0awJWZM2g/fQo0bgwvvgjXXAOuQ59sdeGaggMnbdEUxyJVKZpROiuAbhEeHx90\n+w7gjtiWJlVt4ZoCgsfiuF2GsYW5dLv3bsjLgxEj4JFHnMVJKijSiVwRqRq60lYCGiUnBlavOmZP\nAdOWz6Tzl5lw4omwYAH06XPYrxHpRK6IVA0FvgAErmo1JSXcsPQdbvtyFknGwsMPw623QlJSzF5L\n/fQi1UOBL2Tm5PP4vO854YeVPDBvEicU/EBuj7NIm/0yHHtsdZcnIjGiJQ7jXGZOPndPW8DV0x/m\n7VfH0WTfXsZcejc/vvy6wl6kjlELP55Zyw9PTObdWU/RtHAPU0/9H9676HrGDOquLheROkiBH69W\nrWLn1X/i+q+/ZOnRaQy//D7WterIJIW9SJ2lwI83hYXwwAN4H3mUxMRkbh84hjdOPhdrXPTTwiIi\ndZoCP558+CGMHg3r1zP3pP7cd9a17DiiKaCFRUTigQI/Hmze7AytnD0b0tJ46b4XuXfvgZa8f0ET\nte5F6jaN0qnLSkrg6afhhBOcRUnuvx+++Ybtp/bC7TKA07JX2IvEB7Xw66rFi2HkSFi2DM47D555\nBo47LnCBlcdrcRsY0bu9wl4kTqiFX9fs2uX0059+OuTnO904778Pxx0HOIuN+KdA9lhNXiYSTxT4\ndYW1MGsWHH88TJkCY8fCqlVwySXOPMf4ly3cHvK0RsmJ1VGtiFQDdenUBWvWwE03wfz5cOqpTou+\ne/dSuy1cUxCY695PLXyR+KHAr8327XMmN3voIUhOhkmT4MYbwe0O2c2/Tu32vftJcJlA6Ce5XZqe\nWCSOKPCrQfgyfhVa1i8z02nVr10LQ4fCP/4BLVtGfK3Rry0NLDqS4DJ0ad2ElIZJDO0ZeVlCEamb\nFPhVLDiAX1+8kev7dmDa5xsoLPYwO3sTE6/oVmYIZ+bks2zxKobPnkir9/8NnTo5wT9gQJmvF7zC\nFECJ19K9bVMmDO4S8/cmIjWiQCEPAAAMzElEQVSbAr+KzczKCwRwkcfLc5+tC3SxFBZ7WLimoFTg\nZ+bkM+ur9bR7cwa3fjqdZE8R60bfxnGP3+d05ZQhMyefjTt/UzeOiAAK/CqVmZPP6vw9IY8Fn0SN\ntORfZk4+U56Yzfj3JnLy1jUsbNeVu88dRd+MM5kQFPb+bqFGyYns2VdMo+TEwDcHAzSs56Z9iyMY\n27+zunFE4pQCv5IFB7E/gCOJOL3BL7/Q8P/+ypvvzWJng8b8+Q/jeOeEvtRPSgj5YHhsXi5TPluH\nJ+jDw+0ygfsW2LvfQ+7WvZXyHkWkdlDgV6LMnHzGzlpGYbEnsFZsJAkuExr21sLs2ewfM5ae27cx\ns/sFPNr7SvYd0Zh+HZuHnGzNzMlnyqdrSx3b4y39YkUeb8QuIxGJDwr8ShR+VWtwq9svwWWYfOUp\nB0J43TrnStl581jbsiN3DP8/clqnkdqoHsO7H8O4gWkhz1+4pqDMD5Jw6r8XiW8K/ErgH/cefFVr\nktvF9X07MGtxHjt/PXCx0wmtGjlhv38/PPYYPPAAJCby6tC/Mr51X7wuN3gtm3fvY9rnG+japmnI\nUM5GyYnUT3QHvkUcc2R98nYWhtTTumkynVMbaRimSJxT4MdYcDdOsLSWDRk3MI2ubZoy6tUllHgt\nCS7D2P6d4ZNPYNQoyM2Fyy5jwag7mZD5E96g4ZRwYBTP8o27At049RPdjOjdngWrt7Hz1yJObN2U\nLbv3B0YCJbld/H1QFwW9iCjwY23hmoKIJ2Zzt+4lMyefjPRUJl95CgvXFHBOMzj74XEwYwZ06AAf\nfADnncf8OStDxs77JbgMjZITnRO0vm6cwmIPs7Ly2Pmb861h864tnH9SKwqLSgDUqheRAAV+jPXp\nlMLs7E0UFntwAf7YDj5hmnF8ChkL/wPX3A6//gp/+xvceSfUr1/qGMFj6Eu8lllZeaXOA/jD3m/d\ntj3Mu/Wsyn6rIlLLaLbMGMtIT2XiFd3ol5ZCeusmJLmdH3FgjP0330CvXs5c9V27wooVcN99gbAP\nPsZVZ7QjtXG9kOMHh7spo4YB6aWnWBARUQu/kixav5PCYg9Jbhf90lIY3qU557z4qLMCVbNmTjfO\nsGGBqYvDZaSnkpGeSqPkRCZ9sjbiPm6XwWUMRR4vCS5DauN6/LFb6ZE8IiKgwI+J8MnPgvvxi0o8\nnLv6K84Z/zhs2uTMZvnQQ3DkkVEd2x/e/1m2ifxf9odcmVvitfRLa0GbZg0ObeI1EYlLxtooB3HH\nWI8ePWx2dna1vHYsBU+GluR2MWmYMw/92FnLaL79J+776Dn6rVkMv/udszDJGWcc1mvNzMrji7U7\nKPJ4qZ/oPuhkayJS9xhjllhre1TkuWrhH6bwydBmZuXx0rCuzN37Oe2m/QOX2+VMXTx2LCQc3o/b\n381ToemURSTuKfBjrHPuMuh2LZ1ycuCii+Cpp6BNm5i+hj/4RUQOhQL/MA3t2Y4v1u7giD0/c9dn\nL3PJikxo1w7eeQcuvLC6yxMRCVDgH6aM41P4d/L3tP/nvSQX/gq33w533w0NGlR3aSIiIcodh2+M\nSTbGLDbGfGOM+c4Yc2+EfeoZY94wxqw1xmQZY46tjGJrpKwsThz/Fxp0/R2u5cudETgKexGpgaJp\n4e8HzrHW7jXGJAKfG2M+sNYuCtrnOuBna21HY8wQ4BHg8kqot+Y54wz4+GM4++wyx9SLiNQE5bbw\nrcO/ckai71/4WM7BwHTf7beA/sbEUfr166ewF5EaL6qpFYwxbmPMcmAbkGmtzQrbpTWwEcBaWwLs\nBppHOM4NxphsY0x2QUHB4VUuIiKHJKrAt9Z6rLVdgWOA04wxXcJ2idS8LXVFl7V2qrW2h7W2R0qK\nFuIQEalKhzR5mrV2F/ApcF7Ypk1AGwBjTALQBNgZg/pERCRGohmlk2KMaeq7XR8YAHwftttc4Grf\n7UuAj211zdkgIiIRRTNKpxUw3RjjxvmAeNNa+64xZgKQba2dC7wIzDDGrMVp2Q+ptIpFRKRCyg18\na+0KoFuEx8cH3d4HXBrb0kREJJa0AIqISJxQ4IuIxAkFvohInFDgi4jECQW+iEicUOCLiMQJBb6I\nSJxQ4IuIxAkFvohInFDgi4jECQW+iEicUOCLiMQJBb6ISJxQ4IuIxAkFvohInFDgi4jECQW+iEic\nUOCLiMQJBb6ISJxQ4IuIxAkFvohInFDgi4jECQW+iEicUOCLiMQJBb6ISJxQ4IuIxAkFvohInFDg\ni4jECQW+iEicUOCLiMQJBb6ISJxQ4IuIxAkFvohInFDgi4jECQW+iEicKDfwjTFtjDGfGGNWGWO+\nM8bcHGGfs40xu40xy33/xldOuSIiUlEJUexTAvzVWrvUGNMIWGKMybTW5oTtt9Bae2HsSxQRkVgo\nt4Vvrd1irV3qu70HWAW0ruzCREQktqJp4QcYY44FugFZETafYYz5BvgJuM1a+12E598A3OC7u98Y\ns/KQqq1ZWgDbq7uIw6D6q1dtrr821w61v/60ij7RWGuj29GYhsBnwAPW2rfDtjUGvNbavcaY84Gn\nrbWdyjletrW2RwXrrnaqv3qp/upTm2uH+K4/qlE6xphE4F/Aa+FhD2Ct/cVau9d3+30g0RjToiIF\niYhI5YhmlI4BXgRWWWufKGOflr79MMac5jvujlgWKiIihyeaPvxewHDgW2PMct9jdwJtAay1U4BL\ngFHGmBKgEBhiy+8rmlqxkmsM1V+9VH/1qc21QxzXH3UfvoiI1G660lZEJE4o8EVE4kSVBb4xppkx\nJtMYs8b33yMj7NPVGPOVbwqHFcaYy6uqvrIYY84zxuQaY9YaY26PsL2eMeYN3/Ys37UKNUYU9f/F\nGJPj+3l/ZIxpVx11RlJe7UH7XWKMscaYGjXULpr6jTGX+X7+3xljZlZ1jQcTxe9OW9+0K8t8vz/n\nV0edkRhjphljtpV1rY9xTPS9txXGmO5VXePBRFH/MF/dK4wxXxpjTo7qwNbaKvkHPArc7rt9O/BI\nhH06A518t48GtgBNq6rGCPW4gXVAByAJ+AZID9vnJmCK7/YQ4I3qqreC9fcDGvhuj6op9UdTu2+/\nRsACYBHQo7rrPsSffSdgGXCk7/5R1V33IdY/FRjlu50O/FDddQfV1hfoDqwsY/v5wAeAAU4Hsqq7\n5kOs/8yg35vfR1t/VXbpDAam+25PB/4YvoO1drW1do3v9k/ANiClyios7TRgrbV2vbW2CHgd530E\nC35fbwH9/UNUa4By67fWfmKt/c13dxFwTBXXWJZofvYA9+E0JvZVZXFRiKb+64FJ1tqfAay126q4\nxoOJpn4LNPbdboJzlX2NYK1dAOw8yC6DgVesYxHQ1BjTqmqqK1959Vtrv/T/3nAIf7dVGfip1tot\n4MzPAxx1sJ194/mTcFoZ1aU1sDHo/iZKzyMU2MdaWwLsBppXSXXli6b+YNfhtHpqgnJrN8Z0A9pY\na9+tysKiFM3PvjPQ2RjzhTFmkTHmvCqrrnzR1P934EpjzCbgfeDPVVNaTBzq30ZNFvXf7SHNpVMe\nY8x8oGWETXcd4nFaATOAq6213ljUVkGRWurh41ij2ae6RF2bMeZKoAdwVqVWFL2D1m6McQFPAtdU\nVUGHKJqffQJOt87ZOC20hcaYLtbaXZVcWzSiqf8K4GVr7T+MMWcAM3z1V+ffbLRq8t9t1Iwx/XAC\nv3c0+8c08K21A8raZozJN8a0stZu8QV6xK+vvnl53gP+5vuqVZ02AW2C7h9D6a+t/n02GWMScL7a\nHuyrZFWKpn6MMQNwPpTPstbur6LaylNe7Y2ALsCnvh60lsBcY8wga212lVVZtmh/dxZZa4uBDcaY\nXJwPgK+rpsSDiqb+64DzAKy1XxljknEmJqtJXVNliepvoyYzxvwOeAH4vbU2qpkNqrJLZy5wte/2\n1cCc8B2MMUnAv3H61mZXYW1l+RroZIxp76ttCM77CBb8vi4BPra+Myk1QLn1+7pFngMG1bA+5IPW\nbq3dba1tYa091lp7LE4/Zk0Je4jud+c/OCfN8c091RlYX6VVli2a+n8E+gMYY04AkoGCKq2y4uYC\nV/lG65wO7PZ3OdcGxpi2wNvAcGvt6qifWIVnnZsDHwFrfP9t5nu8B/CC7/aVQDGwPOhf1+o4Sx5U\n9/nAapxzCXf5HpuAEy7g/JLPBtYCi4EO1VlvBeqfD+QH/bznVnfN0dYetu+n1KBROlH+7A3wBJAD\nfIszJUm1130I9acDX+CM4FkOnFvdNQfVPgtnlF8xTmv+OmAkMDLoZz/J996+rYG/O+XV/wLwc9Df\nbXY0x9XUCiIicUJX2oqIxAkFvohInFDgi4jECQW+iEicUOCLiMQJBb6ISJxQ4IuIxIn/B4vBBsHp\naRg3AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x216ffdcd358>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig = plt.figure()\n",
    "ax = fig.add_subplot(111)\n",
    "ax.scatter(xMat[:,1].flatten().A[0],yMat.T[:,0].flatten().A[0],s=10)\n",
    "xCopy = xMat.copy()\n",
    "xCopy.sort(0)\n",
    "yHat = xCopy*ws\n",
    "ax.plot(xCopy[:,1],yHat,c=\"red\")\n",
    "plt.xlim(-0.2,1.2)\n",
    "plt.ylim(2.5,5)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1.        , 0.98647356],\n",
       "       [0.98647356, 1.        ]])"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 之前sort过\n",
    "yHat = xMat*ws\n",
    "np.corrcoef(yHat.T,yMat)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### (LWLR) Locally Weighted Linear Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def lwlr(testPoint,xArr,yArr,k=1.0):\n",
    "    xMat = np.mat(xArr); yMat = np.mat(yArr).T\n",
    "    m = np.shape(xMat)[0]\n",
    "    weights = np.mat(np.eye((m)))\n",
    "    for j in range(m):                      #next 2 lines create weights matrix\n",
    "        diffMat = testPoint - xMat[j,:]     #\n",
    "        weights[j,j] = np.exp(diffMat*diffMat.T/(-2.0*k**2))\n",
    "    xTx = xMat.T * (weights * xMat)\n",
    "\n",
    "    if np.linalg.det(xTx) == 0.0:\n",
    "        print (\"This matrix is singular, cannot do inverse\")\n",
    "        return\n",
    "    ws = xTx.I * (xMat.T * (weights * yMat))\n",
    "    print(np.shape(ws),np.shape(xTx),np.shape(xMat.T),np.shape(weights),np.shape(xMat),np.shape(yMat))\n",
    "\n",
    "    return testPoint * ws"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def lwlrTest(testArr,xArr,yArr,k=1.0):  #loops over all the data points and applies lwlr to each one\n",
    "    m = np.shape(testArr)[0]\n",
    "    yHat = np.zeros(m)\n",
    "    for i in range(m):\n",
    "        yHat[i] = lwlr(testArr[i],xArr,yArr,k)\n",
    "    return yHat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2, 1) (2, 2) (2, 200) (200, 200) (200, 2) (200, 1)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "matrix([[3.12204471]])"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lwlr(xArr[0],xArr,yArr,1.0)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "yHat = lwlrTest(xArr,xArr,yArr,0.003)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def lwlrTestPlot(xArr,yArr,k=1.0):  #same thing as lwlrTest except it sorts X first\n",
    "    yHat = np.zeros(np.shape(yArr))       #easier for plotting\n",
    "    xCopy = np.mat(xArr)\n",
    "    xCopy.sort(0)\n",
    "    for i in range(np.shape(xArr)[0]):\n",
    "        yHat[i] = lwlr(xCopy[i],xArr,yArr,k)\n",
    "    return yHat,xCopy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "xMat = np.mat(xArr)\n",
    "srtInd = xMat[:,1].argsort(0)\n",
    "xSort = xMat[srtInd][:,0,:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXwAAAD8CAYAAAB0IB+mAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3XlcVNX/x/HXQVBcAPddU1NzX4pc\nskzLQi21bP2296vM9jLzm/VNvrRYfctsz0zbN9MsNU1zSVMBFRMX3DV3VEQBUXbO748ZFRFlwIEB\n5v18PObBnbnn3vsZHvCZz5x77rnGWouIiJR9Pp4OQEREiocSvoiIl1DCFxHxEkr4IiJeQglfRMRL\nKOGLiHgJlxK+MWaHMWatMSbaGBOVx3pjjHnfGLPVGLPGGHOx+0MVEZHz4VuAtr2ttYfOsq4f0ML5\n6Ap84vwpIiIlhLu6dAYBX1uHSKCqMaaem/YtIiJu4GqFb4E/jDEW+NRaOz7X+gbA7hzP9zhfi83Z\nyBgzBBgCULly5UtatWpVqKBFRLzVypUrD1lraxVmW1cTfg9r7T5jTG1grjFmo7X2rxzrTR7bnDFn\ng/ODYjxAcHCwjYo643SAiIicgzFmZ2G3dalLx1q7z/nzIPAL0CVXkz1AoxzPGwL7ChuUiIi4X74J\n3xhT2RgTcGIZuBZYl6vZdOAe52idbkCitTYWEREpMVzp0qkD/GKMOdH+e2vtbGPMUABr7ThgFtAf\n2AocB+4vmnBFRKSw8k341trtQMc8Xh+XY9kCj7k3NBERcSddaSsi4iWU8EVEvIQSvoiIl1DCFxHx\nEkr4IiJeQglfRMRLKOGLiHgJJXwRES+hhC8i4iWU8EVEvIQSvoiIl1DCFxHxEkr4IiJeQglfRMRL\nKOGLiHgJJXwRES+hhC8i4iWU8EVEvIQSvoiIl1DCFxHxEkr4IlKyRURA376On3JefD0dgIjIOYWF\nwZw5juXZsz0bSymnhC8iJVto6Ok/pdDUpSMiJVv37o7Kvnt319qrC+isVOGLSMkSEUFm2Mv8/fhI\n/jyQwYFlq3jxju7U6NXDte3VBXRWqvBFpPicpfoOn/4X79wXyqhP5/HQ11F0bnUfty45yvjNx5la\nvTVDv4hkRcgteVftufa57LEXGHvbc8QdyzjVXlU/oApfRIpRzP8+4tVqPRn/ymgCZs0AIDUji4cX\nHyK5TjCBWxKpUbMR/f9eSK+tK+hR05c5mUG8fNld3FKvFTd8EU5oh0uoVrm8Y4fjx8Pjj0NGhuP5\n7NkMi8lkb5MrydoXy/CwMEeVr6ofUIUvIsXolR53E3FBRyb3vp2svv0gIoJ5Gw5wtFwFvt42ndU3\n1WfBinG8Oes9+sVvIvDtN7gleRvLP7yHp5dN5rcarbhm7CJmrN7H/A0HeGfKCia17kV8YA0IDSU+\nOY29CSkAzOl8jeNEb0QEJCRA165ef+JXFb6IFJtEv0pAEi/HV2Vi01u45+Pp/NWxF3XTk7ls2P2O\nE7M5R+V07w4BAVTMTOPplI2EPHkFz01ZzRM/rHK0ufgGAK6rfJyPundnzcaDAPRvX5dZa2Fb8/bs\ne+QZPmkykCczttLN1RO/ZZQSvogUi6TUDDbtTzr5vFF2Cq83uBwOZTL0798pF74RIiNPjco5YexY\nR5dMaCit6wXyy6M9mH3nU9QMX8QFHVsSesvzzNl4kINHU4nenYCPgWHXXMSstfuZE7OfOcE3sToh\ni0MBF/OHB953SaKELyLFImrHYbItfLb9NzrP/IGaV3Rjw39/4vdnXuOeFb9C+1Z5b5jrA8CvnA8D\nnroDErbAC8/yfPNWzF1/gMmPvcrqTj1pWSeA5rWr0KFhEG/P2US2heqVy7MlOZ3ktEyqVPDetKc+\nfBEpFpHbD1O+nA9XXNWZmhV9YfBgWtcLZNhTN1Dzim6OSt5VOcbmX1irCpcd3c3XAS1Zte8oHRoG\nARDSti7Z1tH85dZ+WAtr9yQWwTsrPZTwRaRYRG6Pp1Pjqvj/OhXi42HqVMeKgl5YlZNzuOWzVQ5x\nvLw/ib7+dGxUFYC+7eoCMHjtfC577TkAVoeN8eqhmd773UZEik1Sagbr9iby+FUt3DtVgnO45SVR\nUczIKs+Xgx+nf7trALiwVhV+n/Rvmu3eRIXKlWiclsjqo85tvHRopip8ESlyJ/rvuzWtfn4VfW6h\noRASAqNH06RrB/7btSbVbhp4sopvPfJJKlQNgrfeomN1P1Y3aAWDB5//cUsplyt8Y0w5IArYa629\nPte6+4C3gL3Olz601k5wV5AiUrqd6L/v3Liae3ec84TukCGOq2lzXmA1ZIjjAdx50//RZ8M27NaK\nGOdr3qYgXTpPARuAwLOsn2Stffz8QxKRsmbZ9ng6NapKxfLlivZA5+gu6jb8oZPDO72VS106xpiG\nwHWAqnYRKZCjqRms3ZtIt2bVi/5g5+oucmdXUinlah/+u8AIIPscbW4yxqwxxkwxxjTKq4ExZogx\nJsoYExUXF1fQWEWkFIraccTRf9+shqdD8Xr5JnxjzPXAQWvtynM0mwE0sdZ2AOYBX+XVyFo73lob\nbK0NrlWrVqECFpHSJWLJWspnZ9E5drOnQ/F6rlT4PYCBxpgdwI/AVcaYb3M2sNbGW2vTnE8/Ay5x\na5QiUipZa/kjZj9ddq2h4msvezocr5fvSVtr7UhgJIAxphcw3Fp7V842xph61tpY59OBOE7uioiX\nstZy81t/UGXHNnYEXsBQn8NefbK0pCj0hVfGmJeBKGvtdOBJY8xAIBM4DNznnvBEpDSK2ZfEysOZ\nEHgBvjaLkM/egBNz2IvHFCjhW2sXAgudy6NyvH7yW4CIyLTovfgZICuLy+r6n7phiXiUplYQEbfK\nzrbMWB3Lla1q80D1FBp89iF0D/Dq4ZAlhRK+iLjV8h2H2Z+Uysj+rej+/AOOK1/Tk7x2/pqSRAlf\nRNxq+up9VPQrxzVt6rh3ojQ5b0r4IuI26ZnZzFobyzVt6lCpvO+Zd68Sj1LCF5FCy8q2bItLJnpX\nAtF7Evh7w14SjmcxyD8p/42l2Cnhi0ih/WvMHyyPzwQgwN+XjnH/8HzUfHpFH4Mbr/RwdJKbEr6I\nFEpSagYrDmVw07r5POobS9NffsBnWSRsm6Y++xJKCV9ECmXtnkSsMQyqlMyFw54CH6M++xJOCV9E\nCiV6dwIAHb/+GCr6eTgacYVucShSVjhv6F1cN+letSuBZrUqE6RkX2qowhcpK5w39AZO61ZJTsuk\ncvlyGGPcdihrLdG7E+jZsqbb9ilFTxW+SCmXtiScv255iMh+t5Pat/9pJ0w3HzhK8Mtz+PWOp91a\n+e9LTOVQchqdGlV12z6l6Cnhi5Ri2dmWx75byT0X3sDtsbV4o1xzWLsWcFTho6atIzULog5nOr4B\n5Oj2yQ4PZ/fAWwv1QbD8j2UAdDqy263vR4qWEr5IaZKrn/79BVuYF9SMYbGR9N2+nMktLif9Py8B\njikOIrcfpoIPbGzRyVH5O7t9bFgYz30ZTs/Wd7PyrU8LFMKizXG8uOIIFxzZR6sPXnf7W5Sio4Qv\nUpqc6KcPC2Pu+gO8O28Lgy9uwBNfvsxN3S/kWIVKLH3hfxxNzeC1mRvo0DCIW7tcwMbqjcju2s2R\n9ENCeL/3vfxcvTXlrOXtWsGkLw0/81h5nASe+vceHvhyBRdUq8hPe2dTftRLxfjm5Xwp4YuUJqGh\n7L4yhCUpFXjm86W0DyrH6BvbY4zhyi7NqZOezITMurw3bwtxyWm8MqgdbesHciw9izkx+5lR6QI+\nHfkRY48EctPaeYyK+I6Iak0Z9NNmYvYlnnaoVW+NY3dkNISFYa1l3KJtDPtpNV2aVmfSsKup89vP\nmvK4lDHWWo8cODg42EZFRXnk2CKl1U8/LmRE9DEAGibs56cNk6i/6A/Hyr59GXekCm/0vh8fA7dd\n2pjXB7dn7Z5EBny45LT9NDt+iFlR4/G/717mzlvFC60HcCQDHr+qOY9VPET8G2Po1e4+2h3dz4Rb\n2vDi6uPM3JfOgAblefuRq6jgW66437o4GWNWWmuDC7OthmWKlBIZWdmMj9xNo9R0hi39nms2R1Cl\nc4dTDUJD+derr/OBycYvI40RNRwTmLVrEMin/8ykfNQKajRryKFDSVyyYRn+vS6HIUO4Zghcejyd\n/06P4d15W/jjeBx+1XqQan2IqlKfPouPk5iUwoi/vmFo1aP4PHGNh34Dcr6U8EVKgcyl4Tz99XK2\nVmvBJ7Gz6PfEbTDV7/Q5a7p3J2jmdD685WEqRS6l2rqGcOVsjDGEPH03hG2H3Zth/XoIDDxt26qV\nyvPu7Z3p174ez/6wkuRKtbipUXn+iM2g6qH9fHlhJu2qHtUcOaWcEr5IKfDGxAXMrN2ZF7fOpV/y\nNmh/FwwZkmfb3sPug7CdZ3wYMHs2dOvmeN66dZ797yFt69L6md5MXLKdYddcxMhbbyVg9m9U6HO1\n5sgpA5TwRUq4uKNpfF23M7fEreehPZGwzDEG/qwJ+FwTmI0d6xjpc45KvfHm1YR9Ega1Q+HFEZCZ\nosq+jFDCFynhvvloKhlZVXikYw34v/wT9jm5Mptl7ikaVNmXGUr4IiXcn1vi6ZKyk2YT34ZDh4o+\nAes+tGWWxuGLlGAp6VlsqNWE4MP/wOjRxXPQE98CNMa+zFGFL1KCrdmTQCaGzu+8DG3qeDocKeVU\n4YuUYKucNxnp3FizUsr5U8IXKaESj2cwdckWmqYeocbavz0djpQB6tIRKYESj2dw18Rl7EhMZ/z0\nsbCxlkbLyHlTwhcpYVLSs7j7/flsOpzOuPpH6NWylkbMiFso4YuUMN8t28mahCw+/fV1rmoWpMpe\n3EZ9+CIlSFpmFp8t3k73mr6ENAtSZS9upQpfpASZ+vdeDiSlMeaBrjBclb24lyp8kRIiMyubcYu2\n0bFhED2a1/B0OFIGKeGLlBAz18ayM/44j/ZujjHG0+FIGeRywjfGlDPGrDLG/JbHugrGmEnGmK3G\nmGXGmCbuDFKkTIuIILtvXz6euZYWtatwTWtdUStFoyAV/lPAhrOsewA4Yq1tDowF3jzfwES8RlgY\n87cdYdPRLB7tfSE+PqrupWi4lPCNMQ2B64AJZ2kyCPjKuTwFuNroO6mIS+Kff4k3r3uchpV8GNCh\nvqfDkTLM1Qr/XWAEkH2W9Q2A3QDW2kwgETjjrJMxZogxJsoYExUXF1eIcEXcKzktkxfGzePQ9TdC\nRESxHz8zK5vBi5PY6l+dZ1b+gu/yZcUeg3iPfBO+MeZ64KC1duW5muXxmj3jBWvHW2uDrbXBtWrV\nKkCYIkVj8eY4vt+RxqR4P8eNP4pRWmYWL01bx85j2bzx+/vcNP2zYo9BvIsrFX4PYKAxZgfwI3CV\nMebbXG32AI0AjDG+QBBw2I1xihSJ9bFJAMy8tP8ZFzl98/2fTLv9SWx4uNuPe/DPpdz6zFf8sHw3\nD+8K57bMPdC1qy60kiKVb8K31o601ja01jYBbgcWWGvvytVsOnCvc/lmZ5szKnyRkiZmnyPhr69U\ni+3N2598PSk1g9DVyTzVJISHvlnJwaOpji6fvn3d0vUz5ss/2VChGuNmv8PIH0ZjGjWCyEjddESK\nVKHH4RtjXjbGDHQ+nQjUMMZsBYYBz7sjOJGilJyWyYqtcfRM2gnAzJnLTyb0yG3xZBsfbo2PYXG1\nZlw79i+mvfc9ds6c8+52SUrN4Jc67bll7Tz6ZuyHkBBV9lIsCjS1grV2IbDQuTwqx+upwC3uDEyk\nqP20YjdHMy3Dfn2PlMHD+W1Vea5fvpamYWEsfeRtKvqV49VPhvPwkeMMn7yap5r05UCvBIYMvuK8\njrt6TgTpPr708zkMEyaoqpdioyttxStlZVu+CP+HS6r70qlDU67v3pxNFWvSe8h43rj5ORZvPUTX\nZtUp7+vDhbWqMGXoZbQ/foB59dvB1Knndew1P/8BQHufY0r2UqyU8MUrzV1/gN2HU3igXweYPZt+\nA04l3nFbU9ked4zLm9c8+Vo5H8PFbRsTU78l2aNG5bVLAKy1jPl8PisH3wcREew5cpx+o2fTbvhU\nuj/9PQf/XEr0xT1pmnqEoP+MLMq3KHIGJXzxPhERfD5uOg0q+nCt88bgtQP9CchMpXLacR7ZsgD/\n7Ax6p+w7bbOOl7biWLnyrGnU5qy7Xrg5jg82p/JLciWOv/IaD329kj1HjnNz9BwO+VXi7e+Xsua4\nDx27tVV1L8VO0yOL11n7v49ZftHt/OfPb/G9turJxLs04n2y//6bquV9eHra+1RYczVce2qK4qtb\n16G8rw9fhe/g96PxhK/ZybhBLWlw9eWAo7ofO3czALEXtuG51jeyMTaJz3vUoPfi1fjuq8WERt0h\nKY0ODXVTcil+qvDF60y8+h4qp6dwa8Svp424CXz7Dape2QPeeosKfa4+Y+RMUHQUffat45dVe/ls\ny3Fi/Gvy+XcLT66fv+Ega/YkUsHXh4XVmjFzXzrP921F7xuuhMhInpgQSrVKfgB0bBRULO9VJCdV\n+FLmZWZlE7E9nq0Hk1m/L4lf96Rzf6uqBPa+4vSk3r37qdsJDhly5o7Cwhi2IobmbXpyc7l4xrS8\nlkn1O/N0agZVKvjyztzNXFCjEl0qZzJ5VzqDGpZnSM9mJzcPqujHfy4qz4cRB2m7eyNccFkRv3OR\n0xlPXR8VHBxso6KiPHJs8S7PfTyXybvSAQj09+WiugG8c2snGlWvVLAdRUTAM884lseOZU2j1gz8\ncCkv7f2LBhe3ZeiBGozpXJnqU35galoQb2XE4P/7zNP30bcvzJnjGHuve9VKIRhjVlprgwuzrSp8\nKdMW/rqIybvSuT9qGo/6x1FzxtTC31yke3fH1bBOHYDg5H18FtiGCpszaZa5m0FfTMM3dBS9w8Ly\nvpjqxGu60Eo8QH34UmalZmTxwl+xND+0i+fXTKfWiyMcyd6NUyQ80Lsl+wNqsjOoDi/tXYxv6KhT\nXUN5jcI51zqRIqaEL2VHrkS+encC+8oHMCJpDRWm/XIqyYaFObpV3DAz5bU39uTOQ2v579xx9M6O\nVyKXEk1dOlJ2nEjkALNnn5wJs9PEsRDgf6qdG7tVyvkYXutSHX6NgIdHn/f+RIqSEr6UHbkS+fp9\nSdSsUoHaOZM9nD4axx2mToX4eMfPvEb3iJQQSvhSdjgT+bTJC4m95wWiL+pJm/rVi/64OhErpYQS\nvpQpS7ce4pmoo2TX7wFHs7j64sCiP6i7vzGIFBGdtJUyY39iKk/+sIpmAb5M2vIzdwYkc/O4MI/c\nq1akJFKFL2XGmD82kZyWyaQnr6B57X50PXGRU9oRVeAiqMKXMmL34eP88vce/rUjguYD+ziq+tBQ\n3U1KJAdV+FImfLJoGz5ZmQyd9jEkxzuGaM6ercpeJAdV+FLqxSamMCVqD7fERlM3OR4CA1XVi+RB\nCV9KvU9/XEp2ZgaPdGt4alIyXfEqcgZ16Uip9tfmOL7ZnsLNa+bRcPVmdeGInIMSvpRaG2Yv5tH5\nh2hR0fAf8w+MUjeOyLko4Uup9NfmOB6fd5DKKcf4YuM0AmbN8HRIIiWe+vClVLHWMmHxdu77Yjn1\nq1dmyu7fqPfCcE+HJVIqqMKXUmXGlEW8uvIY/er58fbQq6hcoa+nQxIpNVThi1tkh4e77aYi57Js\nTiQBqcl89Ps7VK6gekWkIJTwpfCcNxzZPncJbX49yLKYPW65qci5rGtwEe0O78Jn8OAiPY5IWaSE\nL4XnvOHIoq+mk+rjxw/97i/SC54ys7LZmOJDuz0bHXPPi0iB6DuxFMqBpFT+fmgk/RISWFahNgBz\n6rTl2MWXUrmIjrk+Nok0H1861K4ET2kIpkhBqcIX1+S4X2xWtuXWTyN4ZEUycxt3Znnl+lyYepiU\njCzmxOzPcxt3+ClqNz4Guo5/S1fSihSCEr64JseNv7+O2MHO+OMAfND6Wg5XCmJIt0Y0rFaR8bPX\nsfTmB9n6xxKOvjLabTcLX7Mnge8id3HPwWhqx6w67/2JeCMlfHGNc6rhPc/9h7fmbOLKpJ0MWL+I\nNWnlAeh2zaXc0KkBG5OyuLP5jfRZkEj7DkP57bbHztqvn740nHm3DnWM8IG8vxFERJDWvQcvjf6J\nmmlHGfbt6CI/MSxSVinhi0syunTl1cffYcCiJEhJ4bW2FbjCJxGAOv6GxtUrcUPnBgBctSua9+wG\n6qcfZUqXgWd2vzgT+y9vfc2DzQbwwae/O17P8S3ihNjRb3Nri5tZHdiA0L++ILBze82EKVJIOmkr\nLpkTs58JS/7h2oRtDJ3xMQ3bN6GHXxUAusRuwpj+NK9dhembJtFyxo/4Vw0kpv1AvvC9gcSUDIIq\n+p3amTOxL/i/twF4t143grceokceNwN/uc/DbNlzjE9nv0PI6gWO2TDVfy9SKEr44pKVkevxz87g\no87++O1oAqGhNACGj5vJFbeFnGzXYcQjkHIQBg+m7+yljPcpx/wNBxh8ccNTOwsNJQMfltZrxaB2\n9Vm/L4mnflzFY72b0/S9r+l1kWPUz5YDR5l/MIs7rmhJyOBXHR8Uqu5FCs1Ya8/dwBh/4C+gAo4P\niCnW2tBcbe4D3gL2Ol/60Fo74Vz7DQ4OtlFRUYUMW4qTDQ/n2q/WUSMpnh+PLHJ5CuLsbEuPNxfQ\nrkEQY2/rxMbYJNbHJrHtYDLNa1fhpWkxjLvrEprXrsz1HywhNSMbgFUvXcPehBTunrgM36xMfl71\nJY1HPqPKXgQwxqy01gYXZltXKvw04CprbbIxxg9YYoz53VobmavdJGvt44UJQkq2qDGfsaXFzTy4\ndjaMcb3C9vExhNSwfBWzn/ahc8hdWvga6NG8BgH+foQNbMu/f14LwLNjZrAiuwqB/n58Fz6RxjMm\nQ3qS5roXOU/5nrS1DsnOp37Ox7m/FkiZ8l3vOwjITGUAcWeuzGes/f2/jefGdQsYFhvBxHuDiRh5\nFSFxGwHotmstAasc3/Juu7Qxf/40gjtW/c6C4/7UKJfNpIe70WTk07oRuYi7WGvzfQDlgGggGXgz\nj/X3AbHAGmAK0Ogs+xkCRAFRjRs3tlKChYdbGxJiDyxYYlu8MMuG3jnKWrC2a9fT24WEOF4PCTnn\nfmx4+MmXonr0tSH3f2BX121++v4+/dQeCahm377iLnvguhuL4E2JlH5AlHUhb+f1cGlYprU2y1rb\nCWgIdDHGtMvVZAbQxFrbAZgHfHWW/Yy31gZba4Nr1apVgI8lKTYRERztP4BbvoqmTZsH6TIngfSs\nbO7Ysjjv9s7x+WetwLt3P+Mes5dkHmH2F0/QYf/W09sOGULVOTN5tlIctV98zk1vSEROyPek7Rkb\nGBMKHLPWvn2W9eWAw9baoHPtRydtSybbty/Pm5ZM6hjCwCObuOjIXlqsWsq1NYCqVR2J/XxPnkZE\nwDPPOJbHjtXJWJECKNKTtsaYWkCGtTbBGFMR6AO8matNPWttrPPpQGBDYYIRz/vo9hFM2pjC4zsX\nM/zJQY4Xw2Lck+hzcteHh4i4zJVROvWAr5yVuw/wk7X2N2PMyzj6kqYDTxpjBgKZwGEcffpSyqRn\nZvPF9lSu2hXNsz++CUeiHd0x7h4dc+KKWtDIG5FilG/Ct9auATrn8fqoHMsjgZHuDU2K29z1B4hP\nt9y97BdMjRpFNzImjytqRaTo6UpbOenHFbtoUNGHni1qQegHRdfdcuJErogUK02eJgBsPXiUxVsO\ncev2cMqFjlLfukgZpApfsNYSOj2GgKw07pjyIRxYoQpcpAxShS9Mn7KIpVvjea52CrWu6Kq+dZEy\nShW+l4vZl8jI5UfodGA7d0bPU2UvUoYp4Xux1Iwshk4IJygjhfE7fqfcm694OiQRKULq0vFiny/9\nh93Hsxnzy5vUruSrE7UiZZwqfC8V9+dSPp51kD4BWVzWup767UW8gBK+l3r3u8WkVmvNCysnqd9e\nxEuoS8cLpaRn8XOd9txyZCPNnn/K0+GISDFRwvdCS35bQmoWXDegu/rtRbyIEr6Xsdby5fwNVE1J\nousnb3g6HBEpRkr4Xubz7xayNKARw5dPxm/wDZ4OR0SKkRK+F0lKzeC96MP02hbFnZG/wNSpng5J\nRIqREr4X+XLpDpJ8/Rl+ZBWmq6ZQEPE2GpbpDSIiSHhuJBO6Ps41jQJpN+9XT0ckIh6gCt8TIiKg\nb1/HTyBh0VIibn4AGx5eJIfb+OaH3NjmDo77+PHUoq+L5BgiUvIp4XvCM8/AnDkkPjeSMX9s4vLf\nDvKv5oP59qN8+tRzfVC4YtP+o9zY9g6SKwfy/ZpvaTfisfMMXkRKK3XpeEA2ENbnYX5udzXJC7bS\nLzCL5NhdhDXuTasdh7m0SfXTN4iIcNwHNiEBli1zvObC1bGZS8P5948bqFitHjP+ewN1g253/5sR\nkVJDFX4xip61mG/uHM7w65/lq0sG0Gt7FLM+f4JPlk7gwzEP0qhGZR759m9iE1NO3zDnTb9DQvI+\n2Xqi+h8//uS3gDcmLiC6cl1CZ31A3ZBeBfpmICJljyr8ohYRwdFXRjN6wFP8sDMNGvWGZPjXunmM\nnj8O06wZhIYSVNGP8Xdfwg0fLWXot38zaUg3/P3KOfaR46bftls3jDFnHIMBAzh69Di3NRxI44oX\n03T8H0yoE8x9MX8waKXz20BYmObNEfFiqvCL2O7X3yGkyWAm7UhhSHN/lq37nM0f38HrM9/FVKgA\nMTEnpzdoUSeAMbd2YvXuBEZNW4cND2fTDXeQbS07v/uZayPTeWP2xjMPEhYG8fG82v9xNtZszJ/N\nu/BJnWBC4jby4vVtoWtXx0PDMEW8mir8ohQRwRcBrYirUoPJPYO45Lqe8ODV0LYtrF8PDRqcsUnf\ndnV54qrmfLBgKzuS97K81Z34/nqITJ+FAKRnZjOyX+vTNwoN5fegC5nU9Eoe6XUhD37wPFmL/qL2\nsSOwNwQiI4vhzYpISaeEXwS+/u5PNvy5nLt2hPNDh3u4NnE7l1z39KkGEyY4qvKzVNzP9GlJTMxO\nFtCAKw5vY3VAfcofT+Wa1L38QDu2xSVTcW0007+cxeauvdmQ6c/GZv3p3KgqT/dpQYWgf0P8fsfO\nVNWLiJOx1nrkwMHBwTYqKspY33nnAAAKB0lEQVQjxy5K4dsOcef4SKwxVMxKJygzlekrJ1L7zVcK\nNDPl0f4DmLU3jUH1/Uga8QL+b71J+oh/03PBUXpU92Hbxp1sr1qfOunJtGzbhC6HtnPX+89TLfRF\nGDKkCN+hiHiSMWaltTa4MNuqD9+N4pPTePrHaJoGlKNn0k6y/cozfsfv1F4831HRF0DASy9wWz0f\n/Ef9h9pbYghcHk7NzTE8cHlT5u7PYHvV+nwxewzLWiXyzeRQnnj9Eart2wUvvFBE705ESjt16bhJ\ndrbl2cmrSUjJ4MtHe9C8dgiHj6VTd311SDlY8K6V7t1PjagZMADi42HECB66rCc/dX6QHnGb6f3J\na6eGbLZpA76+MHq0+9+ciJQJSvhuMnHJPyzcFMcrg9rSpn4gRERQ90Q//fkOhRw92lG516lD4O8z\nWHzkEH5BgUCP04Zs6mYmInIuSvhusHp3Am/O3khI2zrc1e0Cx4s5L5Y634Q/ZIjj4bzitnxCwun7\n1th6EXGBEv55yszK5skvI6iTmsj/GgeduigqZ+XtLie6eU5MtaAROCJSAEr452nzgWR2Hsvm7bkT\nCAo/dmrMe84+eHcryn2LSJmlUTrnqWnNyny7+lt6byt7Q0xFpGxRwj9PFcuX4/JRT1KjZ3cYO9bT\n4YiInJW6dNxBXSwiUgrkW+EbY/yNMcuNMauNMTHGmDOuIDLGVDDGTDLGbDXGLDPGNCmKYEVEpPBc\n6dJJA66y1nYEOgF9jTHdcrV5ADhirW0OjAXedG+YIiJyvvJN+NYh2fnUz/nIPQHPIOAr5/IU4Gpz\nxqTtIiLiSS6dtDXGlDPGRAMHgbnW2mW5mjQAdgNYazOBRKBGHvsZYoyJMsZExcXFnV/kIiJSIC4l\nfGttlrW2E9AQ6GKMaZerSV7V/BnTcFprx1trg621wbVq1Sp4tCIiUmgFGpZprU0AFgJ9c63aAzQC\nMMb4AkHAYTfEJyIibuLKKJ1axpiqzuWKQB8g9332pgP3OpdvBhZYT020LyIieXJlHH494CtjTDkc\nHxA/WWt/M8a8DERZa6cDE4FvjDFbcVT2txdZxCIiUij5Jnxr7Rqgcx6vj8qxnArc4t7QRETEnTS1\ngoiIl1DCFxHxEkr4IiJeQglfRMRLKOGLiHgJJXwRES+hhC8i4iWU8EVEvIQSvoiIl1DCFxHxEkr4\nIiJeQglfRMRLKOGLiHgJJXwRES+hhC8i4iWU8EVEvIQSvoiIl1DCFxHxEkr4IiJeQglfRMRLKOGL\niHgJJXwRES+hhC8i4iWU8EVEvIQSvoiIl1DCFxHxEkr4IiJeQglfRMRLKOGLiHgJJXwRES+hhC8i\n4iWU8EVEvIQSvoiIl1DCFxHxEkr4IiJeIt+Eb4xpZIz50xizwRgTY4x5Ko82vYwxicaYaOdjVNGE\nKyIiheXrQptM4Flr7d/GmABgpTFmrrV2fa52i62117s/RBERcYd8K3xrbay19m/n8lFgA9CgqAMT\nERH3cqXCP8kY0wToDCzLY3V3Y8xqYB8w3Fobk8f2Q4Ahzqdpxph1BYq2ZKkJHPJ0EOdB8XtWaY6/\nNMcOpT/+iwq7obHWutbQmCrAIuA1a+3UXOsCgWxrbbIxpj/wnrW2RT77i7LWBhcybo9T/J6l+D2n\nNMcO3h2/S6N0jDF+wM/Ad7mTPYC1Nslam+xcngX4GWNqFiYgEREpGq6M0jHARGCDtfads7Sp62yH\nMaaLc7/x7gxURETOjyt9+D2Au4G1xpho52svAI0BrLXjgJuBR4wxmUAKcLvNv69ofOFCLjEUv2cp\nfs8pzbGDF8fvch++iIiUbrrSVkTESyjhi4h4iWJL+MaY6saYucaYLc6f1fJo08kYE+GcwmGNMea2\n4orvbIwxfY0xm4wxW40xz+exvoIxZpJz/TLntQolhgvxDzPGrHf+vucbYy7wRJx5yS/2HO1uNsZY\nY0yJGmrnSvzGmFudv/8YY8z3xR3jubjwt9PYOe3KKuffT39PxJkXY8znxpiDZ7vWxzi873xva4wx\nFxd3jOfiQvx3OuNeY4wJN8Z0dGnH1tpieQD/A553Lj8PvJlHm5ZAC+dyfSAWqFpcMeYRTzlgG9AM\nKA+sBtrkavMoMM65fDswyVPxFjL+3kAl5/IjJSV+V2J3tgsA/gIigWBPx13A330LYBVQzfm8tqfj\nLmD844FHnMttgB2ejjtHbD2Bi4F1Z1nfH/gdMEA3YJmnYy5g/Jfl+Lvp52r8xdmlMwj4yrn8FXBD\n7gbW2s3W2i3O5X3AQaBWsUV4pi7AVmvtdmttOvAjjveRU873NQW4+sQQ1RIg3/ittX9aa487n0YC\nDYs5xrNx5XcP8AqOYiK1OINzgSvxPwR8ZK09AmCtPVjMMZ6LK/FbINC5HITjKvsSwVr7F3D4HE0G\nAV9bh0igqjGmXvFEl7/84rfWhp/4u6EA/7fFmfDrWGtjwTE/D1D7XI2d4/nL46gyPKUBsDvH8z2c\nOY/QyTbW2kwgEahRLNHlz5X4c3oAR9VTEuQbuzGmM9DIWvtbcQbmIld+9y2BlsaYpcaYSGNM32KL\nLn+uxP9f4C5jzB5gFvBE8YTmFgX93yjJXP6/LdBcOvkxxswD6uax6sUC7qce8A1wr7U22x2xFVJe\nlXrucayutPEUl2MzxtwFBANXFmlErjtn7MYYH2AscF9xBVRArvzufXF06/TCUaEtNsa0s9YmFHFs\nrnAl/n8BX1prxxhjugPfOOP35P+sq0ry/63LjDG9cST8y11p79aEb63tc7Z1xpgDxph61tpYZ0LP\n8+urc16emcB/nF+1PGkP0CjH84ac+bX1RJs9xhhfHF9tz/VVsji5Ej/GmD44PpSvtNamFVNs+ckv\n9gCgHbDQ2YNWF5hujBlorY0qtijPztW/nUhrbQbwjzFmE44PgBXFE+I5uRL/A0BfAGtthDHGH8fE\nZCWpa+psXPrfKMmMMR2ACUA/a61LMxsUZ5fOdOBe5/K9wLTcDYwx5YFfcPStTS7G2M5mBdDCGNPU\nGdvtON5HTjnf183AAus8k1IC5Bu/s1vkU2BgCetDPmfs1tpEa21Na20Ta20THP2YJSXZg2t/O7/i\nOGmOc+6plsD2Yo3y7FyJfxdwNYAxpjXgD8QVa5SFNx24xzlapxuQeKLLuTQwxjQGpgJ3W2s3u7xh\nMZ51rgHMB7Y4f1Z3vh4MTHAu3wVkANE5Hp08cZY8R9z9gc04ziW86HztZRzJBRx/5JOBrcByoJkn\n4y1E/POAAzl+39M9HbOrsedqu5ASNErHxd+9Ad4B1gNrcUxJ4vG4CxB/G2ApjhE80cC1no45R+w/\n4Bjll4Gjmn8AGAoMzfG7/8j53taWwL+d/OKfABzJ8X8b5cp+NbWCiIiX0JW2IiJeQglfRMRLKOGL\niHgJJXwRES+hhC8i4iWU8EVEvIQSvoiIl/h/7XRd8bz54+wAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x216ffdd3518>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig = plt.figure()\n",
    "ax = fig.add_subplot(111)\n",
    "ax.plot(xSort[:,1],yHat[srtInd])\n",
    "ax.scatter(xMat[:,1].flatten().A[0],np.mat(yArr).T.flatten().A[0],s=2,c=\"red\")\n",
    "plt.xlim(-0.2,1.2)\n",
    "plt.ylim(2.5,5)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### EX : Abalone"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def rssError(yArr, yHatArr):\n",
    "    return ((yArr-yHatArr)**2).sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "abX, abY = loadDataSet('./Data/CH8/abalone.txt')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "yHat01 = lwlrTest(abX[0:99],abX[0:99],abY[0:99],0.1)\n",
    "yHat1 = lwlrTest(abX[0:99],abX[0:99],abY[0:99],1)\n",
    "yHat10 = lwlrTest(abX[0:99],abX[0:99],abY[0:99],10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "56.811168577945885"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rssError(abY[0:99],yHat01.T)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "429.8905618702014"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rssError(abY[0:99],yHat1.T)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "549.118170882589"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rssError(abY[0:99],yHat10.T)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "yHat01 = lwlrTest(abX[100:199],abX[0:99],abY[0:99],0.1)\n",
    "yHat1 = lwlrTest(abX[100:199],abX[0:99],abY[0:99],1)\n",
    "yHat10 = lwlrTest(abX[100:199],abX[0:99],abY[0:99],10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "60571.25199167674"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rssError(abY[100:199],yHat01.T)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "573.5261441896967"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rssError(abY[100:199],yHat1.T)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "517.5711905380425"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rssError(abY[100:199],yHat10.T)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "518.6363153239507"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ws = standRegres(abX[0:99],abY[0:99])\n",
    "yHat = np.mat(abX[100:199])*ws\n",
    "rssError(abY[100:199],yHat.T.A)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(200, 2)"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.shape(xArr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(200,)"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.shape(yArr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "matrix([[0.29446352, 0.58838891, 0.20384418, 0.92659794, 0.74334729],\n",
       "        [0.1639852 , 0.92876611, 0.6643704 , 0.114225  , 0.54832251],\n",
       "        [0.41538914, 0.40006275, 0.62295329, 0.43937438, 0.17229928],\n",
       "        [0.17215863, 0.02087312, 0.80113956, 0.84828326, 0.64198167]])"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tmp_x = np.mat(np.random.rand(4,5))\n",
    "tmp_x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "matrix([[2., 0., 0., 0.],\n",
       "        [0., 2., 0., 0.],\n",
       "        [0., 0., 2., 0.],\n",
       "        [0., 0., 0., 2.]])"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tmp_w = np.mat(np.eye(4)*2)\n",
    "tmp_w"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "matrix([[0.58892705, 0.3279704 , 0.83077829, 0.34431725],\n",
       "        [1.17677781, 1.85753221, 0.80012551, 0.04174624],\n",
       "        [0.40768836, 1.3287408 , 1.24590659, 1.60227911],\n",
       "        [1.85319587, 0.22845   , 0.87874877, 1.69656652],\n",
       "        [1.48669458, 1.09664501, 0.34459856, 1.28396335]])"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tmp_x.T*tmp_w"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "matrix([[0.63157329, 0.99067636, 1.13132542, 1.24026227, 0.98179874],\n",
       "        [0.99067636, 2.73858776, 2.00585411, 1.6895438 , 2.05794268],\n",
       "        [1.13132542, 2.00585411, 3.02567175, 2.4361446 , 2.27493516],\n",
       "        [1.24026227, 1.6895438 , 2.4361446 , 3.56853084, 2.7434048 ],\n",
       "        [0.98179874, 2.05794268, 2.27493516, 2.7434048 , 2.59010055]])"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tmp_x.T*tmp_w*tmp_x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "matrix([[ 3.92305579e+15, -1.71344015e+15, -6.33518719e+14,\n",
       "         -2.42792929e+15,  3.00239975e+15],\n",
       "        [-1.71344015e+15,  7.48364874e+14,  2.76696653e+14,\n",
       "          1.06042630e+15, -1.31133294e+15],\n",
       "        [-6.33518719e+14,  2.76696653e+14,  1.02304425e+14,\n",
       "          3.92076671e+14, -4.84845626e+14],\n",
       "        [-2.42792929e+15,  1.06042630e+15,  3.92076671e+14,\n",
       "          1.50261453e+15, -1.85814699e+15],\n",
       "        [ 3.00239975e+15, -1.31133294e+15, -4.84845626e+14,\n",
       "         -1.85814699e+15,  2.29780170e+15]])"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(tmp_x.T*tmp_w*tmp_x).I"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Shrinkage"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Ridge Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def ridgeRegres(xMat, yMat, lam=0.2):\n",
    "    xTx = xMat.T*xMat\n",
    "    denom = xTx + np.eye(np.shape(xMat)[1])*lam\n",
    "    if np.linalg.det(denom) == 0:\n",
    "        print(\"This matrix is singular, cannot do inverse\")\n",
    "        return\n",
    "    ws = denom.I*(xMat.T*yMat)\n",
    "    return ws"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def ridgeTest(xArr, yArr):\n",
    "    xMat = np.mat(xArr)\n",
    "    yMat = np.mat(yArr).T\n",
    "    yMean = np.mean(yMat,0)\n",
    "    yMat = yMat - yMean\n",
    "    xMeans = np.mean(xMat, 0)\n",
    "    xVar = np.var(xMat,0)\n",
    "    xMat = (xMat - xMeans)/xVar\n",
    "    numTestPts = 30\n",
    "    wMat = np.zeros((numTestPts,np.shape(xMat)[1]))\n",
    "    for i in range(numTestPts):\n",
    "        ws = ridgeRegres(xMat, yMat, np.exp(i-10))\n",
    "        wMat[i,:] = ws.T\n",
    "    return wMat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "xArr, yArr = loadDataSet(\"./Data/CH8/abalone.txt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ridgeWeight = ridgeTest(xArr,yArr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX8AAAD8CAYAAACfF6SlAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3Xl8XHd97//X55xZNNq30eLd8pLY\nsR3bUZyQOGQpoUlY0pQQQn+UUkpzaaEtl9vfD/q793eh/NpbbvtruW2h0GyFQFvgFggBEsjmBQhJ\nLMdOYseJLduyLVu2JMvapdnO5/fHjMayLK8aaTQznyecx1nnnO/Jkd/zne985xxRVYwxxhQWJ9sF\nMMYYM/Ms/I0xpgBZ+BtjTAGy8DfGmAJk4W+MMQXIwt8YYwqQhb8xxhQgC39jjClAFv7GGFOAfNku\nwLnU1tbqokWLsl0MY4zJKdu3b+9W1fCFtpu14b9o0SJaWlqyXQxjjMkpInLoYrazZh9jjClAFv7G\nGFOALPyNMaYAWfgbY0wBsvA3xpgCZOFvjDEFyMLfGGMK0Kzt53+5YpEEr/zsorq5Ti+Z5l3KmQeQ\nSY6XXCbpF4/fRlIzIoI4qXlJzQuIkxqnlruu4LgOjivJwZecPnO5g8/v4C9yCYZ8uH4nfRxjzOyS\nd+EfjyZoeaotu4WwxyID4DiCP+QSKPIlh5BLIDQ27aO8pojK+mIq64qpCIdw/fZB1JiZknfhHyoL\n8Imv3pbtYkwr1QnvLjrJpOq46dMrFD09ral9pcZj8+qdudzzFPWURFzxEoqX8NLjRGJsmeLFPeIx\nj9honMhInOhoglhqHB2NEx2NM9wXpff4MJHhOKNDsXS5RaCspij5RpB6Q6isD1FZV0xpdRGOY58g\njMmkvAv/QnBWU4pMNjn7wzIyEqevc5jeE6mhc4TeE8N07O8gFkmktwuV+bni+kZW3thIVUNJFkts\nTP6w8DdZEwz5qFtYTt3C8jOWqyrD/VH6Ooc5dXyYw7t7eO25I+x85jANTRWs3NjIkvV1BIrsz9eY\nyyVnNSHMEs3NzWo3djNjhvujvPliB3t+2UHviWH8QZdlzXWsuHEO9YvL7YtlY1JEZLuqNl9wOwt/\nk0tUleP7+3jjhQ5aW04Qj3pUzylhxQ2NXHF9A6HSQLaLaExWWfibvBcdibOv5QR7XujgxMF+HJ/w\n9g8s56qb5ma7aMZkzcWGvzWampwVCPm46qa5XHXTXE4eHeSF77ey+V/fIjqSYN07F2S7eMbMatax\n2uSFmrml3PUHa1h6TR0vfL+Vl544cHaXWGNMmtX8Td5wfQ63/95V+IMuLU+2ERtNcOP7l9qXwcZM\nYso1fxGZLyKbRGSPiOwWkT+ZZBsRkX8QkVYReU1E1k/1uMZMxnGEWz90JWtuncerzx9h87fexPPs\nE4AxE2Wi5h8H/ouqviIiZcB2EXlGVd8Yt82dwLLUcB3w1dTYmIwTR9h43zICIR8tT7YRjSR4x++u\nxHWtldOYMVMOf1XtADpS0wMisgeYC4wP/7uBxzTZCPuiiFSKSGPqtcZknIhw3Xub8AddfvWD/cQj\nCX79gVX4/G62i2bMrJDRqpCILALWAS9NWDUXODJuvj21zJhptf7XF3LzB5fTtuskP/7ya0RH49ku\nkjGzQsbCX0RKge8Bn1LV/omrJ3nJWQ2xIvKAiLSISEtXV1emimYK3Kqb5/GOj6zk2L5envj7nWfc\nUM6YQpWR8BcRP8ng/1dV/f4km7QD88fNzwOOTdxIVR9U1WZVbQ6Hw5komjEAXHFdA3f8/iq6Dg/w\n+Jd2MNwfzXaRjMmqTPT2EeARYI+q/t05NnsC+HCq18/1QJ+195uZ1rQuzLv+cA19J4b58ZdfRa0X\nkClgmaj53wj8NnCbiOxMDXeJyMdF5OOpbZ4EDgCtwEPAH2bguMZcsgVX1XDLh66k6/AAra90Zrs4\nxmRNJnr7/IIL3Dw+1cvnE1M9ljGZsOzaerY/1ca2n7SxZH2dPSjGFCTr+GwKjuMI1757Mac6hti/\n3Wr/pjBZ+JuCtHR9HdVzStj2k4P2C2BTkCz8TUESR7j2XYs5dXyY1pYT2S6OMTPOwt8UrCXrwtTM\nLWHbT9rwEl62i2PMjLLwNwVrrPbfe2KYfdus9m8Ki4W/KWhNa8PUzCu12r8pOBb+pqCJI2x492L6\nukbY+7LV/k3hsPA3BW/x1bXUzi9l25NW+zeFw8LfFDyRZO2/v2uEt146nu3iGDMjLPyNARatqSW8\noIyWJ9tIWO3fFAALf2MYV/vvHuWtX1nt3+Q/C39jUhaurqFuYar2H7fav8lvFv7GpIgIG97TxEDP\nKG/+yu44bvKbhb8x4yy4qpr6xeW0PGW1f5PfLPyNGWes7X+wJ8KeF6z2b/KXhb8xE8xfWU1DUznb\nn2ojEbPav8lPFv7GTJCs/TcxeCrCG78861HTxuQFC39jJjFvRRWNSyvY/lQb8Vgi28UxJuMs/I2Z\nhEjyaV9DfVH2bbOnfZn8Y+FvzDnMu6KKsuoi9u+w8Df5x8LfmHMQEZrWhznyRg+RkXi2i2NMRln4\nG3MeS9bV4SWUtte6s10UYzIqI+EvIo+KSKeI7DrH+ltEpE9EdqaG/56J4xoz3RoWl1NSEeDAjq5s\nF8WYjMpUzf/rwB0X2Obnqro2NXwhQ8c1ZlqJIzStDXN490liEev1Y/JHRsJfVbcCPZnYlzGzzZL1\ndcRjHod2ncx2UYzJmJls83+biLwqIk+JyFUzeFxjpqRxWSWhMr/1+jF5ZabC/xVgoapeDfwj8Phk\nG4nIAyLSIiItXV3WxmpmB8cRFl8d5tDrJ+0HXyZvzEj4q2q/qg6mpp8E/CJSO8l2D6pqs6o2h8Ph\nmSiaMRdlybowsUiCI29Y66bJDzMS/iLSICKSmt6QOq41oJqcMfeKKoLFPvZbrx+TJ3yZ2ImI/Dtw\nC1ArIu3A5wA/gKp+DbgX+AMRiQMjwP2qqpk4tjEzwfU5LF5Ty4FXu0nEPVyf/UTG5LaMhL+qfvAC\n678MfDkTxzImW5rW1/Hmi8dpf+sUC6+qyXZxjJkSq74Yc5Hmr6jCH3Q58Ir1+jG5z8LfmIvk87ss\nWl3DgVe78RL2kBeT2yz8jbkETevqGB2Mcay1L9tFMWZKLPyNuQQLV9Xg8zvst6Yfk+Ms/I25BP6g\ny4JVNRzY2YV61mHN5C4Lf2Mu0ZJ1YYb7ohw/YE0/JndZ+BtziRatrsXxif3gy+Q0C39jLlEg5GPB\nimr27+jEfqtocpWFvzGXoWldHYM9EToPDWS7KMZcFgt/Yy7D4qtrcRzhgN3m2eQoC39jLkNRiZ+5\nV1Sy/5Uua/oxOcnC35jL1LSujr6uEU4eHcp2UYy5ZBb+xlymprVhRLAffJmcZOFvzGUqLg/QuLTS\nunyanGThb8wULFkf5lTHEKeOW9OPyS0W/sZMQdPaOgD2v2K1f5NbLPyNmYLSqiD1i8vZb10+TY6x\n8Ddmipasr6P7yCB9XSPZLooxF83C35gpWrIuDGC1f5NTLPyNmaLy2hDhBWUcsF4/JodY+BuTAU3r\nwpw42M/gqdFsF8WYi2Lhb0wGnG76sdq/yQ0ZCX8ReVREOkVk1znWi4j8g4i0ishrIrI+E8c1Zrao\naiihqrHEmn5MzshUzf/rwB3nWX8nsCw1PAB8NUPHNWbWWLIuTEdrL8P90WwXxZgLykj4q+pWoOc8\nm9wNPKZJLwKVItKYiWMbM1ssWR9GFQ6+arV/M/vNVJv/XODIuPn21LIziMgDItIiIi1dXfYPyOSW\nmrmllIdD1u5vcsJMhb9Msuysm6Cr6oOq2qyqzeFweAaKZUzmiAhL1oU5+uYpRodi2S6OMec1U+Hf\nDswfNz8PODZDxzZmxjStC+N5Stvr3dkuijHnNVPh/wTw4VSvn+uBPlXtmKFjGzNj6heWU1oVtBu9\nmVnPl4mdiMi/A7cAtSLSDnwO8AOo6teAJ4G7gFZgGPjdTBzXmNlGHKFpbZjdPz9GdDROoCgj/8SM\nybiM/GWq6gcvsF6BT2TiWMbMdkvWh3ltUzuHdp1kWXN9totjzKTsF77GZFjDkkpCZX77wZeZ1Sz8\njckwxxEWrw3Ttusk8Wgi28UxZlIW/sZMgyXrwsQjCQ6/cb7fPhqTPRb+xkyDuVdUESz2cWCnNf2Y\n2cnC35hp4LoOi9bU0vZaN4m4l+3iGHMWC39jpsmSdWEiw3GOvnUq20Ux5iwW/sZMk/krq/EHXfZb\n04+ZhSz8jZkmPr/LwtU1HNzZheeddSsrY7LKwt+YadS0NszIQIyO1t5sF8WYM1j4GzONFq6qwfU7\ndptnM+tY+BszjQJFPhasrObAji7Umn7MLGLhb8w0W7IuzFBvhBOH+rNdFGPSLPyNmWYLV9fiOMIB\nu82zmUUs/I2ZZkUlfuZdWcX+HZ0kb3BrTPZZ+BszA5rWhenvHuXk0cFsF8UYwMLfmBmx+OowItgT\nvsysYeFvzAwoLg/QuLTSunyaWcPC35gZsmR9mFMdQ5w6PpTtohhj4W/MTGlaWwdgtX8zK1j4GzND\nSquC1C8ut8c7mlnBwt+YGdS0LkzX4QH6u0eyXRRT4DIS/iJyh4i8JSKtIvLZSdZ/RES6RGRnavhY\nJo5rTK5Zur4OBHZtPZrtopgCN+XwFxEX+ApwJ7AS+KCIrJxk0++o6trU8PBUj2tMLiqvDbGsuZ7X\nN7czMhDNdnFMActEzX8D0KqqB1Q1CnwbuDsD+zUmL137rkUkYh47njmc7aKYApaJ8J8LHBk3355a\nNtH7ROQ1EfkPEZmfgeMak5OqGkpYdm2y9j/cb7V/kx2ZCH+ZZNnEG5j8CFikqmuAZ4FvTLojkQdE\npEVEWrq6rEeEyV/NdyVr/zut9m+yJBPh3w6Mr8nPA46N30BVT6pqJDX7EHDNZDtS1QdVtVlVm8Ph\ncAaKZszslK79b7Hav8mOTIT/NmCZiCwWkQBwP/DE+A1EpHHc7HuBPRk4rjE5baz2b23/JhumHP6q\nGgc+CfyMZKh/V1V3i8gXROS9qc3+WER2i8irwB8DH5nqcY3JdVUNJSzbUM8ua/s3WSCz9f7izc3N\n2tLSku1iGDOtek8M82+ff5Gr37GAG9+3NNvFMXlARLaravOFtrNf+BqTRZX1xSzf0GC1fzPjLPyN\nybLmuxaRiHvsePpQtotiCoiFvzFZlq79bzlqtX8zY3zZLoAxJln73/vycbY91cryt1fR09NDT08P\n0WiUYDBIUVERwWBw0ulAIIDjWD3OXBoLf2Nm2PDwMCdPnuTUqVPpkO/p6eFUYxeb9kTYNK4jtIhc\n1EPfi4qKmD9/PkuXLmXp0qXU1NRM4xmYfGDhb8w0SyQStLe309rayr59+zh+/PgZ68vLy6muruaK\nK67g4MsDNK2cxw13raS6uppAIEAsFmN0dJRIJEIkEpl0enBwkIMHD7Jv3z4AqqqqWLZsGUuXLmXR\nokUEAoFsnLqZxSz8jZkGAwMD6bA/cOAAo6OjiAjz58/ntttuo76+nurqaiorK/H7/enXPTvyBvu3\nd1J+bzXBYBCAQCBw0eHd09OTPu4rr7zCyy+/jOu6LFy4kKVLl7Js2TJqa2sRmeyuLKaQWD9/YzJg\nrHa/b98+Wltb07X70tLSdA28qamJUCh03v30dg7zb59/iTW3zWPjvcumVKZYLMbhw4fTbwbd3d0A\nhMNhNm7cyKpVq3Bdd0rHMLPPxfbzt/A35jINDw/T2trK3r17aW1tPaN2Pxb4DQ0Nl1zLfu7rb9C6\nvZMP/cXbKKkIZqy8vb297Nu3j23bttHZ2UlVVRU33XQTa9asweezRoB8YeFvTIapKt3d3ezdu5e9\ne/dy+PBhVJXi4mKWL1/OsmXLLqp2fyHp2v+t89j4/qnV/ifjeR5vvfUWW7dupaOjg4qKCjZu3Mja\ntWvPaIIyucnC35gMiMfjHDp0KB34p06dAqC+vp7ly5ezfPly5s6dm/Guls994w32tXTy2xmu/Y+n\nqrS2trJlyxba29spKyvjhhtu4JprrrEviHOYhb8xlyiRSNDV1UVHR0d6OH78OLFYDNd1aWpqSgd+\nRUXFtJZlrPa/6u1zefv9y6f1WKrKwYMH2bp1K21tbRQXF3PDDTdw7bXXpr90NrnjYsPfGvpMQYrF\nYnR2dp4R9CdOnCCRSADg9/tpaGhg3bp1LFmyhMWLF89obbiyrpiVG+fw+uZ2ikr9XPuuRdPWQ0dE\naGpqoqmpiUOHDrF161aeffZZfvnLX/K2t72N6667zt4E8pDV/E1eicViDA4OpoehoaEz5seGvr6+\n9I+nioqKaGxspLGxkYaGBhobG6mpqcn6r2YTCY/N33yTN188zqqb53LTB5bjODPTRfPo0aNs2bKF\nvXv3EgqFuOGGG9iwYYO9CeSAgm32icfj7N+/fxpKlH8u9tpP3G78/GTrxpZdaFpV8TzvnMPY+ng8\nTjweJxaLnXOIx+NEo1Gi0cnvjVNcXExpaWl6qKioSAd+ZWXlrO33rqr86gf72fH0YZasD3P7716F\n65+5N6WjR4+yefNm9u3bl24O2rBhg30nMIsVbPgPDQ3xN3/zN9NQIjPTHMdJD4FAAL/fj8/nw+/3\nn3MoKSmhpKTkjKAvKSnJ+f7sO545zAvfa2XuFVXc9fHVBEIz22Lb3t7Opk2b2L9/P8XFxWzcuJHm\n5mZ7E5iFCjb8R2OjvLz35WkoUe6Yci12kpfL2EI5/7L08SW1fvw4Va6x+9WIIziOg4ggjiCSmk9N\nj1HO/huddNnETyHoWcvHL5s4rejp5Up62cTtx+9z7DVjHEmejyMODuOmxUmvd8Qh4AYIOkGCviBF\nbhEBN0CRr4iAEzjn9XvrxQ6ef+xNauaV8u5PXk1x+cwH75EjR9i0aRMHDhygpKQk/SZgXURnj4IN\n/57RHm7+zs3TUCJjZkbQDSbfDNwiiv3FVAYr00N15wKCzzfhligLPiiEGyqoClZRWVRJdVF1+k1m\nuh06dIjNmzdz8OBBSktLuf7661m1ahWVlZUzcnxzbgUb/tFElBc7XpyGEplskkk+jkxWQx7bbuKn\nkvGvT38CGfvf+HmRM8aTbT+2bPwxBEl/YvDUwyP1nYV6eOqhaHqc8BJEE1EiXoRIPEIkMcmQWj4U\nG6I30ktfpI9TkVP0RfooP1XPnXseIOHEeXLF1zhZcgwAn+OjvriexpJGGksaaShpoKGkIT3fWNpI\nib9kKpfhLG1tbWzevJm2tjYA5s2bx1VXXcXKlSunvTusmVzBhr8x+W40PsrhQyfY+mAbsdEEDe+L\nMVrXw4mhE3QMdXB86DgdQx10DneS0MQZry0LlNFY0sic0jnMK53HnNI5Z0yXBcouq0w9PT3s3r2b\n3bt3p+9rtGDBgvQbQVnZ5e3XXDoLf2Py3EDPKD/6h530d4/yzt+7iqZ14TPWx7043SPd6TeDjqEO\njg0e4/jQcY4OHuXo4FFG4iNnvKYsUHbGm0JDcQP1JfXUF9fTUNJAbagWn3P+L5u7u7vTbwSdnZ0A\nLFy4kFWrVrFixQpKS0sz+x/CnMHC35gCMDoY48dfeZXOtn7mrahm0eoaFq2upbz2wvcXUlV6I70c\nGzzG0cGj6fHY9LGhY2e9OTjiUFtUm35DqC+pp6G4gdriWqqLqqkpqqG6qJrKokr8jp/Ozs70G8HY\nXUWrq6upq6ujrq6O+vp66urqqK6uzvkeWbPFjIa/iNwB/D3gAg+r6hcnrA8CjwHXACeBD6hq2/n2\nebnh3zcS4+Pf3H7JrzMzb8qdks7TK2niuvHfD0h6GTiSfEVytZDqqJQaJ+cdR3BEcFPbJ+fBTS13\nRNLTfp/gdxx8ruB3Hfyu4HMc/D4HvyP4UstCfpfigI9QwKU4NSSnfYT8Lu4l/JgrFknQ8mQbB3Z2\n0XtiGIDqOSUsWlPL4jW11C0qv6wfh6kqfaO9nOhtp6vvGF39HZzsP05PfyenBjvpHeiib7CbRGQU\nX4Lk4JGeLpMQ5U4x5RKiREL4nXJGpZJRp4hhCTCMm75QDkolUINQDVR7SoUIxT5f8jGVgQDi9yeH\nidMBP05JCW55OU5pKW55OW5ZGVJcPGt/vzGdZuz2DiLiAl8BbgfagW0i8oSqvjFus98DTqnqUhG5\nH/ifwAemeuzJaCTC6td/Ph27Nhk0WVfNDOz0jH3rhOXjJ0WTW2mqS2f6tTrxtYqniurY9Oll8XHT\nCqiX/FFawjvd/TP9RoMi4wokqTKcXqepbZNjn0DAFYp8QrHPIeRzCPmEkE8och1CLgR9Qsh1CLhQ\n4whzHCVSE6RjtIrjnTXs+OkAr/z0EAEi1Mtx6r12woljuPFRNBZLDtHo6emJQzQK8TgANanh0gyl\nhjMlBDwH4q5LX3kZfZWV9FdUMFBeweHyCvaFimHszSoRxxmMEBwdpSgSoWh0lODYeDSSnI6M4ovF\n8cdi+OJxfPHktCOCO/amUFaGW1aGU16GW1GBW16RHFeU41ZU4Iwtq6xIby95/lzkTPxSZAPQqqoH\nAETk28DdwPjwvxv4fGr6P4Avi4joNLQ5lXpR7nnm0Uzv1phZwUNQETwET5LTMXGIIiTEIeE4+ByX\nua5LfaCEgcor6K9YzrHSJRxxFyJunBKvG18ggs8fwV8cwa8R/Izg1xECOkxABwnoIP5EP0IMcYDU\nkPxNxth8alpOzyPJT0vqJMekyiiOD5EylHI8pwiVIEqAEgkSIkCDBIEAQoB4zGUIjxHxiBAnInEi\nRXGioQTDxOmVGBHieHL++BAFnzq4KvjGBk9wEuCejOF2deN4XTiquJ7iJDwcTY09D0m+pad6c6XG\nounzFscB10lNC+K6qf8WDuI66fWO44Dr4rgOOA7qpD5Vpr47EdeB1CfH5H9Dh1BFOTfc9/5p+zuC\nzIT/XODIuPl24LpzbaOqcRHpI1mR6M7A8c/gVlSw5NlnM71bkyMu6VP+pO1GF1gmp+vyp5eNXy2n\ntznXePx24wdOB+b4QZxkaJCIINEhEpFBhgZOMdjfy/BgH6OD/USG+5ChHnzDPcjoKfyRXoKxXhbF\nt1Ca+BGl3jAnY0s5GLmWU/H5xDREVEMMepVENURUS1Auv6ZbpBBSISQQktPjYgeKHKFoYrOTnh6r\nKjH1iGucuBcjplGKvFESXhTFB6kusuCNe5lHHI+oeEQdj4SjyU8UAgkHEpKcT4gSFyUhXnLsJscR\nFE+UBB4JUTw84ggqkGy9nsr3D15yUCB+eXuoOhTihvumUISLkInwn+yf28S35IvZBhF5AHgAkt3E\nLqswsUECWz51Wa81M20aGv2ZGLRnB++Zr5dLHDNhGWeu01RtUUmNz0i5ZN//aBGJqIvGXLyYixf3\n4cV8aNyPFw/gJQJ4iSCeVwSeIDqK6AhCBCGKSBSIUkyUEiKIxBAiODKI4wzjBAWn2IdTXodTthAp\nroTiauqDVZT5yhl2Kxj1VzDsq2TIKWfYLWM4LoyOxhkdjhMZiTE6Eic2mkAV3IRSHPMIxTxCUY/i\nWGqIJodQ7Oy3jZgDI0GXkYBDb9Bh2OcxMtzJUP9hIiMniQ33EBvqJhYfJqGx0y/0BXBKqnBKKnFK\nyxBfAFw/4voRnx9xA+BLzY8tc3zgnA5rRVIV9vHXXPAp+FSTHwm8sXY+LzmtiniKxOOIF0MSMVAP\n8RKgcdTzUPWQ1Bj1Upc0+QlB1Us266XGSqpZb9zlH9cumf7LVFKfKias9vm8s/9eMywT4d8OzB83\nPw84do5t2kXEB1QAPRN3pKoPAg9C8gvfyyqNejDUdVkvNTNoyi1+k7xe9cx1kzb6nxnGlzRO7+pc\n6zT9ppDQUuKJeuKJhuQ4Xkc8UUc8Xody7tsyiBPBcaM4bhQJxBAXVCvx1I+qLzl4Dppw0GTV9uyd\nRID+cfsMusk3g2I/TsiHz+9QqlDiKWGNgteNepqssKompxU07uENRPGGJ1RfHcGtCODWhfBVFuFW\nBtODryI5lqDLyfbDHNzRwtEd2zj21h68RIJAqJjqOXOpnd9AWe1qysN1lNWGKa+to7w2TFFpWUF+\nSZsNmQj/bcAyEVkMHAXuB35rwjZPAL8D/Aq4F3h+Otr7AQhVwQObp2XXxoynCY/4yVHincPEuoaJ\nd40Q704OZwSmI/iqi/DVhgjWhvDVFuGUBHBCLk6RDyeUHCToQ9xLCz5VhbiisQTeSDw5DMfxhmNn\nTg+PrYvhDcWSbctj7fJjbfU+wHGSPYNS7c/u4opkqFcGcatSQV8WQCbpPRSLjHJk9+sc+PE2Du5s\nob8r2cc/vGARze/5TRava2bOsitxrEvnrDDl8E+14X8S+BnJhrJHVXW3iHwBaFHVJ4BHgG+KSCvJ\nGv/9Uz2uMTPFiySIdw0T6xpJBn3nMPHOYeInR083HwBueQBfOERodS2+2mJ84RC+2hC+qmDyS71p\nICLgF8Tv4BTP/M3VVJW9L/6C3Zuf5cju14nHoviCQRauXst1v3Efi9c1U1ZTO+PlMhdmP/IyBc2L\nJEj0R0j0R5NDXwSvP3p6WV+ERN+4ZwQ44KsJ4QsX468rxlcXwh9Ojp1gYT0Y79jeN9n82EN07HuL\nivoGlqzfwOJ1zcxbuRqf3eUza+wxjibn6NgXZKc74CeXeYomxsYeJMbNx71kG3UiuU4jHl40gUbi\nyelIHI0k8CIJNJpIT3vDMRJ9UTSSOKscEnRxywO4FUGCSyrx1YaSQR8O4asJIb787v99If1dnWz9\nt6/z1gtbKams4tc//iesvPk2HMeac3JJ3oV/YijGiS8V8C98L+mD3EVufJ7NzvrgmP5C9PzLdPyy\n8aGfaQIScJNfegZT44CLP1xM0dIqnPJAMujLg8kvMcsDBVeDv1jRkWFeevx/s/0njyMI17/vfq59\n7/sIFF34VhJm9sm7v3LxCaGrLv23iHllOnpLnGeXk/bOOKOX3Zm3Vx7rHnm6TzvpeyqklzkT5l1J\n/nDGTX1R6Z5ehpt6GIwr44LehwRdxO9M+uWkuXiel2DXpmf45Xe+xXBfLytuupWN93+Y8trwhV9s\nZq28C38n6KPqnmXZLoYxeeHQazvZ/M2H6T7cxpwrVvIb/9f/Q+PSK7JdLJMBeRf+xpipG+o9xdMP\n/iMHtr9Mebied3/qsyy//kZbC08hAAANR0lEQVTrg59HLPyNMWfoaH2LJ/72fzA6OMhNv/UR1t/5\nXnz2oPa8Y+FvjEnbtekZnn3knyiprOaD/+/fULeoKdtFMtPEwt8YQyIeZ/NjD7PzZz9mwaqrefen\nPkOorDzbxTLTyMLfmAI33NfLj770Rdr37OKad9/D23/rI3YLhgJg4W9MATu+fx8//Nu/ZHRggLv+\n6E9ZsfGWbBfJzBALf2MK1O4tz/HMQ1+mpLKK+7/w19QvXpLtIpkZZOFvTIFJxONs+dYj7HjqR8y/\nag3v/tRnKC6vyHaxzAyz8DemgAz39/GjL/0V7W/sYv1dd3Pzhz5q7fsFysLfmAJxvHUvT/zdXzHS\n38edn/g0K99+W7aLZLLIwt+YAvD680/z3CP/RElVdbJ9v2lptotksszC35g8Fo/F2PQv/8xrz/2U\nBavX8q4//j+tfd8AFv7G5K2Bk9088Xf/g+Ote9lw973ceP9v2z33TZqFvzF56Mgbr/Pj//U/iUUi\nvOfTf8by627MdpHMLGPhb0weUVVeefKHbPnWo1Q2zOG+//5X1Mybn+1imVnIwt+YPBEbHeXpB/+R\nN3+5haXXXs8df/hpgsXF2S6WmaUs/I3JA73HO/jh3/4l3UcOsfH+D7Ph7nsRp7CfNWzOz8LfmBym\nquza9AxbvvkIIsL7Pvt5Fq29JtvFMjlgSuEvItXAd4BFQBtwn6qemmS7BPB6avawqr53Ksc1xsDJ\no0d49qGv0L5nF3OvXMmdn/g0FXUN2S6WyRFTrfl/FnhOVb8oIp9NzX9mku1GVHXtFI9ljCHZd//l\nx7/Ly4//b3zBILc/8EesvvV2a+Yxl2Sq4X83cEtq+hvAZiYPf2NMBhx543WeeegrnDrWzpU33swt\nH/4YJZVV2S6WyUFTDf96Ve0AUNUOEak7x3ZFItICxIEvqurjk20kIg8ADwAsWLBgikUzJn+MDPSz\n5VuPsnvzs1TU1fObf/bnLLa2fTMFFwx/EXkWmKwh8b9ewnEWqOoxEWkCnheR11V1/8SNVPVB4EGA\n5uZmvYT9G5OXVJU9v9jM5sceZnRwgGvvvpe3ve9+/MGibBfN5LgLhr+qvuNc60TkhIg0pmr9jUDn\nOfZxLDU+ICKbgXXAWeFvjDmt93gHzz7yTxx6bQeNS6/g9v/2F4QXLs52sUyemGqzzxPA7wBfTI1/\nOHEDEakChlU1IiK1wI3AX0/xuMbkrVhklJd/+B9se+J7uD4/v/bRP2DN7XfYfXlMRk01/L8IfFdE\nfg84DLwfQESagY+r6seAFcA/i4gHOCTb/N+Y4nGNyTuqSmvLi2z+xkP0d3Vy5Y03c/OHPkppdU22\ni2by0JTCX1VPAr82yfIW4GOp6ReA1VM5jjH57lTHUZ7/+oO07dxO7fyF3Pe5v2L+SvtnY6aP/cLX\nmCyKjY7y0uPfpeVH38f1B7jlw7/P2l9/F67P/mma6WV/YcZkgarS+vKv2PTYQwx0d7Hyplt5+4c+\nan32zYyx8DdmhvUcO8rz//I1Dr22g/CCRdz153/KvCuvynaxTIGx8DdmhnhegpYf/YAXvvstXH+A\nWz/yn1j7zrtwXOvFY2aehb8xM+DU8WP89Ctf4tjePSy77gZ+7aN/YE08Jqss/I2ZRqrKq888xZZv\nPYLr83HXH/0pV954MyKS7aKZAmfhb8w0GTjZzc++9vccem0Hi65ezzs//seUVddmu1jGABb+xmSc\nqvLmLzbz3L98jUQ8zjs+9oesecedVts3s4qFvzEZNNzfx7MPf4V9L73AnOUruOMT/5mqhjnZLpYx\nZ7HwNyZD9m9/iaf/+R+JDA1y0299hOb33GP34zGzloW/MVM03N/H1m89yu4tzxFeuJh7/9tfEF6w\nKNvFMua8LPyNuUyqyhtbn2fzNx8hOjzEdfd8gLfdez+uz5/tohlzQRb+xlyGUx1Hefbhf+LwrleZ\ns3wFt//+J6i12r7JIRb+xlyCRDzGtie+z4vf/zY+fyDZk+fX7rCHp5ucY+FvzEU6+uYbPPPQlznZ\nfpjl12/k1o88QGlVdbaLZcxlsfA35gJGhwb5+b9+ndee+ylltWHu+cznaFp/bbaLZcyUWPgbcw6e\nl2Dvr37B5sceZrivj2ve9RvccN//QaAolO2iGTNlFv7GTDDQ082uTc/w+vNPM9DdRX3TUu75zOeo\nb1qa7aIZkzEW/sYAXiLBwZ0tvPbsTzm4YzuqHgtWr+XmD32UZRtusNsum7xj4W8KWn9XJ69veppd\nzz/N4Kkeiisqufbu97H61ndS2dCY7eIZM20s/E3BicdiHHxlG689/zPaXn0FgMVXr+e2j36cpvUb\n7Pm5piBM6a9cRN4PfB5YAWxQ1ZZzbHcH8PeACzysql+cynGNuVgjgwN0tR2k69ABOtsO0NV2gJNH\nj+AlEpRW13D9b97P6ltvpzxcl+2iGjOjplrF2QX8JvDP59pARFzgK8DtQDuwTUSeUNU3pnhsY4Dk\nbRZioyMM9fXSfeQQXW0H6Gw7SGfbfga6u9LblVRVU7dwMU3XbGDOFStYtGa9teWbgjWl8FfVPcCF\n7lO+AWhV1QOpbb8N3A1Y+Bco9Tw8z8OLx4nHosRjURKxOIlYlHgsRiIWJRGLpaZjREdHGOnvZ3Sw\nn5GBfkb6+xkZHEhOD/QzOtBPIh5P71/EoWrOXOZesZLwOxdTt6iJ8MLF9thEY8aZicbNucCRcfPt\nwHXTdbCRwQG+87nPTNfuL4qqZvPgZy+64LY6btG4rXVspKCa2lyT2+jp5agmN9XkOs/zIBXwmho8\nz0M1OX25RByKSksJlZUTKi+noq6BhiXLCZWXEyotI1ReQc28+dTOX4g/WHTZxzGmEFww/EXkWaBh\nklX/VVV/eBHHmOxjwaR5JCIPAA8ALFiw4CJ2fTbHcaiZO/+yXptR2Xxq0yTHPmdpUtue69NberlI\nch8iqWWS/H9qOjlKrhPHxXGc1LSDOE5yfmyQ5Lzr9+P6/fj8gfR0et7nxxfw4/r8+ItChMrLKSou\nsXvoGJMhFwx/VX3HFI/RDoxP43nAsXMc60HgQYDm5ubLqj4Hi0t4z6f/7HJeaowxBWMmqlHbgGUi\nslhEAsD9wBMzcFxjjDHnMKXwF5F7RKQdeBvwExH5WWr5HBF5EkBV48AngZ8Be4DvquruqRXbGGPM\nVEy1t88PgB9MsvwYcNe4+SeBJ6dyLGOMMZlj354ZY0wBsvA3xpgCZOFvjDEFyMLfGGMKkIW/McYU\nIMnqrQjOQ0S6gENT2EUt0J2h4swG+XY+kH/nlG/nA/l3Tvl2PnD2OS1U1fCFXjRrw3+qRKRFVZuz\nXY5Mybfzgfw7p3w7H8i/c8q384HLPydr9jHGmAJk4W+MMQUon8P/wWwXIMPy7Xwg/84p384H8u+c\n8u184DLPKW/b/I0xxpxbPtf8jTHGnEPehb+I3CEib4lIq4h8NtvlyQQRaROR10Vkp4i0ZLs8l0pE\nHhWRThHZNW5ZtYg8IyL7UuOcesbiOc7p8yJyNHWddorIXefbx2wiIvNFZJOI7BGR3SLyJ6nlOXmd\nznM+uXyNikTkZRF5NXVOf55avlhEXkpdo++kbp1/4f3lU7NP6mHxexn3sHjgg7n+sHgRaQOaVTUn\n+yeLyNuBQeAxVV2VWvbXQI+qfjH1Jl2lqtl9/uYlOMc5fR4YVNX/L5tluxwi0gg0quorIlIGbAd+\nA/gIOXidznM+95G710iAElUdFBE/8AvgT4BPA99X1W+LyNeAV1X1qxfaX77V/NMPi1fVKDD2sHiT\nRaq6FeiZsPhu4Bup6W+Q/IeZM85xTjlLVTtU9ZXU9ADJZ2/MJUev03nOJ2dp0mBq1p8aFLgN+I/U\n8ou+RvkW/pM9LD6nL3iKAk+LyPbUc47zQb2qdkDyHypQl+XyZMonReS1VLNQTjSRTCQii4B1wEvk\nwXWacD6Qw9dIRFwR2Ql0As8A+4He1EOz4BIyL9/C/6IfFp9jblTV9cCdwCdSTQ5m9vkqsARYC3QA\nf5vd4lw6ESkFvgd8SlX7s12eqZrkfHL6GqlqQlXXknwW+gZgxWSbXcy+8i38L/ph8bkk9WQ0VLWT\n5JPTNmS3RBlxItUuO9Y+25nl8kyZqp5I/eP0gIfIseuUakf+HvCvqvr91OKcvU6TnU+uX6MxqtoL\nbAauBypFZOypjBedefkW/nn3sHgRKUl9YYWIlADvBHad/1U54Qngd1LTvwP8MItlyYixkEy5hxy6\nTqkvEx8B9qjq341blZPX6Vznk+PXKCwilanpEPAOkt9lbALuTW120dcor3r7AKS6bv0vwAUeVdW/\nzHKRpkREmjj9nGQf8G+5dk4i8u/ALSTvPngC+BzwOPBdYAFwGHi/qubMF6jnOKdbSDYnKNAG/Kex\n9vLZTkQ2Aj8HXge81OL/m2Q7ec5dp/OczwfJ3Wu0huQXui7Jivt3VfULqYz4NlAN7AA+pKqRC+4v\n38LfGGPMheVbs48xxpiLYOFvjDEFyMLfGGMKkIW/McYUIAt/Y4wpQBb+xhhTgCz8jTGmAFn4G2NM\nAfr/AfEvICh2KzpGAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x21685550278>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig = plt.figure()\n",
    "ax = fig.add_subplot(111)\n",
    "ax.plot(ridgeWeight)\n",
    "# plt.xlim(0,20)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### LASSO"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "计算复杂度高"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Stagewise"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def stageWise(xArr, yArr, eps=0.01, numIt=100):\n",
    "    xMat = np.mat(xArr)\n",
    "    yMat = np.mat(yArr).T\n",
    "    yMean = np.mean(yMat,0)\n",
    "    yMat = yMat - yMean\n",
    "    m,n = np.shape(xMat)\n",
    "    returnMat = np.zeros((numIt,n))\n",
    "    ws = np.zeros((n,1))\n",
    "    wsTest = ws.copy()\n",
    "    wsMax = ws.copy()\n",
    "    for i in range(numIt):\n",
    "        print(ws.T)\n",
    "        lowerstError = np.inf\n",
    "        for j in range(n):\n",
    "            for sign in [-1,1]:\n",
    "                wsTest = ws.copy()\n",
    "                wsTest[j] += eps*sign\n",
    "                yTest = xMat*wsTest\n",
    "                rssE = rssError(yMat.A, yTest.A)\n",
    "                if rssE < lowerstError:\n",
    "                    lowerstError = rssE\n",
    "                    wsMax = wsTest\n",
    "        ws = wsMax.copy()\n",
    "        returnMat[i,:] = ws.T\n",
    "    return returnMat    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "xArr, yArr = loadDataSet(\"./Data/CH8/abalone.txt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0. 0. 0. 0. 0. 0. 0. 0.]]\n",
      "[[0.   0.   0.   0.   0.01 0.   0.   0.  ]]\n",
      "[[0.   0.   0.   0.   0.02 0.   0.   0.  ]]\n",
      "[[0.   0.   0.   0.   0.03 0.   0.   0.  ]]\n",
      "[[0.   0.   0.   0.   0.04 0.   0.   0.  ]]\n",
      "[[0.   0.   0.   0.   0.05 0.   0.   0.  ]]\n",
      "[[0.   0.   0.   0.   0.06 0.   0.   0.  ]]\n",
      "[[0.   0.   0.   0.   0.07 0.   0.   0.  ]]\n",
      "[[0.   0.   0.   0.   0.08 0.   0.   0.  ]]\n",
      "[[0.   0.   0.   0.   0.09 0.   0.   0.  ]]\n",
      "[[0.  0.  0.  0.  0.1 0.  0.  0. ]]\n",
      "[[0.   0.   0.   0.   0.11 0.   0.   0.  ]]\n",
      "[[0.   0.   0.   0.   0.12 0.   0.   0.  ]]\n",
      "[[0.   0.   0.   0.   0.13 0.   0.   0.  ]]\n",
      "[[0.   0.   0.   0.   0.14 0.   0.   0.  ]]\n",
      "[[0.   0.   0.   0.   0.15 0.   0.   0.  ]]\n",
      "[[0.   0.   0.   0.   0.16 0.   0.   0.  ]]\n",
      "[[0.   0.   0.   0.   0.17 0.   0.   0.  ]]\n",
      "[[0.   0.   0.   0.   0.18 0.   0.   0.  ]]\n",
      "[[0.   0.   0.   0.   0.19 0.   0.   0.  ]]\n",
      "[[0.  0.  0.  0.  0.2 0.  0.  0. ]]\n",
      "[[0.   0.   0.   0.   0.21 0.   0.   0.  ]]\n",
      "[[0.   0.   0.   0.   0.22 0.   0.   0.  ]]\n",
      "[[0.   0.   0.   0.   0.23 0.   0.   0.  ]]\n",
      "[[0.   0.   0.   0.   0.24 0.   0.   0.  ]]\n",
      "[[0.   0.   0.   0.   0.25 0.   0.   0.  ]]\n",
      "[[0.   0.   0.   0.   0.26 0.   0.   0.  ]]\n",
      "[[0.   0.   0.   0.   0.27 0.   0.   0.  ]]\n",
      "[[0.   0.   0.   0.   0.28 0.   0.   0.  ]]\n",
      "[[0.   0.   0.   0.   0.29 0.   0.   0.  ]]\n",
      "[[0.  0.  0.  0.  0.3 0.  0.  0. ]]\n",
      "[[0.   0.   0.   0.   0.31 0.   0.   0.  ]]\n",
      "[[0.   0.   0.   0.   0.32 0.   0.   0.  ]]\n",
      "[[0.   0.   0.   0.   0.33 0.   0.   0.  ]]\n",
      "[[0.   0.   0.   0.   0.34 0.   0.   0.  ]]\n",
      "[[0.   0.   0.   0.   0.35 0.   0.   0.  ]]\n",
      "[[0.   0.   0.   0.   0.36 0.   0.   0.  ]]\n",
      "[[0.   0.   0.   0.   0.37 0.   0.   0.  ]]\n",
      "[[0.   0.   0.   0.   0.38 0.   0.   0.  ]]\n",
      "[[0.   0.   0.   0.   0.39 0.   0.   0.  ]]\n",
      "[[0.  0.  0.  0.  0.4 0.  0.  0. ]]\n",
      "[[0.   0.   0.   0.   0.41 0.   0.   0.  ]]\n",
      "[[0.   0.   0.   0.   0.42 0.   0.   0.  ]]\n",
      "[[0.   0.   0.   0.   0.43 0.   0.   0.  ]]\n",
      "[[0.   0.   0.   0.   0.44 0.   0.   0.  ]]\n",
      "[[0.   0.   0.   0.   0.45 0.   0.   0.  ]]\n",
      "[[0.   0.   0.   0.   0.46 0.   0.   0.  ]]\n",
      "[[0.   0.   0.   0.   0.47 0.   0.   0.  ]]\n",
      "[[0.   0.   0.   0.   0.48 0.   0.   0.  ]]\n",
      "[[0.   0.   0.   0.   0.49 0.   0.   0.  ]]\n",
      "[[0.  0.  0.  0.  0.5 0.  0.  0. ]]\n",
      "[[0.   0.   0.   0.   0.51 0.   0.   0.  ]]\n",
      "[[0.   0.   0.   0.   0.52 0.   0.   0.  ]]\n",
      "[[0.   0.   0.   0.   0.53 0.   0.   0.  ]]\n",
      "[[0.   0.   0.   0.   0.54 0.   0.   0.  ]]\n",
      "[[0.   0.   0.   0.   0.55 0.   0.   0.  ]]\n",
      "[[0.   0.   0.   0.   0.56 0.   0.   0.  ]]\n",
      "[[0.   0.   0.   0.   0.57 0.   0.   0.  ]]\n",
      "[[0.   0.   0.   0.   0.58 0.   0.   0.  ]]\n",
      "[[0.   0.   0.   0.   0.59 0.   0.   0.  ]]\n",
      "[[0.  0.  0.  0.  0.6 0.  0.  0. ]]\n",
      "[[0.   0.   0.   0.   0.61 0.   0.   0.  ]]\n",
      "[[0.   0.   0.   0.   0.62 0.   0.   0.  ]]\n",
      "[[0.   0.   0.   0.   0.63 0.   0.   0.  ]]\n",
      "[[0.   0.   0.   0.   0.64 0.   0.   0.  ]]\n",
      "[[0.   0.   0.   0.   0.65 0.   0.   0.  ]]\n",
      "[[0.   0.   0.   0.   0.66 0.   0.   0.  ]]\n",
      "[[0.   0.   0.   0.   0.67 0.   0.   0.  ]]\n",
      "[[0.   0.   0.   0.   0.68 0.   0.   0.  ]]\n",
      "[[0.   0.   0.   0.   0.69 0.   0.   0.  ]]\n",
      "[[0.  0.  0.  0.  0.7 0.  0.  0. ]]\n",
      "[[0.   0.   0.   0.   0.71 0.   0.   0.  ]]\n",
      "[[0.   0.   0.   0.   0.72 0.   0.   0.  ]]\n",
      "[[0.   0.   0.   0.   0.73 0.   0.   0.  ]]\n",
      "[[0.   0.   0.   0.   0.74 0.   0.   0.  ]]\n",
      "[[0.   0.   0.   0.   0.75 0.   0.   0.  ]]\n",
      "[[0.   0.   0.   0.   0.76 0.   0.   0.  ]]\n",
      "[[ 0.   -0.01  0.    0.    0.76  0.    0.    0.  ]]\n",
      "[[ 0.   -0.02  0.    0.    0.76  0.    0.    0.  ]]\n",
      "[[ 0.   -0.02  0.    0.    0.77  0.    0.    0.  ]]\n",
      "[[ 0.   -0.03  0.    0.    0.77  0.    0.    0.  ]]\n",
      "[[ 0.   -0.04  0.    0.    0.77  0.    0.    0.  ]]\n",
      "[[ 0.   -0.04  0.    0.    0.78  0.    0.    0.  ]]\n",
      "[[ 0.   -0.05  0.    0.    0.78  0.    0.    0.  ]]\n",
      "[[ 0.   -0.05  0.    0.    0.79  0.    0.    0.  ]]\n",
      "[[ 0.   -0.06  0.    0.    0.79  0.    0.    0.  ]]\n",
      "[[ 0.   -0.07  0.    0.    0.79  0.    0.    0.  ]]\n",
      "[[ 0.   -0.07  0.    0.    0.8   0.    0.    0.  ]]\n",
      "[[ 0.   -0.08  0.    0.    0.8   0.    0.    0.  ]]\n",
      "[[ 0.   -0.09  0.    0.    0.8   0.    0.    0.  ]]\n",
      "[[ 0.   -0.09  0.    0.    0.81  0.    0.    0.  ]]\n",
      "[[ 0.   -0.1   0.    0.    0.81  0.    0.    0.  ]]\n",
      "[[ 0.   -0.11  0.    0.    0.81  0.    0.    0.  ]]\n",
      "[[ 0.   -0.11  0.    0.    0.82  0.    0.    0.  ]]\n",
      "[[ 0.   -0.12  0.    0.    0.82  0.    0.    0.  ]]\n",
      "[[ 0.   -0.13  0.    0.    0.82  0.    0.    0.  ]]\n",
      "[[ 0.   -0.13  0.    0.    0.83  0.    0.    0.  ]]\n",
      "[[ 0.   -0.14  0.    0.    0.83  0.    0.    0.  ]]\n",
      "[[ 0.   -0.14  0.    0.    0.84  0.    0.    0.  ]]\n",
      "[[ 0.   -0.15  0.    0.    0.84  0.    0.    0.  ]]\n",
      "[[ 0.   -0.16  0.    0.    0.84  0.    0.    0.  ]]\n",
      "[[ 0.   -0.16  0.    0.    0.85  0.    0.    0.  ]]\n",
      "[[ 0.   -0.17  0.    0.    0.85  0.    0.    0.  ]]\n",
      "[[ 0.   -0.18  0.    0.    0.85  0.    0.    0.  ]]\n",
      "[[ 0.   -0.18  0.    0.    0.86  0.    0.    0.  ]]\n",
      "[[ 0.   -0.19  0.    0.    0.86  0.    0.    0.  ]]\n",
      "[[ 0.   -0.2   0.    0.    0.86  0.    0.    0.  ]]\n",
      "[[ 0.   -0.2   0.    0.    0.87  0.    0.    0.  ]]\n",
      "[[ 0.   -0.21  0.    0.    0.87  0.    0.    0.  ]]\n",
      "[[ 0.   -0.22  0.    0.    0.87  0.    0.    0.  ]]\n",
      "[[ 0.   -0.22  0.    0.    0.88  0.    0.    0.  ]]\n",
      "[[ 0.   -0.23  0.    0.    0.88  0.    0.    0.  ]]\n",
      "[[ 0.   -0.24  0.    0.    0.88  0.    0.    0.  ]]\n",
      "[[ 0.   -0.24  0.    0.    0.89  0.    0.    0.  ]]\n",
      "[[ 0.   -0.25  0.    0.    0.89  0.    0.    0.  ]]\n",
      "[[ 0.   -0.25  0.    0.    0.9   0.    0.    0.  ]]\n",
      "[[ 0.   -0.26  0.    0.    0.9   0.    0.    0.  ]]\n",
      "[[ 0.   -0.27  0.    0.    0.9   0.    0.    0.  ]]\n",
      "[[ 0.   -0.27  0.    0.    0.91  0.    0.    0.  ]]\n",
      "[[ 0.   -0.28  0.    0.    0.91  0.    0.    0.  ]]\n",
      "[[ 0.   -0.29  0.    0.    0.91  0.    0.    0.  ]]\n",
      "[[ 0.   -0.29  0.    0.    0.92  0.    0.    0.  ]]\n",
      "[[ 0.   -0.3   0.    0.    0.92  0.    0.    0.  ]]\n",
      "[[ 0.   -0.31  0.    0.    0.92  0.    0.    0.  ]]\n",
      "[[ 0.   -0.31  0.    0.    0.93  0.    0.    0.  ]]\n",
      "[[ 0.   -0.32  0.    0.    0.93  0.    0.    0.  ]]\n",
      "[[ 0.   -0.33  0.    0.    0.93  0.    0.    0.  ]]\n",
      "[[ 0.   -0.33  0.    0.    0.94  0.    0.    0.  ]]\n",
      "[[ 0.   -0.34  0.    0.    0.94  0.    0.    0.  ]]\n",
      "[[ 0.   -0.35  0.    0.    0.94  0.    0.    0.  ]]\n",
      "[[ 0.   -0.35  0.    0.    0.95  0.    0.    0.  ]]\n",
      "[[ 0.   -0.36  0.    0.    0.95  0.    0.    0.  ]]\n",
      "[[ 0.   -0.36  0.    0.    0.96  0.    0.    0.  ]]\n",
      "[[ 0.   -0.37  0.    0.    0.96  0.    0.    0.  ]]\n",
      "[[ 0.   -0.38  0.    0.    0.96  0.    0.    0.  ]]\n",
      "[[ 0.   -0.38  0.    0.    0.97  0.    0.    0.  ]]\n",
      "[[ 0.   -0.39  0.    0.    0.97  0.    0.    0.  ]]\n",
      "[[ 0.   -0.4   0.    0.    0.97  0.    0.    0.  ]]\n",
      "[[ 0.   -0.4   0.    0.    0.98  0.    0.    0.  ]]\n",
      "[[ 0.   -0.41  0.    0.    0.98  0.    0.    0.  ]]\n",
      "[[ 0.   -0.42  0.    0.    0.98  0.    0.    0.  ]]\n",
      "[[ 0.   -0.42  0.    0.    0.99  0.    0.    0.  ]]\n",
      "[[ 0.   -0.43  0.    0.    0.99  0.    0.    0.  ]]\n",
      "[[ 0.   -0.44  0.    0.    0.99  0.    0.    0.  ]]\n",
      "[[ 0.   -0.44  0.    0.    1.    0.    0.    0.  ]]\n",
      "[[ 0.   -0.45  0.    0.    1.    0.    0.    0.  ]]\n",
      "[[ 0.   -0.45  0.    0.    1.01  0.    0.    0.  ]]\n",
      "[[ 0.   -0.46  0.    0.    1.01  0.    0.    0.  ]]\n",
      "[[ 0.   -0.47  0.    0.    1.01  0.    0.    0.  ]]\n",
      "[[ 0.   -0.47  0.    0.    1.02  0.    0.    0.  ]]\n",
      "[[ 0.   -0.48  0.    0.    1.02  0.    0.    0.  ]]\n",
      "[[ 0.   -0.49  0.    0.    1.02  0.    0.    0.  ]]\n",
      "[[ 0.   -0.49  0.    0.    1.03  0.    0.    0.  ]]\n",
      "[[ 0.   -0.5   0.    0.    1.03  0.    0.    0.  ]]\n",
      "[[ 0.   -0.51  0.    0.    1.03  0.    0.    0.  ]]\n",
      "[[ 0.   -0.51  0.    0.    1.04  0.    0.    0.  ]]\n",
      "[[ 0.   -0.52  0.    0.    1.04  0.    0.    0.  ]]\n",
      "[[ 0.   -0.53  0.    0.    1.04  0.    0.    0.  ]]\n",
      "[[ 0.   -0.53  0.    0.    1.05  0.    0.    0.  ]]\n",
      "[[ 0.   -0.54  0.    0.    1.05  0.    0.    0.  ]]\n",
      "[[ 0.   -0.55  0.    0.    1.05  0.    0.    0.  ]]\n",
      "[[ 0.   -0.55  0.    0.    1.06  0.    0.    0.  ]]\n",
      "[[ 0.   -0.56  0.    0.    1.06  0.    0.    0.  ]]\n",
      "[[ 0.   -0.56  0.    0.    1.07  0.    0.    0.  ]]\n",
      "[[ 0.   -0.57  0.    0.    1.07  0.    0.    0.  ]]\n",
      "[[ 0.   -0.58  0.    0.    1.07  0.    0.    0.  ]]\n",
      "[[ 0.   -0.58  0.    0.    1.08  0.    0.    0.  ]]\n",
      "[[ 0.   -0.59  0.    0.    1.08  0.    0.    0.  ]]\n",
      "[[ 0.   -0.6   0.    0.    1.08  0.    0.    0.  ]]\n",
      "[[ 0.   -0.6   0.    0.    1.09  0.    0.    0.  ]]\n",
      "[[ 0.   -0.61  0.    0.    1.09  0.    0.    0.  ]]\n",
      "[[ 0.   -0.62  0.    0.    1.09  0.    0.    0.  ]]\n",
      "[[ 0.   -0.62  0.    0.    1.1   0.    0.    0.  ]]\n",
      "[[ 0.   -0.63  0.    0.    1.1   0.    0.    0.  ]]\n",
      "[[ 0.   -0.64  0.    0.    1.1   0.    0.    0.  ]]\n",
      "[[ 0.   -0.64  0.    0.    1.11  0.    0.    0.  ]]\n",
      "[[ 0.   -0.65  0.    0.    1.11  0.    0.    0.  ]]\n",
      "[[ 0.   -0.65  0.    0.    1.12  0.    0.    0.  ]]\n",
      "[[ 0.   -0.66  0.    0.    1.12  0.    0.    0.  ]]\n",
      "[[ 0.   -0.67  0.    0.    1.12  0.    0.    0.  ]]\n",
      "[[ 0.   -0.67  0.    0.    1.13  0.    0.    0.  ]]\n",
      "[[ 0.   -0.68  0.    0.    1.13  0.    0.    0.  ]]\n",
      "[[ 0.   -0.69  0.    0.    1.13  0.    0.    0.  ]]\n",
      "[[ 0.   -0.69  0.    0.    1.14  0.    0.    0.  ]]\n",
      "[[ 0.   -0.7   0.    0.    1.14  0.    0.    0.  ]]\n",
      "[[ 0.   -0.71  0.    0.    1.14  0.    0.    0.  ]]\n",
      "[[ 0.   -0.71  0.    0.    1.15  0.    0.    0.  ]]\n",
      "[[ 0.   -0.72  0.    0.    1.15  0.    0.    0.  ]]\n",
      "[[ 0.   -0.73  0.    0.    1.15  0.    0.    0.  ]]\n",
      "[[ 0.   -0.73  0.    0.    1.16  0.    0.    0.  ]]\n",
      "[[ 0.   -0.74  0.    0.    1.16  0.    0.    0.  ]]\n",
      "[[ 0.   -0.75  0.    0.    1.16  0.    0.    0.  ]]\n",
      "[[ 0.   -0.75  0.    0.    1.17  0.    0.    0.  ]]\n",
      "[[ 0.   -0.76  0.    0.    1.17  0.    0.    0.  ]]\n",
      "[[ 0.   -0.76  0.    0.    1.18  0.    0.    0.  ]]\n",
      "[[ 0.   -0.77  0.    0.    1.18  0.    0.    0.  ]]\n",
      "[[ 0.   -0.78  0.    0.    1.18  0.    0.    0.  ]]\n",
      "[[ 0.   -0.78  0.    0.    1.19  0.    0.    0.  ]]\n",
      "[[ 0.   -0.79  0.    0.    1.19  0.    0.    0.  ]]\n",
      "[[ 0.   -0.8   0.    0.    1.19  0.    0.    0.  ]]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[ 0.  ,  0.  ,  0.  , ...,  0.  ,  0.  ,  0.  ],\n",
       "       [ 0.  ,  0.  ,  0.  , ...,  0.  ,  0.  ,  0.  ],\n",
       "       [ 0.  ,  0.  ,  0.  , ...,  0.  ,  0.  ,  0.  ],\n",
       "       ...,\n",
       "       [ 0.  , -0.79,  0.  , ...,  0.  ,  0.  ,  0.  ],\n",
       "       [ 0.  , -0.8 ,  0.  , ...,  0.  ,  0.  ,  0.  ],\n",
       "       [ 0.  , -0.8 ,  0.  , ...,  0.  ,  0.  ,  0.  ]])"
      ]
     },
     "execution_count": 110,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stageWise(xArr, yArr, 0.01, 200)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def regularize(xMat):#regularize by columns\n",
    "    inMat = xMat.copy()\n",
    "    inMeans = np.mean(inMat,0)   #calc mean then subtract it off\n",
    "    inVar = np.var(inMat,0)      #calc variance of Xi then divide by it\n",
    "    inMat = (inMat - inMeans)/inVar\n",
    "    return inMat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "matrix([[ 0.0430442 , -0.02274163,  0.13214087,  0.02075182,  2.22403814,\n",
       "         -0.99895312, -0.11725427,  0.16622915]])"
      ]
     },
     "execution_count": 113,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xMat = np.mat(xArr)\n",
    "yMat = np.mat(yArr).T\n",
    "xMat = regularize(xMat)\n",
    "yM = np.mean(yMat,0)\n",
    "yMat = yMat - yM\n",
    "weights = standRegres(xMat, yMat.T)\n",
    "weights.T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# def stageWiseTest(abX, abY):\n",
    "#     xMat = np.mat(abX)\n",
    "#     yMat = np.mat(abY).T\n",
    "#     xMat = regularize(xMat)\n",
    "#     yMat = regularize(yMat)\n",
    "#     numTestPts = 30\n",
    "#     wMat = np.zeros((numTestPts,np.shape(xMat)[1]))\n",
    "#     for i in range(numTestPts):\n",
    "#         ws = stageWise(xMat, yMat, np.exp(i-10),200)\n",
    "#         wMat[i,:] = ws.T\n",
    "#     return wMat"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Bias and Virance"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "- 方差☞模型之间的差异\n",
    "- 偏差☞模型预测值和数据之间的差异"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### EX : LEGO"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "数据接口变了，提供的是eBay的数据"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### CV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## Sklearn"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### EX1 : Abalone"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### LinearRegression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 221,
   "metadata": {},
   "outputs": [],
   "source": [
    "X, y = loadDataSet('./Data/CH8/abalone.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.model_selection import train_test_split\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LinearRegression(copy_X=True, fit_intercept=True, n_jobs=1, normalize=False)"
      ]
     },
     "execution_count": 149,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(np.array(X),np.array(y),test_size=0.2,random_state = 2018)\n",
    "clf = LinearRegression()\n",
    "clf.fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([  0.06026924,  -2.59687589,  11.9440577 ,  25.74179628,\n",
       "         9.18727754, -19.827113  ,  -9.62773854,   6.15003366])"
      ]
     },
     "execution_count": 150,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred = clf.predict(X_test)\n",
    "clf.coef_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import mean_squared_error"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6.352735940932754"
      ]
     },
     "execution_count": 152,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mean_squared_error(y_test,y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.48671157752582356"
      ]
     },
     "execution_count": 153,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# R2\n",
    "clf.score(X_test,y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### RidgeRegression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 208,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Ridge(alpha=0.1, copy_X=True, fit_intercept=True, max_iter=None,\n",
       "   normalize=False, random_state=None, solver='auto', tol=0.001)"
      ]
     },
     "execution_count": 208,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.linear_model import Ridge\n",
    "clf = Ridge(alpha=0.1)\n",
    "clf.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 209,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6.179758120851155"
      ]
     },
     "execution_count": 209,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred = clf.predict(X_test)\n",
    "mean_squared_error(y_test,y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 210,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([  0.05803489,  -1.4350165 ,  10.93750646,  23.01067895,\n",
       "         8.84244114, -19.51056886,  -8.98434913,   6.82338736])"
      ]
     },
     "execution_count": 210,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf.coef_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "以上\n",
    "- 增大Alpha，系数会衰减\n",
    "- 下面用5折交叉验证"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "RidgeCV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 211,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RidgeCV(alphas=(0.001, 0.01, 0.1), cv=5, fit_intercept=True, gcv_mode=None,\n",
       "    normalize=False, scoring=None, store_cv_values=False)"
      ]
     },
     "execution_count": 211,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.linear_model import RidgeCV\n",
    "clf = RidgeCV(alphas=(0.001,0.01,0.1),cv=5)\n",
    "clf.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 212,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6.332078823180617"
      ]
     },
     "execution_count": 212,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred = clf.predict(X_test)\n",
    "mean_squared_error(y_test,y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 213,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.01"
      ]
     },
     "execution_count": 213,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf.alpha_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "其实0.1最小，但是传0.001进去你看"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### LASSO"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 225,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.linear_model import Lasso\n",
    "clf = Lasso(alpha=0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 226,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8.79519636205887"
      ]
     },
     "execution_count": 226,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf.fit(X_train, y_train)\n",
    "y_pred = clf.predict(X_test)\n",
    "mean_squared_error(y_test,y_pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "LassoCV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 222,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LassoCV\n",
    "clf = LassoCV(alphas=(0.01,0.01,0.1,1),cv=5,copy_X=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 223,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5.882310490550318"
      ]
     },
     "execution_count": 223,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf.fit(X_train, y_train)\n",
    "y_pred = clf.predict(X_test)\n",
    "mean_squared_error(y_test,y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 224,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.01"
      ]
     },
     "execution_count": 224,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf.alpha_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Refs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. [Ridge](http://scikit-learn.org/stable/auto_examples/linear_model/plot_ridge_coeffs.html#sphx-glr-auto-examples-linear-model-plot-ridge-coeffs-py)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": true,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}