Code.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# <center>Predicting Year in Million Song Dataset using Spark</center>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark import SparkConf                                                                                                                 \n",
    "from pyspark.context import SparkContext                                                                                                      \n",
    "from pyspark.sql import SparkSession, SQLContext\n",
    "\n",
    "sc = SparkContext('local')\n",
    "spark = SparkSession(sc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training data points = 361094 \n",
      "Validation data points = 51401 \n",
      "Test data points = 102850\n"
     ]
    }
   ],
   "source": [
    "from pyspark import *\n",
    "from pyspark.sql import *\n",
    "from pyspark.sql.functions import *\n",
    "import pyspark.sql.functions as F\n",
    "from pyspark.sql.types import *\n",
    "from pyspark.ml.feature import *\n",
    "from pyspark.ml.classification import *\n",
    "from pyspark.ml.evaluation import *\n",
    "from pyspark.mllib.regression import LabeledPoint\n",
    "from pyspark.mllib.linalg import Vectors\n",
    "import pandas as pd\n",
    "\n",
    "# Load the dataset\n",
    "data = spark.read.csv('/Users/santoshdaptardar/Documents/ML with Spark/Data/MSD.txt', header=False, inferSchema=True)\n",
    "\n",
    "# Using MinMax Scaler to scale the features from 0-1\n",
    "v_assembler = VectorAssembler(inputCols=data.columns[1:], outputCol=\"features\")\n",
    "v_data = v_assembler.transform(data)\n",
    "scaler = MinMaxScaler(inputCol='features', outputCol='scaled_features')\n",
    "scale_model = scaler.fit(v_data)\n",
    "scaled_data = scale_model.transform(v_data)\n",
    "new_data = scaled_data.select(col('_c0').alias('label'), col('scaled_features').alias('features'))\n",
    "\n",
    "# Get min and max year and normalize the labels by subtracting min year\n",
    "min_year = new_data.agg({'label': 'min'}); max_year = new_data.agg({'label': 'max'})\n",
    "min_year = min_year.take(1)[0][0]; max_year = max_year.take(1)[0][0]\n",
    "normalize = udf(lambda x: x - min_year)\n",
    "final_data = new_data.withColumn('new_label', normalize('label')).drop('label'). \\\n",
    "                    select(F.col('new_label').cast(FloatType()).alias('label'), 'features')\n",
    "\n",
    "# Split data into train(70%), validation(10%) and test set(20%)\n",
    "train, valid, test = final_data.randomSplit([0.7, 0.1, 0.2], seed = 15)\n",
    "print('Training data points =', train.count(), '\\nValidation data points =', valid.count(), \\\n",
    "      '\\nTest data points =', test.count())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RMSE on training data: 9.73110493081151\n",
      "R2 on training data: 0.20844442735444624\n",
      "+------------------+-----+--------------------+\n",
      "|        prediction|label|            features|\n",
      "+------------------+-----+--------------------+\n",
      "| 71.25992013197201|  4.0|[0.42914797029747...|\n",
      "|  73.0329588968724|  5.0|[0.40366040895273...|\n",
      "| 70.63080174332009|  5.0|[0.61111563148754...|\n",
      "| 63.40511271451693|  7.0|[0.34348004703995...|\n",
      "|61.491258193561116|  7.0|[0.41340084229557...|\n",
      "| 65.05011619129431|  7.0|[0.47933516369832...|\n",
      "| 69.80251228885477|  7.0|[0.53848233361241...|\n",
      "| 69.30783498341364|  7.0|[0.56090668492824...|\n",
      "| 63.67824871230167|  8.0|[0.56175240123318...|\n",
      "|62.961593641592316|  9.0|[0.58430195775104...|\n",
      "+------------------+-----+--------------------+\n",
      "only showing top 10 rows\n",
      "\n",
      "R Squared (R2) on validation data = 0.20542184681861064\n",
      "Root Mean Squared Error (RMSE) on validation set = 9.718869730751331\n",
      "\n",
      "objectiveHistory:\n",
      "(51, 2)\n"
     ]
    }
   ],
   "source": [
    "# Using linear regression model\n",
    "from pyspark.ml.regression import LinearRegression\n",
    "lrm = LinearRegression(featuresCol = 'features', labelCol='label', maxIter=50, regParam=0.3, elasticNetParam=0.8)\n",
    "\n",
    "# Fit the model\n",
    "lr_Model = lrm.fit(train)\n",
    "trainingSummary = lr_Model.summary\n",
    "\n",
    "# Summarize the model over the training set and print out some metrics\n",
    "print(\"RMSE on training data:\", trainingSummary.rootMeanSquaredError)\n",
    "print(\"R2 on training data:\", trainingSummary.r2)\n",
    "\n",
    "# Evaluating the model based on validation set.\n",
    "lr_val_predictions = lr_Model.transform(valid)\n",
    "lr_val_predictions.select(\"prediction\",\"label\",\"features\").show(10)\n",
    "\n",
    "from pyspark.ml.evaluation import RegressionEvaluator\n",
    "lr_evaluator = RegressionEvaluator(predictionCol=\"prediction\", labelCol=\"label\", metricName=\"r2\")\n",
    "\n",
    "print(\"R Squared (R2) on validation data =\", lr_evaluator.evaluate(lr_val_predictions))\n",
    "\n",
    "val_result = lr_Model.evaluate(valid)\n",
    "print(\"Root Mean Squared Error (RMSE) on validation set =\", val_result.rootMeanSquaredError)\n",
    "\n",
    "# Obtain the objective per iteration\n",
    "objectiveHistory = trainingSummary.objectiveHistory\n",
    "print(\"\\nobjectiveHistory:\")\n",
    "\n",
    "# Collecting training error for each iteration using a dataframe\n",
    "training_error_df = pd.DataFrame(columns=['iteration', 'training_error'])\n",
    "i = 0\n",
    "\n",
    "for objective in objectiveHistory:\n",
    "    i += 1\n",
    "    training_error_df = training_error_df.append({'iteration': i, 'training_error': 10 ** (objective)}, ignore_index=True)\n",
    "\n",
    "print(training_error_df.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZEAAAEbCAYAAAAbCrnpAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3de5wcZZ3v8c83yZA7IZAYQkIIFxERCSxjQPECUQFdj7h7PIoXhLN6sq6yirpHRRQV5ey67iK7IquoiB5FUVZ3ERGMAiJHuUy4XyVAQLK5kQskEgJJfueP52mnpqe7p7sy03P7vl+vfk33U09VPVVTVb+6PM9TigjMzMzKGDPYBTAzs+HLQcTMzEpzEDEzs9IcRMzMrDQHETMzK81BxMzMShu1QUTSqZKWStokaYOk2ySdOwDzeYukU2ukL5b0pv6eX1mSQtJpLY5znKTT+7kcZ0laIWmHpIvbON/5eR28ocXxjsnjHdKf5bG+SbpM0nV95LlO0mWF3/2+7TRL0kclHVMjveV9bygZlUFE0hnAN4Crgb8E3gX8J/DGAZjdW4BTa6QvBoZMECnpOKDfdkhJncBngfOBo4HPtWO+2UrgpcANLY53ax7voX4uj/WP9wFnFH4PxLbTrI8Cx9RIfynwo/YWpf+MG+wCDJLTgK9FxCcKaT+V9NnBKtDOkDQWGBsRzw52WXbSQfnvVyLiqZ2dWCvrJSK2Aje2Oo9czpbHazdJHcCOiNhelS5gfEQ8U3K6EyNiS3+UcSBExL0DOX1JE8quu4qIGPLbT0MRMeo+wB+BTzWRbyLwj8CjwFbgEeDvC8PfRTpzXQ9sAK4FOgvDLwai6vMZ4Loa6acWxnsPcE+e56PAR6vKdTHQRbqSuQd4DnhFnWUo5r0feCaX+eCqfAGcVpV2GvBgLscy4EOFYZ+psQwXN1iXY/M4j+Xp3QO8vY91dUyN6dSdb731AswGLgIeBrYAvwc+D+xSmO78PK03FNKWA/8EfAh4PP+PfwDsVshzTB7vkKp1+UHg/wBrgTXAV0gHa6rGvTP/T24BFgJPAJ/pY7scA3w8/0+25uU5pSrPdcBlpCveh4DtwN55/T0BvDzPcytwch5nX+A/gKeATcBPgQNqbCcfBs7Ly7asQTk/kufxJLC6zvQq5Xx7Xp6ngJ8Dc6vy7Q1cmf9/y0n7yGXAdX2sq+uAy5rZZvO28mvgaWAd8HVgamH4qXmchXm6W8jHEeAfgLuAzXlb+R6wZ9W2VHP7psV9r7AsTwCHk05ingZuo+o4QLq7spR0zNsA3AS8ql+Pp/05seHyAX5D2rFPAfaok0fAL/LO9FHg1aSg8fVCnrNIO+mrgdcB38kb1n55+P7ANaRbHkflz1zgYOA+4GeF9Jl5nP9NOvidA7yWdLDYWtzISAfLJ0gHj3cCr6ne6aryriUdQN9Bun13F/AHYEIhX1TN43/ltH8m3QL4e2AH8PE8fG7eUVYWlmH/Buv8nLxcnwSOBy7M039bYV19Lqcdm6e3a43p1J1vvfUCvJgUDN4EvCov2wrS1WhluvOpHUQeA64AXp//15uBCwp5jqF2EHksl+f4/D/dRuFkAJhD2vF/CbyBdFB8kLSzf6aP7fcruRwfzcv4BVKQKJb9uryObgPenMu/K+ng8zQpsPx1XtcvAMbnbeQB4K3Afwfuzutp96plWwlcCpwAvL5BOb9E2seOIR3MriTtd9OqyvkH4LfAicBJpIBzZdW+eGtep2+nexteQWtBpNG2czRpP7s0r6uT8/QvK0zr1Lz8D5EC5LHA4XnYRcDbSNvXm4HfAfcCY/Lww4GNpNvolXnvWmbfy3kq/8c7gf9JOv7cSNrXJxX2qWeBLwKL8nJ9CviLfj2etuvAPZQ+wKF5h4n8z7kHOJvCQYu08wfwxianOYZ0e/B+4KxCes2zJdIZ88VVabuSDg6frko/G1hFujUD3WfthzVRrkrelxXS9iEd1N5bSPvThpyXZQXwrappXUA6q5yQf/8TsLyJMuxOOjhWL9eVwAOF35WddEof06s532bXS/4/vZ10BbBLTptP7SDyEDCukHYesKrw+xhqB5Hrq+b5H8CNhd9fJAW8iYW0t+RxP9Og7AfkbfaUqvTvALcUfl9HOqGZVZXvM3keJ1alvzdvE/sV0uaSDkJnVC3brSX2ubGkK/tNwLuqyvkkML2Qdnqez8T8+/X595E1tuFe+1bVfK+jZyCot+38Bri2Km1R8X9b2D4/2MSyzsl5X1lIr3mVSbl9r/J/XFTIc1hOOyH/fjOwrtX/VaufUflgPSLuBF5IOju6gHSm8ymgS9KUnG0RsD4iLq83HUkvlPQTSatJZ4LPkc7qDixZtJcCk4EfSRpX+ZCuZmaRduqKFRFxe5PTXRMRv638iIhHSZe4C+vknwvsRe+HfZeSAt2Lm5xvxSHApDrTO1DSzBan10iv9aLkdEn3StpC+j99j3T2Pa+P6V0bEdsKv+8FnpefMTTyi6rf99Lz//cSYEn0fJ5Qd1sreDUpiPykahv5FXBYfg5UsTQiVteYRpBuGRUtJAWHh/+UKeJx4P+Rbn0VXdlEOZF0lKQlktaRDvhPA1PovX/cEhEbCr8rzzHmFMq2OiJuKpStsg3vNEmTSPveD6vW6Q2kbeWIqlF+VmMar5P0W0lPkpb18Tyo1WNBK/ves6QgWVFZb5Xt7C5gmqRv51ppk1ssS1NGZRCB9CA1In4aEadFxMGk2wnPB96ds+xBuuytSdJU0oFib9I94leQDgx3ABNKFmtG/lu5n1/5XJvT9y7krXVwqGdNnbTZdfJX0qvnUfm9ewvzHojpNVJrvZxOOgP9CemWyULg/XlYX/+rjVW/nyWddIwvMV5xXnuSbj38SaQHtJv7mO4M0pnuk/TcRi4mXWEV/6f1tpEN0buywew6+VfT+//T57YnaR5p/xDpttnRpP1jDb3Xea11RSHfntTfhvvDdNI6vYCe63Qr0EHP/Q6qll/SS0gnAI+TboO9lHS7Clo/FrSyr2yKiB2VH4X/6YT8+wHS9r4fKfA/IemSfj5pG7W1s3qJiG9K+ke6awito/5BFtKGMhd4bUTcX0mUNG0nirE+/30DtXfUBwrfo4XpPq9O2j118q8s5Cmalf+upzXF6a3rh+k1Umu9/A/SLY0zKwmSDu7HeZaxCuixM0uaQDpTb2Q96Uz3aNIVSbXigbXeNlIrfSXwohrps+j9/2lm2zuBdPV5YkT8ESCf3Zc5YVhF/W24P2qGbaS70kutq6z/qvpdvfx/QToheGvk+0iS9ilZln7d9yLiZ8DP8nHpz0m3Y79MevbUL0bllYikXhtkjs7T6D54/wrYvUHjs4n579bCNF5GurdeVH0G2ij9d6SdYq+I6Krx2dRgsRp5Xi5bpZzzgD8Dbq6T/3HSjvM/qtLfQqo9c1eDZajlbtKtjFrT+31ErO09SkPNzrdiIoX/U/aOFufZ324BXitpYiGtmXZK15DOmqfV2UbKVvO+CThC0r6VBElzgJfRetsZSOt8ByngVbyFcieutwCzJB1ZKFtlG25Vr20nB7kbgRfUWafVQaTaROC5SgDJam1fzWy3ze57LYmIJyPiEtLVeL+eQI3WK5G7JP0n6XJ7Dekh3d+RDnTfznmWkBojXiLpbFLtkNmkB2V/TdroNgNfz1cwc0lnMiuq5nU/cGJunf448F95o7wfOF7S8aSz80ciYp2kzwD/ks9kricF+gOBYyPiL0ou7xPAdyV9khSkPpuX++JamSNiRy7H1/L97CWkWid/A3wiuuvF30/auU8lBYonImJ5jemtl3Qe8ElJ20iVCv6S9MD0bSWWp6n5FiwBPiDpJtKD8neQHlAPpvNIt9R+KulLpFs2Hydtg7WuMIB0i0LSV4Ef5O2ui3RgehFwYES8p2R5LgY+Bvxc0lmkZ3yfJm07XysxvUqw+5akb+by/R29b10140rSbeIfSfoY6YSgsg23qt6281HgV5J2kCrDbCI9L/tz4MyI+H2DaS4BTs/b+E9Jgfeddeb955KuIh07Hqg+MWxh3+uTpL8m3TG5ihSYnk8KTt9pdhpNGegn90PxQ9p5f5FX7DOkWjiXAAdV5ZtIupf+ON3tRM4pDD+BtCFuIVW1ez29a4PMIEX/9RRq3pDuU/6SdG876NlO5J2kh4Zb6K7b/eHC8IuBriaX9WK6D9q/z8vx/yjUJoqqGiKFtL8l1VF/llSbrbqu+gTgW6SdOei7nchnSdU5nyU9BHxHVZ5Taa52Vs351lsvpFtE38r/g/WkapZvoGfNm/nUaSfSqIzUr51VvS4/QzpgFdOOzdvNVuB20nO1Z4DT+1h+kZ7zVNoSrSW1b6iu9XRZjXF7laMwbD9SLbJNpIPcFcDz+9pOGpTzZFLQ3kI66Tqyep3WKmeddTqPdDDcQmo79de02E6kr202l+8q0hn/H/M2ei65SnKj7ZMUhP6Qx/sl6YDdY12RHtDfmPMEjduJ9LXv1fw/FqdFCiA/o/s49wipOvj4Ruus1Y/yzGyEyv1PHRIRnYNdFmtM0stJVU0XRcS1feU3GwpG6+0ss0En6QukxoCrSFXDP0W6Mvn1YJbLrBVtfbAuaaxSb7lX1Bg2XtKlkpZJuknS/MKwM3L6A/kZgtlIMJ7U6PAXpC5SfkNqKFb3mYjZUNPW21mSPgx0klqGv6Fq2PuAQyPivZJOIjXNf2uuivl9Ut3+vUj3Gw+Mqo7kzMys/dp2JSJpLqmmwzfqZDmR7ppRlwGvzj2Mngj8IFLjwEdID5vqtbQ2M7M2auczkfNINRim1hk+h1S7gYjYlrsP2COnF7tKfpzu7hB6kLSY1EkekydPPuKggw6qlc3MzGpYunTpExHRUov2tgSR3GBvTUQsrfVmr/4SEReSeoels7Mzurq6BmpWZmYjjqRHWx2nXbezjgbeKGk56X0MiyR9tyrPCnIfNbl7hGmkRnh/Ss/m0rtBn5mZDYK2BJGIOCMi5kbEfFKfLddERHWLzstJ7x6A1IXxNZGe+l8OnJRrb+1LasRTr7sOMzNro0FtJ5K7E+mK1N36N4H/K2kZqVXxSQARcY+kH5Jaj24D3u+aWWZmQ8OIbbHuZyJmZq2RtLTV3i1GZS++ZmbWPxxEzMysNAcRMzMrzUHEzMxKcxAxM7PSHETMzKw0BxEzMyvNQcTMzEpzEDEzs9IcRMzMrDQHETMzK81BxMzMSnMQMTOz0hxEzMysNAcRMzMrzUHEzMxKcxAxM7PSHETMzKy0trxjXdIE4HpgfJ7nZRHx6ao8XwKOzT8nAc+LiN3ysO3AXXnYYxHxxnaU28zMGmtLEAG2AosiYrOkDuAGST+PiBsrGSLiQ5Xvkv4WOLww/paIOKxNZTUzsya15XZWJJvzz478iQajvA34/oAXzMzMdkrbnolIGivpdmANsCQibqqTbx9gX+CaQvIESV2SbpT0pjYU18zMmtC2IBIR2/MtqbnAQkmH1Ml6EumZyfZC2j4R0Qm8HThP0v61RpS0OAebrrVr1/Zr+c3MrLe2186KiI3AtcAJdbKcRNWtrIhYkf8+DFxHz+clxXwXRkRnRHTOnDmz38psZma1tSWISJopqVLTaiLwWuD+GvkOAqYDvyukTZc0Pn+fARwN3NuOcpuZWWPtqp01G/i2pLGkwPXDiLhC0tlAV0RcnvOdBPwgIooP3V8IfE3SjjzuP0SEg4iZ2RCgnsfrkaOzszO6uroGuxhmZsOGpKX5+XPT3GLdzMxKcxAxM7PSHETMzKw0BxEzMyvNQcTMzEpzEDEzs9IcRMzMrDQHETMzK81BxMzMSnMQMTOz0hxEzMysNAcRMzMrzUHEzMxKcxAxM7PSHETMzKw0BxEzMyvNQcTMzEpzEDEzs9IcRMzMrLS2BBFJEyTdLOkOSfdI+myNPKdKWivp9vx5T2HYKZIezJ9T2lFmMzPr27g2zWcrsCgiNkvqAG6Q9POIuLEq36URcVoxQdLuwKeBTiCApZIuj4gNbSm5mZnV1ZYrkUg2558d+RNNjn48sCQi1ufAsQQ4YQCKaWZmLWrbMxFJYyXdDqwhBYWbamT775LulHSZpL1z2hzgD4U8j+e0WvNYLKlLUtfatWv7tfxmZtZb24JIRGyPiMOAucBCSYdUZfkpMD8iDiVdbXy7xDwujIjOiOicOXPmzhfazMwaanvtrIjYCFxL1S2piFgXEVvzz28AR+TvK4C9C1nn5jQzMxtk7aqdNVPSbvn7ROC1wP1VeWYXfr4RuC9/vxo4TtJ0SdOB43KamZkNsnbVzpoNfFvSWFLg+mFEXCHpbKArIi4HPiDpjcA2YD1wKkBErJf0OeCWPK2zI2J9m8ptZmYNKKLZSlLDS2dnZ3R1dQ12MczMhg1JSyOis5Vx3GLdzMxKcxAxM7PSHETMzKw0BxEzMyvNQcTMzEpzEDEzs9IcRMzMrLQ+g4iSeZIccMzMrIdmA8N9gAayIGZmNvz0GUQiNWm/A9h/4ItjZmbDSbN9Z/0SuErSRaR3e/ypr5SI+M5AFMzMzIa+ZoPIsaTu14+vSg/AQcTMbJRqKohExCsGuiBmZjb8NN0VvKRpwJ+TXk27AvhZRDw5UAUzM7Ohr6naWZIWAg8BpwMLgQ8CD0k6cgDLZmZmQ1yzVyL/CpweEd+tJEh6e053IDEzG6WabSfyAuCSqrRLgQP7tzhmZjacNBtElgFvrkr7S+Dh/i2OmZkNJ83ezvoQcIWkDwCPAvOBg4E3NjOypAnA9cD4PM/LIuLTVXk+DLyH9I71tcBfRcSjedh24K6c9bGIaGq+ZmY2sJqt4nuDpP2B/wbsBSwBroiIJ5qcz1ZgUURsltQB3CDp5xFxYyHPbUBnRDwt6W+AfwTemodtiYjDmpyXmZm1SZ9BRNJYUt9ZL46Ii8vMJHedsjn/7MifqMpzbeHnjcA7y8zLzMzap5m+s7aTOl+csDMzkjRW0u3AGmBJRNzUIPu7gZ8Xfk+Q1CXpRklvajCPxTlf19q1a3emuGZm1oRmH6yfC3xf0tGS9sldw8+TNK/ZGUXE9nxLai6wUNIhtfJJeifQCXyxkLxPRHQCbwfOy7fWas3jwojojIjOmTNnNls0MzMrqdkH61/Jf0+oSg9gbCszjIiNkq7N07q7OEzSa4AzgVdFxNbCOCvy34clXQccTmr82NfM4KGH4Jpr4I47YMsWmDgRFiyARYtg//1B7uHezKysZoNIx87MRNJM4LkcQCYCrwW+UJXncOBrwAkRsaaQPh14OiK2SpoBHE166N7Ytm1w9tnwyCMwfjzMmJE+27bB0qXw29/CvvvCaafBHnvszOKZmY1azbzZsPJgfVy+JdXj0+R8ZgPXSroTuIX0TOQKSWdLqlTX/SIwBfiRpNslXZ7TXwh0SboDuBb4h4i4t885rlyZPvvsA7NnQ0dHuuro6Ei/99knDT/nHFi3rsnFMDOzIqWKU31kkh4kVb8dNh0uds6cGV3vf3/fGVetSkHlrLN8a8vMRjVJS/Pz56a17cF623U0eQdu1ixYvjw9OzEzs5a0/cH6kCPBLrukh+8HHDDYpTEzG1ba8mB9yJsxI9XeMjOzljTb7UnNB+i55tSGfi3RYBg7NlX/NTOzljR8JiJpTdXvq6uyPNrvJRoM27en9iNmZtaSvh6sVx9ZX1L1e2RUZ3riidQA0czMWtJXEOmr/m/f9YOHugjYujW1YDczs5Y0W8V3+HnuuebyrV6dWq7vX7M7LjMza6CvB+sTJF1U+D256vf4AShT/xgzJjUknDWrdyPCCFi/PtXIevJJeNGLUvcn7lPLzKwlfQWRL1T9ru6zqu8+rAbL7Nnps3x5agcyY0aqhbVpE/zmN7BxI0ybloLGrru6Ty0zsxKa6vZkOOrs7IyuW27p2Yvvhg1wzz0peCxYALvvXvsqZfVqmDQJzjzTgcTMRo0y3Z4029hweJJSK/QDDkjB4eyz01XHnns2HmfPPdOtsPPPd59aZmYNjNwH69Ueeih1Cz9rVnP53aeWmVmfRk8Queaa9F6RZq8qin1qmZlZTaMniNxxR3q43gr3qWVm1lBTz0QkvavOoK3A48DNEdFkw4xBsmVL60HEfWqZmTXU7IP1xaQuT9aRgsYcYAZwGzAfeFbSmyLi1oEoZL+YODFV4232PSPgPrXMzPrQ7O2sW4GPR8ReEbEwIuYAHwNuAvYCLgK+PEBl7B8LFqQ+slrhPrXMzBpqNoicDPxrVdqXgXdFxA7g74EX1RtZ0gRJN0u6Q9I9kj5bI894SZdKWibpJknzC8POyOkPSDq+yTL3tGhR6iOr2XYx7lPLzKxPzQaRNcDrqtJOANbm7xOAmu8cybYCiyJiAXAYcIKko6ryvBvYEBEHAF8it5aXdDBwEilInQBcIKn1tynuv39qib56dXP53aeWmVmfmg0ipwOXSPq1pO9K+jXwfeCDefhRwAX1Ro5kc/7ZkT/VlwQnAt/O3y8DXi1JOf0HEbE1Ih4BlgELmyx3Nyl1ZTJxYmpIWO+KJCINnzQp5XdDQzOzupoKIhHxc+AA4GLgPtLB/oCcTkRcHRGfajQNSWMl3U66qlkSETdVZZkD/CFPbxvwJLBHMT2rPNivNY/Fkrokda1du7Z3hj32gE9+MvWp9dhjsHJl6u13x470d+XKlD57trs8MTNrQtPdnkTEGuBbZWeUX7F7mKTdgJ9IOiQi7i47vTrzuBC4EFLfWTUz7bFH6sqk2KfWli3pCuWII9yLr5lZC5ptJ7IP8DnS84wpxWERsV8rM4yIjZKuJT3fKAaRFcDewOOSxgHTSFWKK+kVc3NaecU+tczMrLRmr0QuId1SOhN4utWZSJoJPJcDyETgtfTuZv5y4BTgd8CbgWsiIiRdTnoecy6pOvHzgZtbLYOZmfW/ZoPIi4FX5ltSZcwGvp1rVY0BfhgRV0g6G+iKiMuBbwL/V9IyYD2pRhYRcY+kHwL3AtuA9+9EOczMrB819T4RSVcCZ0bEbQNfpP7R2dkZXV1dg10MM7NhYyDfJ/IgcLWky4BVxQERcXYrMzQzs5Gj2SCyO3A1MDV/KkbOaxEjatfY8nvXzczqaiqIRMTJA12QQbVuXXqL4SOPpHeOzJiRPn7vuplZQ3WDiKS5EfF4/j6vXr6IeGwgCtY269bB5z+frjz22afn1UZHR2p4GJEaIp5zjhshmpkVNGqxfl/h+3Lgkfy3+HlkQErVLhHpCmTLlvRe9Xq3qyrvXX/66ZS/2U4czcxGuEZBZFrhewewC939XnUU0oYvv3fdzGyn1A0iuYv3yvft9T7tKeYA8XvXzcx2Stu7PRlS/N51M7Od0pZuT4asZt67HgEbNqTbXqtWpd5+t26FCy901V8zG/Xa1e3J0NTXe9effhpuvhk2boSxY9M7RsaPhzFjXPXXzIzmX0p1A3DoQBZkUDR67/rTT8P118PmzTBtGkydmgLJli0wZ06q+rvPPt1Vf9eta2/ZzcyGgNHd7cmiRelqIqLnLamIdAWybRtMmdIzffv2dPUB3VV/V61KVX/POsu3tsxsVBnd3Z5U3ru+cmUKBhUbNqRbWNOm9cz/xz/CbrvB9Ok904tVf/2OEjMbRUZ3tyeV965//vPpamLWrJT2yCPp1lXlqiIiBZCODli4sPfVRrHqr4OImY0i7vak8t71889PVxO77AL/9V/pofv27enZyPbt6Qpk4cL0cL0WV/01s1Go0ZXIfXTfulpOunVVfcM/gLH9X6w2q37v+vXXp8DR0QF77ZVueU2f3vh5R+Whu5nZKNIoiFR3ezKyFd+7fscd6cqjXtXf6rYj27alar9Tp8KyZW47YmajRt0gUt3tSXuKM0QsWJDagcye3XtYrbYjY8bAU0/Bjh3p+YrbjpjZKNFUOxFJYyW9T9Klkn4l6ZrKp8nx95Z0raR7Jd0j6YM18vxvSbfnz92StkvaPQ9bLumuPGzg33m7aFFqlV7dW2+9tiOQ8h52mNuOmNmo0mxjw3OBDwA3A0cCPwPmkhohNmMb8JGIOBg4Cni/pIOLGSLiixFxWEQcBpwB/Doi1heyHJuHt/T+31IqVX9Xry4WsGfbkeLtqkrV3912S7e5VqyAq6+G17wG3ve+1EXKsmXuQt7MRpxm24m8GTg6IpZLOisi/lnSlcC/NTNyRKwEVubvmyTdB8wB7q0zytuA7zdZtv5Xq+pvrbYjxaq/L35xukqp3OaaPh02bUp5urrgqqtSv1vPe15KmzAhPbSvvPDqmWdSjbBDD03PZZYtgzvv9Gt6zWxIUzRxdixpA7B7RISklcB+EbFF0lMRsWtLM5TmA9cDh0TEUzWGTwIeBw6oXIlIegTYQKoN9rWIuLDOtBcDiwHmzZt3xKOPPtpK0XqrvDa30pBw0ybYddf07KNY9ffFL4ZbbklXKZMndx/kN21KVX+3bk3B5dln01XMwoUpQKxenYLIrFlw1FFp/N/8pjtYvfKVaX7PPQcPPwyPPprmecABKUjVCzi10lsJWgOV12XzuhgK8xvIsg3zkz1JS1u929NsEPkd8IGIuEXSFcBdwJPAqRFxUAsFnAL8GjgnIn5cJ89bgXdGxH8rpM2JiBWSngcsAf42Iq5vNK/Ozs7o6uqHxycRKYC85z3p4XkEjBuXWrjvu28KIpXnJFOm9Bz3mWdSDa699+4OLhs3pv66Zs3qzv/HP/Ycb9KkFKQ6OqCzE+66K403Jt993GWX9Pylq6t3wHnqqd6BaNw4uOmm5oLWQOV12bwuhsL8BrJsu+yS9u2tW4dt5ZqBDCJHAc9FxFJJLwC+RmpD8pGIuK7JwnUAVwBXR8S5DfL9BPhRRFxSZ/hngM0R8U+N5tdvQaTir/4qBYPqs4v161MQmTatd/9by5enAHHIIT3TNm9OZzYTJ3anP/hg+nvggd3TqQ44tdInT+4ZcCpXRJVAVDR5cvpbL2gNVF6XzetiKMxvIMvW0QGveEXKF5ECzKRJcOaZwyqQlAkifT5YlzQWOBC4GyAiHoiIYyLiiBYCiIBvAvf1EUCmAa8C/rOQNlnS1Mp34LhKWdqq0m18teouUiqeeSZ9im1NnnkmnaV0dKRnLMX07dvTbbJnnklplbYoW7akM59iFyzF9L8BhyUAAA+zSURBVDFjUoB59ln46U/Tra8pU1L65Mmp9f2KFd1XQlLv9IHM67J5XQyF+Q1k2aZMSfluvrm7M9c990xB5/zzR3yFmj6DSG4j8uWI2LoT8zkaOBlYVKjG+3pJ75X03kK+vwB+ERHF04NZwA2S7iDVDvtZRFy1E2Upp1638atW1e4KZcOGFBimTu2ZVulna/PmnuljxqRPJbg0Cji10seOTZfelSrHlbzVwale+kDlddm8LobC/AaybJCCysaNPffJYsesI1izVXx/Jun1ZWcSETdEhCLi0Eo13oi4MiK+GhFfLeS7OCJOqhr34YhYkD8viohzypZjp9RrO1JprV5t8+buWlrFtI6OFEh27Oid3tHRHVwaBZxa6ZVnJhs39sxbHZzqpQ9UXpfN62IozG8gywZpnxw7Nt2ZKKZVOmYdwZqt4jsG+LGkG0ivyf3TkTQi/mogCjbk1Os2fty4FBCKZyyQLocnT061OCp27EgbVkTPwFNJr3yHvgNOrfQJE3oGlkreyvdG6QOV12XzuhgK8xvIslVMmpTuTBSNgo5Zm70SeRD4IvA7UvXbFYXP6FBpOzJxYtpQKlcklXufFRHdVyGzZ/d8VjJmTBpeuc9anV4MLn0FnFrpY8b0DCyVvNUBp1b6QOV12bwuhsL8BrJsFWPG9H5uOgo6Zm14JSLpbRHx/Yj4VLsKNKTV6jZ+7lx47LG08WzZ0t12ZMGC1M6keCUyZUp3A8Tiba5KOnQ/QykGnOJzlUbplQ2/Om/le6P0gcrrsnldDIX5DWTZKnbsSHcmirZv766FOUL1dSXytbaUYjipdBt/5plwxBHprKRyj3SvvVI98le9Cg46KG1AxWco06eng//48T2Dy/TpaQPcsaM7uFRqfNQKOPXSn3mm5xVOJW/1lU+t9IHK67J5XQyF+Q1k2SqefrrnrW5IlXEWLOiddwTpK4ioj+Gjk5TaeSxeDBdcAL/8JRx3HMyZ0/3ekenT0xVJpR56RLpamTCh97tJJkxIl71jxnQHl0YBp1b6brulILTbbj3zVgeneukDlddl87oYCvMbyLJB2r+3b0/PTYtpW7fCokWMZH09WB8r6VgaBJOIuKZ/izQM1brNNWNGaqx0/fWwZk16IDd9OrzsZakB0+bNPRsr7bVX9/dJk2oHnEaBaPv21MJ2e6HX/kpwqvTV1Sh9oPK6bF4XQ2F+A1k26O6EtRhcVq9OQWX//RnJGrZYl7QdeJT6QSQiYr+BKNjO6vcW682odJFyzTWpRsaWLelAv3ZtCiJ77AEzZ6azk5tv7tltwpFHpg222MXCS14Ct92Wrjyg+8yo2FfXSGz9O5LL5nUxMtfFKG6x3lcQabmDxaFiUIJIPbWCS7OdwG3YkL6PGwfz5sF++6Xvmzb17tNn6tTa6WPHNhe0BjKvy+Z1MRTmN5BlGz8+PQN59lmYP999Z+UJOogMBbWC0GjoEXUkl83rYmSuC/fi22uCmyJiat0MQ9iICiJmZm3Q7x0wDtcAYmZm7dFsi3UzM7NeHETMzKw0BxEzMyvNQcTMzEpzEDEzs9IcRMzMrDQHETMzK60tQUTS3pKulXSvpHskfbBGnmMkPVl4B/tZhWEnSHpA0jJJH29Hmc3MrG/Nvh53Z20DPhIRt0qaCiyVtCQi7q3K95uIeEMxQdJY4CvAa0lvVbxF0uU1xjUzszZry5VIRKyMiFvz903AfcCcJkdfCCyLiIcj4lngB8CJA1NSMzNrRdufiUiaDxwO3FRj8Esl3SHp55JelNPmAH8o5HmcOgFI0mJJXZK61q5d24+lNjOzWtoaRCRNAf4dOD0inqoafCuwT0QsAL4M/Eer04+ICyOiMyI6Z86cufMFNjOzhtoWRCR1kALI9yLix9XDI+KpiNicv18JdEiaAawA9i5knZvTzMxskLWrdpaAbwL3RcS5dfLsmfMhaWEu2zrgFuD5kvaVtAtwEnB5O8ptZmaNtat21tHAycBdkm7PaZ8A5gFExFeBNwN/I2kbsAU4KdLLTrZJOg24GhgLXBQR97Sp3GZm1kDDl1INZ34plZlZa/r9pVRmZmaNOIiYmVlpDiJmZlaag4iZmZXmIGJmZqU5iJiZWWkOImZmVpqDiJmZleYgYmZmpTmImJlZaQ4iZmZWmoOImZmV5iBiZmalOYiYmVlpDiJmZlaag4iZmZXmIGJmZqU5iJiZWWltCSKS9pZ0raR7Jd0j6YM18rxD0p2S7pL0W0kLCsOW5/TbJfmdt2ZmQ8S4Ns1nG/CRiLhV0lRgqaQlEXFvIc8jwKsiYoOk1wEXAkcWhh8bEU+0qbxmZtaEtgSRiFgJrMzfN0m6D5gD3FvI89vCKDcCc9tRNjMzK6/tz0QkzQcOB25qkO3dwM8LvwP4haSlkhYPXOnMzKwV7bqdBYCkKcC/A6dHxFN18hxLCiIvLyS/PCJWSHoesETS/RFxfY1xFwOLAebNm9fv5Tczs57adiUiqYMUQL4XET+uk+dQ4BvAiRGxrpIeESvy3zXAT4CFtcaPiAsjojMiOmfOnNnfi2BmZlXaVTtLwDeB+yLi3Dp55gE/Bk6OiN8X0ifnh/FImgwcB9w98KU2M7O+tOt21tHAycBdkm7PaZ8A5gFExFeBs4A9gAtSzGFbRHQCs4Cf5LRxwCURcVWbym1mZg20q3bWDYD6yPMe4D010h8GFvQew8zMBptbrJuZWWkOImZmVpqDiJmZleYgYmZmpTmImJlZaQ4iZmZWmoOImZmV5iBiZmalOYiYmVlpDiJmZlaag4iZmZXmIGJmZqU5iJiZWWkOImZmVpqDiJmZleYgYmZmpTmImJlZaQ4iZmZWmoOImZmV1pYgImlvSddKulfSPZI+WCOPJP2rpGWS7pT0Z4Vhp0h6MH9OaUeZzcysb+PaNJ9twEci4lZJU4GlkpZExL2FPK8Dnp8/RwL/BhwpaXfg00AnEHncyyNiQ5vKbmZmdbTlSiQiVkbErfn7JuA+YE5VthOB70RyI7CbpNnA8cCSiFifA8cS4IR2lNvMzBpr15XIn0iaDxwO3FQ1aA7wh8Lvx3NavfRa014MLM4/t0q6e+dLPCTNAJ4Y7EIMIC/f8OblG75e0OoIbQ0ikqYA/w6cHhFP9ff0I+JC4MI8r66I6OzveQwFI3nZwMs33Hn5hi9JXa2O07baWZI6SAHkexHx4xpZVgB7F37PzWn10s3MbJC1q3aWgG8C90XEuXWyXQ68K9fSOgp4MiJWAlcDx0maLmk6cFxOMzOzQdau21lHAycDd0m6Pad9ApgHEBFfBa4EXg8sA54G/mcetl7S54Bb8nhnR8T6JuZ5Yf8Vf8gZycsGXr7hzss3fLW8bIqIgSiImZmNAm6xbmZmpTmImJlZaSMuiEg6QdIDufuUjw92eXaWpIskrSm2eZG0u6QluRuYJbnCwbBUr0uckbCMkiZIulnSHXnZPpvT95V0U95GL5W0y2CXdWdIGivpNklX5N8jZvkkLZd0l6TbK9VfR8K2WSFpN0mXSbpf0n2SXtrq8o2oICJpLPAVUhcqBwNvk3Tw4JZqp11M7xb6Hwd+FRHPB36Vfw9XlS5xDgaOAt6f/2cjYRm3AosiYgFwGHBCrnn4BeBLEXEAsAF49yCWsT98kNQLRcVIW75jI+KwQtuQkbBtVvwLcFVEHAQsIP0fW1u+iBgxH+ClwNWF32cAZwx2ufphueYDdxd+PwDMzt9nAw8Mdhn7cVn/E3jtSFtGYBJwK6lfuCeAcTm9xzY73D6kdlu/AhYBVwAaYcu3HJhRlTYitk1gGvAIuYJV2eUbUVcitNBFyjA3K1IbGoBVwKzBLEx/qeoSZ0QsY77VczuwhtTv20PAxojYlrMM9230POCjwI78ew9G1vIF8AtJS3O3SjBCtk1gX2At8K18O/IbkibT4vKNtCAy6kQ6XRj29bQbdYkznJcxIrZHxGGkM/aFwEGDXKR+I+kNwJqIWDrYZRlAL4+IPyPdIn+/pFcWBw7nbZPUTvDPgH+LiMOBP1J166qZ5RtpQWS0dJGyOvdwTP67ZpDLs1PqdIkzopYxIjYC15Ju7+wmqdLQdzhvo0cDb5S0HPgB6ZbWvzBylo+IWJH/rgF+QjoRGCnb5uPA4xFR6Qz3MlJQaWn5RloQuQV4fq4dsgtwEqk7lZHmcqDycq5TSM8RhqUGXeIM+2WUNFPSbvn7RNKznvtIweTNOduwXDaAiDgjIuZGxHzSvnZNRLyDEbJ8kibn9x+Rb/McB9zNCNg2ASJiFfAHSZWee18N3EuLyzfiWqxLej3pPu1Y4KKIOGeQi7RTJH0fOIbU/fRq0gu6/gP4IanbmEeBt0RzXcEMOZJeDvwGuIvu++qfID0XGdbLKOlQ4NukbXEM8MOIOFvSfqQz992B24B3RsTWwSvpzpN0DPB3EfGGkbJ8eTl+kn+OAy6JiHMk7cEw3zYrJB0GfAPYBXiY1N3UGFpYvhEXRMzMrH1G2u0sMzNrIwcRMzMrzUHEzMxKcxAxM7PSHETMzKw0BxGzIUbS5ly91GzIcxAxq5K7/36NpFMl3TDA87pO0nuKaRExJSIeHsj5mvUXBxGzAVLo+sNsxHIQMavthcBXgZfm20sbASSNl/RPkh6TtFrSV3OXJkg6RtLjkj4maRWpd9Tpkq6QtFbShvx9bs5/DvAK4Pw8j/Nzekg6IH+fJuk7efxHJX1S0pg87FRJN+TybJD0iKTXtX1N2ajmIGJW233Ae4Hf5dtLu+X0fwAOJL1k6gBSN+dnFcbbk9Tdxz7AYtI+9q38ex6wBTgfICLOJHX5clqex2k1yvFl0nsf9gNeBbyL1DVFxZGk9z/MAP4R+Gbuj8ysLRxEzJqUD86LgQ9FxPqI2AT8H1LngxU7gE9HxNaI2BIR6yLi3yPi6Zz/HFIwaGZ+Y/O0z4iITRGxHPhn4ORCtkcj4usRsZ3UT9dshu/7LWwY8j1bs+bNJL2hcGnhZF+kDhYr1kbEM38aKE0CvkR6xXHlXdVTJY3NB/5GZgAdpE7wKh6l50ueVlW+RMTTuVxTml0gs53lKxGz+qp7J32CdDvqRRGxW/5Mi4gpDcb5CPAC4MiI2BWovNRIdfJXz+850q2winkM4/dz2MjjIGJW32pgbn43DRGxA/g68CVJzwOQNEfS8Q2mMZUUeDZK2p3UlX/1PGq2CclXKj8EzpE0VdI+wIeB7+7EMpn1KwcRs/quAe4BVkl6Iqd9DFgG3CjpKeCXpCuNes4DJpKuKm4Erqoa/i/Am3Ptqn+tMf7fkl5b+jBwA3AJcFG5xTHrf36fiJmZleYrETMzK81BxMzMSnMQMTOz0hxEzMysNAcRMzMrzUHEzMxKcxAxM7PSHETMzKy0/w9cHvsihD1KpAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "axes = plt.gca()\n",
    "axes.set_xlim([0,60])\n",
    "axes.set_ylim([2,4])\n",
    "\n",
    "plt.scatter(training_error_df['iteration'], training_error_df['training_error'], \\\n",
    "            s=14**2, c='red', alpha=0.5)\n",
    "axes.set_title(\"Scatter plot of training error and iterations\", fontsize=15)\n",
    "axes.set_xlabel('Iteration', fontsize=12)\n",
    "axes.set_ylabel('Training Error', fontsize=12)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Root Mean Squared Error (RMSE) on test data = 9.744908828244753\n"
     ]
    }
   ],
   "source": [
    "test_result = lr_Model.evaluate(test)\n",
    "print(\"Root Mean Squared Error (RMSE) on test data =\", test_result.rootMeanSquaredError)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}