BusraNur_Orhan_homework3_BigData&AI.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "BusraNur_Orhan_homework3.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyMs0iw/Ygldd6NCyU5SbN2F",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/busranurorhan/Complete-Python-3-Bootcamp/blob/master/BusraNur_Orhan_homework3_BigData%26AI.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "eUxUEjYxnDkr",
        "outputId": "8c7ef017-aedd-4e1c-c171-89c95dea6cad"
      },
      "source": [
        "!apt-get update\n",
        "!apt-get install openjdk-8-jdk-headless -qq > /dev/null\n",
        "import os\n",
        "os.environ[\"JAVA_HOME\"] = \"/usr/lib/jvm/java-8-openjdk-amd64\"\n",
        "!update-alternatives --set java /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java\n",
        "!java -version\n",
        "!pip install pyspark"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\r0% [Working]\r            \rGet:1 http://security.ubuntu.com/ubuntu bionic-security InRelease [88.7 kB]\n",
            "Get:2 https://cloud.r-project.org/bin/linux/ubuntu bionic-cran40/ InRelease [3,626 B]\n",
            "Ign:3 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  InRelease\n",
            "Get:4 http://ppa.launchpad.net/c2d4u.team/c2d4u4.0+/ubuntu bionic InRelease [15.9 kB]\n",
            "Hit:5 http://archive.ubuntu.com/ubuntu bionic InRelease\n",
            "Ign:6 https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64  InRelease\n",
            "Get:7 http://archive.ubuntu.com/ubuntu bionic-updates InRelease [88.7 kB]\n",
            "Hit:8 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  Release\n",
            "Hit:9 https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64  Release\n",
            "Get:10 http://security.ubuntu.com/ubuntu bionic-security/main amd64 Packages [2,365 kB]\n",
            "Hit:11 http://ppa.launchpad.net/cran/libgit2/ubuntu bionic InRelease\n",
            "Get:12 http://security.ubuntu.com/ubuntu bionic-security/universe amd64 Packages [1,430 kB]\n",
            "Get:13 http://archive.ubuntu.com/ubuntu bionic-backports InRelease [74.6 kB]\n",
            "Get:14 http://ppa.launchpad.net/deadsnakes/ppa/ubuntu bionic InRelease [15.9 kB]\n",
            "Hit:15 http://ppa.launchpad.net/graphics-drivers/ppa/ubuntu bionic InRelease\n",
            "Get:16 http://ppa.launchpad.net/c2d4u.team/c2d4u4.0+/ubuntu bionic/main Sources [1,802 kB]\n",
            "Get:19 http://ppa.launchpad.net/c2d4u.team/c2d4u4.0+/ubuntu bionic/main amd64 Packages [922 kB]\n",
            "Get:20 http://archive.ubuntu.com/ubuntu bionic-updates/universe amd64 Packages [2,209 kB]\n",
            "Get:21 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 Packages [2,801 kB]\n",
            "Get:22 http://ppa.launchpad.net/deadsnakes/ppa/ubuntu bionic/main amd64 Packages [40.8 kB]\n",
            "Fetched 11.9 MB in 3s (3,860 kB/s)\n",
            "Reading package lists... Done\n",
            "update-alternatives: using /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java to provide /usr/bin/java (java) in manual mode\n",
            "openjdk version \"1.8.0_292\"\n",
            "OpenJDK Runtime Environment (build 1.8.0_292-8u292-b10-0ubuntu1~18.04-b10)\n",
            "OpenJDK 64-Bit Server VM (build 25.292-b10, mixed mode)\n",
            "Collecting pyspark\n",
            "  Downloading pyspark-3.1.2.tar.gz (212.4 MB)\n",
            "\u001b[K     |████████████████████████████████| 212.4 MB 60 kB/s \n",
            "\u001b[?25hCollecting py4j==0.10.9\n",
            "  Downloading py4j-0.10.9-py2.py3-none-any.whl (198 kB)\n",
            "\u001b[K     |████████████████████████████████| 198 kB 69.1 MB/s \n",
            "\u001b[?25hBuilding wheels for collected packages: pyspark\n",
            "  Building wheel for pyspark (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for pyspark: filename=pyspark-3.1.2-py2.py3-none-any.whl size=212880768 sha256=cfb8d268e7dd12913edfff36c47f6e6b469df2d3bccc5c293f7ea42be396a240\n",
            "  Stored in directory: /root/.cache/pip/wheels/a5/0a/c1/9561f6fecb759579a7d863dcd846daaa95f598744e71b02c77\n",
            "Successfully built pyspark\n",
            "Installing collected packages: py4j, pyspark\n",
            "Successfully installed py4j-0.10.9 pyspark-3.1.2\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "E6a6D0nUT0a2"
      },
      "source": [
        "## Random forest##"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0Dv4TiK7nuOY",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "87c8eabf-06f4-448e-ad07-c33d101c227d"
      },
      "source": [
        "from pyspark.sql import SparkSession\n",
        "from pyspark.ml.feature import StringIndexer, VectorAssembler\n",
        "from pyspark.ml.regression import LinearRegression\n",
        "from pyspark.ml.evaluation import RegressionEvaluator\n",
        "from pyspark.ml.classification import RandomForestClassifier\n",
        "from pyspark.ml.evaluation import MulticlassClassificationEvaluator\n",
        "from pyspark.ml.tuning import CrossValidator,ParamGridBuilder\n",
        "from sklearn.model_selection import train_test_split, GridSearchCV,cross_val_score\n",
        "from sklearn.model_selection import GridSearchCV\n",
        "from sklearn.metrics import mean_squared_error, r2_score\n",
        "import matplotlib.pyplot as plt\n",
        "from sklearn.preprocessing import scale \n",
        "from sklearn import model_selection\n",
        "from sklearn.tree import DecisionTreeRegressor, DecisionTreeClassifier\n",
        "from sklearn.neighbors import KNeighborsRegressor\n",
        "from sklearn.ensemble import BaggingRegressor\n",
        "\n",
        "\n",
        "spark = SparkSession.Builder().appName('Spark DataFrame Introduction').getOrCreate()\n",
        "\n",
        "leafDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','false')\\\n",
        "                        .option('delimiter',',').csv('leaf.csv')\n",
        "\n",
        "leafDF = leafDF.withColumnRenamed(\"_c0\",\"Class\")\n",
        "leafDF = leafDF.withColumnRenamed(\"_c1\",\"specimen_number\")\n",
        "leafDF = leafDF.withColumnRenamed(\"_c2\",\"Eccentricity\")\n",
        "leafDF = leafDF.withColumnRenamed(\"_c3\",\"Aspect_Ratio\")\n",
        "leafDF = leafDF.withColumnRenamed(\"_c4\",\"Elongation\")\n",
        "leafDF = leafDF.withColumnRenamed(\"_c5\",\"Solidity\")\n",
        "leafDF = leafDF.withColumnRenamed(\"_c6\",\"Stochastic_Convexity\")\n",
        "leafDF = leafDF.withColumnRenamed(\"_c7\",\"Isoperimetric_Factor\")\n",
        "leafDF = leafDF.withColumnRenamed(\"_c8\",\"Maximal_Indentation_Depth\")\n",
        "leafDF = leafDF.withColumnRenamed(\"_c9\",\"Lobedness\")\n",
        "leafDF = leafDF.withColumnRenamed(\"_c10\",\"Average_Intensity\")\n",
        "leafDF = leafDF.withColumnRenamed(\"_c11\",\"Average_Contrast\")\n",
        "leafDF = leafDF.withColumnRenamed(\"_c12\",\"Smoothness\")\n",
        "leafDF = leafDF.withColumnRenamed(\"_c13\",\"Third_moment\")\n",
        "leafDF = leafDF.withColumnRenamed(\"_c14\",\"Uniformity\")\n",
        "leafDF = leafDF.withColumnRenamed(\"_c15\",\"Entropy\")\n",
        "\n",
        "#### Asagidaki kodların ikisini de çalıştırıp 340 bulduğum için  data setimizin distinct olup olmadığını anlıyorum###\n",
        "\n",
        "#leafDF.count()\n",
        "#leafDF.distinct().count()\n",
        "\n",
        "### Asagidaki kod ile data setimde null değerler olup olmadığına max&min&standart sapma ile bagimli ve bagimsiz degiskenlerin durunlarını gozlemliyorum##\n",
        "\n",
        "#leafDF.describe().show()\n",
        "\n",
        "\n",
        "#leafDF.printSchema()  #Bu iki kod ile data setimdeki tüm değişkenlerin sayısal olduğundan emin oluyorum\n",
        "#leafDF.dtypes\n",
        "\n",
        "\n",
        "#leafDF = leafDF.dropna()       #data setimdeki eksik gözlemlerden kurtulmak için bu komutu kosturdum ama eksik gozlem yoktu\n",
        "\n",
        "vec = VectorAssembler(inputCols=leafDF.columns[1:15],outputCol='features')\n",
        "leafDF = vec.transform(leafDF)\n",
        "\n",
        "leafDF = leafDF.withColumnRenamed('Class','label')\n",
        "\n",
        "leafDF = leafDF.select('features','label')\n",
        "\n",
        "#leafDF.show()\n",
        "\n",
        "trainDF, testDF = leafDF.randomSplit([0.75,0.25],seed=1234)\n",
        "\n",
        "rf = RandomForestClassifier()\n",
        "eva = MulticlassClassificationEvaluator(metricName='accuracy')\n",
        "myParams = ParamGridBuilder().addGrid(rf.numTrees,[30,40,50,60,100])\\\n",
        "                            .addGrid(rf.maxDepth,[4,5,6])\\\n",
        "                            .addGrid(rf.impurity,['gini','entropy']).build()\n",
        "validator = CrossValidator(parallelism=4, \n",
        "                                 estimator=rf,\n",
        "                                 estimatorParamMaps=myParams,\n",
        "                                 evaluator=eva,\n",
        "                                 numFolds=5)\n",
        "model = validator.fit(trainDF)\n",
        "print(\"Best Model hesaplandı\")\n",
        "print(\"Best NumTrees : \", model.bestModel.getNumTrees)\n",
        "print(\"Best MaxDepth : \", model.bestModel.getMaxDepth())\n",
        "print(\"Best Impurity : \", model.bestModel.getImpurity())\n",
        "resultDF = model.transform(testDF)\n",
        "result = eva.evaluate(resultDF)\n",
        "print('Accuracy',result)"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Best Model hesaplandı\n",
            "Best NumTrees :  30\n",
            "Best MaxDepth :  6\n",
            "Best Impurity :  entropy\n",
            "Accuracy 0.6222222222222222\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ei_9Fi2KT39-"
      },
      "source": [
        "#Decision Tree"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nTCD0sMaT0ne",
        "outputId": "dbcd39a5-0fdc-4d76-e473-317a589b572e"
      },
      "source": [
        "from pyspark.sql import SparkSession\n",
        "from pyspark.ml.feature import VectorAssembler\n",
        "from pyspark.ml.evaluation import RegressionEvaluator\n",
        "from pyspark.ml.classification import DecisionTreeClassifier\n",
        "from pyspark.ml.evaluation import MulticlassClassificationEvaluator\n",
        "from pyspark.ml.tuning import CrossValidator,ParamGridBuilder\n",
        "from sklearn.model_selection import train_test_split, GridSearchCV,cross_val_score\n",
        "from sklearn.model_selection import GridSearchCV\n",
        "from sklearn.metrics import mean_squared_error, r2_score\n",
        "from sklearn import model_selection\n",
        "from pyspark.ml.regression import DecisionTreeRegressor\n",
        "\n",
        "\n",
        "spark = SparkSession.Builder().appName('Spark DataFrame Introduction').getOrCreate()\n",
        "\n",
        "leafDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','false')\\\n",
        "                        .option('delimiter',',').csv('leaf.csv')\n",
        "\n",
        "leafDF = leafDF.withColumnRenamed(\"_c0\",\"Class\")\n",
        "leafDF = leafDF.withColumnRenamed(\"_c1\",\"specimen_number\")\n",
        "leafDF = leafDF.withColumnRenamed(\"_c2\",\"Eccentricity\")\n",
        "leafDF = leafDF.withColumnRenamed(\"_c3\",\"Aspect_Ratio\")\n",
        "leafDF = leafDF.withColumnRenamed(\"_c4\",\"Elongation\")\n",
        "leafDF = leafDF.withColumnRenamed(\"_c5\",\"Solidity\")\n",
        "leafDF = leafDF.withColumnRenamed(\"_c6\",\"Stochastic_Convexity\")\n",
        "leafDF = leafDF.withColumnRenamed(\"_c7\",\"Isoperimetric_Factor\")\n",
        "leafDF = leafDF.withColumnRenamed(\"_c8\",\"Maximal_Indentation_Depth\")\n",
        "leafDF = leafDF.withColumnRenamed(\"_c9\",\"Lobedness\")\n",
        "leafDF = leafDF.withColumnRenamed(\"_c10\",\"Average_Intensity\")\n",
        "leafDF = leafDF.withColumnRenamed(\"_c11\",\"Average_Contrast\")\n",
        "leafDF = leafDF.withColumnRenamed(\"_c12\",\"Smoothness\")\n",
        "leafDF = leafDF.withColumnRenamed(\"_c13\",\"Third_moment\")\n",
        "leafDF = leafDF.withColumnRenamed(\"_c14\",\"Uniformity\")\n",
        "leafDF = leafDF.withColumnRenamed(\"_c15\",\"Entropy\")\n",
        "\n",
        "#### Asagidaki kodların ikisini de çalıştırıp 340 bulduğum için  data setimizin distinct olup olmadığını anlıyorum###\n",
        "\n",
        "#leafDF.count()\n",
        "#leafDF.distinct().count()\n",
        "\n",
        "### Asagidaki kod ile data setimde null değerler olup olmadığına max&min&standart sapma ile bagimli ve bagimsiz degiskenlerin durunlarını gozlemliyorum##\n",
        "\n",
        "#leafDF.describe().show()\n",
        "\n",
        "\n",
        "#leafDF.printSchema()  #Bu iki kod ile data setimdeki tüm değişkenlerin sayısal olduğundan emin oluyorum\n",
        "#leafDF.dtypes\n",
        "\n",
        "\n",
        "#leafDF = leafDF.dropna()       #data setimdeki eksik gözlemlerden kurtulmak için bu komutu kosturdum ama eksik gozlem yoktu\n",
        "\n",
        "vec = VectorAssembler(inputCols=leafDF.columns[1:15],outputCol='features')\n",
        "leafDF = vec.transform(leafDF)\n",
        "\n",
        "leafDF = leafDF.withColumnRenamed('Class','label')\n",
        "\n",
        "leafDF = leafDF.select('features','label')\n",
        "\n",
        "#leafDF.show()\n",
        "\n",
        "trainDF, testDF = leafDF.randomSplit([0.75,0.25],seed=1234)\n",
        "\n",
        "dt = DecisionTreeRegressor(labelCol=\"label\", featuresCol=\"features\")\n",
        "\n",
        "eva = MulticlassClassificationEvaluator(metricName='accuracy')\n",
        "\n",
        "\n",
        "\n",
        "myParams = (ParamGridBuilder()\n",
        "             #.addGrid(dt.maxDepth, [10, 20, 30])            \n",
        "             #.addGrid(dt.maxDepth, [2, 5, 10])\n",
        "             .addGrid(dt.maxBins, [10, 20, 40, 80, 100])            \n",
        "             #.addGrid(dt.maxBins, [10, 20])\n",
        "             .build())\n",
        "\n",
        "# Evaluate model\n",
        "dtevaluator = RegressionEvaluator(predictionCol=\"prediction\", labelCol=\"label\", metricName=\"rmse\")\n",
        "\n",
        "\n",
        "\n",
        "validator = CrossValidator(estimator = dt,\n",
        "                      estimatorParamMaps = myParams,\n",
        "                      evaluator = dtevaluator,\n",
        "                      numFolds = 5)\n",
        "model = validator.fit(trainDF)\n",
        "\n",
        "print(\"Best Model hesaplandı\")\n",
        "\n",
        "print(model)\n",
        "\n",
        "resultDF = model.transform(testDF)\n",
        "result = eva.evaluate(resultDF)\n",
        "print('Accuracy',result)\n",
        "\n"
      ],
      "execution_count": 51,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Best Model hesaplandı\n",
            "CrossValidatorModel_f8f64b5ef5ef\n",
            "Accuracy 0.4444444444444444\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XPc98bvUT72Y"
      },
      "source": [
        "#Burada pandas ile başlayıp GBM classification yapmak istedim ama collabs ve jupyterlab'da yaklaşık yarım saat sürdüğü için sonuc bulamadım. "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fumZzNeXTvFx"
      },
      "source": [
        "import numpy as np\n",
        "import pandas as pd \n",
        "import statsmodels.api as sm\n",
        "import statsmodels.formula.api as smf\n",
        "import seaborn as sns\n",
        "from sklearn.preprocessing import scale \n",
        "from sklearn.model_selection import train_test_split, GridSearchCV, cross_val_score\n",
        "from sklearn.metrics import confusion_matrix, accuracy_score, classification_report\n",
        "from sklearn.metrics import roc_auc_score,roc_curve\n",
        "import statsmodels.formula.api as smf\n",
        "import matplotlib.pyplot as plt\n",
        "from sklearn.neighbors import KNeighborsClassifier\n",
        "from sklearn.discriminant_analysis import LinearDiscriminantAnalysis\n",
        "from sklearn.linear_model import LogisticRegression\n",
        "from sklearn.svm import SVC\n",
        "from sklearn.naive_bayes import GaussianNB\n",
        "from sklearn import tree\n",
        "from sklearn.ensemble import RandomForestClassifier\n",
        "from sklearn.ensemble import GradientBoostingClassifier\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "leafPD = pd.read_csv('leaf.csv', delimiter=',',header=None, names=['Class','specimen_number','Eccentricity','Aspect_Ratio','Elongation','Solidity','Stochastic_Convexity','Isoperimetric_Factor','Maximal_Indentation_Depth','Lobedness','Average_Intensity','Average_Contrast','Smoothness','Third_moment','Uniformity','Entropy'])\n",
        "df = leafPD.copy()\n",
        "df = df.dropna()\n",
        "df.head()\n",
        "\n",
        "y = df[\"Class\"]\n",
        "X = df.drop([\"Class\"], axis=1)\n",
        "\n",
        "X = pd.DataFrame(X)\n",
        "\n",
        "X_train, X_test, y_train, y_test = train_test_split(X, y, \n",
        "                                                    test_size=0.30, \n",
        "                                                    random_state=42)\n",
        "\n",
        "gbm_model = GradientBoostingClassifier().fit(X_train, y_train)\n",
        "\n",
        "y_pred = gbm_model.predict(X_test)\n",
        "accuracy_score(y_test, y_pred)\n",
        "\n",
        "\n",
        "#model tunning#\n",
        "\n",
        "gbm_params = {\"learning_rate\" : [0.001, 0.01, 0.1, 0.05],\n",
        "             \"n_estimators\": [100,500,100],\n",
        "             \"max_depth\": [3,5,10],\n",
        "             \"min_samples_split\": [2,5,10]}\n",
        "\n",
        "gbm = GradientBoostingClassifier()\n",
        "\n",
        "gbm_cv = GridSearchCV(gbm, gbm_params, cv = 10, n_jobs = -1, verbose = 2)    \n",
        "\n",
        "gbm_cv.fit(X_train, y_train)\n",
        "\n",
        "print(\"En iyi parametreler: \" + str(gbm_cv.best_params_))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "F9yJzSqKUUd8"
      },
      "source": [
        "#Burada  MultilayerPerceptronClassifier yapmayı denedim ama aldığım hataya çözüm bulamadım. "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "N54v7_xoRsA2",
        "outputId": "8afcf62e-aea7-4300-acba-52e5ae2ac58a"
      },
      "source": [
        "from pyspark.sql import SparkSession\n",
        "from pyspark.ml.feature import VectorAssembler\n",
        "from pyspark.ml.evaluation import RegressionEvaluator\n",
        "from pyspark.ml.classification import DecisionTreeClassifier\n",
        "from pyspark.ml.evaluation import MulticlassClassificationEvaluator\n",
        "from pyspark.ml.tuning import CrossValidator,ParamGridBuilder\n",
        "from sklearn.model_selection import train_test_split, GridSearchCV,cross_val_score\n",
        "from sklearn.model_selection import GridSearchCV\n",
        "from sklearn.metrics import mean_squared_error, r2_score\n",
        "from sklearn import model_selection\n",
        "from pyspark.ml.regression import DecisionTreeRegressor\n",
        "from pyspark.ml.classification import MultilayerPerceptronClassifier\n",
        "from pyspark.ml.evaluation import MulticlassClassificationEvaluator\n",
        "\n",
        "\n",
        "spark = SparkSession.Builder().appName('Spark DataFrame Introduction').getOrCreate()\n",
        "\n",
        "leafDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','false')\\\n",
        "                        .option('delimiter',',').csv('leaf.csv')\n",
        "\n",
        "leafDF = leafDF.withColumnRenamed(\"_c0\",\"Class\")\n",
        "leafDF = leafDF.withColumnRenamed(\"_c1\",\"specimen_number\")\n",
        "leafDF = leafDF.withColumnRenamed(\"_c2\",\"Eccentricity\")\n",
        "leafDF = leafDF.withColumnRenamed(\"_c3\",\"Aspect_Ratio\")\n",
        "leafDF = leafDF.withColumnRenamed(\"_c4\",\"Elongation\")\n",
        "leafDF = leafDF.withColumnRenamed(\"_c5\",\"Solidity\")\n",
        "leafDF = leafDF.withColumnRenamed(\"_c6\",\"Stochastic_Convexity\")\n",
        "leafDF = leafDF.withColumnRenamed(\"_c7\",\"Isoperimetric_Factor\")\n",
        "leafDF = leafDF.withColumnRenamed(\"_c8\",\"Maximal_Indentation_Depth\")\n",
        "leafDF = leafDF.withColumnRenamed(\"_c9\",\"Lobedness\")\n",
        "leafDF = leafDF.withColumnRenamed(\"_c10\",\"Average_Intensity\")\n",
        "leafDF = leafDF.withColumnRenamed(\"_c11\",\"Average_Contrast\")\n",
        "leafDF = leafDF.withColumnRenamed(\"_c12\",\"Smoothness\")\n",
        "leafDF = leafDF.withColumnRenamed(\"_c13\",\"Third_moment\")\n",
        "leafDF = leafDF.withColumnRenamed(\"_c14\",\"Uniformity\")\n",
        "leafDF = leafDF.withColumnRenamed(\"_c15\",\"Entropy\")\n",
        "\n",
        "#### Asagidaki kodların ikisini de çalıştırıp 340 bulduğum için  data setimizin distinct olup olmadığını anlıyorum###\n",
        "\n",
        "#leafDF.count()\n",
        "#leafDF.distinct().count()\n",
        "\n",
        "### Asagidaki kod ile data setimde null değerler olup olmadığına max&min&standart sapma ile bagimli ve bagimsiz degiskenlerin durunlarını gozlemliyorum##\n",
        "\n",
        "#leafDF.describe().show()\n",
        "\n",
        "\n",
        "#leafDF.printSchema()  #Bu iki kod ile data setimdeki tüm değişkenlerin sayısal olduğundan emin oluyorum\n",
        "#leafDF.dtypes\n",
        "\n",
        "\n",
        "#leafDF = leafDF.dropna()       #data setimdeki eksik gözlemlerden kurtulmak için bu komutu kosturdum ama eksik gozlem yoktu\n",
        "\n",
        "vec = VectorAssembler(inputCols=leafDF.columns[1:15],outputCol='features')\n",
        "leafDF = vec.transform(leafDF)\n",
        "\n",
        "leafDF = leafDF.withColumnRenamed('Class','label')\n",
        "\n",
        "leafDF = leafDF.select('features','label')\n",
        "\n",
        "#leafDF.show()\n",
        "\n",
        "trainDF, testDF = leafDF.randomSplit([0.75,0.25],seed=1234)\n",
        "\n",
        "layers = [4, 5, 4, 3]\n",
        "\n",
        "trainer = MultilayerPerceptronClassifier(maxIter=100, layers=layers, blockSize=128, seed=1234)\n",
        "\n",
        "model = trainer.fit(trainDF)\n",
        "\n",
        "print(\"Best Model hesaplandı\")\n",
        "\n",
        "result = model.transform(testDF)\n",
        "predictionAndLabels = result.select(\"prediction\", \"label\")\n",
        "evaluator = MulticlassClassificationEvaluator(metricName=\"accuracy\")\n",
        "print(\"Test set accuracy = \" + str(evaluator.evaluate(predictionAndLabels)))\n",
        "\n"
      ],
      "execution_count": 69,
      "outputs": [
        {
          "output_type": "error",
          "ename": "Py4JJavaError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mPy4JJavaError\u001b[0m                             Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-69-931d9156f3f7>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     66\u001b[0m \u001b[0mtrainer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mMultilayerPerceptronClassifier\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmaxIter\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m100\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlayers\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlayers\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mblockSize\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m128\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mseed\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1234\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     67\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 68\u001b[0;31m \u001b[0mmodel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtrainer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrainDF\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     69\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     70\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Best Model hesaplandı\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/pyspark/ml/base.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, dataset, params)\u001b[0m\n\u001b[1;32m    159\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcopy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mparams\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_fit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    160\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 161\u001b[0;31m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_fit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    162\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    163\u001b[0m             raise ValueError(\"Params must be either a param map or a list/tuple of param maps, \"\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/pyspark/ml/wrapper.py\u001b[0m in \u001b[0;36m_fit\u001b[0;34m(self, dataset)\u001b[0m\n\u001b[1;32m    333\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    334\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_fit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdataset\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 335\u001b[0;31m         \u001b[0mjava_model\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_fit_java\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    336\u001b[0m         \u001b[0mmodel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_create_model\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mjava_model\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    337\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_copyValues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/pyspark/ml/wrapper.py\u001b[0m in \u001b[0;36m_fit_java\u001b[0;34m(self, dataset)\u001b[0m\n\u001b[1;32m    330\u001b[0m         \"\"\"\n\u001b[1;32m    331\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_transfer_params_to_java\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 332\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_java_obj\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_jdf\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    333\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    334\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_fit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdataset\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/py4j/java_gateway.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args)\u001b[0m\n\u001b[1;32m   1303\u001b[0m         \u001b[0manswer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgateway_client\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msend_command\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcommand\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1304\u001b[0m         return_value = get_return_value(\n\u001b[0;32m-> 1305\u001b[0;31m             answer, self.gateway_client, self.target_id, self.name)\n\u001b[0m\u001b[1;32m   1306\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1307\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mtemp_arg\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtemp_args\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/pyspark/sql/utils.py\u001b[0m in \u001b[0;36mdeco\u001b[0;34m(*a, **kw)\u001b[0m\n\u001b[1;32m    109\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mdeco\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkw\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    110\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 111\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkw\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    112\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mpy4j\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprotocol\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mPy4JJavaError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    113\u001b[0m             \u001b[0mconverted\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mconvert_exception\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjava_exception\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/py4j/protocol.py\u001b[0m in \u001b[0;36mget_return_value\u001b[0;34m(answer, gateway_client, target_id, name)\u001b[0m\n\u001b[1;32m    326\u001b[0m                 raise Py4JJavaError(\n\u001b[1;32m    327\u001b[0m                     \u001b[0;34m\"An error occurred while calling {0}{1}{2}.\\n\"\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 328\u001b[0;31m                     format(target_id, \".\", name), value)\n\u001b[0m\u001b[1;32m    329\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    330\u001b[0m                 raise Py4JError(\n",
            "\u001b[0;31mPy4JJavaError\u001b[0m: An error occurred while calling o438956.fit.\n: org.apache.spark.SparkException: Job aborted due to stage failure: Task 0 in stage 44371.0 failed 1 times, most recent failure: Lost task 0.0 in stage 44371.0 (TID 44371) (99d914e32f2f executor driver): org.apache.spark.SparkException: Failed to execute user defined function(OneHotEncoderModel$$Lambda$3838/2111205862: (double, int) => struct<type:tinyint,size:int,indices:array<int>,values:array<double>>)\n\tat org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)\n\tat org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)\n\tat org.apache.spark.sql.execution.WholeStageCodegenExec$$anon$1.hasNext(WholeStageCodegenExec.scala:755)\n\tat scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458)\n\tat scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458)\n\tat scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458)\n\tat scala.collection.Iterator$GroupedIterator.fill(Iterator.scala:1209)\n\tat scala.collection.Iterator$GroupedIterator.hasNext(Iterator.scala:1215)\n\tat scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458)\n\tat scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458)\n\tat org.apache.spark.storage.memory.MemoryStore.putIterator(MemoryStore.scala:221)\n\tat org.apache.spark.storage.memory.MemoryStore.putIteratorAsValues(MemoryStore.scala:299)\n\tat org.apache.spark.storage.BlockManager.$anonfun$doPutIterator$1(BlockManager.scala:1423)\n\tat org.apache.spark.storage.BlockManager.org$apache$spark$storage$BlockManager$$doPut(BlockManager.scala:1350)\n\tat org.apache.spark.storage.BlockManager.doPutIterator(BlockManager.scala:1414)\n\tat org.apache.spark.storage.BlockManager.getOrElseUpdate(BlockManager.scala:1237)\n\tat org.apache.spark.rdd.RDD.getOrCompute(RDD.scala:384)\n\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:335)\n\tat org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90)\n\tat org.apache.spark.scheduler.Task.run(Task.scala:131)\n\tat org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:497)\n\tat org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1439)\n\tat org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:500)\n\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)\n\tat java.lang.Thread.run(Thread.java:748)\nCaused by: org.apache.spark.SparkException: Unseen value: 36.0. To handle unseen values, set Param handleInvalid to keep.\n\tat org.apache.spark.ml.feature.OneHotEncoderModel.$anonfun$encoder$1(OneHotEncoder.scala:275)\n\tat org.apache.spark.ml.feature.OneHotEncoderModel.$anonfun$encoder$1$adapted(OneHotEncoder.scala:260)\n\t... 26 more\n\nDriver stacktrace:\n\tat org.apache.spark.scheduler.DAGScheduler.failJobAndIndependentStages(DAGScheduler.scala:2258)\n\tat org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2(DAGScheduler.scala:2207)\n\tat org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2$adapted(DAGScheduler.scala:2206)\n\tat scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:62)\n\tat scala.collection.mutable.ResizableArray.foreach$(ResizableArray.scala:55)\n\tat scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:49)\n\tat org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:2206)\n\tat org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1(DAGScheduler.scala:1079)\n\tat org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1$adapted(DAGScheduler.scala:1079)\n\tat scala.Option.foreach(Option.scala:407)\n\tat org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:1079)\n\tat org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:2445)\n\tat org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2387)\n\tat org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2376)\n\tat org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:49)\n\tat org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:868)\n\tat org.apache.spark.SparkContext.runJob(SparkContext.scala:2196)\n\tat org.apache.spark.SparkContext.runJob(SparkContext.scala:2217)\n\tat org.apache.spark.SparkContext.runJob(SparkContext.scala:2236)\n\tat org.apache.spark.SparkContext.runJob(SparkContext.scala:2261)\n\tat org.apache.spark.rdd.RDD.count(RDD.scala:1253)\n\tat org.apache.spark.mllib.optimization.LBFGS$.runLBFGS(LBFGS.scala:195)\n\tat org.apache.spark.mllib.optimization.LBFGS.optimizeWithLossReturned(LBFGS.scala:154)\n\tat org.apache.spark.ml.ann.FeedForwardTrainer.train(Layer.scala:855)\n\tat org.apache.spark.ml.classification.MultilayerPerceptronClassifier.$anonfun$train$1(MultilayerPerceptronClassifier.scala:228)\n\tat org.apache.spark.ml.util.Instrumentation$.$anonfun$instrumented$1(Instrumentation.scala:191)\n\tat scala.util.Try$.apply(Try.scala:213)\n\tat org.apache.spark.ml.util.Instrumentation$.instrumented(Instrumentation.scala:191)\n\tat org.apache.spark.ml.classification.MultilayerPerceptronClassifier.train(MultilayerPerceptronClassifier.scala:184)\n\tat org.apache.spark.ml.classification.MultilayerPerceptronClassifier.train(MultilayerPerceptronClassifier.scala:93)\n\tat org.apache.spark.ml.Predictor.fit(Predictor.scala:151)\n\tat org.apache.spark.ml.Predictor.fit(Predictor.scala:115)\n\tat sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\n\tat sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\n\tat sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\n\tat java.lang.reflect.Method.invoke(Method.java:498)\n\tat py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\n\tat py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)\n\tat py4j.Gateway.invoke(Gateway.java:282)\n\tat py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\n\tat py4j.commands.CallCommand.execute(CallCommand.java:79)\n\tat py4j.GatewayConnection.run(GatewayConnection.java:238)\n\tat java.lang.Thread.run(Thread.java:748)\nCaused by: org.apache.spark.SparkException: Failed to execute user defined function(OneHotEncoderModel$$Lambda$3838/2111205862: (double, int) => struct<type:tinyint,size:int,indices:array<int>,values:array<double>>)\n\tat org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)\n\tat org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)\n\tat org.apache.spark.sql.execution.WholeStageCodegenExec$$anon$1.hasNext(WholeStageCodegenExec.scala:755)\n\tat scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458)\n\tat scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458)\n\tat scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458)\n\tat scala.collection.Iterator$GroupedIterator.fill(Iterator.scala:1209)\n\tat scala.collection.Iterator$GroupedIterator.hasNext(Iterator.scala:1215)\n\tat scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458)\n\tat scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458)\n\tat org.apache.spark.storage.memory.MemoryStore.putIterator(MemoryStore.scala:221)\n\tat org.apache.spark.storage.memory.MemoryStore.putIteratorAsValues(MemoryStore.scala:299)\n\tat org.apache.spark.storage.BlockManager.$anonfun$doPutIterator$1(BlockManager.scala:1423)\n\tat org.apache.spark.storage.BlockManager.org$apache$spark$storage$BlockManager$$doPut(BlockManager.scala:1350)\n\tat org.apache.spark.storage.BlockManager.doPutIterator(BlockManager.scala:1414)\n\tat org.apache.spark.storage.BlockManager.getOrElseUpdate(BlockManager.scala:1237)\n\tat org.apache.spark.rdd.RDD.getOrCompute(RDD.scala:384)\n\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:335)\n\tat org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90)\n\tat org.apache.spark.scheduler.Task.run(Task.scala:131)\n\tat org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:497)\n\tat org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1439)\n\tat org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:500)\n\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)\n\t... 1 more\nCaused by: org.apache.spark.SparkException: Unseen value: 36.0. To handle unseen values, set Param handleInvalid to keep.\n\tat org.apache.spark.ml.feature.OneHotEncoderModel.$anonfun$encoder$1(OneHotEncoder.scala:275)\n\tat org.apache.spark.ml.feature.OneHotEncoderModel.$anonfun$encoder$1$adapted(OneHotEncoder.scala:260)\n\t... 26 more\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 476
        },
        "id": "8Z9zvBcfob-T",
        "outputId": "1ca2a342-ce6a-420f-be8c-80542c91c24e"
      },
      "source": [
        "import numpy as np\n",
        "import pandas as pd \n",
        "import statsmodels.api as sm\n",
        "import statsmodels.formula.api as smf\n",
        "import seaborn as sns\n",
        "from sklearn.preprocessing import scale \n",
        "from sklearn.model_selection import train_test_split, GridSearchCV, cross_val_score\n",
        "from sklearn.metrics import confusion_matrix, accuracy_score, classification_report\n",
        "from sklearn.metrics import roc_auc_score,roc_curve\n",
        "import statsmodels.formula.api as smf\n",
        "import matplotlib.pyplot as plt\n",
        "from sklearn.neighbors import KNeighborsClassifier\n",
        "from sklearn.discriminant_analysis import LinearDiscriminantAnalysis\n",
        "from sklearn.linear_model import LogisticRegression\n",
        "from sklearn.svm import SVC\n",
        "from sklearn.naive_bayes import GaussianNB\n",
        "from sklearn import tree\n",
        "from sklearn.ensemble import RandomForestClassifier\n",
        "from sklearn.ensemble import GradientBoostingClassifier\n",
        "from xgboost import XGBClassifier\n",
        "from lightgbm import LGBMClassifier\n",
        "\n",
        "\n",
        "leafPD = pd.read_csv('leaf.csv', delimiter=',',header=None, names=['Class','specimen_number','Eccentricity','Aspect_Ratio','Elongation','Solidity','Stochastic_Convexity','Isoperimetric_Factor','Maximal_Indentation_Depth','Lobedness','Average_Intensity','Average_Contrast','Smoothness','Third_moment','Uniformity','Entropy'])\n",
        "df = leafPD.copy()\n",
        "df = df.dropna()\n",
        "df.head()\n",
        "\n",
        "y = df[\"Class\"]\n",
        "X = df.drop([\"Class\"], axis=1)\n",
        "\n",
        "X = pd.DataFrame(X)\n",
        "\n",
        "X_train, X_test, y_train, y_test = train_test_split(X, y, \n",
        "                                                    test_size=0.30, \n",
        "                                                    random_state=42)\n",
        "\n",
        "gbm_model = GradientBoostingClassifier().fit(X_train, y_train)\n",
        "\n",
        "y_pred = gbm_model.predict(X_test)\n",
        "accuracy_score(y_test, y_pred)\n",
        "\n",
        "\n",
        "#model tunning#\n",
        "\n",
        "gbm_params = {\"learning_rate\" : [0.001, 0.01, 0.1, 0.05],\n",
        "             \"n_estimators\": [100,500,100],\n",
        "             \"max_depth\": [3,5,10],\n",
        "             \"min_samples_split\": [2,5,10]}\n",
        "\n",
        "gbm = GradientBoostingClassifier()\n",
        "\n",
        "gbm_cv = GridSearchCV(gbm, gbm_params, cv = 10, n_jobs = -1, verbose = 2)    \n",
        "\n",
        "gbm_cv.fit(X_train, y_train)\n",
        "\n",
        "print(\"En iyi parametreler: \" + str(gbm_cv.best_params_))\n",
        "\n"
      ],
      "execution_count": 66,
      "outputs": [
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Fitting 10 folds for each of 108 candidates, totalling 1080 fits\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/model_selection/_split.py:667: UserWarning: The least populated class in y has only 4 members, which is less than n_splits=10.\n",
            "  % (min_groups, self.n_splits)), UserWarning)\n",
            "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 2 concurrent workers.\n",
            "[Parallel(n_jobs=-1)]: Done  37 tasks      | elapsed:  4.5min\n",
            "[Parallel(n_jobs=-1)]: Done 158 tasks      | elapsed: 22.6min\n"
          ]
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-66-e37dd8a1cf42>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     53\u001b[0m \u001b[0mgbm_cv\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mGridSearchCV\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgbm\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgbm_params\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcv\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m10\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn_jobs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverbose\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     54\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 55\u001b[0;31m \u001b[0mgbm_cv\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_train\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     56\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     57\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"En iyi parametreler: \"\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgbm_cv\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbest_params_\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/sklearn/model_selection/_search.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, X, y, groups, **fit_params)\u001b[0m\n\u001b[1;32m    708\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0mresults\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    709\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 710\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_run_search\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mevaluate_candidates\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    711\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    712\u001b[0m         \u001b[0;31m# For multi-metric evaluation, store the best_index_, best_params_ and\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/sklearn/model_selection/_search.py\u001b[0m in \u001b[0;36m_run_search\u001b[0;34m(self, evaluate_candidates)\u001b[0m\n\u001b[1;32m   1149\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_run_search\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mevaluate_candidates\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1150\u001b[0m         \u001b[0;34m\"\"\"Search all candidates in param_grid\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1151\u001b[0;31m         \u001b[0mevaluate_candidates\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mParameterGrid\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparam_grid\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1152\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1153\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/sklearn/model_selection/_search.py\u001b[0m in \u001b[0;36mevaluate_candidates\u001b[0;34m(candidate_params)\u001b[0m\n\u001b[1;32m    687\u001b[0m                                \u001b[0;32mfor\u001b[0m \u001b[0mparameters\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtest\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    688\u001b[0m                                in product(candidate_params,\n\u001b[0;32m--> 689\u001b[0;31m                                           cv.split(X, y, groups)))\n\u001b[0m\u001b[1;32m    690\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    691\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mout\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m<\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/joblib/parallel.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, iterable)\u001b[0m\n\u001b[1;32m   1052\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1053\u001b[0m             \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_backend\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mretrieval_context\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1054\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mretrieve\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1055\u001b[0m             \u001b[0;31m# Make sure that we get a last message telling us we are done\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1056\u001b[0m             \u001b[0melapsed_time\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_start_time\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/joblib/parallel.py\u001b[0m in \u001b[0;36mretrieve\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    931\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    932\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_backend\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'supports_timeout'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 933\u001b[0;31m                     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_output\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mextend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mjob\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    934\u001b[0m                 \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    935\u001b[0m                     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_output\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mextend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mjob\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/joblib/_parallel_backends.py\u001b[0m in \u001b[0;36mwrap_future_result\u001b[0;34m(future, timeout)\u001b[0m\n\u001b[1;32m    540\u001b[0m         AsyncResults.get from multiprocessing.\"\"\"\n\u001b[1;32m    541\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 542\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mfuture\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    543\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mCfTimeoutError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    544\u001b[0m             \u001b[0;32mraise\u001b[0m \u001b[0mTimeoutError\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/lib/python3.7/concurrent/futures/_base.py\u001b[0m in \u001b[0;36mresult\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m    428\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__get_result\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    429\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 430\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_condition\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    431\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    432\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_state\u001b[0m \u001b[0;32min\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mCANCELLED\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mCANCELLED_AND_NOTIFIED\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/lib/python3.7/threading.py\u001b[0m in \u001b[0;36mwait\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m    294\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m    \u001b[0;31m# restore state no matter what (e.g., KeyboardInterrupt)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    295\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mtimeout\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 296\u001b[0;31m                 \u001b[0mwaiter\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0macquire\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    297\u001b[0m                 \u001b[0mgotit\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    298\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    }
  ]
}