BusraNur_Orhan_PostAssessment_All_Question_son.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "BusraNur_Orhan_PostAssessment_All_Question.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyMAknSUSL+jItgePH5RQA1g",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/busranurorhan/Complete-Python-3-Bootcamp/blob/master/BusraNur_Orhan_PostAssessment_All_Question_son.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KUCg7jsFp46H",
        "outputId": "f38d3ac0-1ae7-47ec-b0d4-0ea30c16a549"
      },
      "source": [
        "!apt-get update\n",
        "!apt-get install openjdk-8-jdk-headless -qq > /dev/null\n",
        "import os\n",
        "os.environ[\"JAVA_HOME\"] = \"/usr/lib/jvm/java-8-openjdk-amd64\"\n",
        "!update-alternatives --set java /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java\n",
        "!java -version\n",
        "!pip install pyspark"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\r0% [Working]\r            \rGet:1 http://security.ubuntu.com/ubuntu bionic-security InRelease [88.7 kB]\n",
            "\r0% [Connecting to archive.ubuntu.com (91.189.88.142)] [1 InRelease 14.2 kB/88.7\r                                                                               \rGet:2 https://cloud.r-project.org/bin/linux/ubuntu bionic-cran40/ InRelease [3,626 B]\n",
            "Ign:3 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  InRelease\n",
            "Get:4 http://ppa.launchpad.net/c2d4u.team/c2d4u4.0+/ubuntu bionic InRelease [15.9 kB]\n",
            "Hit:5 http://archive.ubuntu.com/ubuntu bionic InRelease\n",
            "Ign:6 https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64  InRelease\n",
            "Get:7 http://archive.ubuntu.com/ubuntu bionic-updates InRelease [88.7 kB]\n",
            "Hit:8 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  Release\n",
            "Hit:9 https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64  Release\n",
            "Hit:10 http://ppa.launchpad.net/cran/libgit2/ubuntu bionic InRelease\n",
            "Get:11 http://archive.ubuntu.com/ubuntu bionic-backports InRelease [74.6 kB]\n",
            "Get:12 http://security.ubuntu.com/ubuntu bionic-security/universe amd64 Packages [1,431 kB]\n",
            "Get:13 http://ppa.launchpad.net/deadsnakes/ppa/ubuntu bionic InRelease [15.9 kB]\n",
            "Get:14 http://security.ubuntu.com/ubuntu bionic-security/main amd64 Packages [2,365 kB]\n",
            "Hit:15 http://ppa.launchpad.net/graphics-drivers/ppa/ubuntu bionic InRelease\n",
            "Get:16 http://ppa.launchpad.net/c2d4u.team/c2d4u4.0+/ubuntu bionic/main Sources [1,802 kB]\n",
            "Get:19 http://ppa.launchpad.net/c2d4u.team/c2d4u4.0+/ubuntu bionic/main amd64 Packages [922 kB]\n",
            "Get:20 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 Packages [2,801 kB]\n",
            "Get:21 http://archive.ubuntu.com/ubuntu bionic-updates/universe amd64 Packages [2,210 kB]\n",
            "Get:22 http://ppa.launchpad.net/deadsnakes/ppa/ubuntu bionic/main amd64 Packages [40.8 kB]\n",
            "Fetched 11.9 MB in 3s (3,949 kB/s)\n",
            "Reading package lists... Done\n",
            "update-alternatives: using /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java to provide /usr/bin/java (java) in manual mode\n",
            "openjdk version \"1.8.0_292\"\n",
            "OpenJDK Runtime Environment (build 1.8.0_292-8u292-b10-0ubuntu1~18.04-b10)\n",
            "OpenJDK 64-Bit Server VM (build 25.292-b10, mixed mode)\n",
            "Collecting pyspark\n",
            "  Downloading pyspark-3.1.2.tar.gz (212.4 MB)\n",
            "\u001b[K     |████████████████████████████████| 212.4 MB 73 kB/s \n",
            "\u001b[?25hCollecting py4j==0.10.9\n",
            "  Downloading py4j-0.10.9-py2.py3-none-any.whl (198 kB)\n",
            "\u001b[K     |████████████████████████████████| 198 kB 68.2 MB/s \n",
            "\u001b[?25hBuilding wheels for collected packages: pyspark\n",
            "  Building wheel for pyspark (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for pyspark: filename=pyspark-3.1.2-py2.py3-none-any.whl size=212880768 sha256=eaa2d0e0aae52bacf28b3232946656f5a499c77826b6301cc583149c0ec3358e\n",
            "  Stored in directory: /root/.cache/pip/wheels/a5/0a/c1/9561f6fecb759579a7d863dcd846daaa95f598744e71b02c77\n",
            "Successfully built pyspark\n",
            "Installing collected packages: py4j, pyspark\n",
            "Successfully installed py4j-0.10.9 pyspark-3.1.2\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4wxW5Wbkfige"
      },
      "source": [
        "#######   QUESTION-1     ######"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "msKvHxQUf7fd",
        "outputId": "94a3af81-fa11-4388-d7b0-7fc0b0e7bcd8",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "from pyspark.sql import SparkSession\n",
        "from pyspark.sql.functions import UserDefinedFunction\n",
        "from pyspark.sql.types import FloatType,IntegerType\n",
        "\n",
        "spark = SparkSession.Builder().appName('Spark DataFrame Introduction').getOrCreate()\n",
        "\n",
        "autoDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','true')\\\n",
        "                        .option('delimiter',',').csv('auto-mpg.data.txt')\n",
        "#autoDF.show(3)\n",
        "\n",
        "#autoDF.describe().show()  #null deger yok\n",
        "\n",
        "#autoDF.printSchema()\n",
        "\n",
        "\n",
        "autoDF = autoDF.select('mpg','origin','name')\n",
        "\n",
        "#autoDF.show()\n",
        "\n",
        "\n",
        "\n",
        "def USA(ori):\n",
        "    ori = \"\"\n",
        "    if ori == 1 :\n",
        "       return float(ori)\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "USAUDF=UserDefinedFunction(USA,IntegerType())\n",
        "EuropeUDF=UserDefinedFunction(Europe,IntegerType())\n",
        "AsiaUDF=UserDefinedFunction(Asia,IntegerType())\n",
        "\n",
        "autoDF=autoDF.withColumn('USA_column',USAUDF('mpg'))\n",
        "autoDF.show()\n",
        "\n"
      ],
      "execution_count": 93,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+----+------+--------------------+----------+\n",
            "| mpg|origin|                name|USA_column|\n",
            "+----+------+--------------------+----------+\n",
            "|18.0|     1|chevrolet chevell...|      null|\n",
            "|15.0|     1|   buick skylark 320|      null|\n",
            "|18.0|     1|  plymouth satellite|      null|\n",
            "|16.0|     1|       amc rebel sst|      null|\n",
            "|17.0|     1|         ford torino|      null|\n",
            "|15.0|     1|    ford galaxie 500|      null|\n",
            "|14.0|     1|    chevrolet impala|      null|\n",
            "|14.0|     1|   plymouth fury iii|      null|\n",
            "|14.0|     1|    pontiac catalina|      null|\n",
            "|15.0|     1|  amc ambassador dpl|      null|\n",
            "|15.0|     1| dodge challenger se|      null|\n",
            "|14.0|     1|  plymouth 'cuda 340|      null|\n",
            "|15.0|     1|chevrolet monte c...|      null|\n",
            "|14.0|     1|buick estate wago...|      null|\n",
            "|24.0|     3|toyota corona mar...|      null|\n",
            "|22.0|     1|     plymouth duster|      null|\n",
            "|18.0|     1|          amc hornet|      null|\n",
            "|21.0|     1|       ford maverick|      null|\n",
            "|27.0|     3|        datsun pl510|      null|\n",
            "|26.0|     2|volkswagen 1131 d...|      null|\n",
            "+----+------+--------------------+----------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gyMwsTQ9Deg0",
        "outputId": "db819bca-c9dc-48b0-c05f-5c21b1e91a4a"
      },
      "source": [
        "from pyspark.sql import SparkSession\n",
        "from pyspark.sql.functions import UserDefinedFunction\n",
        "from pyspark.sql.types import FloatType,IntegerType\n",
        "\n",
        "spark = SparkSession.Builder().appName('Spark DataFrame Introduction').getOrCreate()\n",
        "\n",
        "autoDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','true')\\\n",
        "                        .option('delimiter',',').csv('auto-mpg.data.txt')\n",
        "#autoDF.show(3)\n",
        "\n",
        "#autoDF.describe().show()  #null deger yok\n",
        "\n",
        "autoDF.printSchema()\n",
        "\n",
        "\n",
        "autoDF = autoDF.select('mpg','origin','name')\n",
        "\n",
        "#autoDF.show()\n",
        "\n",
        "\n",
        "\n",
        "def USA(x):\n",
        "    if x == 1:\n",
        "     return x\n",
        "\n",
        "def Europe(x):\n",
        "    if x ==2:\n",
        "     return x\n",
        "\n",
        "def Asia(x):\n",
        "    if x ==3:\n",
        "     return x\n",
        "\n",
        "USAUDF=UserDefinedFunction(USA,IntegerType())\n",
        "EuropeUDF=UserDefinedFunction(Europe,IntegerType())\n",
        "AsiaUDF=UserDefinedFunction(Asia,IntegerType())\n",
        "\n",
        "# cozum 1\n",
        "autoDF1= autoDF.withColumn('USA_column',USAUDF('mpg'))\n",
        "autoDF1 = autoDF1.sort('mpg',ascending=False)\n",
        "autoDF1.show()\n",
        "\n",
        "# cozum 2\n",
        "\n",
        "autoDF2= autoDF.withColumn('Europe_column',EuropeUDF('mpg'))\n",
        "autoDF2 = autoDF2.sort('mpg',ascending=False)\n",
        "autoDF2.show()\n",
        "\n",
        "\n",
        "# cozum 2\n",
        "\n",
        "autoDF3= autoDF.withColumn('Asia_column',AsiaUDF('mpg'))\n",
        "autoDF3 = autoDF3.sort('mpg',ascending=False)\n",
        "autoDF3.show()"
      ],
      "execution_count": 102,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "root\n",
            " |-- mpg: double (nullable = true)\n",
            " |-- cylinders: integer (nullable = true)\n",
            " |-- displacement: double (nullable = true)\n",
            " |-- horsepower: string (nullable = true)\n",
            " |-- weight: integer (nullable = true)\n",
            " |-- acceleration: double (nullable = true)\n",
            " |-- model_year: integer (nullable = true)\n",
            " |-- origin: integer (nullable = true)\n",
            " |-- name: string (nullable = true)\n",
            "\n",
            "+----+------+--------------------+----------+\n",
            "| mpg|origin|                name|USA_column|\n",
            "+----+------+--------------------+----------+\n",
            "|46.6|     3|           mazda glc|      null|\n",
            "|44.6|     3| honda civic 1500 gl|      null|\n",
            "|44.3|     2|vw rabbit c (diesel)|      null|\n",
            "|44.0|     2|           vw pickup|      null|\n",
            "|43.4|     2|  vw dasher (diesel)|      null|\n",
            "|43.1|     2|volkswagen rabbit...|      null|\n",
            "|41.5|     2|           vw rabbit|      null|\n",
            "|40.9|     2|renault lecar deluxe|      null|\n",
            "|40.8|     3|          datsun 210|      null|\n",
            "|39.4|     3|      datsun b210 gx|      null|\n",
            "|39.1|     3|      toyota starlet|      null|\n",
            "|39.0|     1|      plymouth champ|      null|\n",
            "|38.1|     3|toyota corolla te...|      null|\n",
            "|38.0|     1|plymouth horizon ...|      null|\n",
            "|38.0|     1|oldsmobile cutlas...|      null|\n",
            "|38.0|     3|         honda civic|      null|\n",
            "|38.0|     3|       datsun 310 gx|      null|\n",
            "|37.7|     3|       toyota tercel|      null|\n",
            "|37.3|     2|  fiat strada custom|      null|\n",
            "|37.2|     3|          datsun 310|      null|\n",
            "+----+------+--------------------+----------+\n",
            "only showing top 20 rows\n",
            "\n",
            "+----+------+--------------------+-------------+\n",
            "| mpg|origin|                name|Europe_column|\n",
            "+----+------+--------------------+-------------+\n",
            "|46.6|     3|           mazda glc|         null|\n",
            "|44.6|     3| honda civic 1500 gl|         null|\n",
            "|44.3|     2|vw rabbit c (diesel)|         null|\n",
            "|44.0|     2|           vw pickup|         null|\n",
            "|43.4|     2|  vw dasher (diesel)|         null|\n",
            "|43.1|     2|volkswagen rabbit...|         null|\n",
            "|41.5|     2|           vw rabbit|         null|\n",
            "|40.9|     2|renault lecar deluxe|         null|\n",
            "|40.8|     3|          datsun 210|         null|\n",
            "|39.4|     3|      datsun b210 gx|         null|\n",
            "|39.1|     3|      toyota starlet|         null|\n",
            "|39.0|     1|      plymouth champ|         null|\n",
            "|38.1|     3|toyota corolla te...|         null|\n",
            "|38.0|     1|plymouth horizon ...|         null|\n",
            "|38.0|     1|oldsmobile cutlas...|         null|\n",
            "|38.0|     3|         honda civic|         null|\n",
            "|38.0|     3|       datsun 310 gx|         null|\n",
            "|37.7|     3|       toyota tercel|         null|\n",
            "|37.3|     2|  fiat strada custom|         null|\n",
            "|37.2|     3|          datsun 310|         null|\n",
            "+----+------+--------------------+-------------+\n",
            "only showing top 20 rows\n",
            "\n",
            "+----+------+--------------------+-----------+\n",
            "| mpg|origin|                name|Asia_column|\n",
            "+----+------+--------------------+-----------+\n",
            "|46.6|     3|           mazda glc|       null|\n",
            "|44.6|     3| honda civic 1500 gl|       null|\n",
            "|44.3|     2|vw rabbit c (diesel)|       null|\n",
            "|44.0|     2|           vw pickup|       null|\n",
            "|43.4|     2|  vw dasher (diesel)|       null|\n",
            "|43.1|     2|volkswagen rabbit...|       null|\n",
            "|41.5|     2|           vw rabbit|       null|\n",
            "|40.9|     2|renault lecar deluxe|       null|\n",
            "|40.8|     3|          datsun 210|       null|\n",
            "|39.4|     3|      datsun b210 gx|       null|\n",
            "|39.1|     3|      toyota starlet|       null|\n",
            "|39.0|     1|      plymouth champ|       null|\n",
            "|38.1|     3|toyota corolla te...|       null|\n",
            "|38.0|     1|plymouth horizon ...|       null|\n",
            "|38.0|     1|oldsmobile cutlas...|       null|\n",
            "|38.0|     3|         honda civic|       null|\n",
            "|38.0|     3|       datsun 310 gx|       null|\n",
            "|37.7|     3|       toyota tercel|       null|\n",
            "|37.3|     2|  fiat strada custom|       null|\n",
            "|37.2|     3|          datsun 310|       null|\n",
            "+----+------+--------------------+-----------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ta5Gzd_pfWBv"
      },
      "source": [
        "#######   QUESTION-2          ^######"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wJp58wa6dZ9n",
        "outputId": "8fe979c1-ea44-4d5f-c3d4-203a3a9fb948"
      },
      "source": [
        "########   QUESTION-3          ######\n",
        "from pyspark.sql import SparkSession\n",
        "from pyspark.sql.functions import UserDefinedFunction\n",
        "from pyspark.sql.types import FloatType,IntegerType,StringType\n",
        "\n",
        "spark = SparkSession.Builder().appName('Spark DataFrame Introduction').getOrCreate()\n",
        "\n",
        "autoDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','true')\\\n",
        "                        .option('delimiter',',').csv('auto-mpg.data.txt')\n",
        "\n",
        "#autoDF.show()\n",
        "\n",
        "def cartypeAta(enlem):\n",
        "    cartype = \"\"\n",
        "    if (enlem > 0 and enlem < 7):\n",
        "        \n",
        "        cartype  = \"Fast Car\"\n",
        "    elif (enlem > 7 and enlem < 12):   \n",
        "        cartype  = \"Average Car\"\n",
        "    else:\n",
        "        cartype  = \"Slow Car\"\n",
        "    \n",
        "    return cartype\n",
        "\n",
        "\n",
        "cartypeAtaUDF = UserDefinedFunction(cartypeAta,StringType())\n",
        "\n",
        "autoDF = autoDF.withColumn('Car-Type',cartypeAtaUDF('acceleration'))\n",
        "\n",
        "autoDF = autoDF.sort('Car-Type',ascending=False)\n",
        "autoDF.show()"
      ],
      "execution_count": 80,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+----+---------+------------+----------+------+------------+----------+------+--------------------+--------+\n",
            "| mpg|cylinders|displacement|horsepower|weight|acceleration|model_year|origin|                name|Car-Type|\n",
            "+----+---------+------------+----------+------+------------+----------+------+--------------------+--------+\n",
            "|24.0|        4|       107.0|        90|  2430|        14.5|        70|     2|         audi 100 ls|Slow Car|\n",
            "|13.0|        8|       400.0|       170|  4746|        12.0|        71|     1|ford country squi...|Slow Car|\n",
            "|25.0|        4|       104.0|        95|  2375|        17.5|        70|     2|            saab 99e|Slow Car|\n",
            "|11.0|        8|       318.0|       210|  4382|        13.5|        70|     1|          dodge d200|Slow Car|\n",
            "| 9.0|        8|       304.0|       193|  4732|        18.5|        70|     1|            hi 1200d|Slow Car|\n",
            "|16.0|        8|       304.0|       150|  3433|        12.0|        70|     1|       amc rebel sst|Slow Car|\n",
            "|27.0|        4|        97.0|        88|  2130|        14.5|        71|     3|        datsun pl510|Slow Car|\n",
            "|22.0|        6|       198.0|        95|  2833|        15.5|        70|     1|     plymouth duster|Slow Car|\n",
            "|28.0|        4|       140.0|        90|  2264|        15.5|        71|     1| chevrolet vega 2300|Slow Car|\n",
            "|21.0|        6|       200.0|        85|  2587|        16.0|        70|     1|       ford maverick|Slow Car|\n",
            "|25.0|        4|       113.0|        95|  2228|        14.0|        71|     3|       toyota corona|Slow Car|\n",
            "|26.0|        4|        97.0|        46|  1835|        20.5|        70|     2|volkswagen 1131 d...|Slow Car|\n",
            "|25.0|        4|        98.0|         ?|  2046|        19.0|        71|     1|          ford pinto|Slow Car|\n",
            "|18.0|        8|       307.0|       130|  3504|        12.0|        70|     1|chevrolet chevell...|Slow Car|\n",
            "|19.0|        6|       232.0|       100|  2634|        13.0|        71|     1|         amc gremlin|Slow Car|\n",
            "|21.0|        6|       199.0|        90|  2648|        15.0|        70|     1|         amc gremlin|Slow Car|\n",
            "|16.0|        6|       225.0|       105|  3439|        15.5|        71|     1|plymouth satellit...|Slow Car|\n",
            "|10.0|        8|       307.0|       200|  4376|        15.0|        70|     1|           chevy c20|Slow Car|\n",
            "|17.0|        6|       250.0|       100|  3329|        15.5|        71|     1|chevrolet chevell...|Slow Car|\n",
            "|24.0|        4|       113.0|        95|  2372|        15.0|        70|     3|toyota corona mar...|Slow Car|\n",
            "+----+---------+------------+----------+------+------------+----------+------+--------------------+--------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6F03bR00cBg9"
      },
      "source": [
        "#######   QUESTION-3          ^######"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SQFE-78gPD0j",
        "outputId": "2cbb93d4-1366-49b5-baf7-14e8b737116c"
      },
      "source": [
        "#Question 3##\n",
        "\n",
        "from pyspark.sql import SparkSession\n",
        "from pyspark.ml.feature import StringIndexer,OneHotEncoder,VectorAssembler # böle bir indexleme var şehire göre modelimi girdisi etkilensin istiyoruz f(a,b,c)= ax +by +e bu x'i etkilicek\n",
        "from pyspark.ml.regression import LinearRegression\n",
        "from pyspark.ml.evaluation import RegressionEvaluator\n",
        "from pyspark.ml.evaluation import MulticlassClassificationEvaluator\n",
        "from pyspark.ml.tuning import CrossValidator, ParamGridBuilder\n",
        "from pyspark.ml.feature import Imputer\n",
        "\n",
        "spark = SparkSession.Builder().appName('Spark DataFrame Introduction').getOrCreate()\n",
        "spark = SparkSession.Builder().appName('SparkML Analysis').getOrCreate()\n",
        "\n",
        "\n",
        "autoDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','true')\\\n",
        "                        .option('delimiter',',').csv('auto-mpg.data.txt')\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "#autoDF.describe().show() ### kod ile data setimde null değerler olup olmadığına max&min&standart sapma ile bagimli ve bagimsiz degiskenlerin durunlarını gozlemliyorum , null değer gözlemlemedim##\n",
        "\n",
        "#autoDF.count()\n",
        "\n",
        "\n",
        "#autoDF.printSchema()  #Bu iki kod ile data setimdeki tüm değişkenlerin sayısal olduğundan emin oluyorum\n",
        "\n",
        "\n",
        "autoDF = autoDF.dropna()       #data setimdeki eksik gözlemlerden kurtulmak için bu komutu kosturdum ama eksik gozlem yoktu\n",
        "\n",
        "autoDF = autoDF.withColumn('horsepower',autoDF['horsepower'].cast('float'))  #horsepoer column string'di onu float'a donusturdum\n",
        "\n",
        "autoDF = autoDF.drop('name')\n",
        "#autoDF .show(50)\n",
        "\n",
        "#autoDF.printSchema()\n",
        "\n",
        "meanImputer = Imputer(inputCol='horsepower',outputCol='horsepower_imputed',strategy='mean')\n",
        "autoDF = meanImputer.fit(autoDF).transform(autoDF)\n",
        "\n",
        "\n",
        "indexer = StringIndexer(inputCol='origin',outputCol='indexedorigin') ##Linear modellerde bu kulanılmamalı (Tree-based lerde kötü sonuç vermeyebilir)\n",
        "\n",
        "autoDF = indexer.fit(autoDF).transform(autoDF)\n",
        "\n",
        "encoder = OneHotEncoder(inputCol='indexedorigin',outputCol='encodedorigin')\n",
        "\n",
        "autoDF = encoder.fit(autoDF).transform(autoDF)\n",
        "\n",
        "\n",
        "#autoDF.show(50)\n",
        "\n",
        "autoDF = autoDF.withColumnRenamed('mpg','label') #label atama\n",
        "\n",
        "\n",
        "vec =VectorAssembler(inputCols=['cylinders','displacement','weight','acceleration','model_year','origin','horsepower_imputed','encodedorigin'],outputCol='features') #feature selection\n",
        "autoDF =vec.transform(autoDF)\n",
        "\n",
        "#autoDF.show()\n",
        "\n",
        "autoDF= autoDF.select('features','label')\n",
        "\n",
        "autoDF.show()\n",
        "\n",
        "trainDF, testDF = autoDF.randomSplit([0.8,0.2])\n",
        "\n",
        "lr = LinearRegression()\n",
        "\n",
        "model = lr.fit(trainDF)\n",
        "resultDF = model.transform(testDF)\n",
        "\n",
        "eva = RegressionEvaluator(metricName='rmse')\n",
        "res = eva.evaluate(resultDF)\n",
        "print('RMSE : ',res)\n",
        "\n",
        "\n",
        "#sonuc : RMSE :  3.295092970511488 çıktı"
      ],
      "execution_count": 79,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+--------------------+-----+\n",
            "|            features|label|\n",
            "+--------------------+-----+\n",
            "|[8.0,307.0,3504.0...| 18.0|\n",
            "|[8.0,350.0,3693.0...| 15.0|\n",
            "|[8.0,318.0,3436.0...| 18.0|\n",
            "|[8.0,304.0,3433.0...| 16.0|\n",
            "|[8.0,302.0,3449.0...| 17.0|\n",
            "|[8.0,429.0,4341.0...| 15.0|\n",
            "|[8.0,454.0,4354.0...| 14.0|\n",
            "|[8.0,440.0,4312.0...| 14.0|\n",
            "|[8.0,455.0,4425.0...| 14.0|\n",
            "|[8.0,390.0,3850.0...| 15.0|\n",
            "|[8.0,383.0,3563.0...| 15.0|\n",
            "|[8.0,340.0,3609.0...| 14.0|\n",
            "|[8.0,400.0,3761.0...| 15.0|\n",
            "|[8.0,455.0,3086.0...| 14.0|\n",
            "|[4.0,113.0,2372.0...| 24.0|\n",
            "|[6.0,198.0,2833.0...| 22.0|\n",
            "|[6.0,199.0,2774.0...| 18.0|\n",
            "|[6.0,200.0,2587.0...| 21.0|\n",
            "|[4.0,97.0,2130.0,...| 27.0|\n",
            "|[4.0,97.0,1835.0,...| 26.0|\n",
            "+--------------------+-----+\n",
            "only showing top 20 rows\n",
            "\n",
            "RMSE :  3.295092970511488\n"
          ]
        }
      ]
    }
  ]
}