Sınavcalisma_10_10_2021_2.30.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Sınavcalisma.9.10.2021_417.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyNjID5RhtjPvb4kbPuYvr7h",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/busranurorhan/Complete-Python-3-Bootcamp/blob/master/S%C4%B1navcalisma_10_10_2021_2.30.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0f1oYeHdaXk0",
        "outputId": "969fcd3e-e835-4ba0-d9a9-6e7f0744fc2b"
      },
      "source": [
        "!apt-get update\n",
        "!apt-get install openjdk-8-jdk-headless -qq > /dev/null\n",
        "import os\n",
        "os.environ[\"JAVA_HOME\"] = \"/usr/lib/jvm/java-8-openjdk-amd64\"\n",
        "!update-alternatives --set java /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java\n",
        "!java -version\n",
        "!pip install pyspark"
      ],
      "execution_count": 41,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\r0% [Working]\r            \rGet:1 http://security.ubuntu.com/ubuntu bionic-security InRelease [88.7 kB]\n",
            "\r0% [Connecting to archive.ubuntu.com (91.189.88.142)] [1 InRelease 14.2 kB/88.7\r0% [Connecting to archive.ubuntu.com (91.189.88.142)] [Waiting for headers] [Co\r                                                                               \rHit:2 https://cloud.r-project.org/bin/linux/ubuntu bionic-cran40/ InRelease\n",
            "\r0% [Connecting to archive.ubuntu.com (91.189.88.142)] [Connecting to ppa.launch\r0% [1 InRelease gpgv 88.7 kB] [Connecting to archive.ubuntu.com (91.189.88.142)\r                                                                               \rIgn:3 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  InRelease\n",
            "\r0% [1 InRelease gpgv 88.7 kB] [Waiting for headers] [Waiting for headers] [Wait\r                                                                               \rHit:4 http://archive.ubuntu.com/ubuntu bionic InRelease\n",
            "Hit:5 http://ppa.launchpad.net/c2d4u.team/c2d4u4.0+/ubuntu bionic InRelease\n",
            "Ign:6 https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64  InRelease\n",
            "Get:7 http://archive.ubuntu.com/ubuntu bionic-updates InRelease [88.7 kB]\n",
            "Hit:8 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  Release\n",
            "Hit:9 https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64  Release\n",
            "Hit:10 http://ppa.launchpad.net/cran/libgit2/ubuntu bionic InRelease\n",
            "Hit:11 http://ppa.launchpad.net/deadsnakes/ppa/ubuntu bionic InRelease\n",
            "Get:12 http://archive.ubuntu.com/ubuntu bionic-backports InRelease [74.6 kB]\n",
            "Hit:13 http://ppa.launchpad.net/graphics-drivers/ppa/ubuntu bionic InRelease\n",
            "Fetched 252 kB in 2s (110 kB/s)\n",
            "Reading package lists... Done\n",
            "openjdk version \"1.8.0_292\"\n",
            "OpenJDK Runtime Environment (build 1.8.0_292-8u292-b10-0ubuntu1~18.04-b10)\n",
            "OpenJDK 64-Bit Server VM (build 25.292-b10, mixed mode)\n",
            "Requirement already satisfied: pyspark in /usr/local/lib/python3.7/dist-packages (3.1.2)\n",
            "Requirement already satisfied: py4j==0.10.9 in /usr/local/lib/python3.7/dist-packages (from pyspark) (0.10.9)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "m1S3-1uZgoE4",
        "outputId": "dfe3dc10-a15f-44dd-91a2-7ff637397e2a"
      },
      "source": [
        "from pyspark.sql import SparkSession\n",
        "spark = SparkSession.Builder().appName('Spark DataFrame Introduction').getOrCreate()\n",
        "pointsDF = spark.read.option('inferSchema','true').csv('points.txt')\n",
        "pointsDF = pointsDF.withColumnRenamed('_c0','x') #Kolon isimlerini değiştiriyoruz\n",
        "pointsDF = pointsDF.withColumnRenamed('_c1','y') #Kolon isimlerini değiştiriyoruz\n",
        "pointsDF = pointsDF.withColumn(\"x\",pointsDF[\"x\"].cast(\"float\"))  #Kolonun veri tipi değiştiriyoruz\n",
        "pointsDF =  pointsDF.withColumn(\"2xy\", 2* pointsDF['x']+pointsDF['y']) #Yeni kolon ekliyoruz\n",
        "pointsDF.show()\n",
        "pointsDF.printSchema()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+----+---+----+\n",
            "|   x|  y| 2xy|\n",
            "+----+---+----+\n",
            "| 5.0|  5|15.0|\n",
            "|15.0| 17|47.0|\n",
            "|15.0| 16|46.0|\n",
            "| 5.0|  6|16.0|\n",
            "| 6.0|  6|18.0|\n",
            "|16.0| 16|48.0|\n",
            "| 8.0|  8|24.0|\n",
            "| 7.0|  6|20.0|\n",
            "|17.0| 17|51.0|\n",
            "| 6.0|  7|19.0|\n",
            "| 7.0|  8|22.0|\n",
            "|18.0| 18|54.0|\n",
            "| 8.0|  6|22.0|\n",
            "|17.0| 16|50.0|\n",
            "| 5.0|  8|18.0|\n",
            "|17.0| 16|50.0|\n",
            "| 8.0|  8|24.0|\n",
            "|16.0| 17|49.0|\n",
            "|17.0| 18|52.0|\n",
            "|18.0| 16|52.0|\n",
            "+----+---+----+\n",
            "only showing top 20 rows\n",
            "\n",
            "root\n",
            " |-- x: float (nullable = true)\n",
            " |-- y: integer (nullable = true)\n",
            " |-- 2xy: float (nullable = true)\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lI228ck7g-O3"
      },
      "source": [
        "from pyspark.sql import SparkSession\n",
        "spark = SparkSession.Builder().appName('Spark DataFrame Introduction').getOrCreate()\n",
        "pointsDF = spark.read.option('inferSchema','true').option('header','false').csv('points.txt')\n",
        "#pointsDF.printSchema()\n",
        "#pointsDF.show()\n",
        "\n",
        "#print( pointsDF.rdd.map(lambda x : tuple(x)).take(5) )\n",
        "\n",
        "\n",
        "pointsRDD = pointsDF.rdd\n",
        "\n",
        "\n",
        "\n",
        "df = pointsRDD.toDF()\n",
        "\n",
        "df.show()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gQT4fl-px9xJ",
        "outputId": "886af77b-8326-462a-ce75-0c3dc47157cf"
      },
      "source": [
        "from pyspark.sql import SparkSession\n",
        "from pyspark.sql.functions import UserDefinedFunction\n",
        "from pyspark.sql.types import FloatType\n",
        "\n",
        "spark = SparkSession.Builder().appName('Spark DataFrame Introduction').getOrCreate()\n",
        "\n",
        "dollarDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','false')\\\n",
        "                        .option('delimiter','\\t').csv('DollarDataset.txt')\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "dollarDF = dollarDF.withColumnRenamed('_c0','id')\n",
        "dollarDF = dollarDF.withColumnRenamed('_c1','tarih')\n",
        "dollarDF = dollarDF.withColumnRenamed('_c2','fiyat')\n",
        "\n",
        "#dollarDF.show() \n",
        "\n",
        "#dollarDF.count() #17776\n",
        "\n",
        "\n",
        "dollarDF = dollarDF.filter(dollarDF['fiyat'].isNotNull())  #bu islemden sonra index bouldu\n",
        "\n",
        "\n",
        "#dollarDF.show(100) \n",
        "\n",
        "#dollarDF.count()  #17190 demekki 586 null values\n",
        "\n",
        "\n",
        "dollarDF = dollarDF.drop('id') #index column2ını drop ediyorum\n",
        "#dollarDF.show() \n",
        "\n",
        "rdd = dollarDF.rdd.zipWithIndex() #index'i en son'a ekledi\n",
        "\n",
        "#rdd.collect()\n",
        "\n",
        "\n",
        "dollarDF = rdd.map(lambda x : (x[1],x[0][0],x[0][1])).toDF()\n",
        "\n",
        "\n",
        "#dollarDF.show() #baslıklar bozudlu\n",
        "\n",
        "# data frame çevirince _1 _2 otomatik atanıyor\n",
        "dollarDF = dollarDF.withColumnRenamed('_1','id')\n",
        "dollarDF = dollarDF.withColumnRenamed('_2','tarih')\n",
        "dollarDF = dollarDF.withColumnRenamed('_3','fiyat')\n",
        "\n",
        "#dollarDF.show()\n",
        "#dollarDF.printSchema()\n",
        "\n",
        "def stringtoFloat(f):\n",
        "  f=f.replace('.','')\n",
        "  f=f.replace(',','.')\n",
        "  return float(f)\n",
        "\n",
        "stringtoFloatUDF = UserDefinedFunction(stringtoFloat,FloatType())  #eger yeni bir kolon ekleyeeksek ve bu mevcut kolanla işlemler yapılıp elde edilefcekse bu uderdefinedfunction yapmamız lazım\n",
        "\n",
        "dollarDF = dollarDF.withColumn(\"fiyat\",stringtoFloatUDF('fiyat'))\n",
        "\n",
        "#dollarDF.show()\n",
        "\n",
        "dollarDF2 = dollarDF\n",
        "dollarDF2 = dollarDF2.withColumn('id',dollarDF2['id']+1)\n",
        "dollarDF2 = dollarDF2.withColumnRenamed('tarih','tarih2')\n",
        "dollarDF2 = dollarDF2.withColumnRenamed('fiyat','fiyat2')\n",
        "\n",
        "#dollarDF2.show()\n",
        "\n",
        "joinedDF = dollarDF2.join(dollarDF,'id')\n",
        "\n",
        "joinedDF = joinedDF.sort('id')\n",
        "\n",
        "joinedDF = joinedDF.withColumn('oran',((joinedDF[\"fiyat\"]-joinedDF[\"fiyat2\"])*100)/joinedDF['fiyat2'])\n",
        "\n",
        "joinedDF = joinedDF.sort('oran',ascending=False)\n",
        "joinedDF.show()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+-----+----------+---------+----------+---------+------------------+\n",
            "|   id|    tarih2|   fiyat2|     tarih|    fiyat|              oran|\n",
            "+-----+----------+---------+----------+---------+------------------+\n",
            "| 2683|19-08-1960|      2.8|22-08-1960|      9.0| 221.4285751994776|\n",
            "| 7589|24-01-1980|     35.0|25-01-1980|     70.0|             100.0|\n",
            "| 5201|07-08-1970|      9.0|10-08-1970|    14.85| 65.00000678168402|\n",
            "|12912|22-02-2001| 685391.0|23-02-2001| 957879.0| 39.75657690281898|\n",
            "|11172|05-04-1994| 23031.84|06-04-1994| 31988.89| 38.88986153848502|\n",
            "| 7433|11-06-1979|     26.5|12-06-1979|     35.0|32.075471698113205|\n",
            "| 7108|28-02-1978|    19.25|01-03-1978|     25.0| 29.87012987012987|\n",
            "|11173|06-04-1994| 31988.89|07-04-1994| 39853.13| 24.58427901483376|\n",
            "|11125|26-01-1994| 15155.63|27-01-1994|  17215.5|13.591451969845416|\n",
            "|12913|23-02-2001| 957879.0|26-02-2001|1072988.0|12.017071049683729|\n",
            "|11186|25-04-1994| 30658.56|26-04-1994| 33925.01|10.654287715190682|\n",
            "|12936|03-04-2001|1114738.0|04-04-2001|1232604.0|10.573426222125738|\n",
            "| 6997|20-09-1977|     17.5|21-09-1977|    19.25|              10.0|\n",
            "|12935|02-04-2001|1020560.0|03-04-2001|1114738.0| 9.228070863055578|\n",
            "|13010|17-07-2001|1366647.0|18-07-2001|1482571.0| 8.482365965754141|\n",
            "|11211|02-06-1994| 30623.63|03-06-1994|  33181.5| 8.352598926775999|\n",
            "|11270|24-08-1994|  32583.7|25-08-1994| 35291.28| 8.309620883812805|\n",
            "|14843|22-10-2008|     1.52|23-10-2008|     1.63| 7.236843450908203|\n",
            "|12933|29-03-2001| 992354.0|30-03-2001|1056544.0| 6.468457828557148|\n",
            "|12922|14-03-2001| 941519.0|15-03-2001| 999788.0| 6.188828903081085|\n",
            "+-----+----------+---------+----------+---------+------------------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8PxbLwzpEEst",
        "outputId": "f62b01e5-7e7c-4deb-8807-e93e244772ce"
      },
      "source": [
        "from pyspark.sql import SparkSession\n",
        "from pyspark.sql.functions import UserDefinedFunction\n",
        "from pyspark.sql.types import FloatType\n",
        "\n",
        "spark = SparkSession.Builder().appName('Spark DataFrame Introduction').getOrCreate()\n",
        "\n",
        "depremDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','true')\\\n",
        "                        .option('delimiter','\\t').csv('EarthquakeDataset.txt')\n",
        "\n",
        "\n",
        "depremDF = depremDF.select(\"Latitude\",\"Longitude\",\"xM\",\"Location\")\n",
        "depremDF.show()\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+--------+---------+---+--------------------+\n",
            "|Latitude|Longitude| xM|            Location|\n",
            "+--------+---------+---+--------------------+\n",
            "| 37.8435|  26.7775|3.8|          EGE DENIZI|\n",
            "| 36.4918|  28.7092|3.9|             AKDENIZ|\n",
            "|  37.796|  26.4165|4.4|ONIKI ADALAR (AKD...|\n",
            "| 38.4172|  39.1482|3.9|KAVAKKOY-SIVRICE ...|\n",
            "|  36.475|  28.7852|4.2|             AKDENIZ|\n",
            "|  38.505|   39.218|5.6|KAVAKTEPE- (ELAZI...|\n",
            "| 38.5305|  39.2102|4.1|KAVAKTEPE- (ELAZI...|\n",
            "| 38.0033|  42.9642|3.9|KORULU-CATAK (VAN...|\n",
            "| 36.5213|   28.695|3.7|             AKDENIZ|\n",
            "| 37.8888|  27.6075|3.7|DAGKARAAGAC-GERME...|\n",
            "| 35.7737|   36.447|3.9|              SURIYE|\n",
            "| 37.8967|  43.8062|4.5|AYDEMIR-BASKALE (...|\n",
            "|  37.126|  28.5738|4.0|ARICILAR-ULA (MUG...|\n",
            "| 37.1175|  31.0832|3.7|ETLER-SERIK (ANTA...|\n",
            "| 35.7012|  26.2608|3.6|             AKDENIZ|\n",
            "| 35.7032|  26.2382|4.6|             AKDENIZ|\n",
            "|  35.704|  26.2952|3.8|             AKDENIZ|\n",
            "|  35.691|  26.2818|3.9|             AKDENIZ|\n",
            "| 38.8573|  43.5082|4.7|ERMISLER- (VAN) [...|\n",
            "|  37.683|  27.0502|3.5|MILLI PARK-KUSADA...|\n",
            "+--------+---------+---+--------------------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "l2t5f_VZFo-Y",
        "outputId": "128936ac-3da7-4175-e5fe-a37cf9b3ea0a"
      },
      "source": [
        "from pyspark.sql import SparkSession\n",
        "from pyspark.sql.functions import UserDefinedFunction\n",
        "from pyspark.sql.types import FloatType,StringType\n",
        "\n",
        "spark = SparkSession.Builder().appName('Spark DataFrame Introduction').getOrCreate()\n",
        "\n",
        "depremDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','true')\\\n",
        "                        .option('delimiter','\\t').csv('EarthquakeDataset.txt')\n",
        "\n",
        "\n",
        "depremDF = depremDF.select(\"Latitude\",\"Longitude\",\"xM\",\"Location\")\n",
        "\n",
        "\n",
        "def bolgeAta(enlem,boylam):\n",
        "    bolge = \"\"\n",
        "    if (enlem >= 39):\n",
        "        if (boylam <=31):\n",
        "            bolge = \"Bölge 1\"\n",
        "        elif (boylam <= 36):\n",
        "            bolge = \"Bölge 2\"\n",
        "        elif (boylam <= 41):\n",
        "            bolge = \"Bölge 3\"\n",
        "        else:\n",
        "            bolge = \"Bölge 4\"\n",
        "    else:\n",
        "        if (boylam <=31):\n",
        "            bolge = \"Bölge 5\"\n",
        "        elif (boylam <= 36):\n",
        "            bolge = \"Bölge 6\"\n",
        "        elif (boylam <= 41):\n",
        "            bolge = \"Bölge 7\"\n",
        "        else:\n",
        "            bolge = \"Bölge 8\"\n",
        "    return bolge\n",
        "\n",
        "bolgeAtaUDF = UserDefinedFunction(bolgeAta,StringType())\n",
        "depremDF = depremDF.withColumn('Bolge',bolgeAtaUDF('Latitude','Longitude'))\n",
        "depremDF = depremDF.groupBy('Bolge').count()\n",
        "depremDF = depremDF.sort('count',ascending=False)\n",
        "depremDF.show(9)\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+-------+-----+\n",
            "|  Bolge|count|\n",
            "+-------+-----+\n",
            "|Bölge 5| 7946|\n",
            "|Bölge 1| 2827|\n",
            "|Bölge 8| 1547|\n",
            "|Bölge 7| 1335|\n",
            "|Bölge 6| 1277|\n",
            "|Bölge 2|  994|\n",
            "|Bölge 3|  982|\n",
            "|Bölge 4|  752|\n",
            "+-------+-----+\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6t5gs7wUNyMj",
        "outputId": "2c31b225-96c3-40ac-9126-def32389104b"
      },
      "source": [
        "from pyspark.sql import SparkSession\n",
        "spark = SparkSession.Builder().appName('Spark DataFrame Introduction').getOrCreate()\n",
        "worldDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','false')\\\n",
        "                        .option('delimiter',',').csv('world.txt')\n",
        "\n",
        "\n",
        "worldDF = worldDF.withColumnRenamed('_c0','ulke')\n",
        "worldDF = worldDF.withColumnRenamed('_c1','kita')\n",
        "worldDF = worldDF.withColumnRenamed('_c2','baskent')\n",
        "worldDF = worldDF.withColumnRenamed('_c3','nufus')\n",
        "\n",
        "#worldDF.show()\n",
        "\n",
        "#Dünyada hangi kıtada kaç ülke vardır ?\n",
        "cozum1DF = worldDF.groupBy('kita').count()\n",
        "cozum1DF = cozum1DF.sort('count',ascending=False)\n",
        "cozum1DF.show()\n",
        "#Kıtalardaki ülkelerinin ortalama nüfusu (kıta bazında) nedir ?\n",
        "cozum2DF = worldDF.groupBy('kita').mean()\n",
        "cozum2DF.show()\n",
        "cozum2DF = cozum2DF.sort('avg(nufus)',ascending=False)\n",
        "cozum2DF = cozum2DF.withColumn('ortalama nufus',cozum2DF['avg(nufus)'].cast('int'))\n",
        "cozum2DF.show()\n",
        "\n",
        "cozum3DF = worldDF.groupBy('kita').sum('nufus')\n",
        "cozum3DF = cozum3DF.sort('sum(nufus)',ascending=False)\n",
        "cozum3DF.show()\n",
        "\n",
        "#Kıtaların toplam nüfusu"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+-------------+-----+\n",
            "|         kita|count|\n",
            "+-------------+-----+\n",
            "|       Africa|   55|\n",
            "|         Asia|   49|\n",
            "|       Europe|   46|\n",
            "|North America|   23|\n",
            "|       Oceana|   13|\n",
            "|South America|   12|\n",
            "+-------------+-----+\n",
            "\n",
            "+-------------+--------------------+\n",
            "|         kita|          avg(nufus)|\n",
            "+-------------+--------------------+\n",
            "|       Europe|1.4397671369565217E7|\n",
            "|       Africa|1.6516486363636363E7|\n",
            "|North America| 2.222895156521739E7|\n",
            "|South America|       3.124975725E7|\n",
            "|       Oceana|  2462989.3076923075|\n",
            "|         Asia| 8.127662059183674E7|\n",
            "+-------------+--------------------+\n",
            "\n",
            "+-------------+--------------------+--------------+\n",
            "|         kita|          avg(nufus)|ortalama nufus|\n",
            "+-------------+--------------------+--------------+\n",
            "|         Asia| 8.127662059183674E7|      81276620|\n",
            "|South America|       3.124975725E7|      31249757|\n",
            "|North America| 2.222895156521739E7|      22228951|\n",
            "|       Africa|1.6516486363636363E7|      16516486|\n",
            "|       Europe|1.4397671369565217E7|      14397671|\n",
            "|       Oceana|  2462989.3076923075|       2462989|\n",
            "+-------------+--------------------+--------------+\n",
            "\n",
            "+-------------+----------+\n",
            "|         kita|sum(nufus)|\n",
            "+-------------+----------+\n",
            "|         Asia|3982554409|\n",
            "|       Africa| 908406750|\n",
            "|       Europe| 662292883|\n",
            "|North America| 511265886|\n",
            "|South America| 374997087|\n",
            "|       Oceana|  32018861|\n",
            "+-------------+----------+\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0K5R9-fsUL44",
        "outputId": "c8cf4bc8-4ec0-4058-87e2-5c8b490f79a2"
      },
      "source": [
        "from pyspark.sql import SparkSession\n",
        "spark = SparkSession.Builder().appName('Spark DataFrame Introduction').getOrCreate()\n",
        "worldDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','false')\\\n",
        "                        .option('delimiter',',').csv('world.txt')\n",
        "\n",
        "worldDF = worldDF.withColumnRenamed('_c0','ulke')\n",
        "worldDF = worldDF.withColumnRenamed('_c1','kita')\n",
        "worldDF = worldDF.withColumnRenamed('_c2','baskent')\n",
        "worldDF = worldDF.withColumnRenamed('_c3','nufus')\n",
        "\n",
        "worldDF.createOrReplaceTempView('world')\n",
        "\n",
        "cozumxDF = spark.sql('select * from world ')\n",
        "cozum1DF = spark.sql('select kita,count(*) as count from world group by kita order by count desc')\n",
        "cozumxDF.show()\n",
        "cozum1DF.show()"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+-------------------+-------------+----------------+---------+\n",
            "|               ulke|         kita|         baskent|    nufus|\n",
            "+-------------------+-------------+----------------+---------+\n",
            "|        Afghanistan|         Asia|           Kabul| 29863010|\n",
            "|            Albania|       Europe|          Tirana|  3129678|\n",
            "|            Algeria|       Africa|         Algiers| 32853800|\n",
            "|            Andorra|       Europe|Andorra la Vella|    67151|\n",
            "|             Angola|       Africa|          Luanda| 15941390|\n",
            "|Antigua and Barbuda|North America|      St. John's|    81479|\n",
            "|          Argentina|South America|    Buenos Aires| 38747150|\n",
            "|            Armenia|         Asia|         Yerevan|  3016312|\n",
            "|          Australia|       Oceana|        Canberra| 20155130|\n",
            "|            Austria|       Europe|          Vienna|  8189444|\n",
            "|         Azerbaijan|         Asia|            Baku|  8410801|\n",
            "|            Bahamas|North America|          Nassau|   323063|\n",
            "|            Bahrain|         Asia|          Manama|   726617|\n",
            "|         Bangladesh|         Asia|           Dhaka|141822300|\n",
            "|           Barbados|North America|      Bridgetown|   269556|\n",
            "|            Belarus|       Europe|           Minsk|  9755106|\n",
            "|            Belgium|       Europe|        Brussels| 10419050|\n",
            "|             Belize|North America|        Belmopan|   269736|\n",
            "|              Benin|       Africa|         Cotonou|  8438853|\n",
            "|             Bhutan|         Asia|         Thimphu|  2162546|\n",
            "+-------------------+-------------+----------------+---------+\n",
            "only showing top 20 rows\n",
            "\n",
            "+-------------+-----+\n",
            "|         kita|count|\n",
            "+-------------+-----+\n",
            "|       Africa|   55|\n",
            "|         Asia|   49|\n",
            "|       Europe|   46|\n",
            "|North America|   23|\n",
            "|       Oceana|   13|\n",
            "|South America|   12|\n",
            "+-------------+-----+\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "o1KNNT-e3kgF",
        "outputId": "28a5b45c-de12-4c65-ac0a-ffbdcfbd6fec"
      },
      "source": [
        "from pyspark.sql import SparkSession\n",
        "\n",
        "spark = SparkSession.Builder().appName('Spark DataFrame Introduction').getOrCreate()\n",
        "actorsDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','true')\\\n",
        "                        .option('delimiter',',').csv('join-actors_BNO.txt')\n",
        "\n",
        "seriesDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','true')\\\n",
        "                        .option('delimiter',',').csv('join-series_BNO.txt')\n",
        "seriesDF.show()\n",
        "actorsDF.show(25)\n",
        "\n",
        "#Without SQL\n",
        "joinedDF = actorsDF.join(seriesDF,'movieid','outer')\n",
        "joinedDF1 = actorsDF.join(seriesDF,'movieid','right')\n",
        "joinedDF2 = actorsDF.join(seriesDF,'movieid','left') #left , right ,inner ,outer\n",
        "joinedDF3 = actorsDF.join(seriesDF,'movieid','inner')\n",
        "\n",
        "#joinedDF = seriesDF.join(actorsDF,'movieid','outer')\n",
        "#joinedDF1 = seriesDF.join(actorsDF,'movieid','right')\n",
        "#joinedDF2 = seriesDF.join(actorsDF,'movieid','left') #left , right ,inner ,outer\n",
        "#joinedDF3 = seriesDF.join(actorsDF,'movieid','inner')\n",
        "\n",
        "\n",
        "joinedDF= joinedDF.sort(\"movieid\",ascending=True) \n",
        "\n",
        "joinedDF1= joinedDF1.sort(\"movieid\",ascending=True) \n",
        "joinedDF2= joinedDF2.sort(\"movieid\",ascending=True) \n",
        "joinedDF3= joinedDF3.sort(\"movieid\",ascending=True) \n",
        "\n",
        "joinedDF.show(26)\n",
        "joinedDF1.show(26)\n",
        "joinedDF2.show(26)\n",
        "joinedDF3.show(26)\n",
        "\n",
        "'''\n",
        "\n",
        "#with SQl\n",
        "\n",
        "actorsDF.createOrReplaceTempView('actors')\n",
        "seriesDF.createOrReplaceTempView('series')\n",
        "#sqlDF = spark.sql('select a.*,s.* from actors a,series s where s.movieid=a.movieid')\n",
        "sqlDF = spark.sql('select a.*,s.* from actors a full outer join series s on s.movieid=a.movieid')\n",
        "sqlDF.show(40)\n",
        "'''"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+-------+------------------+\n",
            "|movieid|         moviename|\n",
            "+-------+------------------+\n",
            "|      1|     Lethal Weapon|\n",
            "|      2|     The Last Ship|\n",
            "|      3|Person of Interest|\n",
            "|      4|   Game of Thrones|\n",
            "|      5|      Flashforward|\n",
            "|      6|        Revolution|\n",
            "|      7|              Lost|\n",
            "|      8|  Leyle ile Mecnun|\n",
            "|      9|               BNO|\n",
            "+-------+------------------+\n",
            "\n",
            "+---+--------------------+-------+\n",
            "| id|                name|movieid|\n",
            "+---+--------------------+-------+\n",
            "|  1|           Eric Dane|      2|\n",
            "|  2|        Adam Baldwin|      2|\n",
            "|  3|     Charles Parnell|      2|\n",
            "|  4|         Billy Burke|      6|\n",
            "|  5|    Tracy Spiridakos|      6|\n",
            "|  6|  Giancarlo Esposito|      6|\n",
            "|  7|        Damon Wayans|      1|\n",
            "|  8|     Clayne Crawford|      1|\n",
            "|  9|      Joseph Fiennes|      5|\n",
            "| 10|      Jack Davenport|      5|\n",
            "| 11|    Zachary Knighton|      5|\n",
            "| 12|        Jim Caviezel|      3|\n",
            "| 13|     Michael Emerson|      3|\n",
            "| 14|    Taraji P. Henson|      3|\n",
            "| 15|      Peter Dinklage|      4|\n",
            "| 16|       Emilia Clarke|      4|\n",
            "| 17|       Kit Harington|      4|\n",
            "| 18|Nikolaj Coster-Wa...|      4|\n",
            "| 19|     Ahmet Demirelli|   null|\n",
            "| 20|       Josh Holloway|      7|\n",
            "| 21|    Evangeline Lilly|      7|\n",
            "| 22|         Matthew Fox|      7|\n",
            "| 23|                null|      9|\n",
            "+---+--------------------+-------+\n",
            "\n",
            "+-------+----+--------------------+------------------+\n",
            "|movieid|  id|                name|         moviename|\n",
            "+-------+----+--------------------+------------------+\n",
            "|   null|  19|     Ahmet Demirelli|              null|\n",
            "|      1|   7|        Damon Wayans|     Lethal Weapon|\n",
            "|      1|   8|     Clayne Crawford|     Lethal Weapon|\n",
            "|      2|   3|     Charles Parnell|     The Last Ship|\n",
            "|      2|   1|           Eric Dane|     The Last Ship|\n",
            "|      2|   2|        Adam Baldwin|     The Last Ship|\n",
            "|      3|  12|        Jim Caviezel|Person of Interest|\n",
            "|      3|  13|     Michael Emerson|Person of Interest|\n",
            "|      3|  14|    Taraji P. Henson|Person of Interest|\n",
            "|      4|  15|      Peter Dinklage|   Game of Thrones|\n",
            "|      4|  18|Nikolaj Coster-Wa...|   Game of Thrones|\n",
            "|      4|  16|       Emilia Clarke|   Game of Thrones|\n",
            "|      4|  17|       Kit Harington|   Game of Thrones|\n",
            "|      5|  10|      Jack Davenport|      Flashforward|\n",
            "|      5|   9|      Joseph Fiennes|      Flashforward|\n",
            "|      5|  11|    Zachary Knighton|      Flashforward|\n",
            "|      6|   6|  Giancarlo Esposito|        Revolution|\n",
            "|      6|   4|         Billy Burke|        Revolution|\n",
            "|      6|   5|    Tracy Spiridakos|        Revolution|\n",
            "|      7|  20|       Josh Holloway|              Lost|\n",
            "|      7|  21|    Evangeline Lilly|              Lost|\n",
            "|      7|  22|         Matthew Fox|              Lost|\n",
            "|      8|null|                null|  Leyle ile Mecnun|\n",
            "|      9|  23|                null|               BNO|\n",
            "+-------+----+--------------------+------------------+\n",
            "\n",
            "+-------+----+--------------------+------------------+\n",
            "|movieid|  id|                name|         moviename|\n",
            "+-------+----+--------------------+------------------+\n",
            "|      1|   8|     Clayne Crawford|     Lethal Weapon|\n",
            "|      1|   7|        Damon Wayans|     Lethal Weapon|\n",
            "|      2|   3|     Charles Parnell|     The Last Ship|\n",
            "|      2|   2|        Adam Baldwin|     The Last Ship|\n",
            "|      2|   1|           Eric Dane|     The Last Ship|\n",
            "|      3|  13|     Michael Emerson|Person of Interest|\n",
            "|      3|  12|        Jim Caviezel|Person of Interest|\n",
            "|      3|  14|    Taraji P. Henson|Person of Interest|\n",
            "|      4|  15|      Peter Dinklage|   Game of Thrones|\n",
            "|      4|  17|       Kit Harington|   Game of Thrones|\n",
            "|      4|  18|Nikolaj Coster-Wa...|   Game of Thrones|\n",
            "|      4|  16|       Emilia Clarke|   Game of Thrones|\n",
            "|      5|  10|      Jack Davenport|      Flashforward|\n",
            "|      5|   9|      Joseph Fiennes|      Flashforward|\n",
            "|      5|  11|    Zachary Knighton|      Flashforward|\n",
            "|      6|   5|    Tracy Spiridakos|        Revolution|\n",
            "|      6|   4|         Billy Burke|        Revolution|\n",
            "|      6|   6|  Giancarlo Esposito|        Revolution|\n",
            "|      7|  22|         Matthew Fox|              Lost|\n",
            "|      7|  21|    Evangeline Lilly|              Lost|\n",
            "|      7|  20|       Josh Holloway|              Lost|\n",
            "|      8|null|                null|  Leyle ile Mecnun|\n",
            "|      9|  23|                null|               BNO|\n",
            "+-------+----+--------------------+------------------+\n",
            "\n",
            "+-------+---+--------------------+------------------+\n",
            "|movieid| id|                name|         moviename|\n",
            "+-------+---+--------------------+------------------+\n",
            "|   null| 19|     Ahmet Demirelli|              null|\n",
            "|      1|  8|     Clayne Crawford|     Lethal Weapon|\n",
            "|      1|  7|        Damon Wayans|     Lethal Weapon|\n",
            "|      2|  1|           Eric Dane|     The Last Ship|\n",
            "|      2|  3|     Charles Parnell|     The Last Ship|\n",
            "|      2|  2|        Adam Baldwin|     The Last Ship|\n",
            "|      3| 14|    Taraji P. Henson|Person of Interest|\n",
            "|      3| 12|        Jim Caviezel|Person of Interest|\n",
            "|      3| 13|     Michael Emerson|Person of Interest|\n",
            "|      4| 18|Nikolaj Coster-Wa...|   Game of Thrones|\n",
            "|      4| 15|      Peter Dinklage|   Game of Thrones|\n",
            "|      4| 16|       Emilia Clarke|   Game of Thrones|\n",
            "|      4| 17|       Kit Harington|   Game of Thrones|\n",
            "|      5| 10|      Jack Davenport|      Flashforward|\n",
            "|      5|  9|      Joseph Fiennes|      Flashforward|\n",
            "|      5| 11|    Zachary Knighton|      Flashforward|\n",
            "|      6|  6|  Giancarlo Esposito|        Revolution|\n",
            "|      6|  5|    Tracy Spiridakos|        Revolution|\n",
            "|      6|  4|         Billy Burke|        Revolution|\n",
            "|      7| 20|       Josh Holloway|              Lost|\n",
            "|      7| 21|    Evangeline Lilly|              Lost|\n",
            "|      7| 22|         Matthew Fox|              Lost|\n",
            "|      9| 23|                null|               BNO|\n",
            "+-------+---+--------------------+------------------+\n",
            "\n",
            "+-------+---+--------------------+------------------+\n",
            "|movieid| id|                name|         moviename|\n",
            "+-------+---+--------------------+------------------+\n",
            "|      1|  7|        Damon Wayans|     Lethal Weapon|\n",
            "|      1|  8|     Clayne Crawford|     Lethal Weapon|\n",
            "|      2|  1|           Eric Dane|     The Last Ship|\n",
            "|      2|  2|        Adam Baldwin|     The Last Ship|\n",
            "|      2|  3|     Charles Parnell|     The Last Ship|\n",
            "|      3| 13|     Michael Emerson|Person of Interest|\n",
            "|      3| 14|    Taraji P. Henson|Person of Interest|\n",
            "|      3| 12|        Jim Caviezel|Person of Interest|\n",
            "|      4| 18|Nikolaj Coster-Wa...|   Game of Thrones|\n",
            "|      4| 16|       Emilia Clarke|   Game of Thrones|\n",
            "|      4| 15|      Peter Dinklage|   Game of Thrones|\n",
            "|      4| 17|       Kit Harington|   Game of Thrones|\n",
            "|      5| 10|      Jack Davenport|      Flashforward|\n",
            "|      5| 11|    Zachary Knighton|      Flashforward|\n",
            "|      5|  9|      Joseph Fiennes|      Flashforward|\n",
            "|      6|  5|    Tracy Spiridakos|        Revolution|\n",
            "|      6|  6|  Giancarlo Esposito|        Revolution|\n",
            "|      6|  4|         Billy Burke|        Revolution|\n",
            "|      7| 20|       Josh Holloway|              Lost|\n",
            "|      7| 21|    Evangeline Lilly|              Lost|\n",
            "|      7| 22|         Matthew Fox|              Lost|\n",
            "|      9| 23|                null|               BNO|\n",
            "+-------+---+--------------------+------------------+\n",
            "\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "\"\\n\\n#with SQl\\n\\nactorsDF.createOrReplaceTempView('actors')\\nseriesDF.createOrReplaceTempView('series')\\n#sqlDF = spark.sql('select a.*,s.* from actors a,series s where s.movieid=a.movieid')\\nsqlDF = spark.sql('select a.*,s.* from actors a full outer join series s on s.movieid=a.movieid')\\nsqlDF.show(40)\\n\""
            ]
          },
          "metadata": {},
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Icx0rmZEC1in",
        "outputId": "6b566117-d244-4724-8227-a8a3efca1642"
      },
      "source": [
        "\n",
        "#points-null-vakues.txt dosyasında;\n",
        " # --- ilk kolondaki null değerleri 1 ile \n",
        " # --- ikinci kolondaki null değerleri 0 ile değiştiriniz!! (UserDefinedFunction)\n",
        " # --- kısayol\n",
        " # --- ilk kolondaki null değerleri ortalama ile\n",
        " # --- ikinci kolondaki medyan ile değişitiriniz\n",
        "\n",
        "from pyspark.sql import SparkSession\n",
        "from pyspark.sql.functions import UserDefinedFunction\n",
        "from pyspark.sql.types import IntegerType\n",
        "\n",
        "spark = SparkSession.Builder().appName('Spark DataFrame Introduction').getOrCreate()\n",
        "pointsDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','false')\\\n",
        "                        .option('delimiter',',').csv('points-null-values.txt')\n",
        "pointsDF = pointsDF.withColumnRenamed('_c0','x')\n",
        "pointsDF = pointsDF.withColumnRenamed('_c1','y')\n",
        "\n",
        "pointsDF.show(25)\n",
        "\n",
        "\n",
        "def replaceNullColumn1(x):\n",
        "  if x is None:\n",
        "    x=999\n",
        "  return x\n",
        "\n",
        "def replaceNullColumn2(x):\n",
        "  if x is None:\n",
        "    x=777\n",
        "  return x\n",
        "\n",
        "replaceUDF1=UserDefinedFunction(replaceNullColumn1,IntegerType())\n",
        "replaceUDF2=UserDefinedFunction(replaceNullColumn2,IntegerType())\n",
        "\n",
        "pointsDF = pointsDF.withColumn('X_',replaceUDF1('x'))\n",
        "pointsDF = pointsDF.withColumn('Y_',replaceUDF2('y'))\n",
        "\n",
        "pointsDF.show(50)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+--------+--------+--------+--------+\n",
            "|       x|       y|      X_|      Y_|\n",
            "+--------+--------+--------+--------+\n",
            "|       5|       5|       5|       5|\n",
            "|     105|     107|     105|     107|\n",
            "|     105|     106|     105|     106|\n",
            "|    null|       6|     999|       6|\n",
            "|       6|       6|       6|       6|\n",
            "|     106|    null|     106|     777|\n",
            "|       7|       7|       7|       7|\n",
            "|    null|       6|     999|       6|\n",
            "|     107|     107|     107|     107|\n",
            "|       6|       7|       6|       7|\n",
            "|       7|    null|       7|     777|\n",
            "|     108|     108|     108|     108|\n",
            "|       8|       6|       8|       6|\n",
            "|    null|     108|     999|     108|\n",
            "|       5|       8|       5|       8|\n",
            "|     107|     106|     107|     106|\n",
            "|       8|       8|       8|       8|\n",
            "|    null|     107|     999|     107|\n",
            "|     107|     108|     107|     108|\n",
            "|     108|     106|     108|     106|\n",
            "|10000008|10000006|10000008|10000006|\n",
            "+--------+--------+--------+--------+\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JII_l1DSEns3",
        "outputId": "88a8e7e2-d78f-4b7c-c54f-7540f9832cb2"
      },
      "source": [
        "#Kısa Yol (Sabit sayı ile değiştirme)\n",
        "from pyspark.sql import SparkSession\n",
        "from pyspark.sql.functions import UserDefinedFunction\n",
        "from pyspark.sql.types import IntegerType\n",
        "\n",
        "spark = SparkSession.Builder().appName('Spark DataFrame Introduction').getOrCreate()\n",
        "pointsDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','false')\\\n",
        "                        .option('delimiter',',').csv('points-null-values.txt')\n",
        "pointsDF = pointsDF.withColumnRenamed('_c0','x')\n",
        "pointsDF = pointsDF.withColumnRenamed('_c1','y')\n",
        "\n",
        "pointsDF.describe().show()\n",
        "\n",
        "\n",
        "#pointsDF = pointsDF.fillna(9999,'x')\n",
        "#pointsDF = pointsDF.fillna(8888,'y')\n",
        "pointsDF.show()"
      ],
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+-------+-----------------+-----------------+\n",
            "|summary|                x|                y|\n",
            "+-------+-----------------+-----------------+\n",
            "|  count|               17|               19|\n",
            "|   mean|         588289.0|526369.8947368421|\n",
            "| stddev|2425344.472743975|2294145.690034356|\n",
            "|    min|                5|                5|\n",
            "|    max|         10000008|         10000006|\n",
            "+-------+-----------------+-----------------+\n",
            "\n",
            "+----+----+\n",
            "|   x|   y|\n",
            "+----+----+\n",
            "|   5|   5|\n",
            "| 105| 107|\n",
            "| 105| 106|\n",
            "|null|   6|\n",
            "|   6|   6|\n",
            "| 106|null|\n",
            "|   7|   7|\n",
            "|null|   6|\n",
            "| 107| 107|\n",
            "|   6|   7|\n",
            "|   7|null|\n",
            "| 108| 108|\n",
            "|   8|   6|\n",
            "|null| 108|\n",
            "|   5|   8|\n",
            "| 107| 106|\n",
            "|   8|   8|\n",
            "|null| 107|\n",
            "| 107| 108|\n",
            "| 108| 106|\n",
            "+----+----+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IKiP-kJLE4TX",
        "outputId": "78c0dc14-dd46-4eac-9bed-f52dd2325e4c"
      },
      "source": [
        "from pyspark.sql import SparkSession\n",
        "from pyspark.sql.functions import UserDefinedFunction\n",
        "from pyspark.sql.types import IntegerType\n",
        "\n",
        "spark = SparkSession.Builder().appName('Spark DataFrame Introduction').getOrCreate()\n",
        "pointsDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','false')\\\n",
        "                        .option('delimiter',',').csv('points-null-values.txt')\n",
        "pointsDF = pointsDF.withColumnRenamed('_c0','x')\n",
        "pointsDF = pointsDF.withColumnRenamed('_c1','y')\n",
        "pointsDF.show(40)\n",
        "pointsDF.describe().show()  #aşağıdaki cıktıda standart sapma ortalama mean değerden cok fazla demekki aykırı degerler var o zaman medıan kullanmak daha iyidir"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+--------+--------+\n",
            "|       x|       y|\n",
            "+--------+--------+\n",
            "|       5|       5|\n",
            "|     105|     107|\n",
            "|     105|     106|\n",
            "|    null|       6|\n",
            "|       6|       6|\n",
            "|     106|    null|\n",
            "|       7|       7|\n",
            "|    null|       6|\n",
            "|     107|     107|\n",
            "|       6|       7|\n",
            "|       7|    null|\n",
            "|     108|     108|\n",
            "|       8|       6|\n",
            "|    null|     108|\n",
            "|       5|       8|\n",
            "|     107|     106|\n",
            "|       8|       8|\n",
            "|    null|     107|\n",
            "|     107|     108|\n",
            "|     108|     106|\n",
            "|10000008|10000006|\n",
            "+--------+--------+\n",
            "\n",
            "+-------+-----------------+-----------------+\n",
            "|summary|                x|                y|\n",
            "+-------+-----------------+-----------------+\n",
            "|  count|               17|               19|\n",
            "|   mean|         588289.0|526369.8947368421|\n",
            "| stddev|2425344.472743975|2294145.690034356|\n",
            "|    min|                5|                5|\n",
            "|    max|         10000008|         10000006|\n",
            "+-------+-----------------+-----------------+\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "iinmq1S_Eh-Y",
        "outputId": "a6b910ae-5cd1-48ee-cba3-1ad6e1683ada"
      },
      "source": [
        "#Kısa Yol (Ortalama ile değiştirme)\n",
        "from pyspark.sql import SparkSession\n",
        "from pyspark.ml.feature import Imputer\n",
        "\n",
        "spark = SparkSession.Builder().appName('Spark DataFrame Introduction').getOrCreate()\n",
        "pointsDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','false')\\\n",
        "                        .option('delimiter',',').csv('points-null-values.txt')\n",
        "pointsDF = pointsDF.withColumnRenamed('_c0','x')\n",
        "pointsDF = pointsDF.withColumnRenamed('_c1','y')\n",
        "pointsDF.show(26)\n",
        "\n",
        "pointsDF.describe().show()\n",
        "\n",
        "#Imputer kullanırken öncelikle outlier var mı diye bakarız\n",
        "#Eğer outlier yok ise ortalama (Mean) ile değiştirmek mantıklıdır\n",
        "#Eğer outlier varsa ve silmek istemiyorsak medyan (median) ile değiştirmek mantıklıdır\n",
        "\n",
        "meanImputer = Imputer(inputCol='x',outputCol='imputedXMean',strategy='mean')\n",
        "pointsDF = meanImputer.fit(pointsDF).transform(pointsDF)\n",
        "\n",
        "medianImputer = Imputer(inputCol='y',outputCol='imputedYMedian',strategy='median')\n",
        "pointsDF = medianImputer.fit(pointsDF).transform(pointsDF)\n",
        "\n",
        "pointsDF.show(100)\n"
      ],
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+--------+--------+\n",
            "|       x|       y|\n",
            "+--------+--------+\n",
            "|       5|       5|\n",
            "|     105|     107|\n",
            "|     105|     106|\n",
            "|    null|       6|\n",
            "|       6|       6|\n",
            "|     106|    null|\n",
            "|       7|       7|\n",
            "|    null|       6|\n",
            "|     107|     107|\n",
            "|       6|       7|\n",
            "|       7|    null|\n",
            "|     108|     108|\n",
            "|       8|       6|\n",
            "|    null|     108|\n",
            "|       5|       8|\n",
            "|     107|     106|\n",
            "|       8|       8|\n",
            "|    null|     107|\n",
            "|     107|     108|\n",
            "|     108|     106|\n",
            "|10000008|10000006|\n",
            "+--------+--------+\n",
            "\n",
            "+-------+-----------------+-----------------+\n",
            "|summary|                x|                y|\n",
            "+-------+-----------------+-----------------+\n",
            "|  count|               17|               19|\n",
            "|   mean|         588289.0|526369.8947368421|\n",
            "| stddev|2425344.472743975|2294145.690034356|\n",
            "|    min|                5|                5|\n",
            "|    max|         10000008|         10000006|\n",
            "+-------+-----------------+-----------------+\n",
            "\n",
            "+--------+--------+------------+--------------+\n",
            "|       x|       y|imputedXMean|imputedYMedian|\n",
            "+--------+--------+------------+--------------+\n",
            "|       5|       5|           5|             5|\n",
            "|     105|     107|         105|           107|\n",
            "|     105|     106|         105|           106|\n",
            "|    null|       6|      588289|             6|\n",
            "|       6|       6|           6|             6|\n",
            "|     106|    null|         106|           106|\n",
            "|       7|       7|           7|             7|\n",
            "|    null|       6|      588289|             6|\n",
            "|     107|     107|         107|           107|\n",
            "|       6|       7|           6|             7|\n",
            "|       7|    null|           7|           106|\n",
            "|     108|     108|         108|           108|\n",
            "|       8|       6|           8|             6|\n",
            "|    null|     108|      588289|           108|\n",
            "|       5|       8|           5|             8|\n",
            "|     107|     106|         107|           106|\n",
            "|       8|       8|           8|             8|\n",
            "|    null|     107|      588289|           107|\n",
            "|     107|     108|         107|           108|\n",
            "|     108|     106|         108|           106|\n",
            "|10000008|10000006|    10000008|      10000006|\n",
            "+--------+--------+------------+--------------+\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mgoSPVq_MYYc"
      },
      "source": [
        "#Homework 2 hint: DataFrame pivot, Recommendation Engine , Collabrative Filtering"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "B6U2xsb5Mfbc"
      },
      "source": [
        "## Categorical Values IMPUTER##"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3N_qI4yaMnPc",
        "outputId": "9e048210-6c17-4545-ea48-6682cca3b5bf"
      },
      "source": [
        "from pyspark.sql import SparkSession\n",
        "from pyspark.sql.functions import UserDefinedFunction\n",
        "from pyspark.sql.types import IntegerType\n",
        "\n",
        "spark = SparkSession.Builder().appName('Spark DataFrame Introduction').getOrCreate()\n",
        "evlerDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','true')\\\n",
        "                        .option('delimiter',';').csv('evler.csv')\n",
        "\n",
        "def lengthFunction(text):\n",
        "  return len(text)\n",
        "\n",
        "#pointsDF = pointsDF.withColumn('sehirX',pointsDF['Sehir']) #bu şekilde çalışıyor ama len almak isteidğimizde hata veriyor. Ç;ünkü tüm kolonu obje olarak\n",
        "# almaya çalışıyor \n",
        "#çarpım dolduğunda bunu satır satır hepsiyle yapıyor ama len denildiğinde bunu anlamıyor lenght metodunu burada çalışmaz UDF yazmamız lazım\n",
        "# yeni eklenecek kolan bu değer İzmir'i\n",
        "\n",
        "IUDF = UserDefinedFunction(lengthFunction,IntegerType())\n",
        "\n",
        "\n",
        "\n",
        "evlerDF = evlerDF.withColumn('sehirX',IUDF('Sehir'))\n",
        "\n",
        "evlerDF.show()\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+--------+---+-----+------+------+\n",
            "|   Sehir|Oda|salon| fiyat|sehirX|\n",
            "+--------+---+-----+------+------+\n",
            "|   izmir|  3|    1|500000|     5|\n",
            "|istanbul|  3|    1|800000|     8|\n",
            "|  ankara|  3|    1|600000|     6|\n",
            "|   izmir|  2|    1|300000|     5|\n",
            "|istanbul|  2|    1|450000|     8|\n",
            "|istanbul|  1|    1|250000|     8|\n",
            "|  ankara|  2|    1|380000|     6|\n",
            "|   izmir|  1|    1|220000|     5|\n",
            "|istanbul|  1|    1|245000|     8|\n",
            "+--------+---+-----+------+------+\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uT9DlToAPMx6"
      },
      "source": [
        "## Categorical Values&One Encoding ##"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "p9U5pmoAPU34",
        "outputId": "bc678e7c-2191-4b89-e84e-d5e998563686"
      },
      "source": [
        "from pyspark.sql import SparkSession\n",
        "from pyspark.ml.feature import StringIndexer,OneHotEncoder,VectorAssembler # böle bir indexleme var şehire göre modelimi girdisi etkilensin istiyoruz f(a,b,c)= ax +by +e bu x'i etkilicek\n",
        "from pyspark.ml.regression import LinearRegression\n",
        "\n",
        "spark = SparkSession.Builder().appName('Spark DataFrame Introduction').getOrCreate()\n",
        "evlerDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','true')\\\n",
        "                        .option('delimiter',';').csv('evler.csv')\n",
        "\n",
        "evlerDF.show()   \n",
        "\n",
        "#machine learning kosturmak için datamız tümuyle sayısal olmalı, bu sebeple sehir isimlerini değiştircez\n",
        "\n",
        "indexer = StringIndexer(inputCol='Sehir',outputCol='indexedSehir') ##Linear modellerde bu kulanılmamalı (Tree-based lerde kötü sonuç vermeyebilir)\n",
        "\n",
        "evlerDF = indexer.fit(evlerDF).transform(evlerDF)\n",
        "\n",
        "evlerDF.show()   \n",
        "#Index işleminden sonra Encode işlemi yaparsak en doğrusu olur (Hem linear hemde tree-based algoritmalar için)\n",
        "\n",
        "encoder = OneHotEncoder(inputCol='indexedSehir',outputCol='encodedSehir')\n",
        "\n",
        "evlerDF = encoder.fit(evlerDF).transform(evlerDF)\n",
        "\n",
        "evlerDF.show()  \n"
      ],
      "execution_count": 44,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+--------+---+-----+------+\n",
            "|   Sehir|Oda|salon| fiyat|\n",
            "+--------+---+-----+------+\n",
            "|   izmir|  3|    1|500000|\n",
            "|istanbul|  3|    1|800000|\n",
            "|  ankara|  3|    1|600000|\n",
            "|   izmir|  2|    1|300000|\n",
            "|istanbul|  2|    1|450000|\n",
            "|istanbul|  1|    1|250000|\n",
            "|  ankara|  2|    1|380000|\n",
            "|   izmir|  1|    1|220000|\n",
            "|istanbul|  1|    1|245000|\n",
            "+--------+---+-----+------+\n",
            "\n",
            "+--------+---+-----+------+------------+\n",
            "|   Sehir|Oda|salon| fiyat|indexedSehir|\n",
            "+--------+---+-----+------+------------+\n",
            "|   izmir|  3|    1|500000|         1.0|\n",
            "|istanbul|  3|    1|800000|         0.0|\n",
            "|  ankara|  3|    1|600000|         2.0|\n",
            "|   izmir|  2|    1|300000|         1.0|\n",
            "|istanbul|  2|    1|450000|         0.0|\n",
            "|istanbul|  1|    1|250000|         0.0|\n",
            "|  ankara|  2|    1|380000|         2.0|\n",
            "|   izmir|  1|    1|220000|         1.0|\n",
            "|istanbul|  1|    1|245000|         0.0|\n",
            "+--------+---+-----+------+------------+\n",
            "\n",
            "+--------+---+-----+------+------------+-------------+\n",
            "|   Sehir|Oda|salon| fiyat|indexedSehir| encodedSehir|\n",
            "+--------+---+-----+------+------------+-------------+\n",
            "|   izmir|  3|    1|500000|         1.0|(2,[1],[1.0])|\n",
            "|istanbul|  3|    1|800000|         0.0|(2,[0],[1.0])|\n",
            "|  ankara|  3|    1|600000|         2.0|    (2,[],[])|\n",
            "|   izmir|  2|    1|300000|         1.0|(2,[1],[1.0])|\n",
            "|istanbul|  2|    1|450000|         0.0|(2,[0],[1.0])|\n",
            "|istanbul|  1|    1|250000|         0.0|(2,[0],[1.0])|\n",
            "|  ankara|  2|    1|380000|         2.0|    (2,[],[])|\n",
            "|   izmir|  1|    1|220000|         1.0|(2,[1],[1.0])|\n",
            "|istanbul|  1|    1|245000|         0.0|(2,[0],[1.0])|\n",
            "+--------+---+-----+------+------------+-------------+\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vxYNA70t-_SK",
        "outputId": "4b94ad70-dc63-42e2-c743-7c14f3978c9b"
      },
      "source": [
        "from pyspark.sql import SparkSession\n",
        "from pyspark.ml.feature import StringIndexer,OneHotEncoder,VectorAssembler # böle bir indexleme var şehire göre modelimi girdisi etkilensin istiyoruz f(a,b,c)= ax +by +e bu x'i etkilicek\n",
        "from pyspark.ml.regression import LinearRegression\n",
        "\n",
        "spark = SparkSession.Builder().appName('Spark DataFrame Introduction').getOrCreate()\n",
        "evlerDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','true')\\\n",
        "                        .option('delimiter',';').csv('evler.csv')\n",
        "\n",
        "evlerDF.show() \n",
        "\n",
        "evlerDF = evlerDF.distinct()\n",
        "\n",
        "#machine learning kosturmak için datamız tümuyle sayısal olmalı, bu sebeple sehir isimlerini değiştircez\n",
        "\n",
        "indexer = StringIndexer(inputCol='Sehir',outputCol='indexedSehir') ##Linear modellerde bu kulanılmamalı (Tree-based lerde kötü sonuç vermeyebilir)\n",
        "\n",
        "evlerDF = indexer.fit(evlerDF).transform(evlerDF)\n",
        "\n",
        "evlerDF.show()   \n",
        "#Index işleminden sonra Encode işlemi yaparsak en doğrusu olur (Hem linear hemde tree-based algoritmalar için)\n",
        "\n",
        "encoder = OneHotEncoder(inputCol='indexedSehir',outputCol='encodedSehir')\n",
        "\n",
        "evlerDF = encoder.fit(evlerDF).transform(evlerDF)\n",
        "\n",
        "evlerDF.show()  "
      ],
      "execution_count": 45,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+--------+---+-----+------+\n",
            "|   Sehir|Oda|salon| fiyat|\n",
            "+--------+---+-----+------+\n",
            "|   izmir|  3|    1|500000|\n",
            "|istanbul|  3|    1|800000|\n",
            "|  ankara|  3|    1|600000|\n",
            "|   izmir|  2|    1|300000|\n",
            "|istanbul|  2|    1|450000|\n",
            "|istanbul|  1|    1|250000|\n",
            "|  ankara|  2|    1|380000|\n",
            "|   izmir|  1|    1|220000|\n",
            "|istanbul|  1|    1|245000|\n",
            "+--------+---+-----+------+\n",
            "\n",
            "+--------+---+-----+------+------------+\n",
            "|   Sehir|Oda|salon| fiyat|indexedSehir|\n",
            "+--------+---+-----+------+------------+\n",
            "|   izmir|  1|    1|220000|         1.0|\n",
            "|   izmir|  3|    1|500000|         1.0|\n",
            "|   izmir|  2|    1|300000|         1.0|\n",
            "|istanbul|  3|    1|800000|         0.0|\n",
            "|  ankara|  3|    1|600000|         2.0|\n",
            "|  ankara|  2|    1|380000|         2.0|\n",
            "|istanbul|  1|    1|245000|         0.0|\n",
            "|istanbul|  1|    1|250000|         0.0|\n",
            "|istanbul|  2|    1|450000|         0.0|\n",
            "+--------+---+-----+------+------------+\n",
            "\n",
            "+--------+---+-----+------+------------+-------------+\n",
            "|   Sehir|Oda|salon| fiyat|indexedSehir| encodedSehir|\n",
            "+--------+---+-----+------+------------+-------------+\n",
            "|   izmir|  1|    1|220000|         1.0|(2,[1],[1.0])|\n",
            "|   izmir|  3|    1|500000|         1.0|(2,[1],[1.0])|\n",
            "|   izmir|  2|    1|300000|         1.0|(2,[1],[1.0])|\n",
            "|istanbul|  3|    1|800000|         0.0|(2,[0],[1.0])|\n",
            "|  ankara|  3|    1|600000|         2.0|    (2,[],[])|\n",
            "|  ankara|  2|    1|380000|         2.0|    (2,[],[])|\n",
            "|istanbul|  1|    1|245000|         0.0|(2,[0],[1.0])|\n",
            "|istanbul|  1|    1|250000|         0.0|(2,[0],[1.0])|\n",
            "|istanbul|  2|    1|450000|         0.0|(2,[0],[1.0])|\n",
            "+--------+---+-----+------+------------+-------------+\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fKf5mxXm7l06",
        "outputId": "adae9f6e-f275-47b0-cb3f-9f1236ba188d"
      },
      "source": [
        "from pyspark.sql import SparkSession\n",
        "from pyspark.ml.feature import StringIndexer,OneHotEncoder,VectorAssembler # böle bir indexleme var şehire göre modelimi girdisi etkilensin istiyoruz f(a,b,c)= ax +by +e bu x'i etkilicek\n",
        "from pyspark.ml.regression import LinearRegression\n",
        "\n",
        "spark = SparkSession.Builder().appName('Spark DataFrame Introduction').getOrCreate()\n",
        "evlerDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','true')\\\n",
        "                        .option('delimiter',';').csv('evler.csv')\n",
        "\n",
        "evlerDF.show()   \n",
        "df = evlerDF.distinct()  #data frame'i distinct yapmak istiyorsak\n",
        "df.show()\n"
      ],
      "execution_count": 35,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+---------------------+\n",
            "|Sehir\tOda\tsalon\tfiyat|\n",
            "+---------------------+\n",
            "|     izmir\t3\t1\t500000|\n",
            "|  istanbul\t3\t1\t800000|\n",
            "|    ankara\t3\t1\t600000|\n",
            "|     izmir\t2\t1\t300000|\n",
            "|  istanbul\t2\t1\t450000|\n",
            "|  istanbul\t1\t1\t250000|\n",
            "|    ankara\t2\t1\t380000|\n",
            "|     izmir\t1\t1\t220000|\n",
            "|  istanbul\t1\t1\t245000|\n",
            "|    ankara\t2\t1\t380000|\n",
            "+---------------------+\n",
            "\n",
            "+---------------------+\n",
            "|Sehir\tOda\tsalon\tfiyat|\n",
            "+---------------------+\n",
            "|     izmir\t3\t1\t500000|\n",
            "|    ankara\t2\t1\t380000|\n",
            "|  istanbul\t2\t1\t450000|\n",
            "|    ankara\t3\t1\t600000|\n",
            "|  istanbul\t1\t1\t250000|\n",
            "|  istanbul\t1\t1\t245000|\n",
            "|     izmir\t1\t1\t220000|\n",
            "|     izmir\t2\t1\t300000|\n",
            "|  istanbul\t3\t1\t800000|\n",
            "+---------------------+\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wLAGEMhIYFWI"
      },
      "source": [
        "#Dummyfication  bu kullanılan yönteme göre yeni column'lar ekleyip diğer column'ı iptal ediyoruz3 şehir varsa son şehir eklenmez bunun için hoca'nın yüklediği excel'e bak\n",
        "#OneHotEncoding\n",
        "\n",
        "\n",
        "# Bu veriyi büyütür çünkü her yeni sehirler için yeni column ortaya çıkarıyoruz, kategorisel bir kalonumuz varsa, plaka kodu da olabilir,şehir ismide\n",
        "# cinsiyet,yaş bunlar\n",
        "\n",
        "#Sparsevector , densevector bunlar distinct eleman sayısı listenin yarısından az ise işe yarıyor buna bakarsın\n",
        "\n",
        "#distinct sayısına nasıl bakılır öğrenmem lazım"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2MiVL6IdYUb9",
        "outputId": "31e8f318-859e-4dc5-e350-9a1309622379"
      },
      "source": [
        "from pyspark.sql import SparkSession\n",
        "from pyspark.ml.feature import StringIndexer,OneHotEncoder,VectorAssembler # böle bir indexleme var şehire göre modelimi girdisi etkilensin istiyoruz f(a,b,c)= ax +by +e bu x'i etkilicek\n",
        "from pyspark.ml.regression import LinearRegression\n",
        "\n",
        "spark = SparkSession.Builder().appName('Spark DataFrame Introduction').getOrCreate()\n",
        "\n",
        "evlerDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','true')\\\n",
        "                        .option('delimiter',';').csv('evler.csv')\n",
        "\n",
        "\n",
        "indexer = StringIndexer(inputCol='Sehir',outputCol='indexedSehir')  ##linear modellerde bu kullanılması\n",
        "indexModel=indexer.fit(evlerDF)\n",
        "evlerDF = indexModel.transform(evlerDF)\n",
        "\n",
        "encoder = OneHotEncoder(inputCol='indexedSehir',outputCol='encodedSehir')\n",
        "#encodeModel=encoder.fit(evlerDF)\n",
        "#evlerDF = encodeModel.transform(evlerDF)\n",
        "\n",
        "evlerDF= encoder.fit(evlerDF).transform(evlerDF)\n",
        "\n",
        "evlerDF = evlerDF.withColumnRenamed('fiyat','label') #label atama\n",
        "evlerDF = evlerDF.select('encodedSehir','Oda','salon','label')\n",
        "\n",
        "vec =VectorAssembler(inputCols=['encodedSehir','Oda','salon'],outputCol='features')\n",
        "evlerDF =vec.transform(evlerDF)\n",
        "#evlerDF.show()\n",
        "\n",
        "evlerDF = evlerDF.select('features','label')\n",
        "#evlerDF.show()\n",
        "\n",
        "lr = LinearRegression()\n",
        "\n",
        "model = lr.fit(evlerDF) ##Makine Öğrenmesinin yapıldığı satır\n",
        "print('Bitti')\n",
        "\n",
        "print(\"Çarpanlar:  \",model.coefficients)\n",
        "print(\"Sabit sayi \", model.intercept)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Bitti\n",
            "Çarpanlar:   [107857.14398708245,-42261.90458170463,215476.19109859318,0.0]\n",
            "Sabit sayi  -48690.477997543654\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6CElb45AayxY"
      },
      "source": [
        "\n",
        "#SparseVector  --> DenseVector\n",
        "#(2,[],[])     --> [0,0]\n",
        "#(2,[1],[1.0]) --> [0,1]\n",
        "#(2,[0],[1.0]  --> [1,0]\n",
        "#[1,2,3,4] --> 4 *  32bit\n",
        "#[11,0,0,0,34,0,23,0,0,0,0,0,....]  --> 1000000 * 32bit --> 4 MB #DenseVector\n",
        "#(1000000, [0,4,6],[11,34,23] )  --> 7 * 32bit #SparseVector"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xnb6WX7WbF6H",
        "outputId": "9bf6f593-8c5a-4ee9-96a6-3540eb81c10f"
      },
      "source": [
        "from pyspark.sql import SparkSession\n",
        "from pyspark.ml.feature import StringIndexer,OneHotEncoder,VectorAssembler # böle bir indexleme var şehire göre modelimi girdisi etkilensin istiyoruz f(a,b,c)= ax +by +e bu x'i etkilicek\n",
        "from pyspark.ml.regression import LinearRegression\n",
        "\n",
        "spark = SparkSession.Builder().appName('Spark DataFrame Introduction').getOrCreate()\n",
        "\n",
        "evlerDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','true')\\\n",
        "                        .option('delimiter',';').csv('evler.csv')\n",
        "\n",
        "\n",
        "indexer = StringIndexer(inputCol='Sehir',outputCol='indexedSehir')  ##linear modellerde bu kullanılması\n",
        "indexModel=indexer.fit(evlerDF)\n",
        "evlerDF = indexModel.transform(evlerDF)\n",
        "\n",
        "encoder = OneHotEncoder(inputCol='indexedSehir',outputCol='encodedSehir')\n",
        "encodeModel=encoder.fit(evlerDF)\n",
        "\n",
        "#evlerDF = encodeModel.transform(evlerDF)\n",
        "\n",
        "evlerDF= encoder.fit(evlerDF).transform(evlerDF)\n",
        "\n",
        "evlerDF = evlerDF.withColumnRenamed('fiyat','label') #label atama\n",
        "evlerDF = evlerDF.select('encodedSehir','Oda','salon','label')\n",
        "\n",
        "vec =VectorAssembler(inputCols=['encodedSehir','Oda','salon'],outputCol='features')\n",
        "evlerDF =vec.transform(evlerDF)\n",
        "#evlerDF.show()\n",
        "\n",
        "evlerDF = evlerDF.select('features','label')\n",
        "#evlerDF.show()\n",
        "\n",
        "lr = LinearRegression()\n",
        "\n",
        "model = lr.fit(evlerDF) ##Makine Öğrenmesinin yapıldığı satır\n",
        "#print('Bitti')\n",
        "\n",
        "print(\"Çarpanlar:  \",model.coefficients)\n",
        "print(\"Sabit sayi \", model.intercept)\n",
        "\n",
        "\n",
        "\n",
        "#fiyatı bilinmeyen evler\n",
        "\n",
        "yenilerDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','true')\\\n",
        "                        .option('delimiter',';').csv('evler-fiyati-bilinmeyen.csv')\n",
        "\n",
        "yenilerDF = indexModel.transform(yenilerDF)\n",
        "yenilerDF = encodeModel.transform(yenilerDF)\n",
        "yenilerDF = vec.transform(yenilerDF)\n",
        "yenilerDF.show()\n",
        "\n",
        "\n",
        "\n",
        "yenilerDF = model.transform(yenilerDF) #Prediction yapıyoruz\n",
        "yenilerDF.show()"
      ],
      "execution_count": 47,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Çarpanlar:   [107857.14398708245,-42261.90458170463,215476.19109859318,0.0]\n",
            "Sabit sayi  -48690.477997543654\n",
            "+--------+---+-----+------------+-------------+-----------------+\n",
            "|   Sehir|Oda|salon|indexedSehir| encodedSehir|         features|\n",
            "+--------+---+-----+------------+-------------+-----------------+\n",
            "|   izmir|  3|    1|         1.0|(2,[1],[1.0])|[0.0,1.0,3.0,1.0]|\n",
            "|istanbul|  3|    1|         0.0|(2,[0],[1.0])|[1.0,0.0,3.0,1.0]|\n",
            "|  ankara|  3|    1|         2.0|    (2,[],[])|[0.0,0.0,3.0,1.0]|\n",
            "+--------+---+-----+------------+-------------+-----------------+\n",
            "\n",
            "+--------+---+-----+------------+-------------+-----------------+-----------------+\n",
            "|   Sehir|Oda|salon|indexedSehir| encodedSehir|         features|       prediction|\n",
            "+--------+---+-----+------------+-------------+-----------------+-----------------+\n",
            "|   izmir|  3|    1|         1.0|(2,[1],[1.0])|[0.0,1.0,3.0,1.0]|555476.1907165312|\n",
            "|istanbul|  3|    1|         0.0|(2,[0],[1.0])|[1.0,0.0,3.0,1.0]|705595.2392853183|\n",
            "|  ankara|  3|    1|         2.0|    (2,[],[])|[0.0,0.0,3.0,1.0]|597738.0952982359|\n",
            "+--------+---+-----+------------+-------------+-----------------+-----------------+\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IFj0NQLXdOqy",
        "outputId": "78026cb0-7ad4-44bf-bd8c-2a90e5575867"
      },
      "source": [
        "from pyspark.sql import SparkSession\n",
        "from pyspark.ml.feature import StringIndexer, VectorAssembler\n",
        "from pyspark.ml.classification import DecisionTreeClassifier\n",
        "\n",
        "spark = SparkSession.Builder().appName('Spark DataFrame Introduction').getOrCreate()\n",
        "irisDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','true')\\\n",
        "                        .option('delimiter',',').csv('iris-dataset.txt')\n",
        "#irisDF.show()\n",
        "\n",
        "#irisDF.describe().show()  #burada column'ların türüne bakıyoruz, labellarda dahil bir veri setinde her bilgi/her kolon sayısal olmak zorunda \n",
        "#irisDF.count()\n",
        "#irisDF.printSchema() #string'mi degilmi bakıyoruz\n",
        "\n",
        "#Label da dahil olmak üzere makine öğrenmesinde kullanılacak her kolon tipi sayısal olmalıdır.\n",
        "#Burada class (label) indexliyoruz\n",
        "\n",
        "indexer = StringIndexer(inputCol='class',outputCol='label')\n",
        "indexerModel = indexer.fit(irisDF) #Yeni kolonu eklemek için gerekli olan bilgi (kaç çiçek türü var) \n",
        "                                    #hesaplamak için tüm kolondaki saatırları tek tek işler\n",
        "\n",
        "irisDF = indexerModel.transform(irisDF)\n",
        "\n",
        "irisDF.show()\n",
        "\n",
        "vec =VectorAssembler(inputCols=['sepal-length', 'sepal-width', 'petal-length', 'petal-width'],outputCol='features')\n",
        "\n",
        "irisDF = vec.transform(irisDF)\n",
        "#irisDF.show()\n",
        "\n",
        "irisDF = irisDF.select('features','label')\n",
        "irisDF.show(5)\n",
        "\n",
        "classifier = DecisionTreeClassifier()  # google da spark classficatiın yazıp bir modeli seçtik\n",
        "model = classifier.fit(irisDF) #Makine Öğrenmesini yapıldığı satır. Buna Model Training adı verilir.\n",
        "print(\"Makine öğrendi\")\n",
        "\n",
        "print(model.toDebugString) #burada feature 2 dediği feature'ın indexini gösteriyor. feature 1 hiç kodda yok bu kolonu drop bile edebiliriz demekki diğer kolanla korelasyonu var dedik\n",
        "#yani baska bir kolon arttığında buda artmıs yada azalmıs. makine öğrenmesine girmeden önce korelasyona bakıp bu kolonu drop edebilirdik \n",
        "#Bu tarz işlere feature engineering denir ilerleyen zamanlarda yapıcaz bunu burada hangi algoritmanın düzgün hesap yaptığını gösteriyoruz"
      ],
      "execution_count": 49,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+------------+-----------+------------+-----------+-----------+-----+\n",
            "|sepal-length|sepal-width|petal-length|petal-width|      class|label|\n",
            "+------------+-----------+------------+-----------+-----------+-----+\n",
            "|         5.1|        3.5|         1.4|        0.2|Iris-setosa|  0.0|\n",
            "|         4.9|        3.0|         1.4|        0.2|Iris-setosa|  0.0|\n",
            "|         4.7|        3.2|         1.3|        0.2|Iris-setosa|  0.0|\n",
            "|         4.6|        3.1|         1.5|        0.2|Iris-setosa|  0.0|\n",
            "|         5.0|        3.6|         1.4|        0.2|Iris-setosa|  0.0|\n",
            "|         5.4|        3.9|         1.7|        0.4|Iris-setosa|  0.0|\n",
            "|         4.6|        3.4|         1.4|        0.3|Iris-setosa|  0.0|\n",
            "|         5.0|        3.4|         1.5|        0.2|Iris-setosa|  0.0|\n",
            "|         4.4|        2.9|         1.4|        0.2|Iris-setosa|  0.0|\n",
            "|         4.9|        3.1|         1.5|        0.1|Iris-setosa|  0.0|\n",
            "|         5.4|        3.7|         1.5|        0.2|Iris-setosa|  0.0|\n",
            "|         4.8|        3.4|         1.6|        0.2|Iris-setosa|  0.0|\n",
            "|         4.8|        3.0|         1.4|        0.1|Iris-setosa|  0.0|\n",
            "|         4.3|        3.0|         1.1|        0.1|Iris-setosa|  0.0|\n",
            "|         5.8|        4.0|         1.2|        0.2|Iris-setosa|  0.0|\n",
            "|         5.7|        4.4|         1.5|        0.4|Iris-setosa|  0.0|\n",
            "|         5.4|        3.9|         1.3|        0.4|Iris-setosa|  0.0|\n",
            "|         5.1|        3.5|         1.4|        0.3|Iris-setosa|  0.0|\n",
            "|         5.7|        3.8|         1.7|        0.3|Iris-setosa|  0.0|\n",
            "|         5.1|        3.8|         1.5|        0.3|Iris-setosa|  0.0|\n",
            "+------------+-----------+------------+-----------+-----------+-----+\n",
            "only showing top 20 rows\n",
            "\n",
            "+-----------------+-----+\n",
            "|         features|label|\n",
            "+-----------------+-----+\n",
            "|[5.1,3.5,1.4,0.2]|  0.0|\n",
            "|[4.9,3.0,1.4,0.2]|  0.0|\n",
            "|[4.7,3.2,1.3,0.2]|  0.0|\n",
            "|[4.6,3.1,1.5,0.2]|  0.0|\n",
            "|[5.0,3.6,1.4,0.2]|  0.0|\n",
            "+-----------------+-----+\n",
            "only showing top 5 rows\n",
            "\n",
            "Makine öğrendi\n",
            "DecisionTreeClassificationModel: uid=DecisionTreeClassifier_07a00c968dd3, depth=5, numNodes=17, numClasses=3, numFeatures=4\n",
            "  If (feature 2 <= 2.45)\n",
            "   Predict: 0.0\n",
            "  Else (feature 2 > 2.45)\n",
            "   If (feature 3 <= 1.75)\n",
            "    If (feature 2 <= 4.95)\n",
            "     If (feature 3 <= 1.65)\n",
            "      Predict: 1.0\n",
            "     Else (feature 3 > 1.65)\n",
            "      Predict: 2.0\n",
            "    Else (feature 2 > 4.95)\n",
            "     If (feature 3 <= 1.55)\n",
            "      Predict: 2.0\n",
            "     Else (feature 3 > 1.55)\n",
            "      If (feature 0 <= 6.75)\n",
            "       Predict: 1.0\n",
            "      Else (feature 0 > 6.75)\n",
            "       Predict: 2.0\n",
            "   Else (feature 3 > 1.75)\n",
            "    If (feature 2 <= 4.85)\n",
            "     If (feature 0 <= 5.95)\n",
            "      Predict: 1.0\n",
            "     Else (feature 0 > 5.95)\n",
            "      Predict: 2.0\n",
            "    Else (feature 2 > 4.85)\n",
            "     Predict: 2.0\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "BL2qoCYNxGW1",
        "outputId": "919f6a5e-f9bd-4c7c-d4eb-781f203dd147"
      },
      "source": [
        "from pyspark.sql import SparkSession\n",
        "from pyspark.ml.feature import StringIndexer, VectorAssembler\n",
        "from pyspark.ml.classification import DecisionTreeClassifier\n",
        "\n",
        "spark = SparkSession.Builder().appName('Spark DataFrame Introduction').getOrCreate()\n",
        "irisDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','true')\\\n",
        "                        .option('delimiter',',').csv('iris-dataset.txt')\n",
        "#irisDF.show()\n",
        "\n",
        "#irisDF.describe().show()  #burada column'ların türüne bakıyoruz, labellarda dahil bir veri setinde her bilgi/her kolon sayısal olmak zorunda \n",
        "#irisDF.count()\n",
        "#irisDF.printSchema() #string'mi degilmi bakıyoruz\n",
        "\n",
        "#Label da dahil olmak üzere makine öğrenmesinde kullanılacak her kolon tipi sayısal olmalıdır.\n",
        "#Burada class (label) indexliyoruz\n",
        "\n",
        "indexer = StringIndexer(inputCol='class',outputCol='label')\n",
        "indexerModel = indexer.fit(irisDF) #Yeni kolonu eklemek için gerekli olan bilgi (kaç çiçek türü var) \n",
        "                                    #hesaplamak için tüm kolondaki saatırları tek tek işler\n",
        "\n",
        "irisDF = indexerModel.transform(irisDF)\n",
        "\n",
        "#irisDF.show()\n",
        "\n",
        "vec =VectorAssembler(inputCols=['sepal-length', 'sepal-width', 'petal-length', 'petal-width'],outputCol='features')\n",
        "\n",
        "irisDF = vec.transform(irisDF)\n",
        "#irisDF.show()\n",
        "\n",
        "irisDF = irisDF.select('features','label')\n",
        "irisDF.show(5)\n",
        "\n",
        "classifier = DecisionTreeClassifier()  # google da spark classficatiın yazıp bir modeli seçtik\n",
        "model = classifier.fit(irisDF) #Makine Öğrenmesini yapıldığı satır. Buna Model Training adı verilir.\n",
        "print(\"Makine öğrendi\")\n",
        "print(model.featureImportances) # aşağıdaki sayılarda 1 görünmedi o 0 çıktı demekki yüzdeler %100 olması lazım"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+-----------------+-----+\n",
            "|         features|label|\n",
            "+-----------------+-----+\n",
            "|[5.1,3.5,1.4,0.2]|  0.0|\n",
            "|[4.9,3.0,1.4,0.2]|  0.0|\n",
            "|[4.7,3.2,1.3,0.2]|  0.0|\n",
            "|[4.6,3.1,1.5,0.2]|  0.0|\n",
            "|[5.0,3.6,1.4,0.2]|  0.0|\n",
            "+-----------------+-----+\n",
            "only showing top 5 rows\n",
            "\n",
            "Makine öğrendi\n",
            "(4,[0,2,3],[0.02666666666666668,0.5507226247987117,0.4226107085346217])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "d-QgK4JrxVXo"
      },
      "source": [
        "#modelimizin ne kadar başarılı olduğunu nasıl anlıyoruz ? Bu model ne kadar başarılı veri setinden alıp bu modele sorabiliriz\n",
        "#ama model hesaplarken kullandığımız veri setini kullanırsak bunlar büyük ihtimalle doğru bulacaktır.\n",
        "#bir kısmını ayırıyorum veririm  buna test dataset diyorum\n",
        "#modeli oluştururken kullanacağım data setine de train dataset diyorum data'nın genelde 70-80%'i data set için kullanılır geri kalan 20-30% teste ayırırız"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-BbLU35HfeM8",
        "outputId": "0c78f818-7419-450a-ef75-8794c3649a16"
      },
      "source": [
        "from pyspark.sql import SparkSession\n",
        "from pyspark.ml.feature import StringIndexer, VectorAssembler\n",
        "from pyspark.ml.classification import RandomForestClassifier\n",
        "\n",
        "spark = SparkSession.Builder().appName('Spark DataFrame Introduction').getOrCreate()\n",
        "irisDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','true')\\\n",
        "                        .option('delimiter',',').csv('iris-dataset.txt')\n",
        "#irisDF.show()\n",
        "\n",
        "#irisDF.describe().show()\n",
        "#irisDF.count()\n",
        "#irisDF.printSchema() #string'mi degilmi bakıyoruz,# burada column'ların türüne bakıyoruz, labellarda dahil bir veri setinde her bilgi/her kolon sayısal olmak zorunda \n",
        "\n",
        "indexer = StringIndexer(inputCol='class',outputCol='label')\n",
        "indexerModel = indexer.fit(irisDF) #Yeni kolonu eklemek için gerekli olan bilgi (kaç çiçek türü var) \n",
        "                                    #hesaplamak için tüm kolondaki saatırları tek tek işler\n",
        "\n",
        "irisDF = indexerModel.transform(irisDF)\n",
        "\n",
        "#irisDF.show()\n",
        "\n",
        "vec =VectorAssembler(inputCols=['sepal-length', 'sepal-width', 'petal-length', 'petal-width'],outputCol='features')\n",
        "\n",
        "irisDF = vec.transform(irisDF)\n",
        "#irisDF.show()\n",
        "\n",
        "irisDF = irisDF.select('features','label')\n",
        "irisDF.show(5)\n",
        "\n",
        "classifier = RandomForestClassifier()\n",
        "model = classifier.fit(irisDF) #Makine Öğrenmesini yapıldığı satır. Buna Model Training adı verilir.\n",
        "print(\"Makine öğrendi\")\n",
        "print(model.featureImportances)\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+-----------------+-----+\n",
            "|         features|label|\n",
            "+-----------------+-----+\n",
            "|[5.1,3.5,1.4,0.2]|  0.0|\n",
            "|[4.9,3.0,1.4,0.2]|  0.0|\n",
            "|[4.7,3.2,1.3,0.2]|  0.0|\n",
            "|[4.6,3.1,1.5,0.2]|  0.0|\n",
            "|[5.0,3.6,1.4,0.2]|  0.0|\n",
            "+-----------------+-----+\n",
            "only showing top 5 rows\n",
            "\n",
            "Makine öğrendi\n",
            "(4,[0,1,2,3],[0.0824681606506779,0.013281914233677052,0.34766202244616545,0.5565879026694797])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "DmFZPYNVgySz",
        "outputId": "a7a281d0-b62e-4ddd-bccd-a839f7c1d01b"
      },
      "source": [
        "from pyspark.sql import SparkSession\n",
        "from pyspark.ml.feature import StringIndexer, VectorAssembler\n",
        "from pyspark.ml.classification import DecisionTreeClassifier\n",
        "from pyspark.ml.evaluation import MulticlassClassificationEvaluator\n",
        "\n",
        "spark = SparkSession.Builder().appName('Spark DataFrame Introduction').getOrCreate()\n",
        "irisDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','true')\\\n",
        "                        .option('delimiter',',').csv('iris-dataset.txt')\n",
        "\n",
        "indexer = StringIndexer(inputCol='class',outputCol='label')\n",
        "indexerModel = indexer.fit(irisDF) #Yeni kolonu eklemek için gerekli olan bilgi (kaç çiçek türü var) \n",
        "                                    #hesaplamak için tüm kolondaki saatırları tek tek işler\n",
        "\n",
        "irisDF = indexerModel.transform(irisDF)\n",
        "\n",
        "vec =VectorAssembler(inputCols=['sepal-length', 'sepal-width', 'petal-length', 'petal-width'],outputCol='features')\n",
        "irisDF = vec.transform(irisDF)\n",
        "irisDF = irisDF.select('features','label')\n",
        "#irisDF.show()\n",
        "\n",
        "\n",
        "trainDF, testDF = irisDF.randomSplit([0.75,0.25])\n",
        "\n",
        "#random eğilde hep aynı yerden böl istiyotsak seed=1234 değeri eklemek lazım hep aynı yerden böl demek\n",
        "\n",
        "##fully random operasyon çok sevilmeyen bir durumdur çünkü bir daha asla aynısı gelmeyecek demektir, bu nedenle bilgisayarların\n",
        "# generate ettiği pseudo random kullanılır. \n",
        "\n",
        "\n",
        "classifier = DecisionTreeClassifier()\n",
        "model = classifier.fit(trainDF) #Makine Öğrenmesini yapıldığı satır. Buna Model Training adı verilir.\n",
        "print(\"Makine öğrendi\")\n",
        "\n",
        "#testDF.show()\n",
        "resultDF = model.transform(testDF) #Prediction\n",
        "resultDF.show()\n",
        "\n",
        "eva = MulticlassClassificationEvaluator(metricName='accuracy')\n",
        "result = eva.evaluate(resultDF)\n",
        "print(\"Accuracy : \",result)"
      ],
      "execution_count": 51,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Makine öğrendi\n",
            "+-----------------+-----+--------------+-------------+----------+\n",
            "|         features|label| rawPrediction|  probability|prediction|\n",
            "+-----------------+-----+--------------+-------------+----------+\n",
            "|[4.6,3.2,1.4,0.2]|  0.0|[36.0,0.0,0.0]|[1.0,0.0,0.0]|       0.0|\n",
            "|[4.8,3.0,1.4,0.1]|  0.0|[36.0,0.0,0.0]|[1.0,0.0,0.0]|       0.0|\n",
            "|[4.8,3.0,1.4,0.3]|  0.0|[36.0,0.0,0.0]|[1.0,0.0,0.0]|       0.0|\n",
            "|[4.8,3.4,1.6,0.2]|  0.0|[36.0,0.0,0.0]|[1.0,0.0,0.0]|       0.0|\n",
            "|[4.9,2.5,4.5,1.7]|  2.0|[0.0,34.0,0.0]|[0.0,1.0,0.0]|       1.0|\n",
            "|[5.0,2.0,3.5,1.0]|  1.0|[0.0,34.0,0.0]|[0.0,1.0,0.0]|       1.0|\n",
            "|[5.0,3.0,1.6,0.2]|  0.0|[36.0,0.0,0.0]|[1.0,0.0,0.0]|       0.0|\n",
            "|[5.0,3.4,1.6,0.4]|  0.0|[36.0,0.0,0.0]|[1.0,0.0,0.0]|       0.0|\n",
            "|[5.0,3.5,1.6,0.6]|  0.0|[36.0,0.0,0.0]|[1.0,0.0,0.0]|       0.0|\n",
            "|[5.1,3.3,1.7,0.5]|  0.0|[36.0,0.0,0.0]|[1.0,0.0,0.0]|       0.0|\n",
            "|[5.1,3.4,1.5,0.2]|  0.0|[36.0,0.0,0.0]|[1.0,0.0,0.0]|       0.0|\n",
            "|[5.1,3.5,1.4,0.3]|  0.0|[36.0,0.0,0.0]|[1.0,0.0,0.0]|       0.0|\n",
            "|[5.2,4.1,1.5,0.1]|  0.0|[36.0,0.0,0.0]|[1.0,0.0,0.0]|       0.0|\n",
            "|[5.4,3.0,4.5,1.5]|  1.0|[0.0,34.0,0.0]|[0.0,1.0,0.0]|       1.0|\n",
            "|[5.4,3.4,1.7,0.2]|  0.0|[36.0,0.0,0.0]|[1.0,0.0,0.0]|       0.0|\n",
            "|[5.4,3.7,1.5,0.2]|  0.0|[36.0,0.0,0.0]|[1.0,0.0,0.0]|       0.0|\n",
            "|[5.6,2.5,3.9,1.1]|  1.0|[0.0,34.0,0.0]|[0.0,1.0,0.0]|       1.0|\n",
            "|[5.7,2.5,5.0,2.0]|  2.0|[0.0,0.0,35.0]|[0.0,0.0,1.0]|       2.0|\n",
            "|[5.7,2.9,4.2,1.3]|  1.0|[0.0,34.0,0.0]|[0.0,1.0,0.0]|       1.0|\n",
            "|[5.7,3.0,4.2,1.2]|  1.0|[0.0,34.0,0.0]|[0.0,1.0,0.0]|       1.0|\n",
            "+-----------------+-----+--------------+-------------+----------+\n",
            "only showing top 20 rows\n",
            "\n",
            "Accuracy :  0.9428571428571428\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vcaPnMlXyAXF",
        "outputId": "337f11dd-b69e-429c-c9e2-bc15bc0d4b01"
      },
      "source": [
        "!pip install matplotlib"
      ],
      "execution_count": 50,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: matplotlib in /usr/local/lib/python3.7/dist-packages (3.2.2)\n",
            "Requirement already satisfied: numpy>=1.11 in /usr/local/lib/python3.7/dist-packages (from matplotlib) (1.19.5)\n",
            "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib) (2.4.7)\n",
            "Requirement already satisfied: python-dateutil>=2.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib) (2.8.2)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib) (1.3.2)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.7/dist-packages (from matplotlib) (0.10.0)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from cycler>=0.10->matplotlib) (1.15.0)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "64UUUYV3nS5k",
        "outputId": "220abc40-aacd-47cb-f2fd-6e7cf1d27082"
      },
      "source": [
        "from pyspark.sql import SparkSession\n",
        "from pyspark.ml.feature import StringIndexer, VectorAssembler\n",
        "from pyspark.ml.classification import RandomForestClassifier\n",
        "from pyspark.ml.evaluation import MulticlassClassificationEvaluator\n",
        "\n",
        "\n",
        "\n",
        "spark = SparkSession.Builder().appName('Spark DataFrame Introduction').getOrCreate()\n",
        "irisDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','true')\\\n",
        "                        .option('delimiter',',').csv('iris-dataset.txt')\n",
        "\n",
        "indexer = StringIndexer(inputCol='class',outputCol='label')\n",
        "indexerModel = indexer.fit(irisDF) \n",
        "irisDF = indexerModel.transform(irisDF)\n",
        "\n",
        "vec =VectorAssembler(inputCols=['sepal-length', 'sepal-width', 'petal-length', 'petal-width'],outputCol='features')\n",
        "irisDF = vec.transform(irisDF)\n",
        "irisDF = irisDF.select('features','label')\n",
        "\n",
        "\n",
        "trainDF, testDF = irisDF.randomSplit([0.75,0.25],seed=123)\n",
        "\n",
        "\n",
        "classifier = RandomForestClassifier()\n",
        "model = classifier.fit(trainDF) #Makine Öğrenmesini yapıldığı satır. Buna Model Training adı verilir.\n",
        "print(\"Makine öğrendi\")\n",
        "\n",
        "#testDF.show()\n",
        "resultDF = model.transform(testDF) #Prediction\n",
        "resultDF.show()\n",
        "\n",
        "eva = MulticlassClassificationEvaluator(metricName='accuracy')\n",
        "result = eva.evaluate(resultDF)\n",
        "print(\"Accuracy : \",result)\n",
        "\n",
        "#Accuracy ye her zaman güvenemeyiz. Verinin düzgün dağılıp dağılmadığınaa bakmalıyız\n",
        "#Düzgün dağılmamışsa precision veya recall u yükseltmeye çalışmalıyız."
      ],
      "execution_count": 52,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Makine öğrendi\n",
            "+-----------------+-----+--------------------+--------------------+----------+\n",
            "|         features|label|       rawPrediction|         probability|prediction|\n",
            "+-----------------+-----+--------------------+--------------------+----------+\n",
            "|[4.4,3.0,1.3,0.2]|  0.0|      [20.0,0.0,0.0]|       [1.0,0.0,0.0]|       0.0|\n",
            "|[4.6,3.2,1.4,0.2]|  0.0|      [20.0,0.0,0.0]|       [1.0,0.0,0.0]|       0.0|\n",
            "|[4.8,3.0,1.4,0.3]|  0.0|      [20.0,0.0,0.0]|       [1.0,0.0,0.0]|       0.0|\n",
            "|[4.8,3.1,1.6,0.2]|  0.0|      [20.0,0.0,0.0]|       [1.0,0.0,0.0]|       0.0|\n",
            "|[4.9,3.0,1.4,0.2]|  0.0|      [20.0,0.0,0.0]|       [1.0,0.0,0.0]|       0.0|\n",
            "|[5.0,2.3,3.3,1.0]|  1.0|[0.0,19.971428571...|[0.0,0.9985714285...|       1.0|\n",
            "|[5.0,3.5,1.3,0.3]|  0.0|      [20.0,0.0,0.0]|       [1.0,0.0,0.0]|       0.0|\n",
            "|[5.0,3.5,1.6,0.6]|  0.0|      [20.0,0.0,0.0]|       [1.0,0.0,0.0]|       0.0|\n",
            "|[5.1,3.3,1.7,0.5]|  0.0|      [19.0,1.0,0.0]|     [0.95,0.05,0.0]|       0.0|\n",
            "|[5.1,3.4,1.5,0.2]|  0.0|      [20.0,0.0,0.0]|       [1.0,0.0,0.0]|       0.0|\n",
            "|[5.1,3.5,1.4,0.2]|  0.0|      [20.0,0.0,0.0]|       [1.0,0.0,0.0]|       0.0|\n",
            "|[5.1,3.8,1.5,0.3]|  0.0|      [20.0,0.0,0.0]|       [1.0,0.0,0.0]|       0.0|\n",
            "|[5.1,3.8,1.9,0.4]|  0.0|      [18.0,2.0,0.0]|       [0.9,0.1,0.0]|       0.0|\n",
            "|[5.4,3.4,1.5,0.4]|  0.0|      [20.0,0.0,0.0]|       [1.0,0.0,0.0]|       0.0|\n",
            "|[5.5,2.5,4.0,1.3]|  1.0|[0.0,19.971428571...|[0.0,0.9985714285...|       1.0|\n",
            "|[5.5,4.2,1.4,0.2]|  0.0|      [20.0,0.0,0.0]|       [1.0,0.0,0.0]|       0.0|\n",
            "|[5.6,2.7,4.2,1.3]|  1.0|[0.0,19.971428571...|[0.0,0.9985714285...|       1.0|\n",
            "|[5.6,2.8,4.9,2.0]|  2.0|      [0.0,2.0,18.0]|       [0.0,0.1,0.9]|       2.0|\n",
            "|[5.8,2.6,4.0,1.2]|  1.0|[0.0,19.971428571...|[0.0,0.9985714285...|       1.0|\n",
            "|[5.8,2.7,5.1,1.9]|  2.0|      [0.0,0.1,19.9]|[0.0,0.005,0.9949...|       2.0|\n",
            "+-----------------+-----+--------------------+--------------------+----------+\n",
            "only showing top 20 rows\n",
            "\n",
            "Accuracy :  0.9696969696969697\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Vargcsu_uif9",
        "outputId": "71549ece-4b72-4c01-ba6b-a196a5848e21"
      },
      "source": [
        "#SparkML-HyperParameterTuning-TrainValidationSplit.ipynb\n",
        "\n",
        "from pyspark.sql import SparkSession\n",
        "from pyspark.ml.feature import StringIndexer, VectorAssembler\n",
        "from pyspark.ml.classification import RandomForestClassifier\n",
        "from pyspark.ml.evaluation import MulticlassClassificationEvaluator\n",
        "from pyspark.ml.tuning import TrainValidationSplit,ParamGridBuilder\n",
        "\n",
        "spark = SparkSession.Builder().appName('Spark DataFrame Introduction').getOrCreate()\n",
        "irisDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','true')\\\n",
        "                        .option('delimiter',',').csv('iris-dataset.txt')\n",
        "indexer = StringIndexer(inputCol='class',outputCol='label')\n",
        "indexerModel = indexer.fit(irisDF)                   \n",
        "irisDF = indexerModel.transform(irisDF)\n",
        "vec =VectorAssembler(inputCols=['sepal-length', 'sepal-width', 'petal-length', 'petal-width'],outputCol='features')\n",
        "irisDF = vec.transform(irisDF)\n",
        "irisDF = irisDF.select('features','label')\n",
        "trainDF, testDF = irisDF.randomSplit([0.8,0.2],seed=123)\n",
        "\n",
        "rf = RandomForestClassifier()\n",
        "eva = MulticlassClassificationEvaluator(metricName='accuracy')\n",
        "\n",
        "myParams = ParamGridBuilder().addGrid(rf.numTrees,[1,2,3])\\\n",
        "                            .addGrid(rf.maxDepth,[4,5,6])\\\n",
        "                            .addGrid(rf.impurity,['gini','entropy']).build()\n",
        "\n",
        "validator = TrainValidationSplit(parallelism=4, \n",
        "                                 estimator=rf,\n",
        "                                 estimatorParamMaps=myParams,\n",
        "                                 trainRatio=0.8,  #TrainDataset / Total Train Daataset ( izim problemimizde %80 in yüzde 80 i)\n",
        "                                 evaluator=eva)\n",
        "\n",
        "model = validator.fit(trainDF)\n",
        "print(\"Best Model hesaplandı\")\n",
        "print(\"Best NumTrees : \", model.bestModel.getNumTrees)\n",
        "print(\"Best MaxDepth : \", model.bestModel.getMaxDepth())\n",
        "print(\"Best Impurity : \", model.bestModel.getImpurity())\n",
        "\n",
        "resultDF = model.transform(testDF)\n",
        "result = eva.evaluate(resultDF)\n",
        "\n",
        "print(\"Accuracy of the best model \",result)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Best Model hesaplandı\n",
            "Best NumTrees :  1\n",
            "Best MaxDepth :  4\n",
            "Best Impurity :  gini\n",
            "Accuracy of the best model  0.9310344827586207\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "yl3oUgKqyI4b",
        "outputId": "9925e26c-89d9-4ca8-f4be-d515203b2a1c"
      },
      "source": [
        "#SparkML-HyperParameterTuning-CrossValidation\n",
        "\n",
        "from pyspark.sql import SparkSession\n",
        "from pyspark.ml.feature import StringIndexer, VectorAssembler\n",
        "from pyspark.ml.classification import RandomForestClassifier\n",
        "from pyspark.ml.evaluation import MulticlassClassificationEvaluator\n",
        "from pyspark.ml.tuning import CrossValidator,ParamGridBuilder\n",
        "spark = SparkSession.Builder().appName('Spark DataFrame Introduction').getOrCreate()\n",
        "irisDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','true')\\\n",
        "                        .option('delimiter',',').csv('iris-dataset.txt')\n",
        "indexer = StringIndexer(inputCol='class',outputCol='label')\n",
        "indexerModel = indexer.fit(irisDF)                   \n",
        "irisDF = indexerModel.transform(irisDF)\n",
        "vec =VectorAssembler(inputCols=['sepal-length', 'sepal-width', 'petal-length', 'petal-width'],outputCol='features')\n",
        "irisDF = vec.transform(irisDF)\n",
        "irisDF = irisDF.select('features','label')\n",
        "trainDF, testDF = irisDF.randomSplit([0.8,0.2],seed=123)\n",
        "rf = RandomForestClassifier()\n",
        "eva = MulticlassClassificationEvaluator(metricName='accuracy')\n",
        "myParams = ParamGridBuilder().addGrid(rf.numTrees,[1,2,3])\\\n",
        "                            .addGrid(rf.maxDepth,[4,5,6])\\\n",
        "                            .addGrid(rf.impurity,['gini','entropy']).build()\n",
        "validator = CrossValidator(parallelism=4, \n",
        "                                 estimator=rf,\n",
        "                                 estimatorParamMaps=myParams,\n",
        "                                 evaluator=eva,\n",
        "                                 numFolds=5)\n",
        "\n",
        "\n",
        "#\n",
        "\n",
        "model = validator.fit(trainDF)\n",
        "print(\"Best Model hesaplandı\")\n",
        "print(\"Best NumTrees : \", model.bestModel.getNumTrees)\n",
        "print(\"Best MaxDepth : \", model.bestModel.getMaxDepth())\n",
        "print(\"Best Impurity : \", model.bestModel.getImpurity())\n",
        "resultDF = model.transform(testDF)\n",
        "result = eva.evaluate(resultDF)\n",
        "print(\"Accuracy of the best model \",result)"
      ],
      "execution_count": 53,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Best Model hesaplandı\n",
            "Best NumTrees :  2\n",
            "Best MaxDepth :  4\n",
            "Best Impurity :  gini\n",
            "Accuracy of the best model  0.9310344827586207\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KemOUnJK07mW",
        "outputId": "07d4f621-d601-4b0e-dbcb-db5f8ba5b3af"
      },
      "source": [
        "#Boston House pricing verisetinde cross validation yaparaak en iyi LinearRegrression parametrelerini bulacağız.\n",
        "\n",
        "from pyspark.sql import SparkSession\n",
        "from pyspark.ml.feature import StringIndexer, VectorAssembler\n",
        "from pyspark.ml.regression import LinearRegression\n",
        "from pyspark.ml.evaluation import RegressionEvaluator\n",
        "from pyspark.ml.tuning import CrossValidator,ParamGridBuilder\n",
        "\n",
        "\n",
        "spark = SparkSession.Builder().appName('Spark DataFrame Introduction').getOrCreate()\n",
        "houseDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','true')\\\n",
        "                        .option('delimiter',',').csv('boston-house-prices.txt')\n",
        "\n",
        "\n",
        "houseDF = houseDF.withColumnRenamed('medv','label')\n",
        "#houseDF.show()\n",
        "\n",
        "#houseDF.describe().show()\n",
        "\n",
        "vec = VectorAssembler(inputCols=houseDF.columns[0:-1],outputCol='features')\n",
        "houseDF=vec.transform(houseDF)\n",
        "houseDF = houseDF.select('features','label')\n",
        "\n",
        "houseDF.show(truncate=False)\n",
        "\n",
        "trainDF, testDF = houseDF.randomSplit([0.8,0.2])\n",
        "\n",
        "lr = LinearRegression()\n",
        "\n",
        "model = lr.fit(trainDF)\n",
        "resultDF = model.transform(testDF)\n",
        "\n",
        "eva = RegressionEvaluator(metricName='rmse')\n",
        "res = eva.evaluate(resultDF)\n",
        "print('RMSE : ',res)\n"
      ],
      "execution_count": 55,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+----------------------------------------------------------------------------+-----+\n",
            "|features                                                                    |label|\n",
            "+----------------------------------------------------------------------------+-----+\n",
            "|[0.00632,18.0,2.31,0.0,0.538,6.575,65.2,4.09,1.0,296.0,15.3,396.9,4.98]     |24.0 |\n",
            "|[0.02731,0.0,7.07,0.0,0.469,6.421,78.9,4.9671,2.0,242.0,17.8,396.9,9.14]    |21.6 |\n",
            "|[0.02729,0.0,7.07,0.0,0.469,7.185,61.1,4.9671,2.0,242.0,17.8,392.83,4.03]   |34.7 |\n",
            "|[0.03237,0.0,2.18,0.0,0.458,6.998,45.8,6.0622,3.0,222.0,18.7,394.63,2.94]   |33.4 |\n",
            "|[0.06905,0.0,2.18,0.0,0.458,7.147,54.2,6.0622,3.0,222.0,18.7,396.9,5.33]    |36.2 |\n",
            "|[0.02985,0.0,2.18,0.0,0.458,6.43,58.7,6.0622,3.0,222.0,18.7,394.12,5.21]    |28.7 |\n",
            "|[0.08829,12.5,7.87,0.0,0.524,6.012,66.6,5.5605,5.0,311.0,15.2,395.6,12.43]  |22.9 |\n",
            "|[0.14455,12.5,7.87,0.0,0.524,6.172,96.1,5.9505,5.0,311.0,15.2,396.9,19.15]  |27.1 |\n",
            "|[0.21124,12.5,7.87,0.0,0.524,5.631,100.0,6.0821,5.0,311.0,15.2,386.63,29.93]|16.5 |\n",
            "|[0.17004,12.5,7.87,0.0,0.524,6.004,85.9,6.5921,5.0,311.0,15.2,386.71,17.1]  |18.9 |\n",
            "|[0.22489,12.5,7.87,0.0,0.524,6.377,94.3,6.3467,5.0,311.0,15.2,392.52,20.45] |15.0 |\n",
            "|[0.11747,12.5,7.87,0.0,0.524,6.009,82.9,6.2267,5.0,311.0,15.2,396.9,13.27]  |18.9 |\n",
            "|[0.09378,12.5,7.87,0.0,0.524,5.889,39.0,5.4509,5.0,311.0,15.2,390.5,15.71]  |21.7 |\n",
            "|[0.62976,0.0,8.14,0.0,0.538,5.949,61.8,4.7075,4.0,307.0,21.0,396.9,8.26]    |20.4 |\n",
            "|[0.63796,0.0,8.14,0.0,0.538,6.096,84.5,4.4619,4.0,307.0,21.0,380.02,10.26]  |18.2 |\n",
            "|[0.62739,0.0,8.14,0.0,0.538,5.834,56.5,4.4986,4.0,307.0,21.0,395.62,8.47]   |19.9 |\n",
            "|[1.05393,0.0,8.14,0.0,0.538,5.935,29.3,4.4986,4.0,307.0,21.0,386.85,6.58]   |23.1 |\n",
            "|[0.7842,0.0,8.14,0.0,0.538,5.99,81.7,4.2579,4.0,307.0,21.0,386.75,14.67]    |17.5 |\n",
            "|[0.80271,0.0,8.14,0.0,0.538,5.456,36.6,3.7965,4.0,307.0,21.0,288.99,11.69]  |20.2 |\n",
            "|[0.7258,0.0,8.14,0.0,0.538,5.727,69.5,3.7965,4.0,307.0,21.0,390.95,11.28]   |18.2 |\n",
            "+----------------------------------------------------------------------------+-----+\n",
            "only showing top 20 rows\n",
            "\n",
            "RMSE :  4.415130561299227\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xdg-5L1S4CK_",
        "outputId": "91d72bc5-5610-4146-d1ac-24c53b4a74d1"
      },
      "source": [
        "from pyspark.sql import SparkSession\n",
        "from pyspark.ml.feature import StringIndexer, VectorAssembler\n",
        "from pyspark.ml.regression import LinearRegression\n",
        "from pyspark.ml.evaluation import RegressionEvaluator\n",
        "from pyspark.ml.tuning import CrossValidator,ParamGridBuilder\n",
        "spark = SparkSession.Builder().appName('Spark DataFrame Introduction').getOrCreate()\n",
        "houseDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','true')\\\n",
        "                        .option('delimiter',',').csv('boston-house-prices.txt')\n",
        "houseDF = houseDF.withColumnRenamed('medv','label')\n",
        "#houseDF.describe().show()\n",
        "vec = VectorAssembler(inputCols=houseDF.columns[0:-1],outputCol='features')\n",
        "houseDF=vec.transform(houseDF)\n",
        "\n",
        "houseDF.show()\n",
        "\n",
        "houseDF = houseDF.select('features','label')\n",
        "trainDF, testDF = houseDF.randomSplit([0.8,0.2],seed=123)\n",
        "\n",
        "\n",
        "lr = LinearRegression()\n",
        "eva = RegressionEvaluator(metricName='rmse')\n",
        "myParams = ParamGridBuilder().addGrid(lr.maxIter,[5,6,7])\\\n",
        "                    .addGrid(lr.elasticNetParam,[0.1,0.2,0.3]).build()\n",
        "validator = CrossValidator(parallelism=4, \n",
        "                                 estimator=lr,\n",
        "                                 estimatorParamMaps=myParams,\n",
        "                                 evaluator=eva,\n",
        "                                 numFolds=5)\n",
        "\n",
        "model = validator.fit(trainDF)\n",
        "print(\"Best Model hesaplandı\")\n",
        "print(\"Best maxIter : \", model.bestModel.getMaxIter())\n",
        "print(\"Best elasticNetParam : \", model.bestModel.getElasticNetParam())\n",
        "\n",
        "resultDF = model.transform(testDF)\n",
        "result = eva.evaluate(resultDF)\n",
        "print(\"Accuracy of the best model \",result)"
      ],
      "execution_count": 80,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+-------+----+-----+----+-----+-----+-----+------+---+---+-------+------+-----+-----+--------------------+\n",
            "|   crim|  zn|indus|chas|  nox|   rm|  age|   dis|rad|tax|ptratio|     b|lstat|label|            features|\n",
            "+-------+----+-----+----+-----+-----+-----+------+---+---+-------+------+-----+-----+--------------------+\n",
            "|0.00632|18.0| 2.31|   0|0.538|6.575| 65.2|  4.09|  1|296|   15.3| 396.9| 4.98| 24.0|[0.00632,18.0,2.3...|\n",
            "|0.02731| 0.0| 7.07|   0|0.469|6.421| 78.9|4.9671|  2|242|   17.8| 396.9| 9.14| 21.6|[0.02731,0.0,7.07...|\n",
            "|0.02729| 0.0| 7.07|   0|0.469|7.185| 61.1|4.9671|  2|242|   17.8|392.83| 4.03| 34.7|[0.02729,0.0,7.07...|\n",
            "|0.03237| 0.0| 2.18|   0|0.458|6.998| 45.8|6.0622|  3|222|   18.7|394.63| 2.94| 33.4|[0.03237,0.0,2.18...|\n",
            "|0.06905| 0.0| 2.18|   0|0.458|7.147| 54.2|6.0622|  3|222|   18.7| 396.9| 5.33| 36.2|[0.06905,0.0,2.18...|\n",
            "|0.02985| 0.0| 2.18|   0|0.458| 6.43| 58.7|6.0622|  3|222|   18.7|394.12| 5.21| 28.7|[0.02985,0.0,2.18...|\n",
            "|0.08829|12.5| 7.87|   0|0.524|6.012| 66.6|5.5605|  5|311|   15.2| 395.6|12.43| 22.9|[0.08829,12.5,7.8...|\n",
            "|0.14455|12.5| 7.87|   0|0.524|6.172| 96.1|5.9505|  5|311|   15.2| 396.9|19.15| 27.1|[0.14455,12.5,7.8...|\n",
            "|0.21124|12.5| 7.87|   0|0.524|5.631|100.0|6.0821|  5|311|   15.2|386.63|29.93| 16.5|[0.21124,12.5,7.8...|\n",
            "|0.17004|12.5| 7.87|   0|0.524|6.004| 85.9|6.5921|  5|311|   15.2|386.71| 17.1| 18.9|[0.17004,12.5,7.8...|\n",
            "|0.22489|12.5| 7.87|   0|0.524|6.377| 94.3|6.3467|  5|311|   15.2|392.52|20.45| 15.0|[0.22489,12.5,7.8...|\n",
            "|0.11747|12.5| 7.87|   0|0.524|6.009| 82.9|6.2267|  5|311|   15.2| 396.9|13.27| 18.9|[0.11747,12.5,7.8...|\n",
            "|0.09378|12.5| 7.87|   0|0.524|5.889| 39.0|5.4509|  5|311|   15.2| 390.5|15.71| 21.7|[0.09378,12.5,7.8...|\n",
            "|0.62976| 0.0| 8.14|   0|0.538|5.949| 61.8|4.7075|  4|307|   21.0| 396.9| 8.26| 20.4|[0.62976,0.0,8.14...|\n",
            "|0.63796| 0.0| 8.14|   0|0.538|6.096| 84.5|4.4619|  4|307|   21.0|380.02|10.26| 18.2|[0.63796,0.0,8.14...|\n",
            "|0.62739| 0.0| 8.14|   0|0.538|5.834| 56.5|4.4986|  4|307|   21.0|395.62| 8.47| 19.9|[0.62739,0.0,8.14...|\n",
            "|1.05393| 0.0| 8.14|   0|0.538|5.935| 29.3|4.4986|  4|307|   21.0|386.85| 6.58| 23.1|[1.05393,0.0,8.14...|\n",
            "| 0.7842| 0.0| 8.14|   0|0.538| 5.99| 81.7|4.2579|  4|307|   21.0|386.75|14.67| 17.5|[0.7842,0.0,8.14,...|\n",
            "|0.80271| 0.0| 8.14|   0|0.538|5.456| 36.6|3.7965|  4|307|   21.0|288.99|11.69| 20.2|[0.80271,0.0,8.14...|\n",
            "| 0.7258| 0.0| 8.14|   0|0.538|5.727| 69.5|3.7965|  4|307|   21.0|390.95|11.28| 18.2|[0.7258,0.0,8.14,...|\n",
            "+-------+----+-----+----+-----+-----+-----+------+---+---+-------+------+-----+-----+--------------------+\n",
            "only showing top 20 rows\n",
            "\n",
            "Best Model hesaplandı\n",
            "Best maxIter :  5\n",
            "Best elasticNetParam :  0.1\n",
            "Accuracy of the best model  3.952468980908394\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ntHtglon9vA9"
      },
      "source": [
        "#SparkML-Clustering"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "1CBRd2cd-V4h",
        "outputId": "24d509ca-cf3d-4ed7-d64b-37109497a34d"
      },
      "source": [
        "from pyspark.sql import SparkSession\n",
        "from pyspark.ml.feature import VectorAssembler\n",
        "from pyspark.ml.clustering import KMeans\n",
        "from pyspark.ml.evaluation import ClusteringEvaluator\n",
        "\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "spark = SparkSession.Builder().appName('SparkML Clustering').getOrCreate()\n",
        "pointsDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','false')\\\n",
        "                        .option('delimiter',',').csv('points.txt')\n",
        "\n",
        "\n",
        "pointsDF = pointsDF.withColumnRenamed('_c0','x')\n",
        "pointsDF = pointsDF.withColumnRenamed('_c1','y')\n",
        "\n",
        "\n",
        "\n",
        "pointsDF.show()\n",
        "\n",
        "#pointsDF.count() # 40 cıktı\n",
        "\n",
        "\n",
        "\n",
        "vec = VectorAssembler(inputCols=['x','y'],outputCol='features')\n",
        "pointsDF = vec.transform(pointsDF)\n",
        "\n",
        "k = 4\n",
        "km = KMeans().setK(k)\n",
        "model = km.fit(pointsDF)\n",
        "print(model.clusterCenters())\n",
        "\n",
        "pointsDF = model.transform(pointsDF) #Her noktayı bulmuş olduğumuz cluster center a atamasını yapıyoruz\n",
        "\n",
        "pointsDF.show()\n",
        "\n",
        "\n",
        "for a in range(k):\n",
        "    x = pointsDF.filter(pointsDF['prediction']==a).select('x').rdd.flatMap(lambda x : list(x)).collect()\n",
        "    y = pointsDF.filter(pointsDF['prediction']==a).select('y').rdd.flatMap(lambda x : list(x)).collect()\n",
        "    plt.scatter(x,y)\n",
        "\n",
        "#x = pointsDF.toPandas()['x']\n",
        "#y = pointsDF.toPandas()['y']\n",
        "\n",
        "\n",
        "centers = model.clusterCenters()\n",
        "\n",
        "for x in range(k):\n",
        "    plt.scatter(centers[x][0],centers[x][1])\n",
        "plt.show()\n",
        "\n",
        "eva = ClusteringEvaluator()\n",
        "silhouetteValue = eva.evaluate(pointsDF)\n",
        "print(\"Silhouette :\",silhouetteValue)\n"
      ],
      "execution_count": 58,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+---+---+\n",
            "|  x|  y|\n",
            "+---+---+\n",
            "|  5|  5|\n",
            "| 15| 17|\n",
            "| 15| 16|\n",
            "|  5|  6|\n",
            "|  6|  6|\n",
            "| 16| 16|\n",
            "|  8|  8|\n",
            "|  7|  6|\n",
            "| 17| 17|\n",
            "|  6|  7|\n",
            "|  7|  8|\n",
            "| 18| 18|\n",
            "|  8|  6|\n",
            "| 17| 16|\n",
            "|  5|  8|\n",
            "| 17| 16|\n",
            "|  8|  8|\n",
            "| 16| 17|\n",
            "| 17| 18|\n",
            "| 18| 16|\n",
            "+---+---+\n",
            "only showing top 20 rows\n",
            "\n",
            "[array([6.5       , 6.91666667]), array([16.6,  6.7]), array([ 6.5  , 16.625]), array([16.6, 16.7])]\n",
            "+---+---+-----------+----------+\n",
            "|  x|  y|   features|prediction|\n",
            "+---+---+-----------+----------+\n",
            "|  5|  5|  [5.0,5.0]|         0|\n",
            "| 15| 17|[15.0,17.0]|         3|\n",
            "| 15| 16|[15.0,16.0]|         3|\n",
            "|  5|  6|  [5.0,6.0]|         0|\n",
            "|  6|  6|  [6.0,6.0]|         0|\n",
            "| 16| 16|[16.0,16.0]|         3|\n",
            "|  8|  8|  [8.0,8.0]|         0|\n",
            "|  7|  6|  [7.0,6.0]|         0|\n",
            "| 17| 17|[17.0,17.0]|         3|\n",
            "|  6|  7|  [6.0,7.0]|         0|\n",
            "|  7|  8|  [7.0,8.0]|         0|\n",
            "| 18| 18|[18.0,18.0]|         3|\n",
            "|  8|  6|  [8.0,6.0]|         0|\n",
            "| 17| 16|[17.0,16.0]|         3|\n",
            "|  5|  8|  [5.0,8.0]|         0|\n",
            "| 17| 16|[17.0,16.0]|         3|\n",
            "|  8|  8|  [8.0,8.0]|         0|\n",
            "| 16| 17|[16.0,17.0]|         3|\n",
            "| 17| 18|[17.0,18.0]|         3|\n",
            "| 18| 16|[18.0,16.0]|         3|\n",
            "+---+---+-----------+----------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAAD4CAYAAAD1jb0+AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAVRElEQVR4nO3df4xd9Xnn8ffD2GDjBjvEQwwB6gQFNxtifmTSkGhpCs4apMQ1qpYsaKOQJhVSVmqMGxGFloJd8UcUsuu6itTKCtSwiUwAuaau1RoWsQUp4GrAYJOCYUMp+Fc8LGsncW2wx8/+ce8Ye5jruWfm3rn363m/JDT3PnPzPY9OZj4+c865z43MRJJUnlM63YAkaWwMcEkqlAEuSYUywCWpUAa4JBVqykRubPbs2Tl37tyJ3KQkFe+ZZ555MzN7h9cnNMDnzp1Lf3//RG5SkooXEf82Ut1TKJJUKANckgplgEtSoQxwSSqUAS5JhRo1wCPinojYExEvHFO7JCKejojnIqI/In67XQ1ueHUDCx9ayPx757PwoYVseHVDuzZVlDufvpOL77uYT9z7CS6+72LufPrOTrckFWnf+vW8ctUCXvzYf+CVqxawb/36YtZv5gh8NXDNsNr3gOWZeQlwe/15y214dQPLfrqMXft3kSS79u9i2U+XTfoQv/PpO/nJtp9wJI8AcCSP8JNtPzHEpYr2rV/Prj+7ncM7d0Imh3fuZNef3d6ykG33+qMGeGY+Abw1vAycUX88E9jZkm6GWfnsSg4OHjyudnDwICufXdmOzRXjwZcfrFSXNLI9K/6CPHh8xuTBg+xZ8RdFrD/WN/LcDGyMiO9T+0fgs41eGBE3ATcBnH/++ZU2snv/7kr1yWLoyLvZuqSRHd61q1K929Yf60XMbwBLM/M8YClwd6MXZuaqzOzLzL7e3ve8E/SE5syYU6k+WZwSI//f1qguaWRTzj67Ur3b1h/rb/yNwNr64weBtlzEXHLZEqb1TDuuNq1nGksuW9KOzRXjuguvq1SXNLKzlt5MTDs+Y2LaNM5aenMR64/1FMpO4HPA/wauAl5pSTfDfOEjXwBq58J379/NnBlzWHLZkqP1yeq2y28Daue8j+QRTolTuO7C647WJTVn5qJFQO1c9eFdu5hy9tmctfTmo/VuXz9G+0zMiFgD/C4wG/gFcAewDVhJ7R+Ag8B/y8xnRttYX19fOsxKkqqJiGcys294fdQj8My8ocG3PjnuriRJY+ZVL0kqlAEuSYUywCWpUAa4JBXKAJekQhngklQoA1zSuJU8krXdvbfThH4qvaSTz9DI1KGpe0MjU4GWvOOwneu3u/d28whc0riUPJK13b23mwEuaVxKHsna7t7bzQCXNC4lj2Rtd+/tZoBLGpduHsm6ZcsWVqxYwbJly1ixYgVbtmxp2drdwIuYFezfvIdfbnyNwb1v0zPrNM64ei4zLj2r021JHdWtI1m3bNnC+vXrOXToEAD79u1jff0Ok/nz509I7+026jjZVip5nOz+zXvYu/YV8tC7H1sWU09h1u9/1BCXutCKFSvYt2/fe+ozZ85k6dKlHeho7BqNk/UUSpN+ufG148IbIA8d4ZcbX+tMQ5JOaKTwPlG9RAZ4kwb3vl2pLqmzZs6cWaleIgO8ST2zTqtUl9RZCxYsYOrUqcfVpk6dyoIFCzrUUesZ4E064+q5xNTjd1dMPYUzrp7bmYYkndD8+fNZtGjR0SPumTNnsmjRoqMXME8G3oXSpKELld6FIpVj/vz5J1VgD2eAVzDj0rMMbEldw1MoklSoUQM8Iu6JiD0R8cKw+h9FxEsR8bOI+F77WpQkjaSZI/DVwDXHFiLiSmAxcHFmfhz4futbq9nw6gYWPrSQ+ffOZ+FDC9nw6oZi1i91bamqkueB71q+nBc/fhEv/tbHePHjF7Fr+fKWrQ3t7X3Uc+CZ+UREzB1W/gbw3cx8u/6aPS3r6BgbXt3Asp8u4+Bgbdzjrv27WPbTZQB84SNf6Or1S11bqqrkeeC7li9n75r73y0MDh59fvYdd4xrbWj/vhnrOfALgSsiYlNE/FNEfGrcnYxg5bMrj4bUkIODB1n57MquX7/UtaWqSp4HvveBByvVq2r3vhnrXShTgDOBy4FPAQ9ExEdyhMEqEXETcBPA+eefX2kju/fvrlSvqp3rl7q2VFXJ88AZHKxWr6jd+2asR+DbgbVZ88/AEWD2SC/MzFWZ2ZeZfb29vZU2MmfGnEr1qtq5fqlrS1WVPA+cnp5q9YravW/GGuDrgCsBIuJC4FTgzZZ0dIwlly1hWs/xs3qn9UxjyWVLun79UteWqurmeeCjmfWl6yrVq2r3vhn1FEpErAF+F5gdEduBO4B7gHvqtxa+A9w40umT8Rq6ILfy2ZXs3r+bOTPmsOSyJS27UNfO9UtdW6qqW+eBN2PoQuXeBx6snTbp6WHWl65ryQVMaP++cR64JHU554FL0knGAJekQhngklQoA1ySCmWAS1KhDHBJKpQBLkmF6voAd2yqJI2sqz9SzbGpktRYVx+BOzZVkhrr6gB3bKokNdbVAe7YVElqrKsD3LGpktRYV1/EdGyqJDXW1QEOtRA3sCXpvbr6FIokqTEDXJIKZYBLUqEMcEkqlAEuSYUywCWpUAa4JBVq1ACPiHsiYk9EvDDC974VERkRs9vTniSpkWaOwFcD1wwvRsR5wELg9Rb3JElqwqgBnplPAG+N8K0VwLeBbHVTkqTRjekceEQsBnZk5vNNvPamiOiPiP6BgYGxbE6SNILKAR4RpwN/AtzezOszc1Vm9mVmX29vb9XNSZIaGMsR+AXAh4HnI+I14Fzg2YhwSLckTaDK0wgzcytw1tDzeoj3ZeabLexLkjSKZm4jXAM8BcyLiO0R8fX2tyVJGs2oR+CZecMo35/bsm4kSU3znZiSVCgDXJIKZYBLUqEMcEkqlAEuSYUywCWpUAa4JBXKAJekQhngklQoA1ySCmWAS1KhDHBJKpQBLkmFMsAlqVAGuCQVygCXpEIZ4JJUKANckgplgEtSoQxwSSqUAS5JhRo1wCPinojYExEvHFO7KyJeiogtEfG3ETGrvW1KkoZr5gh8NXDNsNqjwEWZOR94Gbi1xX1JkkYxaoBn5hPAW8Nqj2Tm4frTp4Fz29CbJOkEWnEO/GvAPzT6ZkTcFBH9EdE/MDDQgs1JkmCcAR4RfwocBn7c6DWZuSoz+zKzr7e3dzybkyQdY8pY/4cR8VXgi8CCzMyWdSRJasqYAjwirgG+DXwuM/+9tS1JkprRzG2Ea4CngHkRsT0ivg78AHgf8GhEPBcRf93mPiVJw4x6BJ6ZN4xQvrsNvUiSKvCdmJJUKANckgplgEtSoQxwSSqUAS5JhTLAJalQBrgkFcoAl6RCGeCSVCgDXJIKZYBLUqEMcEkqlAEuSYUywCWpUAa4JBXKAJekQhngklQoA1ySCmWAS1KhDHBJKpQBLkmFMsAlqVCjBnhE3BMReyLihWNqZ0bEoxHxSv3r+9vbpiRpuGaOwFcD1wyrfQd4LDM/CjxWfy5JmkCjBnhmPgG8Nay8GLi3/vhe4NoW9yVJGsVYz4F/MDN31R/vBj7Y6IURcVNE9EdE/8DAwBg3J0kabtwXMTMzgTzB91dlZl9m9vX29o53c5KkurEG+C8i4myA+tc9rWtJktSMsQb43wE31h/fCDzcmnYkSc1q5jbCNcBTwLyI2B4RXwe+C/yniHgF+Hz9uSRpAk0Z7QWZeUODby1ocS+SpAp8J6YkFcoAl6RCGeCSVCgDXJIKZYBLUqEMcEkqlAEuSYUywCWpUAa4JBXKAJekQhngklQoA1ySCmWAS1KhDHBJKpQBLkmFMsAlqVAGuCQVygCXpEIZ4JJUKANckgplgEtSoUb9VPoTiYilwB8CCWwF/iAzD7aisSHrNu/gro3b2Ln3AOfMms4tV8/j2ks/1MpNtM1t67ayZtMbDGbSE8ENnz6PO6/9RNevLU0qWx6Ax/4c9m2HmefCgtth/peKWH/MR+AR8SHgm0BfZl4E9ADXt6SrunWbd3Dr2q3s2HuABHbsPcCta7eybvOOVm6mLW5bt5UfPf06g5kADGbyo6df57Z1W7t6bWlS2fIArP8m7HsDyNrX9d+s1QtYf7ynUKYA0yNiCnA6sHP8Lb3rro3bOHBo8LjagUOD3LVxWys30xZrNr1Rqd4ta0uTymN/DocOHF87dKBWL2D9MQd4Zu4Avg+8DuwC9mXmI8NfFxE3RUR/RPQPDAxU2sbOvQcq1bvJ0NFxs/VuWVuaVPZtr1bvsvXHcwrl/cBi4MPAOcCMiPjy8Ndl5qrM7MvMvt7e3krbOGfW9Er1btITUaneLWtLk8rMc6vVu2z98ZxC+Tzwr5k5kJmHgLXAZ1vSVd0tV89j+tSe42rTp/Zwy9XzWrmZtrjh0+dVqnfL2tKksuB2mDrsgHDq9Fq9gPXHcxfK68DlEXE6cABYAPS3pKu6obtNSrwLZeiOkHbcKdLOtaVJZehukHbdhdLm9SPHcd40IpYD/wU4DGwG/jAz3270+r6+vuzvb2nGS9JJLyKeycy+4fVx3QeemXcAd4xnDUnS2PhOTEkqlAEuSYUywCWpUAa4JBXKAJekQhngklQoA1ySCjWu+8BPBs7Vllqg4Jnabe+9jSZ1gA/N1R4yNFcbGDHEX960m6ce/jm/futtfuPM0/jM4gu48NNzJqxfqSsNzbweGps6NPMaWhOE7Vy/3b232aQ+hVJlrvbLm3bz+I9f4tdv1SYF/Pqtt3n8xy/x8qbdbe1R6nolz9Rud+9tNqkDvMpc7ace/jmH3zlyXO3wO0d46uGft6U3qRglz9Rud+9tNqlPofREjBjWI83VHjrybrYuTRozz61/ZNgI9Q6v/+KTj/Pk/ffxq//7Ju/7wGyuuP4rfOyKK1uydjeY1EfgVeZq/8aZp4342kZ1adLo0pnaLz75OI+s+gG/enMAMvnVmwM8suoHvPjk4xPXe5tN6gC/89pP8OXLzz96xN0TwZcvP3/EC5ifWXwBU049fndNOfUUPrP4ggnpVepa878Ei/4SZp4HRO3ror9s7UztMaz/5P33cfid4/9CPvzO2zx5/30T13ubjWseeFWlzwP3LhSpHP/9+kUwUr5F8K371098Q+PQlnngk82Fn55jYEuFeN8HZtdOn4xQP1lM6lMokk5eV1z/Faacevw1qimnnsYV13+lQx21nkfgkk5KQ3ebnPAulMIZ4JJOWh+74sqTKrCH8xSKJBXKAJekQhngklSocZ0Dj4hZwA+Bi4AEvpaZT7WisSHrNu/gro3b2Ln3AOfMms4tV8/j2ks/VMT6pa4tVVbyONm//2N4ZjXkIEQPfPKr8MX/0Zq1oa29j/ci5krgHzPzP0fEqcDpLejpqHWbd3Dr2q0cODQIwI69B7h17VaAloRVO9cvdW2pspLHyf79H0P/3e8+z8F3n7cixNu8b8Z8CiUiZgK/A9wNkJnvZObecXd0jLs2bjsaUkMOHBrkro3bun79UteWKit5nOwzq6vVq2rzvhnPOfAPAwPA30TE5oj4YUTMGP6iiLgpIvojon9g4L3vijqRnXsPVKpX1c71S11bqqzkcbI5WK1eVZv3zXgCfApwGfBXmXkpsB/4zvAXZeaqzOzLzL7e3t5KGzhn1vRK9arauX6pa0uVNRq92spxsu1aP3qq1atq874ZT4BvB7Zn5qb684eoBXrL3HL1PKZPPX5HTp/awy1Xz+v69UtdW6qsS8fJNuWTX61Wr6rN+2bMFzEzc3dEvBER8zJzG7AA+JeWdFU3dEGuXXdbtHP9UteWKhu6GNeuu0Tauf7Qhcp23YXS5n0zrnGyEXEJtdsITwVeBf4gM/9fo9eXPk5WkjqhLeNkM/M54D2LSpLaz3diSlKhDHBJKpQBLkmFMsAlqVAGuCQVygCXpEJ1/UeqOTZVkkbW1QHu2FRJaqyrT6E4NlWSGuvqAHdsqiQ11tUB7thUSWqsqwPcsamS1FhXX8R0bKokNdbVAQ61EDewJem9uvoUiiSpMQNckgplgEtSoQxwSSqUAS5JhRrXhxpX3ljEAPBvE7bBamYDb3a6iTEotW+w906x984YT++/mZm9w4sTGuDdLCL6R/rU525Xat9g751i753Rjt49hSJJhTLAJalQBvi7VnW6gTEqtW+w906x985oee+eA5ekQnkELkmFMsAlqVCTPsAjYlZEPBQRL0XEixHxmU731KyIWBoRP4uIFyJiTURM63RPjUTEPRGxJyJeOKZ2ZkQ8GhGv1L++v5M9NtKg97vqPzNbIuJvI2JWJ3tsZKTej/netyIiI2J2J3o7kUZ9R8Qf1ff7zyLie53q70Qa/LxcEhFPR8RzEdEfEb/dim1N+gAHVgL/mJm/BVwMvNjhfpoSER8Cvgn0ZeZFQA9wfWe7OqHVwDXDat8BHsvMjwKP1Z93o9W8t/dHgYsycz7wMnDrRDfVpNW8t3ci4jxgIfD6RDfUpNUM6zsirgQWAxdn5seB73egr2as5r37/HvA8sy8BLi9/nzcJnWAR8RM4HeAuwEy853M3NvZriqZAkyPiCnA6cDODvfTUGY+Abw1rLwYuLf++F7g2gltqkkj9Z6Zj2Tm4frTp4FzJ7yxJjTY7wArgG8DXXkXQ4O+vwF8NzPfrr9mz4Q31oQGvSdwRv3xTFr0uzqpAxz4MDAA/E1EbI6IH0bEjE431YzM3EHtCOR1YBewLzMf6WxXlX0wM3fVH+8GPtjJZsbha8A/dLqJZkXEYmBHZj7f6V4quhC4IiI2RcQ/RcSnOt1QBTcDd0XEG9R+b1vyF9tkD/ApwGXAX2XmpcB+uvfP+OPUzxcvpvaP0DnAjIj4cme7Grus3c/alUeDJxIRfwocBn7c6V6aERGnA39C7c/40kwBzgQuB24BHoiI6GxLTfsGsDQzzwOWUv+rf7wme4BvB7Zn5qb684eoBXoJPg/8a2YOZOYhYC3w2Q73VNUvIuJsgPrXrvyTuJGI+CrwReC/ZjlvqLiA2j/6z0fEa9RO/TwbEXM62lVztgNrs+afgSPUBkSV4EZqv6MADwJexByvzNwNvBERQx9zvwD4lw62VMXrwOURcXr9KGQBhVyAPcbfUfvBpv714Q72UklEXEPtHPLvZea/d7qfZmXm1sw8KzPnZuZcaqF4Wf13odutA64EiIgLgVMpZzLhTuBz9cdXAa+0ZNXMnNT/AZcA/cAWaj8g7+90TxV6Xw68BLwA/E/gtE73dIJe11A7V3+IWmh8HfgAtbtPXgH+F3Bmp/us0Pv/Ad4Anqv/99ed7rPZ3od9/zVgdqf7bHKfnwr8qP7z/ixwVaf7rND7fwSeAZ4HNgGfbMW2fCu9JBVqUp9CkaSSGeCSVCgDXJIKZYBLUqEMcEkqlAEuSYUywCWpUP8fam5AQ0xw9gAAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          }
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Silhouette : 0.9464374120400001\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dQ7rKjAoJpF8",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "47c94996-91cb-4e18-f084-6b1ef5514b1c"
      },
      "source": [
        "import nltk\n",
        "nltk.download('vader_lexicon')\n",
        "\n",
        "\n",
        "from nltk.sentiment.vader import SentimentIntensityAnalyzer\n",
        "\n",
        "cumle1 = \"What a nice weather\"\n",
        "cumle2 = \"Worst movie ever seen \"\n",
        "cumle3 = \"not a bad movie\"\n",
        "sIntensityAnalyzer = SentimentIntensityAnalyzer()\n",
        "\n",
        "print(sIntensityAnalyzer.polarity_scores(cumle1))  # compound 0 dan küçükse olumsuz anlam 0'dan büyükse olumlu 0 'a yakınsa nötr böle kişiler bulunabiliyor\n",
        "print(sIntensityAnalyzer.polarity_scores(cumle2))\n",
        "print(sIntensityAnalyzer.polarity_scores(cumle3)) #ironik cümlleri anlayamaz"
      ],
      "execution_count": 59,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[nltk_data] Downloading package vader_lexicon to /root/nltk_data...\n",
            "{'neg': 0.0, 'neu': 0.417, 'pos': 0.583, 'compound': 0.4215}\n",
            "{'neg': 0.577, 'neu': 0.423, 'pos': 0.0, 'compound': -0.6249}\n",
            "{'neg': 0.0, 'neu': 0.412, 'pos': 0.588, 'compound': 0.431}\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/nltk/twitter/__init__.py:20: UserWarning: The twython library has not been installed. Some functionality from the twitter package will not be available.\n",
            "  warnings.warn(\"The twython library has not been installed. \"\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jb_WvLmTWdzn",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "71838273-21e4-477b-fd2a-6328b9e4772c"
      },
      "source": [
        "trCumle =\"kötü film\" #türkçe olan seyleri 0 olarak algılıyor , ingilizce haric belki başka dilleri destekliyordur\n",
        "#itu nlp turkce olarak çalışıyor, 2. kutuphane zemberek kutuphanesi\n",
        "\n",
        "print(trCumle, sIntensityAnalyzer.polarity_scores(trCumle))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "kötü film {'neg': 0.0, 'neu': 1.0, 'pos': 0.0, 'compound': 0.0}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "a-WnQDXeKbed",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "ace0feb3-80de-4c51-fc9b-66dac7b721ab"
      },
      "source": [
        "from pyspark.sql import SparkSession  \n",
        "from pyspark.ml.feature import Tokenizer,CountVectorizer\n",
        "from pyspark.ml.classification import RandomForestClassifier\n",
        "from pyspark.ml.evaluation import BinaryClassificationEvaluator #label olarak datasetinde sadece 0,1 var, bu yüzden binary kullanıyoruz\n",
        "\n",
        "spark = SparkSession.Builder().appName('SparkML Sentiment Analysis').getOrCreate()\n",
        "yorumDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','false')\\\n",
        "                        .option('delimiter','\\t').csv('movie_turkish_train.txt')\n",
        "yorumDF = yorumDF.withColumnRenamed('_c0','cumle')\n",
        "yorumDF = yorumDF.withColumnRenamed('_c1','label')\n",
        "\n",
        "#yorumDF.show(100) \n",
        "\n",
        "\n",
        "#textde ilk işimiz tokenize etmek\n",
        "#yorumDF.show(truncate=False)\n",
        "\n",
        "\n",
        "tknzr = Tokenizer(inputCol='cumle',outputCol='tokenized')\n",
        "yorumDF = tknzr.transform(yorumDF)\n",
        "\n",
        "#yorumDF.show() \n",
        "\n",
        "\n",
        "vectorizer = CountVectorizer(inputCol='tokenized',outputCol='features')\n",
        "vecModel = vectorizer.fit(yorumDF)\n",
        "yorumDF = vecModel.transform(yorumDF)\n",
        "\n",
        "yorumDF = yorumDF.select('features','label')\n",
        "trainDF,testDF = yorumDF.randomSplit([0.75,0.25],seed=1234)\n",
        "\n",
        "#yorumDF.show() \n",
        "\n",
        "\n",
        "classifer = RandomForestClassifier()\n",
        "\n",
        "#model = classifer.fit(yorumDF) ## Bir datascientist in yapabileceği en gerizekalı hareket\n",
        "\n",
        "model = classifer.fit(trainDF)\n",
        "sonucDF = model.transform(testDF)\n",
        "\n",
        "sonucDF.show()\n",
        "\n",
        "eva = BinaryClassificationEvaluator()\n",
        "basari = eva.evaluate(sonucDF)\n",
        "print(basari)"
      ],
      "execution_count": 61,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----+\n",
            "|cumle                                                                                                                                                                                                                                                                            |label|\n",
            "+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----+\n",
            "|bunu da gördük uzay mekigi yakitindan uzayliya dönüsen insan,begenmedim tavsiye de etmiyorum..                                                                                                                                                                                   |0    |\n",
            "|filmi begenmeyenlere aldanmayin arkadaslar herkesin film anlayisi farklidir bas yapit degil ama izlenmesi gereken bir film....                                                                                                                                                   |0    |\n",
            "|günde 1 film izleyen iyi bir film izleyicisim. gerçekten berbat ötesiydi. reklamlara ve afise inanarak gittim..                                                                                                                                                                  |0    |\n",
            "|afisine bakip aldananlardanim içerigi ve kurgusu çok basit.cadilar zamani bu filme göre daha iyi.(benden ). saygilarimla....                                                                                                                                                     |0    |\n",
            "|sadece insanlarin sinirini bozmak için yapilmis bir film yine de izlemek isteyenler kendileri bilir.                                                                                                                                                                             |0    |\n",
            "|ucuz bir aksiyon filmi.4 puani hakeden bir yapim ancak final sahnesi sebebiyle 6 verdim.filmde senaryo neredeyse yok finalde bu durumu kurtarmaya çalismislar.sinemasal açidan önemsiz bir yapim ancak sadece aksiyon filmi izlemek istiyorum diyorsaniz buyrun afiyet olsun.... |0    |\n",
            "|olmamis diyor puanimizi düsük tutuyoruz. televizyonda seyredilebilir ancak. .                                                                                                                                                                                                    |0    |\n",
            "|kesinlikle çok kötüydü. tam bir gereksizlik döngüsü gibi. kim ölcek kim kalcak zaten apaçik ortada. sadece gereksiz bi vahset seven o film izledigini sanip hiçbir sey izlemeyenlere yönelik olmus. maalesefki ebnde izledim iste..                                              |0    |\n",
            "|ben nasil bir film ya nerdeyse 2 senedir 2009 da girecek yaziyor ama 2011 e girdik hala vizyona girecek yaziyor inanilmaz gerçekten..                                                                                                                                            |0    |\n",
            "|bu yüzler fazla tanidik oldu! klasik türk filmi!!! biri hasta olur iyilestirmek için para bulmalari gerekir ve bu yüzden de kötülük yapmak zorundadirlar... korkunc!.                                                                                                            |0    |\n",
            "|. çok ilginç yaa kendimde bir sorunmu var diye düsünüyorum film de bukadar harika olan nedir anlayamadim gercekten. ilk bölüm acayip sikici ikincisinde bir iki sahne fena degil o kadar ....                                                                                    |0    |\n",
            "|bence bu film hiç güzel degildi su güne kadar gittigim en kötü filmdi.korku diye gittik ama ne korku ne gerilim vardi sadece vahsett:(hiç kimseye gitmesini tavsiye etmem....                                                                                                    |0    |\n",
            "|valla ben begenmedim çok bos bir film... zaman harcamaya degmez......                                                                                                                                                                                                            |0    |\n",
            "|gönül ister milyon dolarlar harcanip bu büyük destani animasyon tarzinda deilde , gerçek efektlerle o izledigimiz 300 , lord of the rings tarzi filmler gibi çeklsin , ama suanki sinema potansiyelimiz buna imkan vermez heralde.                                               |0    |\n",
            "|dün aksam bu filme gittim.film arasinda çikacaktim.tamamen zaman kaybindan baska bisey degil.amerikan milliyetçiligini öven siradan gise filmi.görsel efektler disinda izlenecek birsey yok..                                                                                    |0    |\n",
            "|cok kötü olmus.... amerikan pastasi we benzer fimlere benzetilmesi cok sacma.özellikle bi esprinin defalarca yapilmasi( sakarin sürekli düsmesi vs.)cok kötü.hç tavsiye etmem paraniza yazik!!!.                                                                                 |0    |\n",
            "|igrenç yaaa.acaba bu film dalga geçilmesi için mi yapildi.sanirim bizler hiç bir zaman korku/gerilim filmi yapamicaz.......                                                                                                                                                      |0    |\n",
            "|saçma sapan bir film... bu filme iyi diyenler malkoçoglu filmlerine niye güler anlamak zor...modern malkoçu travolta yapmislar...                                                                                                                                                |0    |\n",
            "|çok güzel komedi filmi olmus. herkese tavsiye ederim, gülme garantisi var..                                                                                                                                                                                                      |0    |\n",
            "|beklentimi yüksek tutmadan izledigim halde bir fiyaskoydu. replikler, sesler, ortam, oyunculuk berbat. korku filmi çekmek her yigdin harci degil..                                                                                                                               |0    |\n",
            "+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----+\n",
            "only showing top 20 rows\n",
            "\n",
            "+--------------------+-----+--------------------+--------------------+----------+\n",
            "|            features|label|       rawPrediction|         probability|prediction|\n",
            "+--------------------+-----+--------------------+--------------------+----------+\n",
            "|(7563,[0,1,2,3,4,...|    0|[10.0608734712590...|[0.50304367356295...|       0.0|\n",
            "|(7563,[0,1,2,3,4,...|    0|[9.78157212240486...|[0.48907860612024...|       1.0|\n",
            "|(7563,[0,1,2,3,20...|    1|[9.83527435662915...|[0.49176371783145...|       1.0|\n",
            "|(7563,[0,1,2,4,6,...|    1|[9.66554912831531...|[0.48327745641576...|       1.0|\n",
            "|(7563,[0,1,2,4,6,...|    0|[10.1266058002056...|[0.50633029001028...|       0.0|\n",
            "|(7563,[0,1,2,4,6,...|    0|[10.3641058002056...|[0.51820529001028...|       0.0|\n",
            "|(7563,[0,1,2,4,6,...|    1|[9.19782074582430...|[0.45989103729121...|       1.0|\n",
            "|(7563,[0,1,2,4,7,...|    1|[7.85256317643292...|[0.39262815882164...|       1.0|\n",
            "|(7563,[0,1,2,4,8,...|    1|[9.48379380527919...|[0.47418969026395...|       1.0|\n",
            "|(7563,[0,1,2,4,9,...|    0|[10.4055130030494...|[0.52027565015247...|       0.0|\n",
            "|(7563,[0,1,2,4,11...|    1|[9.15185756246675...|[0.45759287812333...|       1.0|\n",
            "|(7563,[0,1,2,5,14...|    0|[9.94485047716949...|[0.49724252385847...|       1.0|\n",
            "|(7563,[0,1,2,6,7,...|    1|[9.92888812072639...|[0.49644440603631...|       1.0|\n",
            "|(7563,[0,1,2,6,36...|    0|[11.6221601657904...|[0.58110800828952...|       0.0|\n",
            "|(7563,[0,1,2,8,12...|    1|[10.1544872353562...|[0.50772436176781...|       0.0|\n",
            "|(7563,[0,1,2,8,17...|    0|[9.72912019670532...|[0.48645600983526...|       1.0|\n",
            "|(7563,[0,1,2,8,28...|    1|[9.26845767082541...|[0.46342288354127...|       1.0|\n",
            "|(7563,[0,1,2,9,33...|    1|[9.40918425913449...|[0.47045921295672...|       1.0|\n",
            "|(7563,[0,1,2,11,1...|    1|[8.27846752522287...|[0.41392337626114...|       1.0|\n",
            "|(7563,[0,1,2,12,5...|    1|[9.94485047716949...|[0.49724252385847...|       1.0|\n",
            "+--------------------+-----+--------------------+--------------------+----------+\n",
            "only showing top 20 rows\n",
            "\n",
            "0.7606729822834646\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FHMnAsIpKOZ9",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "2c2a7c9c-3835-410f-b70f-2fb97ec35d2b"
      },
      "source": [
        "yeniDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','false')\\\n",
        "                        .option('delimiter','\\t').csv('deneme-yorum.txt')\n",
        "yeniDF = yeniDF.withColumnRenamed('_c0','cumle')\n",
        "yeniDF = tknzr.transform(yeniDF)\n",
        "yeniDF = vecModel.transform(yeniDF)\n",
        "yeniDF.show()\n",
        "yeniYorumSonucDF = model.transform(yeniDF)\n",
        "yeniYorumSonucDF.select('cumle','prediction').show()"
      ],
      "execution_count": 62,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+--------------------+--------------------+--------------------+\n",
            "|               cumle|           tokenized|            features|\n",
            "+--------------------+--------------------+--------------------+\n",
            "| bu film çok kötüydü|[bu, film, çok, k...|(7563,[1,2,3,350]...|\n",
            "|     berbat bir film| [berbat, bir, film]|(7563,[0,1,68],[1...|\n",
            "|bundan daha güzel...|[bundan, daha, gü...|(7563,[13,376,161...|\n",
            "| fena bir film değil|[fena, bir, film,...|(7563,[0,1,400],[...|\n",
            "|çok kötü bir film...|[çok, kötü, bir, ...|(7563,[0,1,2,19,4...|\n",
            "+--------------------+--------------------+--------------------+\n",
            "\n",
            "+--------------------+----------+\n",
            "|               cumle|prediction|\n",
            "+--------------------+----------+\n",
            "| bu film çok kötüydü|       0.0|\n",
            "|     berbat bir film|       0.0|\n",
            "|bundan daha güzel...|       1.0|\n",
            "| fena bir film değil|       1.0|\n",
            "|çok kötü bir film...|       1.0|\n",
            "+--------------------+----------+\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QRHtbApRTyi2",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "84ffbc45-0502-4bcd-d901-3806972aa045"
      },
      "source": [
        "from pyspark.sql import SparkSession\n",
        "from pyspark.ml.feature import Tokenizer,CountVectorizer\n",
        "from pyspark.ml.classification import MultilayerPerceptronClassifier\n",
        "from pyspark.ml.evaluation import BinaryClassificationEvaluator\n",
        "from pyspark.sql.functions import UserDefinedFunction\n",
        "from pyspark.sql.types import StringType\n",
        "\n",
        "spark = SparkSession.Builder().appName('SparkML Sentiment Analysis').getOrCreate()\n",
        "yorumDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','false')\\\n",
        "                        .option('delimiter','\\t').csv('movie_turkish_train.txt')\n",
        "yorumDF = yorumDF.withColumnRenamed('_c0','cumle')  #buradaki cumle aslında bir kişiden gelen yorum paragrafta olabilir\n",
        "yorumDF = yorumDF.withColumnRenamed('_c1','label')\n",
        "\n",
        "def fixCumle(cumle):\n",
        "    cumle = cumle.replace('1','bir') # gunde 1 ve bir aynı şey 1 ile bir'i replace edecek\n",
        "    #cumle = cumle.replace('.','')\n",
        "    #cumle = cumle.replace('(','')\n",
        "    #cumle = cumle.replace(')','')\n",
        "    #cumle = cumle.strip()  #cumlenin basında ve sonundaki fazlalıkları siliyor\n",
        "    #cumle = cumle.replace('filmi','film')\n",
        "    return cumle\n",
        "\n",
        "# tex'in sonunda ki 0 kötü demek satır 500'den sonra 1 olmus\n",
        "#textDF.show()\n",
        "\n",
        "#textde ilk işimiz tokenize etmek\n",
        "\n",
        "fixTextUDF = UserDefinedFunction(fixCumle,StringType())\n",
        "yorumDF = yorumDF.withColumn('cumle2',fixTextUDF('cumle'))\n",
        "\n",
        "yorumDF.show()\n",
        "\n",
        "tknzr = Tokenizer(inputCol='cumle2',outputCol='tokenized')\n",
        "yorumDF = tknzr.transform(yorumDF)\n",
        "yorumDF.show()\n",
        "\n",
        "kelimeSayisi = 3500\n",
        "vectorizer = CountVectorizer(inputCol='tokenized',outputCol='features',vocabSize=kelimeSayisi)\n",
        "vecModel = vectorizer.fit(yorumDF)\n",
        "yorumDF = vecModel.transform(yorumDF)\n",
        "\n",
        "yorumDF.show()\n",
        "#7563 tane tekil kelime var dedi unic ama mesela film filmi filmleri ayrı ayrı sayıyor insanların günlük 500 kelime ile konustugunu dusunutsek\n",
        "\n",
        "yorumDF = yorumDF.select('features','label')\n",
        "trainDF,testDF = yorumDF.randomSplit([0.75,0.25],seed=1234)\n",
        "\n",
        "\n",
        "classifer = MultilayerPerceptronClassifier(layers=[kelimeSayisi,200,2])\n",
        "\n",
        "#classifer = MultilayerPerceptronClassifier(layers=[kelimesayisi,1700,2]) #son sınıf 2 yapay sinir aglarındaki ortadaki 50 en bastaki 7563 sınıf\n",
        "\n",
        "model = classifer.fit(trainDF)\n",
        "sonucDF = model.transform(testDF)\n",
        "\n",
        "eva = BinaryClassificationEvaluator()\n",
        "basari = eva.evaluate(sonucDF)\n",
        "print(basari)"
      ],
      "execution_count": 64,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+--------------------+-----+--------------------+\n",
            "|               cumle|label|              cumle2|\n",
            "+--------------------+-----+--------------------+\n",
            "|bunu da gördük uz...|    0|bunu da gördük uz...|\n",
            "|filmi begenmeyenl...|    0|filmi begenmeyenl...|\n",
            "|günde 1 film izle...|    0|günde bir film iz...|\n",
            "|afisine bakip ald...|    0|afisine bakip ald...|\n",
            "|sadece insanlarin...|    0|sadece insanlarin...|\n",
            "|ucuz bir aksiyon ...|    0|ucuz bir aksiyon ...|\n",
            "|olmamis diyor pua...|    0|olmamis diyor pua...|\n",
            "|kesinlikle çok kö...|    0|kesinlikle çok kö...|\n",
            "|ben nasil bir fil...|    0|ben nasil bir fil...|\n",
            "|bu yüzler fazla t...|    0|bu yüzler fazla t...|\n",
            "|. çok ilginç yaa ...|    0|. çok ilginç yaa ...|\n",
            "|bence bu film hiç...|    0|bence bu film hiç...|\n",
            "|valla ben begenme...|    0|valla ben begenme...|\n",
            "|gönül ister milyo...|    0|gönül ister milyo...|\n",
            "|dün aksam bu film...|    0|dün aksam bu film...|\n",
            "|cok kötü olmus......|    0|cok kötü olmus......|\n",
            "|igrenç yaaa.acaba...|    0|igrenç yaaa.acaba...|\n",
            "|saçma sapan bir f...|    0|saçma sapan bir f...|\n",
            "|çok güzel komedi ...|    0|çok güzel komedi ...|\n",
            "|beklentimi yüksek...|    0|beklentimi yüksek...|\n",
            "+--------------------+-----+--------------------+\n",
            "only showing top 20 rows\n",
            "\n",
            "+--------------------+-----+--------------------+--------------------+\n",
            "|               cumle|label|              cumle2|           tokenized|\n",
            "+--------------------+-----+--------------------+--------------------+\n",
            "|bunu da gördük uz...|    0|bunu da gördük uz...|[bunu, da, gördük...|\n",
            "|filmi begenmeyenl...|    0|filmi begenmeyenl...|[filmi, begenmeye...|\n",
            "|günde 1 film izle...|    0|günde bir film iz...|[günde, bir, film...|\n",
            "|afisine bakip ald...|    0|afisine bakip ald...|[afisine, bakip, ...|\n",
            "|sadece insanlarin...|    0|sadece insanlarin...|[sadece, insanlar...|\n",
            "|ucuz bir aksiyon ...|    0|ucuz bir aksiyon ...|[ucuz, bir, aksiy...|\n",
            "|olmamis diyor pua...|    0|olmamis diyor pua...|[olmamis, diyor, ...|\n",
            "|kesinlikle çok kö...|    0|kesinlikle çok kö...|[kesinlikle, çok,...|\n",
            "|ben nasil bir fil...|    0|ben nasil bir fil...|[ben, nasil, bir,...|\n",
            "|bu yüzler fazla t...|    0|bu yüzler fazla t...|[bu, yüzler, fazl...|\n",
            "|. çok ilginç yaa ...|    0|. çok ilginç yaa ...|[., çok, ilginç, ...|\n",
            "|bence bu film hiç...|    0|bence bu film hiç...|[bence, bu, film,...|\n",
            "|valla ben begenme...|    0|valla ben begenme...|[valla, ben, bege...|\n",
            "|gönül ister milyo...|    0|gönül ister milyo...|[gönül, ister, mi...|\n",
            "|dün aksam bu film...|    0|dün aksam bu film...|[dün, aksam, bu, ...|\n",
            "|cok kötü olmus......|    0|cok kötü olmus......|[cok, kötü, olmus...|\n",
            "|igrenç yaaa.acaba...|    0|igrenç yaaa.acaba...|[igrenç, yaaa.aca...|\n",
            "|saçma sapan bir f...|    0|saçma sapan bir f...|[saçma, sapan, bi...|\n",
            "|çok güzel komedi ...|    0|çok güzel komedi ...|[çok, güzel, kome...|\n",
            "|beklentimi yüksek...|    0|beklentimi yüksek...|[beklentimi, yüks...|\n",
            "+--------------------+-----+--------------------+--------------------+\n",
            "only showing top 20 rows\n",
            "\n",
            "+--------------------+-----+--------------------+--------------------+--------------------+\n",
            "|               cumle|label|              cumle2|           tokenized|            features|\n",
            "+--------------------+-----+--------------------+--------------------+--------------------+\n",
            "|bunu da gördük uz...|    0|bunu da gördük uz...|[bunu, da, gördük...|(3500,[14,16,27,1...|\n",
            "|filmi begenmeyenl...|    0|filmi begenmeyenl...|[filmi, begenmeye...|(3500,[0,1,5,6,51...|\n",
            "|günde 1 film izle...|    0|günde bir film iz...|[günde, bir, film...|(3500,[0,1,4,11,2...|\n",
            "|afisine bakip ald...|    0|afisine bakip ald...|[afisine, bakip, ...|(3500,[2,3,4,13,3...|\n",
            "|sadece insanlarin...|    0|sadece insanlarin...|[sadece, insanlar...|(3500,[0,1,14,17,...|\n",
            "|ucuz bir aksiyon ...|    0|ucuz bir aksiyon ...|[ucuz, bir, aksiy...|(3500,[0,3,5,36,4...|\n",
            "|olmamis diyor pua...|    0|olmamis diyor pua...|[olmamis, diyor, ...|(3500,[12,178,442...|\n",
            "|kesinlikle çok kö...|    0|kesinlikle çok kö...|[kesinlikle, çok,...|(3500,[0,1,2,7,21...|\n",
            "|ben nasil bir fil...|    0|ben nasil bir fil...|[ben, nasil, bir,...|(3500,[0,1,6,16,2...|\n",
            "|bu yüzler fazla t...|    0|bu yüzler fazla t...|[bu, yüzler, fazl...|(3500,[3,4,14,17,...|\n",
            "|. çok ilginç yaa ...|    0|. çok ilginç yaa ...|[., çok, ilginç, ...|(3500,[0,1,2,9,12...|\n",
            "|bence bu film hiç...|    0|bence bu film hiç...|[bence, bu, film,...|(3500,[1,3,6,8,9,...|\n",
            "|valla ben begenme...|    0|valla ben begenme...|[valla, ben, bege...|(3500,[0,2,29,39,...|\n",
            "|gönül ister milyo...|    0|gönül ister milyo...|[gönül, ister, mi...|(3500,[3,6,21,24,...|\n",
            "|dün aksam bu film...|    0|dün aksam bu film...|[dün, aksam, bu, ...|(3500,[3,30,39,50...|\n",
            "|cok kötü olmus......|    0|cok kötü olmus......|[cok, kötü, olmus...|(3500,[7,19,27,32...|\n",
            "|igrenç yaaa.acaba...|    0|igrenç yaaa.acaba...|[igrenç, yaaa.aca...|(3500,[0,1,3,5,17...|\n",
            "|saçma sapan bir f...|    0|saçma sapan bir f...|[saçma, sapan, bi...|(3500,[0,3,11,30,...|\n",
            "|çok güzel komedi ...|    0|çok güzel komedi ...|[çok, güzel, kome...|(3500,[2,5,8,27,1...|\n",
            "|beklentimi yüksek...|    0|beklentimi yüksek...|[beklentimi, yüks...|(3500,[0,5,33,35,...|\n",
            "+--------------------+-----+--------------------+--------------------+--------------------+\n",
            "only showing top 20 rows\n",
            "\n",
            "0.8918153846153851\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "c_e7obUWWGe3",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "683febc1-af7d-478c-f9ef-e25230aac65b"
      },
      "source": [
        "yeniDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','false')\\\n",
        "                        .option('delimiter','\\t').csv('deneme-yorum.txt')\n",
        "\n",
        "yeniDF = yeniDF.withColumnRenamed('_c0','cumle2')\n",
        "yeniDF.show()\n",
        "yeniDF = tknzr.transform(yeniDF)\n",
        "yeniDF = vecModel.transform(yeniDF)\n",
        "yeniDF.show()\n",
        "yeniYorumSonucDF = model.transform(yeniDF)\n",
        "yeniYorumSonucDF.select('cumle2','prediction').show()"
      ],
      "execution_count": 69,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+--------------------+\n",
            "|              cumle2|\n",
            "+--------------------+\n",
            "| bu film çok kötüydü|\n",
            "|     berbat bir film|\n",
            "|bundan daha güzel...|\n",
            "| fena bir film değil|\n",
            "|çok kötü bir film...|\n",
            "+--------------------+\n",
            "\n",
            "+--------------------+--------------------+--------------------+\n",
            "|              cumle2|           tokenized|            features|\n",
            "+--------------------+--------------------+--------------------+\n",
            "| bu film çok kötüydü|[bu, film, çok, k...|(3500,[1,2,3,349]...|\n",
            "|     berbat bir film| [berbat, bir, film]|(3500,[0,1,68],[1...|\n",
            "|bundan daha güzel...|[bundan, daha, gü...|(3500,[13,374,168...|\n",
            "| fena bir film değil|[fena, bir, film,...|(3500,[0,1,399],[...|\n",
            "|çok kötü bir film...|[çok, kötü, bir, ...|(3500,[0,1,2,19,4...|\n",
            "+--------------------+--------------------+--------------------+\n",
            "\n",
            "+--------------------+----------+\n",
            "|              cumle2|prediction|\n",
            "+--------------------+----------+\n",
            "| bu film çok kötüydü|       0.0|\n",
            "|     berbat bir film|       0.0|\n",
            "|bundan daha güzel...|       0.0|\n",
            "| fena bir film değil|       0.0|\n",
            "|çok kötü bir film...|       0.0|\n",
            "+--------------------+----------+\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Yse-7Raga3uW",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "66cd6913-8115-4953-ed05-1ffb8ad1c939"
      },
      "source": [
        "from pyspark.sql import SparkSession\n",
        "from pyspark.ml.feature import Tokenizer,Word2Vec\n",
        "from pyspark.ml.classification import MultilayerPerceptronClassifier\n",
        "from pyspark.ml.evaluation import BinaryClassificationEvaluator\n",
        "from pyspark.sql.functions import UserDefinedFunction\n",
        "from pyspark.sql.types import StringType\n",
        "\n",
        "spark = SparkSession.Builder().appName('SparkML Sentiment Analysis').getOrCreate()\n",
        "yorumDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','false')\\\n",
        "                        .option('delimiter','\\t').csv('movie_turkish_train.txt')\n",
        "yorumDF = yorumDF.withColumnRenamed('_c0','cumle')\n",
        "yorumDF = yorumDF.withColumnRenamed('_c1','label')\n",
        "\n",
        "def fixCumle(cumle):\n",
        "    cumle = cumle.replace('1','bir')\n",
        "    #cumle = cumle.replace('.','')\n",
        "    #cumle = cumle.replace('(','')\n",
        "    #cumle = cumle.replace(')','')\n",
        "    #cumle = cumle.strip()\n",
        "    #cumle = cumle.replace('filmi','film')\n",
        "    return cumle\n",
        "\n",
        "fixTextUDF = UserDefinedFunction(fixCumle,StringType())\n",
        "yorumDF = yorumDF.withColumn('cumle2',fixTextUDF('cumle'))\n",
        "\n",
        "tknzr = Tokenizer(inputCol='cumle2',outputCol='tokenized')\n",
        "yorumDF = tknzr.transform(yorumDF)\n",
        "#yorumDF.show()\n",
        "\n",
        "kelimeSayisi = 3500\n",
        "vectorizer = Word2Vec(inputCol='tokenized',outputCol='features',vectorSize=kelimeSayisi)\n",
        "vecModel = vectorizer.fit(yorumDF)\n",
        "\n",
        "yorumDF = vecModel.transform(yorumDF)\n",
        "\n",
        "yorumDF = yorumDF.select('features','label')\n",
        "trainDF,testDF = yorumDF.randomSplit([0.75,0.25],seed=1234)\n",
        "\n",
        "classifer = MultilayerPerceptronClassifier(layers=[kelimeSayisi,200,2])\n",
        "\n",
        "model = classifer.fit(trainDF)\n",
        "sonucDF = model.transform(testDF)\n",
        "\n",
        "eva = BinaryClassificationEvaluator()\n",
        "basari = eva.evaluate(sonucDF)\n",
        "print(basari)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "0.6950603596945061\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xz6wKQuDxdU2"
      },
      "source": [
        "## Homework3 ##"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7AAJhN1vxjre"
      },
      "source": [
        "## Homework3 ##\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hyZxQJl6K2_l",
        "outputId": "d3b22c5d-0aff-465d-cc28-101a78c920fb"
      },
      "source": [
        "##Logistic Regression\n",
        "\n",
        "from pyspark.sql import SparkSession\n",
        "from pyspark.ml.feature import StringIndexer, VectorAssembler\n",
        "from pyspark.ml.classification import LogisticRegression\n",
        "from pyspark.ml.evaluation import MulticlassClassificationEvaluator\n",
        "from pyspark.ml.tuning import CrossValidator, ParamGridBuilder\n",
        "\n",
        "spark = SparkSession.Builder().appName('Spark DataFrame Introduction').getOrCreate()\n",
        "spark = SparkSession.Builder().appName('SparkML Analysis').getOrCreate()\n",
        "\n",
        "leafDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','false')\\\n",
        "                        .option('delimiter',',').csv('leaf.csv')\n",
        "\n",
        "leafDF = leafDF.withColumnRenamed(\"_c0\",\"Class\")\n",
        "leafDF = leafDF.withColumnRenamed(\"_c1\",\"specimen_number\")\n",
        "leafDF = leafDF.withColumnRenamed(\"_c2\",\"Eccentricity\")\n",
        "leafDF = leafDF.withColumnRenamed(\"_c3\",\"Aspect_Ratio\")\n",
        "leafDF = leafDF.withColumnRenamed(\"_c4\",\"Elongation\")\n",
        "leafDF = leafDF.withColumnRenamed(\"_c5\",\"Solidity\")\n",
        "leafDF = leafDF.withColumnRenamed(\"_c6\",\"Stochastic_Convexity\")\n",
        "leafDF = leafDF.withColumnRenamed(\"_c7\",\"Isoperimetric_Factor\")\n",
        "leafDF = leafDF.withColumnRenamed(\"_c8\",\"Maximal_Indentation_Depth\")\n",
        "leafDF = leafDF.withColumnRenamed(\"_c9\",\"Lobedness\")\n",
        "leafDF = leafDF.withColumnRenamed(\"_c10\",\"Average_Intensity\")\n",
        "leafDF = leafDF.withColumnRenamed(\"_c11\",\"Average_Contrast\")\n",
        "leafDF = leafDF.withColumnRenamed(\"_c12\",\"Smoothness\")\n",
        "leafDF = leafDF.withColumnRenamed(\"_c13\",\"Third_moment\")\n",
        "leafDF = leafDF.withColumnRenamed(\"_c14\",\"Uniformity\")\n",
        "leafDF = leafDF.withColumnRenamed(\"_c15\",\"Entropy\")\n",
        "\n",
        "#leafDF.show(3)\n",
        "\n",
        "\n",
        "#leafDF.describe().show()\n",
        "\n",
        "#leafDF.count()\n",
        "\n",
        "#leafDF.distinct().count()\n",
        "\n",
        "vec = VectorAssembler(inputCols=leafDF.columns[1:16],outputCol='features')\n",
        "leafDF = vec.transform(leafDF)\n",
        "\n",
        "leafDF = leafDF.withColumnRenamed('Class','label')\n",
        "leafDF = leafDF.select('features','label')\n",
        "\n",
        "trainDF, testDF = leafDF.randomSplit([0.8,0.2], seed=1)\n",
        "classifier = LogisticRegression()\n",
        "eva = MulticlassClassificationEvaluator(metricName='accuracy')\n",
        "\n",
        "\n",
        "myParams = ParamGridBuilder().addGrid(classifier.maxIter, [100,150,200])\\\n",
        "                              .addGrid (classifier.regParam, [0.0,1])\\\n",
        "                              .addGrid (classifier.elasticNetParam,[0.0,1]).build()\n",
        "validator = CrossValidator(parallelism=4,\n",
        "                                 estimator=classifier,\n",
        "                                 estimatorParamMaps=myParams,\n",
        "                                 evaluator=eva,\n",
        "                                 numFolds=4)\n",
        "\n",
        "\n",
        "                              \n",
        "model = validator.fit(trainDF)\n",
        "print(\"Best Model Hesaplandı\")\n",
        "print(\"Best MaxIter : \", model.bestModel.getMaxIter())\n",
        "print(\"Best RegParam : \", model.bestModel.getRegParam())\n",
        "print(\"Best ElasticNetParam : \", model.bestModel.getElasticNetParam())\n",
        "\n",
        "resultDF = model.transform(testDF)\n",
        "result = eva.evaluate(resultDF)\n",
        "print(\"Accuracy of the Best Model: \", result)\n",
        "\n"
      ],
      "execution_count": 91,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Best Model Hesaplandı\n",
            "Best MaxIter :  200\n",
            "Best RegParam :  0.0\n",
            "Best ElasticNetParam :  0.0\n",
            "Accuracy of the Best Model:  0.6323529411764706\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zugHOaQksZn8",
        "outputId": "eab90959-d48a-43d7-c49a-85321bebc2e7"
      },
      "source": [
        "##Logistic Regression\n",
        "\n",
        "from pyspark.sql import SparkSession\n",
        "from pyspark.ml.feature import StringIndexer,VectorAssembler\n",
        "from pyspark.ml.classification import LogisticRegression\n",
        "from pyspark.ml.evaluation import MulticlassClassificationEvaluator\n",
        "from pyspark.ml.tuning import CrossValidator, ParamGridBuilder\n",
        "spark = SparkSession.Builder().appName('SparkML Analysis').getOrCreate()\n",
        "leafDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','false')\\\n",
        "                        .option('delimiter',',').csv('leaf.csv')\n",
        "leafDF = leafDF.withColumnRenamed('_c0','class')\n",
        "leafDF = leafDF.withColumnRenamed('_c1','SpecimenNumber')\n",
        "leafDF = leafDF.withColumnRenamed('_c2','Eccentricity')\n",
        "leafDF = leafDF.withColumnRenamed('_c3','AspectRatio')\n",
        "leafDF = leafDF.withColumnRenamed('_c4','Elongation')\n",
        "leafDF = leafDF.withColumnRenamed('_c5','Solidity')\n",
        "leafDF = leafDF.withColumnRenamed('_c6','StochasticConvexity')\n",
        "leafDF = leafDF.withColumnRenamed('_c7','IsoperimetricFactor')\n",
        "leafDF = leafDF.withColumnRenamed('_c8','MaximalIndentationDepth')\n",
        "leafDF = leafDF.withColumnRenamed('_c9','Lobedness')\n",
        "leafDF = leafDF.withColumnRenamed('_c10','AverageIntensity')\n",
        "leafDF = leafDF.withColumnRenamed('_c11','AverageContrast')\n",
        "leafDF = leafDF.withColumnRenamed('_c12','Smoothness')\n",
        "leafDF = leafDF.withColumnRenamed('_c13','ThirdMoment')\n",
        "leafDF = leafDF.withColumnRenamed('_c14','Uniformity')\n",
        "leafDF = leafDF.withColumnRenamed('_c15','Entropy')\n",
        "leafDF = leafDF.drop('SpecimenNumber')\n",
        "indexer = StringIndexer(inputCol='class',outputCol='label' )\n",
        "indexerModel = indexer.fit(leafDF)\n",
        "leafDF = indexerModel.transform(leafDF)\n",
        "vec = VectorAssembler(inputCols=['Eccentricity','AspectRatio','Elongation','Solidity','StochasticConvexity','IsoperimetricFactor','MaximalIndentationDepth','Lobedness','AverageIntensity','AverageContrast','Smoothness','ThirdMoment','Uniformity','Entropy'], outputCol='features')\n",
        "leafDF = vec.transform(leafDF)\n",
        "leafDF = leafDF.select('features', 'label')\n",
        "trainDF, testDF = leafDF.randomSplit([0.8,0.2], seed=1)\n",
        "classifier = LogisticRegression()\n",
        "eva = MulticlassClassificationEvaluator(metricName='accuracy')\n",
        "myParams = ParamGridBuilder().addGrid(classifier.maxIter, [100,150,200])\\\n",
        "                              .addGrid (classifier.regParam, [0.0,1])\\\n",
        "                              .addGrid (classifier.elasticNetParam,[0.0,1]).build()\n",
        "validator = CrossValidator(parallelism=4,\n",
        "                                 estimator=classifier,\n",
        "                                 estimatorParamMaps=myParams,\n",
        "                                 evaluator=eva,\n",
        "                                 numFolds=4)\n",
        "model = validator.fit(trainDF)\n",
        "print(\"Best Model Hesaplandı\")\n",
        "print(\"Best MaxIter : \", model.bestModel.getMaxIter())\n",
        "print(\"Best RegParam : \", model.bestModel.getRegParam())\n",
        "print(\"Best ElasticNetParam : \", model.bestModel.getElasticNetParam())\n",
        "resultDF = model.transform(testDF)\n",
        "result = eva.evaluate(resultDF)\n",
        "print(\"Accuracy of the Best Model: \", result)\n",
        "\n"
      ],
      "execution_count": 89,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Best Model Hesaplandı\n",
            "Best MaxIter :  100\n",
            "Best RegParam :  0.0\n",
            "Best ElasticNetParam :  0.0\n",
            "Accuracy of the Best Model:  0.7794117647058824\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pNrRQaywxBPP",
        "outputId": "cf2b0174-dca3-4f71-844a-7596dc8abcd5"
      },
      "source": [
        "#Random Forest ##\n",
        "\n",
        "from pyspark.sql import SparkSession\n",
        "from pyspark.ml.feature import StringIndexer, VectorAssembler\n",
        "from pyspark.ml.classification import LogisticRegression\n",
        "from pyspark.ml.evaluation import MulticlassClassificationEvaluator\n",
        "from pyspark.ml.tuning import CrossValidator, ParamGridBuilder\n",
        "\n",
        "spark = SparkSession.Builder().appName('Spark DataFrame Introduction').getOrCreate()\n",
        "spark = SparkSession.Builder().appName('SparkML Analysis').getOrCreate()\n",
        "\n",
        "leafDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','false')\\\n",
        "                        .option('delimiter',',').csv('leaf.csv')\n",
        "\n",
        "leafDF = leafDF.withColumnRenamed(\"_c0\",\"Class\")\n",
        "leafDF = leafDF.withColumnRenamed(\"_c1\",\"specimen_number\")\n",
        "leafDF = leafDF.withColumnRenamed(\"_c2\",\"Eccentricity\")\n",
        "leafDF = leafDF.withColumnRenamed(\"_c3\",\"Aspect_Ratio\")\n",
        "leafDF = leafDF.withColumnRenamed(\"_c4\",\"Elongation\")\n",
        "leafDF = leafDF.withColumnRenamed(\"_c5\",\"Solidity\")\n",
        "leafDF = leafDF.withColumnRenamed(\"_c6\",\"Stochastic_Convexity\")\n",
        "leafDF = leafDF.withColumnRenamed(\"_c7\",\"Isoperimetric_Factor\")\n",
        "leafDF = leafDF.withColumnRenamed(\"_c8\",\"Maximal_Indentation_Depth\")\n",
        "leafDF = leafDF.withColumnRenamed(\"_c9\",\"Lobedness\")\n",
        "leafDF = leafDF.withColumnRenamed(\"_c10\",\"Average_Intensity\")\n",
        "leafDF = leafDF.withColumnRenamed(\"_c11\",\"Average_Contrast\")\n",
        "leafDF = leafDF.withColumnRenamed(\"_c12\",\"Smoothness\")\n",
        "leafDF = leafDF.withColumnRenamed(\"_c13\",\"Third_moment\")\n",
        "leafDF = leafDF.withColumnRenamed(\"_c14\",\"Uniformity\")\n",
        "leafDF = leafDF.withColumnRenamed(\"_c15\",\"Entropy\")\n",
        "\n",
        "#leafDF.show(3)\n",
        "\n",
        "\n",
        "#leafDF.describe().show()\n",
        "\n",
        "#leafDF.count()\n",
        "\n",
        "#leafDF.distinct().count()\n",
        "\n",
        "vec = VectorAssembler(inputCols=leafDF.columns[1:16],outputCol='features')\n",
        "leafDF = vec.transform(leafDF)\n",
        "\n",
        "leafDF = leafDF.withColumnRenamed('Class','label')\n",
        "leafDF = leafDF.select('features','label')\n",
        "\n",
        "trainDF, testDF = leafDF.randomSplit([0.8,0.2], seed=1)\n",
        "\n",
        "classifier = RandomForestClassifier()\n",
        "eva = MulticlassClassificationEvaluator(metricName='accuracy')\n",
        "myParams = ParamGridBuilder().addGrid(classifier.numTrees, [40,50])\\\n",
        "                              .addGrid (classifier.maxDepth, [10,12,14,15])\\\n",
        "                              .addGrid (classifier.impurity,['gini','entropy']).build()\n",
        "validator = CrossValidator(parallelism=4,\n",
        "                                 estimator=classifier,\n",
        "                                 estimatorParamMaps=myParams,\n",
        "                                 evaluator=eva,\n",
        "                                 numFolds=5)\n",
        "model = validator.fit(trainDF)\n",
        "print(\"Best Model Hesaplandı\")\n",
        "print(\"Best NumTrees : \", model.bestModel.getNumTrees)\n",
        "print(\"Best MaxDepth : \", model.bestModel.getMaxDepth())\n",
        "print(\"Best Impurity : \", model.bestModel.getImpurity())\n",
        "resultDF = model.transform(testDF)\n",
        "result = eva.evaluate(resultDF)\n",
        "print(\"Accuracy of the Best Model: \", result)\n",
        "\n",
        "\n",
        "\n"
      ],
      "execution_count": 94,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Best Model Hesaplandı\n",
            "Best NumTrees :  40\n",
            "Best MaxDepth :  12\n",
            "Best Impurity :  entropy\n",
            "Accuracy of the Best Model:  0.75\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cQKjfiu2t636",
        "outputId": "900f7ee2-2852-4e40-e39d-33ee4152a50c"
      },
      "source": [
        "### Naive Bayes###\n",
        "\n",
        "from pyspark.sql import SparkSession\n",
        "from pyspark.ml.feature import StringIndexer, VectorAssembler\n",
        "from pyspark.ml.classification import NaiveBayes\n",
        "from pyspark.ml.evaluation import MulticlassClassificationEvaluator\n",
        "from pyspark.ml.tuning import CrossValidator, ParamGridBuilder\n",
        "\n",
        "spark = SparkSession.Builder().appName('Spark DataFrame Introduction').getOrCreate()\n",
        "spark = SparkSession.Builder().appName('SparkML Analysis').getOrCreate()\n",
        "\n",
        "leafDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','false')\\\n",
        "                        .option('delimiter',',').csv('leaf.csv')\n",
        "\n",
        "leafDF = leafDF.withColumnRenamed(\"_c0\",\"Class\")\n",
        "leafDF = leafDF.withColumnRenamed(\"_c1\",\"specimen_number\")\n",
        "leafDF = leafDF.withColumnRenamed(\"_c2\",\"Eccentricity\")\n",
        "leafDF = leafDF.withColumnRenamed(\"_c3\",\"Aspect_Ratio\")\n",
        "leafDF = leafDF.withColumnRenamed(\"_c4\",\"Elongation\")\n",
        "leafDF = leafDF.withColumnRenamed(\"_c5\",\"Solidity\")\n",
        "leafDF = leafDF.withColumnRenamed(\"_c6\",\"Stochastic_Convexity\")\n",
        "leafDF = leafDF.withColumnRenamed(\"_c7\",\"Isoperimetric_Factor\")\n",
        "leafDF = leafDF.withColumnRenamed(\"_c8\",\"Maximal_Indentation_Depth\")\n",
        "leafDF = leafDF.withColumnRenamed(\"_c9\",\"Lobedness\")\n",
        "leafDF = leafDF.withColumnRenamed(\"_c10\",\"Average_Intensity\")\n",
        "leafDF = leafDF.withColumnRenamed(\"_c11\",\"Average_Contrast\")\n",
        "leafDF = leafDF.withColumnRenamed(\"_c12\",\"Smoothness\")\n",
        "leafDF = leafDF.withColumnRenamed(\"_c13\",\"Third_moment\")\n",
        "leafDF = leafDF.withColumnRenamed(\"_c14\",\"Uniformity\")\n",
        "leafDF = leafDF.withColumnRenamed(\"_c15\",\"Entropy\")\n",
        "\n",
        "#leafDF.show(3)\n",
        "\n",
        "\n",
        "#leafDF.describe().show()\n",
        "\n",
        "#leafDF.count()\n",
        "\n",
        "#leafDF.distinct().count()\n",
        "\n",
        "vec = VectorAssembler(inputCols=leafDF.columns[1:16],outputCol='features')\n",
        "leafDF = vec.transform(leafDF)\n",
        "\n",
        "leafDF = leafDF.withColumnRenamed('Class','label')\n",
        "leafDF = leafDF.select('features','label')\n",
        "\n",
        "trainDF, testDF = leafDF.randomSplit([0.8,0.2], seed=1)\n",
        "\n",
        "eva = MulticlassClassificationEvaluator(metricName='accuracy')\n",
        "\n",
        "classifier = NaiveBayes()\n",
        "\n",
        "myParams = ParamGridBuilder().addGrid(classifier.smoothing, [1.0,2.0])\\\n",
        "                              .addGrid (classifier.modelType, ['gaussian','complement', 'multinomial']).build()\n",
        "validator = CrossValidator(parallelism=4,\n",
        "                                 estimator=classifier,\n",
        "                                 estimatorParamMaps=myParams,\n",
        "                                 evaluator=eva,\n",
        "                                 numFolds=4)\n",
        "\n",
        "\n",
        "model = validator.fit(trainDF)\n",
        "print(\"Best Model Hesaplandı\")\n",
        "print(\"Best Smoothing : \", model.bestModel.getSmoothing())\n",
        "print(\"Best ModelType : \", model.bestModel.getModelType())\n",
        "resultDF = model.transform(testDF)\n",
        "result = eva.evaluate(resultDF)\n",
        "print(\"Accuracy of the Best Model: \", result)"
      ],
      "execution_count": 93,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Best Model Hesaplandı\n",
            "Best Smoothing :  1.0\n",
            "Best ModelType :  complement\n",
            "Accuracy of the Best Model:  0.014705882352941176\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "K25qo7cOwjHX"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}