Sınavcalisma_9_10_2021_22.06.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Sınavcalisma.9.10.2021_417.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyN4uafEll+9tH2idsj9wylg",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/busranurorhan/Complete-Python-3-Bootcamp/blob/master/S%C4%B1navcalisma_9_10_2021_22.06.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0f1oYeHdaXk0",
        "outputId": "1219fefd-c08c-477e-8592-d29d434d64db"
      },
      "source": [
        "!apt-get update\n",
        "!apt-get install openjdk-8-jdk-headless -qq > /dev/null\n",
        "import os\n",
        "os.environ[\"JAVA_HOME\"] = \"/usr/lib/jvm/java-8-openjdk-amd64\"\n",
        "!update-alternatives --set java /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java\n",
        "!java -version\n",
        "!pip install pyspark"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\r0% [Working]\r            \rGet:1 http://security.ubuntu.com/ubuntu bionic-security InRelease [88.7 kB]\n",
            "\r0% [Connecting to archive.ubuntu.com] [1 InRelease 0 B/88.7 kB 0%] [Connected t\r0% [Connecting to archive.ubuntu.com (91.189.88.152)] [Waiting for headers] [Wa\r                                                                               \rGet:2 https://cloud.r-project.org/bin/linux/ubuntu bionic-cran40/ InRelease [3,626 B]\n",
            "\r0% [Connecting to archive.ubuntu.com (91.189.88.152)] [Waiting for headers] [Wa\r0% [1 InRelease gpgv 88.7 kB] [Connecting to archive.ubuntu.com (91.189.88.152)\r                                                                               \rIgn:3 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  InRelease\n",
            "\r0% [1 InRelease gpgv 88.7 kB] [Connecting to archive.ubuntu.com (91.189.88.152)\r                                                                               \rGet:4 http://ppa.launchpad.net/c2d4u.team/c2d4u4.0+/ubuntu bionic InRelease [15.9 kB]\n",
            "\r0% [1 InRelease gpgv 88.7 kB] [Waiting for headers] [4 InRelease 2,572 B/15.9 k\r                                                                               \rIgn:5 https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64  InRelease\n",
            "\r0% [1 InRelease gpgv 88.7 kB] [Waiting for headers] [4 InRelease 8,364 B/15.9 k\r                                                                               \rHit:6 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  Release\n",
            "Hit:7 https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64  Release\n",
            "Hit:8 http://archive.ubuntu.com/ubuntu bionic InRelease\n",
            "Get:9 http://archive.ubuntu.com/ubuntu bionic-updates InRelease [88.7 kB]\n",
            "Get:10 http://security.ubuntu.com/ubuntu bionic-security/universe amd64 Packages [1,431 kB]\n",
            "Hit:11 http://ppa.launchpad.net/cran/libgit2/ubuntu bionic InRelease\n",
            "Get:12 http://security.ubuntu.com/ubuntu bionic-security/main amd64 Packages [2,365 kB]\n",
            "Get:13 http://ppa.launchpad.net/deadsnakes/ppa/ubuntu bionic InRelease [15.9 kB]\n",
            "Get:14 http://archive.ubuntu.com/ubuntu bionic-backports InRelease [74.6 kB]\n",
            "Hit:15 http://ppa.launchpad.net/graphics-drivers/ppa/ubuntu bionic InRelease\n",
            "Get:18 http://ppa.launchpad.net/c2d4u.team/c2d4u4.0+/ubuntu bionic/main Sources [1,802 kB]\n",
            "Get:19 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 Packages [2,801 kB]\n",
            "Get:20 http://ppa.launchpad.net/c2d4u.team/c2d4u4.0+/ubuntu bionic/main amd64 Packages [922 kB]\n",
            "Get:21 http://archive.ubuntu.com/ubuntu bionic-updates/universe amd64 Packages [2,210 kB]\n",
            "Get:22 http://ppa.launchpad.net/deadsnakes/ppa/ubuntu bionic/main amd64 Packages [40.8 kB]\n",
            "Fetched 11.9 MB in 4s (3,200 kB/s)\n",
            "Reading package lists... Done\n",
            "update-alternatives: using /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java to provide /usr/bin/java (java) in manual mode\n",
            "openjdk version \"1.8.0_292\"\n",
            "OpenJDK Runtime Environment (build 1.8.0_292-8u292-b10-0ubuntu1~18.04-b10)\n",
            "OpenJDK 64-Bit Server VM (build 25.292-b10, mixed mode)\n",
            "Collecting pyspark\n",
            "  Downloading pyspark-3.1.2.tar.gz (212.4 MB)\n",
            "\u001b[K     |████████████████████████████████| 212.4 MB 60 kB/s \n",
            "\u001b[?25hCollecting py4j==0.10.9\n",
            "  Downloading py4j-0.10.9-py2.py3-none-any.whl (198 kB)\n",
            "\u001b[K     |████████████████████████████████| 198 kB 63.4 MB/s \n",
            "\u001b[?25hBuilding wheels for collected packages: pyspark\n",
            "  Building wheel for pyspark (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for pyspark: filename=pyspark-3.1.2-py2.py3-none-any.whl size=212880768 sha256=256673b112d3469ea574000b568287263a5c30d045d32d12925ddbf1abf3a669\n",
            "  Stored in directory: /root/.cache/pip/wheels/a5/0a/c1/9561f6fecb759579a7d863dcd846daaa95f598744e71b02c77\n",
            "Successfully built pyspark\n",
            "Installing collected packages: py4j, pyspark\n",
            "Successfully installed py4j-0.10.9 pyspark-3.1.2\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "m1S3-1uZgoE4",
        "outputId": "dfe3dc10-a15f-44dd-91a2-7ff637397e2a"
      },
      "source": [
        "from pyspark.sql import SparkSession\n",
        "spark = SparkSession.Builder().appName('Spark DataFrame Introduction').getOrCreate()\n",
        "pointsDF = spark.read.option('inferSchema','true').csv('points.txt')\n",
        "pointsDF = pointsDF.withColumnRenamed('_c0','x') #Kolon isimlerini değiştiriyoruz\n",
        "pointsDF = pointsDF.withColumnRenamed('_c1','y') #Kolon isimlerini değiştiriyoruz\n",
        "pointsDF = pointsDF.withColumn(\"x\",pointsDF[\"x\"].cast(\"float\"))  #Kolonun veri tipi değiştiriyoruz\n",
        "pointsDF =  pointsDF.withColumn(\"2xy\", 2* pointsDF['x']+pointsDF['y']) #Yeni kolon ekliyoruz\n",
        "pointsDF.show()\n",
        "pointsDF.printSchema()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+----+---+----+\n",
            "|   x|  y| 2xy|\n",
            "+----+---+----+\n",
            "| 5.0|  5|15.0|\n",
            "|15.0| 17|47.0|\n",
            "|15.0| 16|46.0|\n",
            "| 5.0|  6|16.0|\n",
            "| 6.0|  6|18.0|\n",
            "|16.0| 16|48.0|\n",
            "| 8.0|  8|24.0|\n",
            "| 7.0|  6|20.0|\n",
            "|17.0| 17|51.0|\n",
            "| 6.0|  7|19.0|\n",
            "| 7.0|  8|22.0|\n",
            "|18.0| 18|54.0|\n",
            "| 8.0|  6|22.0|\n",
            "|17.0| 16|50.0|\n",
            "| 5.0|  8|18.0|\n",
            "|17.0| 16|50.0|\n",
            "| 8.0|  8|24.0|\n",
            "|16.0| 17|49.0|\n",
            "|17.0| 18|52.0|\n",
            "|18.0| 16|52.0|\n",
            "+----+---+----+\n",
            "only showing top 20 rows\n",
            "\n",
            "root\n",
            " |-- x: float (nullable = true)\n",
            " |-- y: integer (nullable = true)\n",
            " |-- 2xy: float (nullable = true)\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lI228ck7g-O3"
      },
      "source": [
        "from pyspark.sql import SparkSession\n",
        "spark = SparkSession.Builder().appName('Spark DataFrame Introduction').getOrCreate()\n",
        "pointsDF = spark.read.option('inferSchema','true').option('header','false').csv('points.txt')\n",
        "#pointsDF.printSchema()\n",
        "#pointsDF.show()\n",
        "\n",
        "#print( pointsDF.rdd.map(lambda x : tuple(x)).take(5) )\n",
        "\n",
        "\n",
        "pointsRDD = pointsDF.rdd\n",
        "\n",
        "\n",
        "\n",
        "df = pointsRDD.toDF()\n",
        "\n",
        "df.show()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gQT4fl-px9xJ",
        "outputId": "886af77b-8326-462a-ce75-0c3dc47157cf"
      },
      "source": [
        "from pyspark.sql import SparkSession\n",
        "from pyspark.sql.functions import UserDefinedFunction\n",
        "from pyspark.sql.types import FloatType\n",
        "\n",
        "spark = SparkSession.Builder().appName('Spark DataFrame Introduction').getOrCreate()\n",
        "\n",
        "dollarDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','false')\\\n",
        "                        .option('delimiter','\\t').csv('DollarDataset.txt')\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "dollarDF = dollarDF.withColumnRenamed('_c0','id')\n",
        "dollarDF = dollarDF.withColumnRenamed('_c1','tarih')\n",
        "dollarDF = dollarDF.withColumnRenamed('_c2','fiyat')\n",
        "\n",
        "#dollarDF.show() \n",
        "\n",
        "#dollarDF.count() #17776\n",
        "\n",
        "\n",
        "dollarDF = dollarDF.filter(dollarDF['fiyat'].isNotNull())  #bu islemden sonra index bouldu\n",
        "\n",
        "\n",
        "#dollarDF.show(100) \n",
        "\n",
        "#dollarDF.count()  #17190 demekki 586 null values\n",
        "\n",
        "\n",
        "dollarDF = dollarDF.drop('id') #index column2ını drop ediyorum\n",
        "#dollarDF.show() \n",
        "\n",
        "rdd = dollarDF.rdd.zipWithIndex() #index'i en son'a ekledi\n",
        "\n",
        "#rdd.collect()\n",
        "\n",
        "\n",
        "dollarDF = rdd.map(lambda x : (x[1],x[0][0],x[0][1])).toDF()\n",
        "\n",
        "\n",
        "#dollarDF.show() #baslıklar bozudlu\n",
        "\n",
        "# data frame çevirince _1 _2 otomatik atanıyor\n",
        "dollarDF = dollarDF.withColumnRenamed('_1','id')\n",
        "dollarDF = dollarDF.withColumnRenamed('_2','tarih')\n",
        "dollarDF = dollarDF.withColumnRenamed('_3','fiyat')\n",
        "\n",
        "#dollarDF.show()\n",
        "#dollarDF.printSchema()\n",
        "\n",
        "def stringtoFloat(f):\n",
        "  f=f.replace('.','')\n",
        "  f=f.replace(',','.')\n",
        "  return float(f)\n",
        "\n",
        "stringtoFloatUDF = UserDefinedFunction(stringtoFloat,FloatType())  #eger yeni bir kolon ekleyeeksek ve bu mevcut kolanla işlemler yapılıp elde edilefcekse bu uderdefinedfunction yapmamız lazım\n",
        "\n",
        "dollarDF = dollarDF.withColumn(\"fiyat\",stringtoFloatUDF('fiyat'))\n",
        "\n",
        "#dollarDF.show()\n",
        "\n",
        "dollarDF2 = dollarDF\n",
        "dollarDF2 = dollarDF2.withColumn('id',dollarDF2['id']+1)\n",
        "dollarDF2 = dollarDF2.withColumnRenamed('tarih','tarih2')\n",
        "dollarDF2 = dollarDF2.withColumnRenamed('fiyat','fiyat2')\n",
        "\n",
        "#dollarDF2.show()\n",
        "\n",
        "joinedDF = dollarDF2.join(dollarDF,'id')\n",
        "\n",
        "joinedDF = joinedDF.sort('id')\n",
        "\n",
        "joinedDF = joinedDF.withColumn('oran',((joinedDF[\"fiyat\"]-joinedDF[\"fiyat2\"])*100)/joinedDF['fiyat2'])\n",
        "\n",
        "joinedDF = joinedDF.sort('oran',ascending=False)\n",
        "joinedDF.show()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+-----+----------+---------+----------+---------+------------------+\n",
            "|   id|    tarih2|   fiyat2|     tarih|    fiyat|              oran|\n",
            "+-----+----------+---------+----------+---------+------------------+\n",
            "| 2683|19-08-1960|      2.8|22-08-1960|      9.0| 221.4285751994776|\n",
            "| 7589|24-01-1980|     35.0|25-01-1980|     70.0|             100.0|\n",
            "| 5201|07-08-1970|      9.0|10-08-1970|    14.85| 65.00000678168402|\n",
            "|12912|22-02-2001| 685391.0|23-02-2001| 957879.0| 39.75657690281898|\n",
            "|11172|05-04-1994| 23031.84|06-04-1994| 31988.89| 38.88986153848502|\n",
            "| 7433|11-06-1979|     26.5|12-06-1979|     35.0|32.075471698113205|\n",
            "| 7108|28-02-1978|    19.25|01-03-1978|     25.0| 29.87012987012987|\n",
            "|11173|06-04-1994| 31988.89|07-04-1994| 39853.13| 24.58427901483376|\n",
            "|11125|26-01-1994| 15155.63|27-01-1994|  17215.5|13.591451969845416|\n",
            "|12913|23-02-2001| 957879.0|26-02-2001|1072988.0|12.017071049683729|\n",
            "|11186|25-04-1994| 30658.56|26-04-1994| 33925.01|10.654287715190682|\n",
            "|12936|03-04-2001|1114738.0|04-04-2001|1232604.0|10.573426222125738|\n",
            "| 6997|20-09-1977|     17.5|21-09-1977|    19.25|              10.0|\n",
            "|12935|02-04-2001|1020560.0|03-04-2001|1114738.0| 9.228070863055578|\n",
            "|13010|17-07-2001|1366647.0|18-07-2001|1482571.0| 8.482365965754141|\n",
            "|11211|02-06-1994| 30623.63|03-06-1994|  33181.5| 8.352598926775999|\n",
            "|11270|24-08-1994|  32583.7|25-08-1994| 35291.28| 8.309620883812805|\n",
            "|14843|22-10-2008|     1.52|23-10-2008|     1.63| 7.236843450908203|\n",
            "|12933|29-03-2001| 992354.0|30-03-2001|1056544.0| 6.468457828557148|\n",
            "|12922|14-03-2001| 941519.0|15-03-2001| 999788.0| 6.188828903081085|\n",
            "+-----+----------+---------+----------+---------+------------------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8PxbLwzpEEst",
        "outputId": "f62b01e5-7e7c-4deb-8807-e93e244772ce"
      },
      "source": [
        "from pyspark.sql import SparkSession\n",
        "from pyspark.sql.functions import UserDefinedFunction\n",
        "from pyspark.sql.types import FloatType\n",
        "\n",
        "spark = SparkSession.Builder().appName('Spark DataFrame Introduction').getOrCreate()\n",
        "\n",
        "depremDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','true')\\\n",
        "                        .option('delimiter','\\t').csv('EarthquakeDataset.txt')\n",
        "\n",
        "\n",
        "depremDF = depremDF.select(\"Latitude\",\"Longitude\",\"xM\",\"Location\")\n",
        "depremDF.show()\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+--------+---------+---+--------------------+\n",
            "|Latitude|Longitude| xM|            Location|\n",
            "+--------+---------+---+--------------------+\n",
            "| 37.8435|  26.7775|3.8|          EGE DENIZI|\n",
            "| 36.4918|  28.7092|3.9|             AKDENIZ|\n",
            "|  37.796|  26.4165|4.4|ONIKI ADALAR (AKD...|\n",
            "| 38.4172|  39.1482|3.9|KAVAKKOY-SIVRICE ...|\n",
            "|  36.475|  28.7852|4.2|             AKDENIZ|\n",
            "|  38.505|   39.218|5.6|KAVAKTEPE- (ELAZI...|\n",
            "| 38.5305|  39.2102|4.1|KAVAKTEPE- (ELAZI...|\n",
            "| 38.0033|  42.9642|3.9|KORULU-CATAK (VAN...|\n",
            "| 36.5213|   28.695|3.7|             AKDENIZ|\n",
            "| 37.8888|  27.6075|3.7|DAGKARAAGAC-GERME...|\n",
            "| 35.7737|   36.447|3.9|              SURIYE|\n",
            "| 37.8967|  43.8062|4.5|AYDEMIR-BASKALE (...|\n",
            "|  37.126|  28.5738|4.0|ARICILAR-ULA (MUG...|\n",
            "| 37.1175|  31.0832|3.7|ETLER-SERIK (ANTA...|\n",
            "| 35.7012|  26.2608|3.6|             AKDENIZ|\n",
            "| 35.7032|  26.2382|4.6|             AKDENIZ|\n",
            "|  35.704|  26.2952|3.8|             AKDENIZ|\n",
            "|  35.691|  26.2818|3.9|             AKDENIZ|\n",
            "| 38.8573|  43.5082|4.7|ERMISLER- (VAN) [...|\n",
            "|  37.683|  27.0502|3.5|MILLI PARK-KUSADA...|\n",
            "+--------+---------+---+--------------------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "l2t5f_VZFo-Y",
        "outputId": "128936ac-3da7-4175-e5fe-a37cf9b3ea0a"
      },
      "source": [
        "from pyspark.sql import SparkSession\n",
        "from pyspark.sql.functions import UserDefinedFunction\n",
        "from pyspark.sql.types import FloatType,StringType\n",
        "\n",
        "spark = SparkSession.Builder().appName('Spark DataFrame Introduction').getOrCreate()\n",
        "\n",
        "depremDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','true')\\\n",
        "                        .option('delimiter','\\t').csv('EarthquakeDataset.txt')\n",
        "\n",
        "\n",
        "depremDF = depremDF.select(\"Latitude\",\"Longitude\",\"xM\",\"Location\")\n",
        "\n",
        "\n",
        "def bolgeAta(enlem,boylam):\n",
        "    bolge = \"\"\n",
        "    if (enlem >= 39):\n",
        "        if (boylam <=31):\n",
        "            bolge = \"Bölge 1\"\n",
        "        elif (boylam <= 36):\n",
        "            bolge = \"Bölge 2\"\n",
        "        elif (boylam <= 41):\n",
        "            bolge = \"Bölge 3\"\n",
        "        else:\n",
        "            bolge = \"Bölge 4\"\n",
        "    else:\n",
        "        if (boylam <=31):\n",
        "            bolge = \"Bölge 5\"\n",
        "        elif (boylam <= 36):\n",
        "            bolge = \"Bölge 6\"\n",
        "        elif (boylam <= 41):\n",
        "            bolge = \"Bölge 7\"\n",
        "        else:\n",
        "            bolge = \"Bölge 8\"\n",
        "    return bolge\n",
        "\n",
        "bolgeAtaUDF = UserDefinedFunction(bolgeAta,StringType())\n",
        "depremDF = depremDF.withColumn('Bolge',bolgeAtaUDF('Latitude','Longitude'))\n",
        "depremDF = depremDF.groupBy('Bolge').count()\n",
        "depremDF = depremDF.sort('count',ascending=False)\n",
        "depremDF.show(9)\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+-------+-----+\n",
            "|  Bolge|count|\n",
            "+-------+-----+\n",
            "|Bölge 5| 7946|\n",
            "|Bölge 1| 2827|\n",
            "|Bölge 8| 1547|\n",
            "|Bölge 7| 1335|\n",
            "|Bölge 6| 1277|\n",
            "|Bölge 2|  994|\n",
            "|Bölge 3|  982|\n",
            "|Bölge 4|  752|\n",
            "+-------+-----+\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6t5gs7wUNyMj",
        "outputId": "03208876-0b58-4ce7-b5f1-52c6be8a775e"
      },
      "source": [
        "from pyspark.sql import SparkSession\n",
        "spark = SparkSession.Builder().appName('Spark DataFrame Introduction').getOrCreate()\n",
        "worldDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','false')\\\n",
        "                        .option('delimiter',',').csv('world.txt')\n",
        "\n",
        "\n",
        "worldDF = worldDF.withColumnRenamed('_c0','ulke')\n",
        "worldDF = worldDF.withColumnRenamed('_c1','kita')\n",
        "worldDF = worldDF.withColumnRenamed('_c2','baskent')\n",
        "worldDF = worldDF.withColumnRenamed('_c3','nufus')\n",
        "\n",
        "#worldDF.show()\n",
        "\n",
        "#Dünyada hangi kıtada kaç ülke vardır ?\n",
        "cozum1DF = worldDF.groupBy('kita').count()\n",
        "cozum1DF = cozum1DF.sort('count',ascending=False)\n",
        "cozum1DF.show()\n",
        "#Kıtalardaki ülkelerinin ortalama nüfusu (kıta bazında) nedir ?\n",
        "cozum2DF = worldDF.groupBy('kita').mean()\n",
        "cozum2DF = cozum2DF.sort('avg(nufus)',ascending=False)\n",
        "cozum2DF = cozum2DF.withColumn('ortalama nufus',cozum2DF['avg(nufus)'].cast('int'))\n",
        "cozum2DF.show()\n",
        "\n",
        "cozum3DF = worldDF.groupBy('kita').sum('nufus')\n",
        "cozum3DF = cozum3DF.sort('sum(nufus)',ascending=False)\n",
        "cozum3DF.show()\n",
        "\n",
        "#Kıtaların toplam nüfusu"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+-------------+-----+\n",
            "|         kita|count|\n",
            "+-------------+-----+\n",
            "|       Africa|   55|\n",
            "|         Asia|   49|\n",
            "|       Europe|   46|\n",
            "|North America|   23|\n",
            "|       Oceana|   13|\n",
            "|South America|   12|\n",
            "+-------------+-----+\n",
            "\n",
            "+-------------+--------------------+--------------+\n",
            "|         kita|          avg(nufus)|ortalama nufus|\n",
            "+-------------+--------------------+--------------+\n",
            "|         Asia| 8.127662059183674E7|      81276620|\n",
            "|South America|       3.124975725E7|      31249757|\n",
            "|North America| 2.222895156521739E7|      22228951|\n",
            "|       Africa|1.6516486363636363E7|      16516486|\n",
            "|       Europe|1.4397671369565217E7|      14397671|\n",
            "|       Oceana|  2462989.3076923075|       2462989|\n",
            "+-------------+--------------------+--------------+\n",
            "\n",
            "+-------------+----------+\n",
            "|         kita|sum(nufus)|\n",
            "+-------------+----------+\n",
            "|         Asia|3982554409|\n",
            "|       Africa| 908406750|\n",
            "|       Europe| 662292883|\n",
            "|North America| 511265886|\n",
            "|South America| 374997087|\n",
            "|       Oceana|  32018861|\n",
            "+-------------+----------+\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0K5R9-fsUL44",
        "outputId": "f1edc35e-50ce-4ddc-92d1-934706783c3c"
      },
      "source": [
        "from pyspark.sql import SparkSession\n",
        "spark = SparkSession.Builder().appName('Spark DataFrame Introduction').getOrCreate()\n",
        "worldDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','false')\\\n",
        "                        .option('delimiter',',').csv('world.txt')\n",
        "\n",
        "worldDF = worldDF.withColumnRenamed('_c0','ulke')\n",
        "worldDF = worldDF.withColumnRenamed('_c1','kita')\n",
        "worldDF = worldDF.withColumnRenamed('_c2','baskent')\n",
        "worldDF = worldDF.withColumnRenamed('_c3','nufus')\n",
        "\n",
        "worldDF.createOrReplaceTempView('world')\n",
        "\n",
        "cozum1DF = spark.sql('select kita,count(*) as count from world group by kita order by count desc')\n",
        "cozum1DF.show()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+-------------+-----+\n",
            "|         kita|count|\n",
            "+-------------+-----+\n",
            "|       Africa|   55|\n",
            "|         Asia|   49|\n",
            "|       Europe|   46|\n",
            "|North America|   23|\n",
            "|       Oceana|   13|\n",
            "|South America|   12|\n",
            "+-------------+-----+\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "o1KNNT-e3kgF",
        "outputId": "164cd4e8-35f3-465d-88c4-0ff93d6ba970"
      },
      "source": [
        "from pyspark.sql import SparkSession\n",
        "\n",
        "spark = SparkSession.Builder().appName('Spark DataFrame Introduction').getOrCreate()\n",
        "actorsDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','true')\\\n",
        "                        .option('delimiter',',').csv('join-actors_BNO.txt')\n",
        "\n",
        "seriesDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','true')\\\n",
        "                        .option('delimiter',',').csv('join-series_BNO.txt')\n",
        "seriesDF.show()\n",
        "actorsDF.show(25)\n",
        "\n",
        "'''\n",
        "#Without SQL\n",
        "joinedDF = actorsDF.join(seriesDF,'movieid','outer')\n",
        "joinedDF1 = actorsDF.join(seriesDF,'movieid','right')\n",
        "joinedDF2 = actorsDF.join(seriesDF,'movieid','left') #left , right ,inner ,outer\n",
        "joinedDF3 = actorsDF.join(seriesDF,'movieid','inner')\n",
        "\n",
        "#joinedDF = seriesDF.join(actorsDF,'movieid','outer')\n",
        "#joinedDF1 = seriesDF.join(actorsDF,'movieid','right')\n",
        "#joinedDF2 = seriesDF.join(actorsDF,'movieid','left') #left , right ,inner ,outer\n",
        "#joinedDF3 = seriesDF.join(actorsDF,'movieid','inner')\n",
        "\n",
        "\n",
        "joinedDF= joinedDF.sort(\"movieid\",ascending=True) \n",
        "\n",
        "joinedDF1= joinedDF1.sort(\"movieid\",ascending=True) \n",
        "joinedDF2= joinedDF2.sort(\"movieid\",ascending=True) \n",
        "joinedDF3= joinedDF3.sort(\"movieid\",ascending=True) \n",
        "\n",
        "joinedDF.show(26)\n",
        "joinedDF1.show(26)\n",
        "joinedDF2.show(26)\n",
        "joinedDF3.show(26)\n",
        "\n",
        "'''\n",
        "\n",
        "#with SQl\n",
        "\n",
        "actorsDF.createOrReplaceTempView('actors')\n",
        "seriesDF.createOrReplaceTempView('series')\n",
        "#sqlDF = spark.sql('select a.*,s.* from actors a,series s where s.movieid=a.movieid')\n",
        "sqlDF = spark.sql('select a.*,s.* from actors a full outer join series s on s.movieid=a.movieid')\n",
        "sqlDF.show(40)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+-------+------------------+\n",
            "|movieid|         moviename|\n",
            "+-------+------------------+\n",
            "|      1|     Lethal Weapon|\n",
            "|      2|     The Last Ship|\n",
            "|      3|Person of Interest|\n",
            "|      4|   Game of Thrones|\n",
            "|      5|      Flashforward|\n",
            "|      6|        Revolution|\n",
            "|      7|              Lost|\n",
            "|      8|  Leyle ile Mecnun|\n",
            "|      9|               BNO|\n",
            "+-------+------------------+\n",
            "\n",
            "+---+--------------------+-------+\n",
            "| id|                name|movieid|\n",
            "+---+--------------------+-------+\n",
            "|  1|           Eric Dane|      2|\n",
            "|  2|        Adam Baldwin|      2|\n",
            "|  3|     Charles Parnell|      2|\n",
            "|  4|         Billy Burke|      6|\n",
            "|  5|    Tracy Spiridakos|      6|\n",
            "|  6|  Giancarlo Esposito|      6|\n",
            "|  7|        Damon Wayans|      1|\n",
            "|  8|     Clayne Crawford|      1|\n",
            "|  9|      Joseph Fiennes|      5|\n",
            "| 10|      Jack Davenport|      5|\n",
            "| 11|    Zachary Knighton|      5|\n",
            "| 12|        Jim Caviezel|      3|\n",
            "| 13|     Michael Emerson|      3|\n",
            "| 14|    Taraji P. Henson|      3|\n",
            "| 15|      Peter Dinklage|      4|\n",
            "| 16|       Emilia Clarke|      4|\n",
            "| 17|       Kit Harington|      4|\n",
            "| 18|Nikolaj Coster-Wa...|      4|\n",
            "| 19|     Ahmet Demirelli|   null|\n",
            "| 20|       Josh Holloway|      7|\n",
            "| 21|    Evangeline Lilly|      7|\n",
            "| 22|         Matthew Fox|      7|\n",
            "| 23|                null|      9|\n",
            "+---+--------------------+-------+\n",
            "\n",
            "+----+--------------------+-------+-------+------------------+\n",
            "|  id|                name|movieid|movieid|         moviename|\n",
            "+----+--------------------+-------+-------+------------------+\n",
            "|  19|     Ahmet Demirelli|   null|   null|              null|\n",
            "|   7|        Damon Wayans|      1|      1|     Lethal Weapon|\n",
            "|   8|     Clayne Crawford|      1|      1|     Lethal Weapon|\n",
            "|   4|         Billy Burke|      6|      6|        Revolution|\n",
            "|   5|    Tracy Spiridakos|      6|      6|        Revolution|\n",
            "|   6|  Giancarlo Esposito|      6|      6|        Revolution|\n",
            "|  12|        Jim Caviezel|      3|      3|Person of Interest|\n",
            "|  13|     Michael Emerson|      3|      3|Person of Interest|\n",
            "|  14|    Taraji P. Henson|      3|      3|Person of Interest|\n",
            "|   9|      Joseph Fiennes|      5|      5|      Flashforward|\n",
            "|  10|      Jack Davenport|      5|      5|      Flashforward|\n",
            "|  11|    Zachary Knighton|      5|      5|      Flashforward|\n",
            "|  23|                null|      9|      9|               BNO|\n",
            "|  15|      Peter Dinklage|      4|      4|   Game of Thrones|\n",
            "|  16|       Emilia Clarke|      4|      4|   Game of Thrones|\n",
            "|  17|       Kit Harington|      4|      4|   Game of Thrones|\n",
            "|  18|Nikolaj Coster-Wa...|      4|      4|   Game of Thrones|\n",
            "|null|                null|   null|      8|  Leyle ile Mecnun|\n",
            "|  20|       Josh Holloway|      7|      7|              Lost|\n",
            "|  21|    Evangeline Lilly|      7|      7|              Lost|\n",
            "|  22|         Matthew Fox|      7|      7|              Lost|\n",
            "|   1|           Eric Dane|      2|      2|     The Last Ship|\n",
            "|   2|        Adam Baldwin|      2|      2|     The Last Ship|\n",
            "|   3|     Charles Parnell|      2|      2|     The Last Ship|\n",
            "+----+--------------------+-------+-------+------------------+\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Icx0rmZEC1in",
        "outputId": "6b566117-d244-4724-8227-a8a3efca1642"
      },
      "source": [
        "\n",
        "#points-null-vakues.txt dosyasında;\n",
        " # --- ilk kolondaki null değerleri 1 ile \n",
        " # --- ikinci kolondaki null değerleri 0 ile değiştiriniz!! (UserDefinedFunction)\n",
        " # --- kısayol\n",
        " # --- ilk kolondaki null değerleri ortalama ile\n",
        " # --- ikinci kolondaki medyan ile değişitiriniz\n",
        "\n",
        "from pyspark.sql import SparkSession\n",
        "from pyspark.sql.functions import UserDefinedFunction\n",
        "from pyspark.sql.types import IntegerType\n",
        "\n",
        "spark = SparkSession.Builder().appName('Spark DataFrame Introduction').getOrCreate()\n",
        "pointsDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','false')\\\n",
        "                        .option('delimiter',',').csv('points-null-values.txt')\n",
        "pointsDF = pointsDF.withColumnRenamed('_c0','x')\n",
        "pointsDF = pointsDF.withColumnRenamed('_c1','y')\n",
        "\n",
        "#pointsDF.show(25)\n",
        "\n",
        "\n",
        "def replaceNullColumn1(x):\n",
        "  if x is None:\n",
        "    x=999\n",
        "  return x\n",
        "\n",
        "def replaceNullColumn2(x):\n",
        "  if x is None:\n",
        "    x=777\n",
        "  return x\n",
        "\n",
        "replaceUDF1=UserDefinedFunction(replaceNullColumn1,IntegerType())\n",
        "replaceUDF2=UserDefinedFunction(replaceNullColumn2,IntegerType())\n",
        "\n",
        "pointsDF = pointsDF.withColumn('X_',replaceUDF1('x'))\n",
        "pointsDF = pointsDF.withColumn('Y_',replaceUDF2('y'))\n",
        "\n",
        "pointsDF.show(50)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+--------+--------+--------+--------+\n",
            "|       x|       y|      X_|      Y_|\n",
            "+--------+--------+--------+--------+\n",
            "|       5|       5|       5|       5|\n",
            "|     105|     107|     105|     107|\n",
            "|     105|     106|     105|     106|\n",
            "|    null|       6|     999|       6|\n",
            "|       6|       6|       6|       6|\n",
            "|     106|    null|     106|     777|\n",
            "|       7|       7|       7|       7|\n",
            "|    null|       6|     999|       6|\n",
            "|     107|     107|     107|     107|\n",
            "|       6|       7|       6|       7|\n",
            "|       7|    null|       7|     777|\n",
            "|     108|     108|     108|     108|\n",
            "|       8|       6|       8|       6|\n",
            "|    null|     108|     999|     108|\n",
            "|       5|       8|       5|       8|\n",
            "|     107|     106|     107|     106|\n",
            "|       8|       8|       8|       8|\n",
            "|    null|     107|     999|     107|\n",
            "|     107|     108|     107|     108|\n",
            "|     108|     106|     108|     106|\n",
            "|10000008|10000006|10000008|10000006|\n",
            "+--------+--------+--------+--------+\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JII_l1DSEns3",
        "outputId": "99bc03f4-da90-46d3-b2b6-8c4768674590"
      },
      "source": [
        "#Kısa Yol (Sabit sayı ile değiştirme)\n",
        "from pyspark.sql import SparkSession\n",
        "from pyspark.sql.functions import UserDefinedFunction\n",
        "from pyspark.sql.types import IntegerType\n",
        "\n",
        "spark = SparkSession.Builder().appName('Spark DataFrame Introduction').getOrCreate()\n",
        "pointsDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','false')\\\n",
        "                        .option('delimiter',',').csv('points-null-values.txt')\n",
        "pointsDF = pointsDF.withColumnRenamed('_c0','x')\n",
        "pointsDF = pointsDF.withColumnRenamed('_c1','y')\n",
        "\n",
        "pointsDF = pointsDF.fillna(9999,'x')\n",
        "pointsDF = pointsDF.fillna(8888,'y')\n",
        "pointsDF.show()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+----+----+\n",
            "|   x|   y|\n",
            "+----+----+\n",
            "|   5|   5|\n",
            "| 105| 107|\n",
            "| 105| 106|\n",
            "|9999|   6|\n",
            "|   6|   6|\n",
            "| 106|8888|\n",
            "|   7|   7|\n",
            "|9999|   6|\n",
            "| 107| 107|\n",
            "|   6|   7|\n",
            "|   7|8888|\n",
            "| 108| 108|\n",
            "|   8|   6|\n",
            "|9999| 108|\n",
            "|   5|   8|\n",
            "| 107| 106|\n",
            "|   8|   8|\n",
            "|9999| 107|\n",
            "| 107| 108|\n",
            "| 108| 106|\n",
            "+----+----+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IKiP-kJLE4TX",
        "outputId": "78c0dc14-dd46-4eac-9bed-f52dd2325e4c"
      },
      "source": [
        "from pyspark.sql import SparkSession\n",
        "from pyspark.sql.functions import UserDefinedFunction\n",
        "from pyspark.sql.types import IntegerType\n",
        "\n",
        "spark = SparkSession.Builder().appName('Spark DataFrame Introduction').getOrCreate()\n",
        "pointsDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','false')\\\n",
        "                        .option('delimiter',',').csv('points-null-values.txt')\n",
        "pointsDF = pointsDF.withColumnRenamed('_c0','x')\n",
        "pointsDF = pointsDF.withColumnRenamed('_c1','y')\n",
        "pointsDF.show(40)\n",
        "pointsDF.describe().show()  #aşağıdaki cıktıda standart sapma ortalama mean değerden cok fazla demekki aykırı degerler var o zaman medıan kullanmak daha iyidir"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+--------+--------+\n",
            "|       x|       y|\n",
            "+--------+--------+\n",
            "|       5|       5|\n",
            "|     105|     107|\n",
            "|     105|     106|\n",
            "|    null|       6|\n",
            "|       6|       6|\n",
            "|     106|    null|\n",
            "|       7|       7|\n",
            "|    null|       6|\n",
            "|     107|     107|\n",
            "|       6|       7|\n",
            "|       7|    null|\n",
            "|     108|     108|\n",
            "|       8|       6|\n",
            "|    null|     108|\n",
            "|       5|       8|\n",
            "|     107|     106|\n",
            "|       8|       8|\n",
            "|    null|     107|\n",
            "|     107|     108|\n",
            "|     108|     106|\n",
            "|10000008|10000006|\n",
            "+--------+--------+\n",
            "\n",
            "+-------+-----------------+-----------------+\n",
            "|summary|                x|                y|\n",
            "+-------+-----------------+-----------------+\n",
            "|  count|               17|               19|\n",
            "|   mean|         588289.0|526369.8947368421|\n",
            "| stddev|2425344.472743975|2294145.690034356|\n",
            "|    min|                5|                5|\n",
            "|    max|         10000008|         10000006|\n",
            "+-------+-----------------+-----------------+\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "iinmq1S_Eh-Y",
        "outputId": "72ad6264-a5f5-4763-c88b-0d79b4b88fb6"
      },
      "source": [
        "#Kısa Yol (Ortalama ile değiştirme)\n",
        "from pyspark.sql import SparkSession\n",
        "from pyspark.ml.feature import Imputer\n",
        "\n",
        "spark = SparkSession.Builder().appName('Spark DataFrame Introduction').getOrCreate()\n",
        "pointsDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','false')\\\n",
        "                        .option('delimiter',',').csv('points-null-values.txt')\n",
        "pointsDF = pointsDF.withColumnRenamed('_c0','x')\n",
        "pointsDF = pointsDF.withColumnRenamed('_c1','y')\n",
        "pointsDF.show(26)\n",
        "\n",
        "#Imputer kullanırken öncelikle outlier var mı diye bakarız\n",
        "#Eğer outlier yok ise ortalama (Mean) ile değiştirmek mantıklıdır\n",
        "#Eğer outlier varsa ve silmek istemiyorsak medyan (median) ile değiştirmek mantıklıdır\n",
        "\n",
        "meanImputer = Imputer(inputCol='x',outputCol='imputedXMean',strategy='mean')\n",
        "pointsDF = meanImputer.fit(pointsDF).transform(pointsDF)\n",
        "\n",
        "medianImputer = Imputer(inputCol='y',outputCol='imputedYMedian',strategy='median')\n",
        "pointsDF = medianImputer.fit(pointsDF).transform(pointsDF)\n",
        "\n",
        "pointsDF.show(100)\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+--------+--------+\n",
            "|       x|       y|\n",
            "+--------+--------+\n",
            "|       5|       5|\n",
            "|     105|     107|\n",
            "|     105|     106|\n",
            "|    null|       6|\n",
            "|       6|       6|\n",
            "|     106|    null|\n",
            "|       7|       7|\n",
            "|    null|       6|\n",
            "|     107|     107|\n",
            "|       6|       7|\n",
            "|       7|    null|\n",
            "|     108|     108|\n",
            "|       8|       6|\n",
            "|    null|     108|\n",
            "|       5|       8|\n",
            "|     107|     106|\n",
            "|       8|       8|\n",
            "|    null|     107|\n",
            "|     107|     108|\n",
            "|     108|     106|\n",
            "|10000008|10000006|\n",
            "+--------+--------+\n",
            "\n",
            "+--------+--------+------------+--------------+\n",
            "|       x|       y|imputedXMean|imputedYMedian|\n",
            "+--------+--------+------------+--------------+\n",
            "|       5|       5|           5|             5|\n",
            "|     105|     107|         105|           107|\n",
            "|     105|     106|         105|           106|\n",
            "|    null|       6|      588289|             6|\n",
            "|       6|       6|           6|             6|\n",
            "|     106|    null|         106|           106|\n",
            "|       7|       7|           7|             7|\n",
            "|    null|       6|      588289|             6|\n",
            "|     107|     107|         107|           107|\n",
            "|       6|       7|           6|             7|\n",
            "|       7|    null|           7|           106|\n",
            "|     108|     108|         108|           108|\n",
            "|       8|       6|           8|             6|\n",
            "|    null|     108|      588289|           108|\n",
            "|       5|       8|           5|             8|\n",
            "|     107|     106|         107|           106|\n",
            "|       8|       8|           8|             8|\n",
            "|    null|     107|      588289|           107|\n",
            "|     107|     108|         107|           108|\n",
            "|     108|     106|         108|           106|\n",
            "|10000008|10000006|    10000008|      10000006|\n",
            "+--------+--------+------------+--------------+\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mgoSPVq_MYYc"
      },
      "source": [
        "#Homework 2 hint: DataFrame pivot, Recommendation Engine , Collabrative Filtering"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "B6U2xsb5Mfbc"
      },
      "source": [
        "## Categorical Values IMPUTER##"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3N_qI4yaMnPc",
        "outputId": "9e048210-6c17-4545-ea48-6682cca3b5bf"
      },
      "source": [
        "from pyspark.sql import SparkSession\n",
        "from pyspark.sql.functions import UserDefinedFunction\n",
        "from pyspark.sql.types import IntegerType\n",
        "\n",
        "spark = SparkSession.Builder().appName('Spark DataFrame Introduction').getOrCreate()\n",
        "evlerDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','true')\\\n",
        "                        .option('delimiter',';').csv('evler.csv')\n",
        "\n",
        "def lengthFunction(text):\n",
        "  return len(text)\n",
        "\n",
        "#pointsDF = pointsDF.withColumn('sehirX',pointsDF['Sehir']) #bu şekilde çalışıyor ama len almak isteidğimizde hata veriyor. Ç;ünkü tüm kolonu obje olarak\n",
        "# almaya çalışıyor \n",
        "#çarpım dolduğunda bunu satır satır hepsiyle yapıyor ama len denildiğinde bunu anlamıyor lenght metodunu burada çalışmaz UDF yazmamız lazım\n",
        "# yeni eklenecek kolan bu değer İzmir'i\n",
        "\n",
        "IUDF = UserDefinedFunction(lengthFunction,IntegerType())\n",
        "\n",
        "\n",
        "\n",
        "evlerDF = evlerDF.withColumn('sehirX',IUDF('Sehir'))\n",
        "\n",
        "evlerDF.show()\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+--------+---+-----+------+------+\n",
            "|   Sehir|Oda|salon| fiyat|sehirX|\n",
            "+--------+---+-----+------+------+\n",
            "|   izmir|  3|    1|500000|     5|\n",
            "|istanbul|  3|    1|800000|     8|\n",
            "|  ankara|  3|    1|600000|     6|\n",
            "|   izmir|  2|    1|300000|     5|\n",
            "|istanbul|  2|    1|450000|     8|\n",
            "|istanbul|  1|    1|250000|     8|\n",
            "|  ankara|  2|    1|380000|     6|\n",
            "|   izmir|  1|    1|220000|     5|\n",
            "|istanbul|  1|    1|245000|     8|\n",
            "+--------+---+-----+------+------+\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uT9DlToAPMx6"
      },
      "source": [
        "## Categorical Values&One Encoding ##"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "p9U5pmoAPU34",
        "outputId": "01672f10-bd31-4369-cd38-29fb40bacec8"
      },
      "source": [
        "from pyspark.sql import SparkSession\n",
        "from pyspark.ml.feature import StringIndexer,OneHotEncoder,VectorAssembler # böle bir indexleme var şehire göre modelimi girdisi etkilensin istiyoruz f(a,b,c)= ax +by +e bu x'i etkilicek\n",
        "from pyspark.ml.regression import LinearRegression\n",
        "\n",
        "spark = SparkSession.Builder().appName('Spark DataFrame Introduction').getOrCreate()\n",
        "evlerDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','true')\\\n",
        "                        .option('delimiter',';').csv('evler.csv')\n",
        "\n",
        "evlerDF.show()   \n",
        "\n",
        "#machine learning kosturmak için datamız tümuyle sayısal olmalı, bu sebeple sehir isimlerini değiştircez\n",
        "\n",
        "indexer = StringIndexer(inputCol='Sehir',outputCol='indexedSehir') ##Linear modellerde bu kulanılmamalı (Tree-based lerde kötü sonuç vermeyebilir)\n",
        "\n",
        "evlerDF = indexer.fit(evlerDF).transform(evlerDF)\n",
        "\n",
        "evlerDF.show()   \n",
        "#Index işleminden sonra Encode işlemi yaparsak en doğrusu olur (Hem linear hemde tree-based algoritmalar için)\n",
        "\n",
        "encoder = OneHotEncoder(inputCol='indexedSehir',outputCol='encodedSehir')\n",
        "\n",
        "evlerDF = encoder.fit(evlerDF).transform(evlerDF)\n",
        "\n",
        "evlerDF.show()  "
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+--------+---+-----+------+\n",
            "|   Sehir|Oda|salon| fiyat|\n",
            "+--------+---+-----+------+\n",
            "|   izmir|  3|    1|500000|\n",
            "|istanbul|  3|    1|800000|\n",
            "|  ankara|  3|    1|600000|\n",
            "|   izmir|  2|    1|300000|\n",
            "|istanbul|  2|    1|450000|\n",
            "|istanbul|  1|    1|250000|\n",
            "|  ankara|  2|    1|380000|\n",
            "|   izmir|  1|    1|220000|\n",
            "|istanbul|  1|    1|245000|\n",
            "+--------+---+-----+------+\n",
            "\n",
            "+--------+---+-----+------+------------+\n",
            "|   Sehir|Oda|salon| fiyat|indexedSehir|\n",
            "+--------+---+-----+------+------------+\n",
            "|   izmir|  3|    1|500000|         1.0|\n",
            "|istanbul|  3|    1|800000|         0.0|\n",
            "|  ankara|  3|    1|600000|         2.0|\n",
            "|   izmir|  2|    1|300000|         1.0|\n",
            "|istanbul|  2|    1|450000|         0.0|\n",
            "|istanbul|  1|    1|250000|         0.0|\n",
            "|  ankara|  2|    1|380000|         2.0|\n",
            "|   izmir|  1|    1|220000|         1.0|\n",
            "|istanbul|  1|    1|245000|         0.0|\n",
            "+--------+---+-----+------+------------+\n",
            "\n",
            "+--------+---+-----+------+------------+-------------+\n",
            "|   Sehir|Oda|salon| fiyat|indexedSehir| encodedSehir|\n",
            "+--------+---+-----+------+------------+-------------+\n",
            "|   izmir|  3|    1|500000|         1.0|(2,[1],[1.0])|\n",
            "|istanbul|  3|    1|800000|         0.0|(2,[0],[1.0])|\n",
            "|  ankara|  3|    1|600000|         2.0|    (2,[],[])|\n",
            "|   izmir|  2|    1|300000|         1.0|(2,[1],[1.0])|\n",
            "|istanbul|  2|    1|450000|         0.0|(2,[0],[1.0])|\n",
            "|istanbul|  1|    1|250000|         0.0|(2,[0],[1.0])|\n",
            "|  ankara|  2|    1|380000|         2.0|    (2,[],[])|\n",
            "|   izmir|  1|    1|220000|         1.0|(2,[1],[1.0])|\n",
            "|istanbul|  1|    1|245000|         0.0|(2,[0],[1.0])|\n",
            "+--------+---+-----+------+------------+-------------+\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wLAGEMhIYFWI"
      },
      "source": [
        "#Dummyfication  bu kullanılan yönteme göre yeni column'lar ekleyip diğer column'ı iptal ediyoruz3 şehir varsa son şehir eklenmez bunun için hoca'nın yüklediği excel'e bak\n",
        "#OneHotEncoding\n",
        "\n",
        "\n",
        "# Bu veriyi büyütür çünkü her yeni sehirler için yeni column ortaya çıkarıyoruz, kategorisel bir kalonumuz varsa, plaka kodu da olabilir,şehir ismide\n",
        "# cinsiyet,yaş bunlar\n",
        "\n",
        "#Sparsevector , densevector bunlar distinct eleman sayısı listenin yarısından az ise işe yarıyor buna bakarsın\n",
        "\n",
        "#distinct sayısına nasıl bakılır öğrenmem lazım"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2MiVL6IdYUb9",
        "outputId": "31e8f318-859e-4dc5-e350-9a1309622379"
      },
      "source": [
        "from pyspark.sql import SparkSession\n",
        "from pyspark.ml.feature import StringIndexer,OneHotEncoder,VectorAssembler # böle bir indexleme var şehire göre modelimi girdisi etkilensin istiyoruz f(a,b,c)= ax +by +e bu x'i etkilicek\n",
        "from pyspark.ml.regression import LinearRegression\n",
        "\n",
        "spark = SparkSession.Builder().appName('Spark DataFrame Introduction').getOrCreate()\n",
        "\n",
        "evlerDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','true')\\\n",
        "                        .option('delimiter',';').csv('evler.csv')\n",
        "\n",
        "\n",
        "indexer = StringIndexer(inputCol='Sehir',outputCol='indexedSehir')  ##linear modellerde bu kullanılması\n",
        "indexModel=indexer.fit(evlerDF)\n",
        "evlerDF = indexModel.transform(evlerDF)\n",
        "\n",
        "encoder = OneHotEncoder(inputCol='indexedSehir',outputCol='encodedSehir')\n",
        "#encodeModel=encoder.fit(evlerDF)\n",
        "#evlerDF = encodeModel.transform(evlerDF)\n",
        "\n",
        "evlerDF= encoder.fit(evlerDF).transform(evlerDF)\n",
        "\n",
        "evlerDF = evlerDF.withColumnRenamed('fiyat','label') #label atama\n",
        "evlerDF = evlerDF.select('encodedSehir','Oda','salon','label')\n",
        "\n",
        "vec =VectorAssembler(inputCols=['encodedSehir','Oda','salon'],outputCol='features')\n",
        "evlerDF =vec.transform(evlerDF)\n",
        "#evlerDF.show()\n",
        "\n",
        "evlerDF = evlerDF.select('features','label')\n",
        "#evlerDF.show()\n",
        "\n",
        "lr = LinearRegression()\n",
        "\n",
        "model = lr.fit(evlerDF) ##Makine Öğrenmesinin yapıldığı satır\n",
        "print('Bitti')\n",
        "\n",
        "print(\"Çarpanlar:  \",model.coefficients)\n",
        "print(\"Sabit sayi \", model.intercept)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Bitti\n",
            "Çarpanlar:   [107857.14398708245,-42261.90458170463,215476.19109859318,0.0]\n",
            "Sabit sayi  -48690.477997543654\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6CElb45AayxY"
      },
      "source": [
        "\n",
        "#SparseVector  --> DenseVector\n",
        "#(2,[],[])     --> [0,0]\n",
        "#(2,[1],[1.0]) --> [0,1]\n",
        "#(2,[0],[1.0]  --> [1,0]\n",
        "#[1,2,3,4] --> 4 *  32bit\n",
        "#[11,0,0,0,34,0,23,0,0,0,0,0,....]  --> 1000000 * 32bit --> 4 MB #DenseVector\n",
        "#(1000000, [0,4,6],[11,34,23] )  --> 7 * 32bit #SparseVector"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xnb6WX7WbF6H",
        "outputId": "fb03438c-9603-49dc-d645-dccf5c15c6af"
      },
      "source": [
        "from pyspark.sql import SparkSession\n",
        "from pyspark.ml.feature import StringIndexer,OneHotEncoder,VectorAssembler # böle bir indexleme var şehire göre modelimi girdisi etkilensin istiyoruz f(a,b,c)= ax +by +e bu x'i etkilicek\n",
        "from pyspark.ml.regression import LinearRegression\n",
        "\n",
        "spark = SparkSession.Builder().appName('Spark DataFrame Introduction').getOrCreate()\n",
        "\n",
        "evlerDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','true')\\\n",
        "                        .option('delimiter',';').csv('evler.csv')\n",
        "\n",
        "\n",
        "indexer = StringIndexer(inputCol='Sehir',outputCol='indexedSehir')  ##linear modellerde bu kullanılması\n",
        "indexModel=indexer.fit(evlerDF)\n",
        "evlerDF = indexModel.transform(evlerDF)\n",
        "\n",
        "encoder = OneHotEncoder(inputCol='indexedSehir',outputCol='encodedSehir')\n",
        "encodeModel=encoder.fit(evlerDF)\n",
        "\n",
        "#evlerDF = encodeModel.transform(evlerDF)\n",
        "\n",
        "evlerDF= encoder.fit(evlerDF).transform(evlerDF)\n",
        "\n",
        "evlerDF = evlerDF.withColumnRenamed('fiyat','label') #label atama\n",
        "evlerDF = evlerDF.select('encodedSehir','Oda','salon','label')\n",
        "\n",
        "vec =VectorAssembler(inputCols=['encodedSehir','Oda','salon'],outputCol='features')\n",
        "evlerDF =vec.transform(evlerDF)\n",
        "#evlerDF.show()\n",
        "\n",
        "evlerDF = evlerDF.select('features','label')\n",
        "#evlerDF.show()\n",
        "\n",
        "lr = LinearRegression()\n",
        "\n",
        "model = lr.fit(evlerDF) ##Makine Öğrenmesinin yapıldığı satır\n",
        "#print('Bitti')\n",
        "\n",
        "print(\"Çarpanlar:  \",model.coefficients)\n",
        "print(\"Sabit sayi \", model.intercept)\n",
        "\n",
        "\n",
        "\n",
        "#fiyatı bilinmeyen evler\n",
        "\n",
        "yenilerDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','true')\\\n",
        "                        .option('delimiter',';').csv('evler-fiyati-bilinmeyen.csv')\n",
        "\n",
        "yenilerDF = indexModel.transform(yenilerDF)\n",
        "yenilerDF = encodeModel.transform(yenilerDF)\n",
        "yenilerDF = vec.transform(yenilerDF)\n",
        "#yenilerDF.show()\n",
        "\n",
        "\n",
        "\n",
        "yenilerDF = model.transform(yenilerDF) #Prediction yapıyoruz\n",
        "yenilerDF.show()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Çarpanlar:   [107857.14398708245,-42261.90458170463,215476.19109859318,0.0]\n",
            "Sabit sayi  -48690.477997543654\n",
            "+--------+---+-----+------------+-------------+-----------------+-----------------+\n",
            "|   Sehir|Oda|salon|indexedSehir| encodedSehir|         features|       prediction|\n",
            "+--------+---+-----+------------+-------------+-----------------+-----------------+\n",
            "|   izmir|  3|    1|         1.0|(2,[1],[1.0])|[0.0,1.0,3.0,1.0]|555476.1907165312|\n",
            "|istanbul|  3|    1|         0.0|(2,[0],[1.0])|[1.0,0.0,3.0,1.0]|705595.2392853183|\n",
            "|  ankara|  3|    1|         2.0|    (2,[],[])|[0.0,0.0,3.0,1.0]|597738.0952982359|\n",
            "+--------+---+-----+------------+-------------+-----------------+-----------------+\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IFj0NQLXdOqy",
        "outputId": "1099884c-c099-4687-9cd9-57c410b78e3f"
      },
      "source": [
        "from pyspark.sql import SparkSession\n",
        "from pyspark.ml.feature import StringIndexer, VectorAssembler\n",
        "from pyspark.ml.classification import DecisionTreeClassifier\n",
        "\n",
        "spark = SparkSession.Builder().appName('Spark DataFrame Introduction').getOrCreate()\n",
        "irisDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','true')\\\n",
        "                        .option('delimiter',',').csv('iris-dataset.txt')\n",
        "#irisDF.show()\n",
        "\n",
        "#irisDF.describe().show()  #burada column'ların türüne bakıyoruz, labellarda dahil bir veri setinde her bilgi/her kolon sayısal olmak zorunda \n",
        "#irisDF.count()\n",
        "#irisDF.printSchema() #string'mi degilmi bakıyoruz\n",
        "\n",
        "#Label da dahil olmak üzere makine öğrenmesinde kullanılacak her kolon tipi sayısal olmalıdır.\n",
        "#Burada class (label) indexliyoruz\n",
        "\n",
        "indexer = StringIndexer(inputCol='class',outputCol='label')\n",
        "indexerModel = indexer.fit(irisDF) #Yeni kolonu eklemek için gerekli olan bilgi (kaç çiçek türü var) \n",
        "                                    #hesaplamak için tüm kolondaki saatırları tek tek işler\n",
        "\n",
        "irisDF = indexerModel.transform(irisDF)\n",
        "\n",
        "#irisDF.show()\n",
        "\n",
        "vec =VectorAssembler(inputCols=['sepal-length', 'sepal-width', 'petal-length', 'petal-width'],outputCol='features')\n",
        "\n",
        "irisDF = vec.transform(irisDF)\n",
        "#irisDF.show()\n",
        "\n",
        "irisDF = irisDF.select('features','label')\n",
        "irisDF.show(5)\n",
        "\n",
        "classifier = DecisionTreeClassifier()  # google da spark classficatiın yazıp bir modeli seçtik\n",
        "model = classifier.fit(irisDF) #Makine Öğrenmesini yapıldığı satır. Buna Model Training adı verilir.\n",
        "print(\"Makine öğrendi\")\n",
        "\n",
        "print(model.toDebugString) #burada feature 2 dediği feature'ın indexini gösteriyor. feature 1 hiç kodda yok bu kolonu drop bile edebiliriz demekki diğer kolanla korelasyonu var dedik\n",
        "#yani baska bir kolon arttığında buda artmıs yada azalmıs. makine öğrenmesine girmeden önce korelasyona bakıp bu kolonu drop edebilirdik \n",
        "#Bu tarz işlere feature engineering denir ilerleyen zamanlarda yapıcaz bunu burada hangi algoritmanın düzgün hesap yaptığını gösteriyoruz"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+-----------------+-----+\n",
            "|         features|label|\n",
            "+-----------------+-----+\n",
            "|[5.1,3.5,1.4,0.2]|  0.0|\n",
            "|[4.9,3.0,1.4,0.2]|  0.0|\n",
            "|[4.7,3.2,1.3,0.2]|  0.0|\n",
            "|[4.6,3.1,1.5,0.2]|  0.0|\n",
            "|[5.0,3.6,1.4,0.2]|  0.0|\n",
            "+-----------------+-----+\n",
            "only showing top 5 rows\n",
            "\n",
            "Makine öğrendi\n",
            "DecisionTreeClassificationModel: uid=DecisionTreeClassifier_a22ef284dbde, depth=5, numNodes=17, numClasses=3, numFeatures=4\n",
            "  If (feature 2 <= 2.45)\n",
            "   Predict: 0.0\n",
            "  Else (feature 2 > 2.45)\n",
            "   If (feature 3 <= 1.75)\n",
            "    If (feature 2 <= 4.95)\n",
            "     If (feature 3 <= 1.65)\n",
            "      Predict: 1.0\n",
            "     Else (feature 3 > 1.65)\n",
            "      Predict: 2.0\n",
            "    Else (feature 2 > 4.95)\n",
            "     If (feature 3 <= 1.55)\n",
            "      Predict: 2.0\n",
            "     Else (feature 3 > 1.55)\n",
            "      If (feature 0 <= 6.75)\n",
            "       Predict: 1.0\n",
            "      Else (feature 0 > 6.75)\n",
            "       Predict: 2.0\n",
            "   Else (feature 3 > 1.75)\n",
            "    If (feature 2 <= 4.85)\n",
            "     If (feature 0 <= 5.95)\n",
            "      Predict: 1.0\n",
            "     Else (feature 0 > 5.95)\n",
            "      Predict: 2.0\n",
            "    Else (feature 2 > 4.85)\n",
            "     Predict: 2.0\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "BL2qoCYNxGW1",
        "outputId": "919f6a5e-f9bd-4c7c-d4eb-781f203dd147"
      },
      "source": [
        "from pyspark.sql import SparkSession\n",
        "from pyspark.ml.feature import StringIndexer, VectorAssembler\n",
        "from pyspark.ml.classification import DecisionTreeClassifier\n",
        "\n",
        "spark = SparkSession.Builder().appName('Spark DataFrame Introduction').getOrCreate()\n",
        "irisDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','true')\\\n",
        "                        .option('delimiter',',').csv('iris-dataset.txt')\n",
        "#irisDF.show()\n",
        "\n",
        "#irisDF.describe().show()  #burada column'ların türüne bakıyoruz, labellarda dahil bir veri setinde her bilgi/her kolon sayısal olmak zorunda \n",
        "#irisDF.count()\n",
        "#irisDF.printSchema() #string'mi degilmi bakıyoruz\n",
        "\n",
        "#Label da dahil olmak üzere makine öğrenmesinde kullanılacak her kolon tipi sayısal olmalıdır.\n",
        "#Burada class (label) indexliyoruz\n",
        "\n",
        "indexer = StringIndexer(inputCol='class',outputCol='label')\n",
        "indexerModel = indexer.fit(irisDF) #Yeni kolonu eklemek için gerekli olan bilgi (kaç çiçek türü var) \n",
        "                                    #hesaplamak için tüm kolondaki saatırları tek tek işler\n",
        "\n",
        "irisDF = indexerModel.transform(irisDF)\n",
        "\n",
        "#irisDF.show()\n",
        "\n",
        "vec =VectorAssembler(inputCols=['sepal-length', 'sepal-width', 'petal-length', 'petal-width'],outputCol='features')\n",
        "\n",
        "irisDF = vec.transform(irisDF)\n",
        "#irisDF.show()\n",
        "\n",
        "irisDF = irisDF.select('features','label')\n",
        "irisDF.show(5)\n",
        "\n",
        "classifier = DecisionTreeClassifier()  # google da spark classficatiın yazıp bir modeli seçtik\n",
        "model = classifier.fit(irisDF) #Makine Öğrenmesini yapıldığı satır. Buna Model Training adı verilir.\n",
        "print(\"Makine öğrendi\")\n",
        "print(model.featureImportances) # aşağıdaki sayılarda 1 görünmedi o 0 çıktı demekki yüzdeler %100 olması lazım"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+-----------------+-----+\n",
            "|         features|label|\n",
            "+-----------------+-----+\n",
            "|[5.1,3.5,1.4,0.2]|  0.0|\n",
            "|[4.9,3.0,1.4,0.2]|  0.0|\n",
            "|[4.7,3.2,1.3,0.2]|  0.0|\n",
            "|[4.6,3.1,1.5,0.2]|  0.0|\n",
            "|[5.0,3.6,1.4,0.2]|  0.0|\n",
            "+-----------------+-----+\n",
            "only showing top 5 rows\n",
            "\n",
            "Makine öğrendi\n",
            "(4,[0,2,3],[0.02666666666666668,0.5507226247987117,0.4226107085346217])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "d-QgK4JrxVXo"
      },
      "source": [
        "#modelimizin ne kadar başarılı olduğunu nasıl anlıyoruz ? Bu model ne kadar başarılı veri setinden alıp bu modele sorabiliriz\n",
        "#ama model hesaplarken kullandığımız veri setini kullanırsak bunlar büyük ihtimalle doğru bulacaktır.\n",
        "#bir kısmını ayırıyorum veririm  buna test dataset diyorum\n",
        "#modeli oluştururken kullanacağım data setine de train dataset diyorum data'nın genelde 70-80%'i data set için kullanılır geri kalan 20-30% teste ayırırız"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-BbLU35HfeM8",
        "outputId": "0c78f818-7419-450a-ef75-8794c3649a16"
      },
      "source": [
        "from pyspark.sql import SparkSession\n",
        "from pyspark.ml.feature import StringIndexer, VectorAssembler\n",
        "from pyspark.ml.classification import RandomForestClassifier\n",
        "\n",
        "spark = SparkSession.Builder().appName('Spark DataFrame Introduction').getOrCreate()\n",
        "irisDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','true')\\\n",
        "                        .option('delimiter',',').csv('iris-dataset.txt')\n",
        "#irisDF.show()\n",
        "\n",
        "#irisDF.describe().show()\n",
        "#irisDF.count()\n",
        "#irisDF.printSchema() #string'mi degilmi bakıyoruz,# burada column'ların türüne bakıyoruz, labellarda dahil bir veri setinde her bilgi/her kolon sayısal olmak zorunda \n",
        "\n",
        "indexer = StringIndexer(inputCol='class',outputCol='label')\n",
        "indexerModel = indexer.fit(irisDF) #Yeni kolonu eklemek için gerekli olan bilgi (kaç çiçek türü var) \n",
        "                                    #hesaplamak için tüm kolondaki saatırları tek tek işler\n",
        "\n",
        "irisDF = indexerModel.transform(irisDF)\n",
        "\n",
        "#irisDF.show()\n",
        "\n",
        "vec =VectorAssembler(inputCols=['sepal-length', 'sepal-width', 'petal-length', 'petal-width'],outputCol='features')\n",
        "\n",
        "irisDF = vec.transform(irisDF)\n",
        "#irisDF.show()\n",
        "\n",
        "irisDF = irisDF.select('features','label')\n",
        "irisDF.show(5)\n",
        "\n",
        "classifier = RandomForestClassifier()\n",
        "model = classifier.fit(irisDF) #Makine Öğrenmesini yapıldığı satır. Buna Model Training adı verilir.\n",
        "print(\"Makine öğrendi\")\n",
        "print(model.featureImportances)\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+-----------------+-----+\n",
            "|         features|label|\n",
            "+-----------------+-----+\n",
            "|[5.1,3.5,1.4,0.2]|  0.0|\n",
            "|[4.9,3.0,1.4,0.2]|  0.0|\n",
            "|[4.7,3.2,1.3,0.2]|  0.0|\n",
            "|[4.6,3.1,1.5,0.2]|  0.0|\n",
            "|[5.0,3.6,1.4,0.2]|  0.0|\n",
            "+-----------------+-----+\n",
            "only showing top 5 rows\n",
            "\n",
            "Makine öğrendi\n",
            "(4,[0,1,2,3],[0.0824681606506779,0.013281914233677052,0.34766202244616545,0.5565879026694797])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "DmFZPYNVgySz",
        "outputId": "ef2b23ab-45f9-4006-8839-cb2e8e55ad60"
      },
      "source": [
        "from pyspark.sql import SparkSession\n",
        "from pyspark.ml.feature import StringIndexer, VectorAssembler\n",
        "from pyspark.ml.classification import DecisionTreeClassifier\n",
        "from pyspark.ml.evaluation import MulticlassClassificationEvaluator\n",
        "\n",
        "spark = SparkSession.Builder().appName('Spark DataFrame Introduction').getOrCreate()\n",
        "irisDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','true')\\\n",
        "                        .option('delimiter',',').csv('iris-dataset.txt')\n",
        "\n",
        "indexer = StringIndexer(inputCol='class',outputCol='label')\n",
        "indexerModel = indexer.fit(irisDF) #Yeni kolonu eklemek için gerekli olan bilgi (kaç çiçek türü var) \n",
        "                                    #hesaplamak için tüm kolondaki saatırları tek tek işler\n",
        "\n",
        "irisDF = indexerModel.transform(irisDF)\n",
        "\n",
        "vec =VectorAssembler(inputCols=['sepal-length', 'sepal-width', 'petal-length', 'petal-width'],outputCol='features')\n",
        "irisDF = vec.transform(irisDF)\n",
        "irisDF = irisDF.select('features','label')\n",
        "#irisDF.show()\n",
        "\n",
        "\n",
        "trainDF, testDF = irisDF.randomSplit([0.75,0.25])\n",
        "\n",
        "#random eğilde hep aynı yerden böl istiyotsak seed=1234 değeri eklemek lazım hep aynı yerden böl demek\n",
        "\n",
        "##fully random operasyon çok sevilmeyen bir durumdur çünkü bir daha asla aynısı gelmeyecek demektir, bu nedenle bilgisayarların\n",
        "# generate ettiği pseudo random kullanılır. \n",
        "\n",
        "\n",
        "classifier = DecisionTreeClassifier()\n",
        "model = classifier.fit(trainDF) #Makine Öğrenmesini yapıldığı satır. Buna Model Training adı verilir.\n",
        "print(\"Makine öğrendi\")\n",
        "\n",
        "#testDF.show()\n",
        "resultDF = model.transform(testDF) #Prediction\n",
        "resultDF.show()\n",
        "\n",
        "eva = MulticlassClassificationEvaluator(metricName='accuracy')\n",
        "result = eva.evaluate(resultDF)\n",
        "print(\"Accuracy : \",result)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Makine öğrendi\n",
            "+-----------------+-----+--------------+-------------+----------+\n",
            "|         features|label| rawPrediction|  probability|prediction|\n",
            "+-----------------+-----+--------------+-------------+----------+\n",
            "|[4.5,2.3,1.3,0.3]|  0.0|[39.0,0.0,0.0]|[1.0,0.0,0.0]|       0.0|\n",
            "|[4.8,3.4,1.9,0.2]|  0.0|[39.0,0.0,0.0]|[1.0,0.0,0.0]|       0.0|\n",
            "|[4.9,3.1,1.5,0.1]|  0.0|[39.0,0.0,0.0]|[1.0,0.0,0.0]|       0.0|\n",
            "|[5.0,3.4,1.5,0.2]|  0.0|[39.0,0.0,0.0]|[1.0,0.0,0.0]|       0.0|\n",
            "|[5.0,3.4,1.6,0.4]|  0.0|[39.0,0.0,0.0]|[1.0,0.0,0.0]|       0.0|\n",
            "|[5.0,3.5,1.6,0.6]|  0.0|[39.0,0.0,0.0]|[1.0,0.0,0.0]|       0.0|\n",
            "|[5.1,3.7,1.5,0.4]|  0.0|[39.0,0.0,0.0]|[1.0,0.0,0.0]|       0.0|\n",
            "|[5.2,4.1,1.5,0.1]|  0.0|[39.0,0.0,0.0]|[1.0,0.0,0.0]|       0.0|\n",
            "|[5.4,3.4,1.5,0.4]|  0.0|[39.0,0.0,0.0]|[1.0,0.0,0.0]|       0.0|\n",
            "|[5.5,2.3,4.0,1.3]|  1.0|[0.0,39.0,0.0]|[0.0,1.0,0.0]|       1.0|\n",
            "|[5.5,4.2,1.4,0.2]|  0.0|[39.0,0.0,0.0]|[1.0,0.0,0.0]|       0.0|\n",
            "|[5.6,2.9,3.6,1.3]|  1.0|[0.0,39.0,0.0]|[0.0,1.0,0.0]|       1.0|\n",
            "|[5.7,3.8,1.7,0.3]|  0.0|[39.0,0.0,0.0]|[1.0,0.0,0.0]|       0.0|\n",
            "|[5.8,2.7,3.9,1.2]|  1.0|[0.0,39.0,0.0]|[0.0,1.0,0.0]|       1.0|\n",
            "|[5.8,2.7,4.1,1.0]|  1.0|[0.0,39.0,0.0]|[0.0,1.0,0.0]|       1.0|\n",
            "|[5.9,3.0,4.2,1.5]|  1.0|[0.0,39.0,0.0]|[0.0,1.0,0.0]|       1.0|\n",
            "|[6.1,2.9,4.7,1.4]|  1.0|[0.0,39.0,0.0]|[0.0,1.0,0.0]|       1.0|\n",
            "|[6.3,2.5,5.0,1.9]|  2.0|[0.0,0.0,32.0]|[0.0,0.0,1.0]|       2.0|\n",
            "|[6.3,2.7,4.9,1.8]|  2.0|[0.0,0.0,32.0]|[0.0,0.0,1.0]|       2.0|\n",
            "|[6.3,3.3,6.0,2.5]|  2.0|[0.0,0.0,32.0]|[0.0,0.0,1.0]|       2.0|\n",
            "+-----------------+-----+--------------+-------------+----------+\n",
            "only showing top 20 rows\n",
            "\n",
            "Accuracy :  1.0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vcaPnMlXyAXF",
        "outputId": "d0b96457-a1bf-4494-9400-33b8f1f088f0"
      },
      "source": [
        "!pip install matplotlib"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: matplotlib in /usr/local/lib/python3.7/dist-packages (3.2.2)\n",
            "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib) (2.4.7)\n",
            "Requirement already satisfied: python-dateutil>=2.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib) (2.8.2)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib) (1.3.2)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.7/dist-packages (from matplotlib) (0.10.0)\n",
            "Requirement already satisfied: numpy>=1.11 in /usr/local/lib/python3.7/dist-packages (from matplotlib) (1.19.5)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from cycler>=0.10->matplotlib) (1.15.0)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "64UUUYV3nS5k",
        "outputId": "ec66c54f-d3f2-4ef4-f9a0-33413ee70055"
      },
      "source": [
        "from pyspark.sql import SparkSession\n",
        "from pyspark.ml.feature import StringIndexer, VectorAssembler\n",
        "from pyspark.ml.classification import RandomForestClassifier\n",
        "from pyspark.ml.evaluation import MulticlassClassificationEvaluator\n",
        "\n",
        "\n",
        "\n",
        "spark = SparkSession.Builder().appName('Spark DataFrame Introduction').getOrCreate()\n",
        "irisDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','true')\\\n",
        "                        .option('delimiter',',').csv('iris-dataset.txt')\n",
        "\n",
        "indexer = StringIndexer(inputCol='class',outputCol='label')\n",
        "indexerModel = indexer.fit(irisDF) \n",
        "irisDF = indexerModel.transform(irisDF)\n",
        "\n",
        "vec =VectorAssembler(inputCols=['sepal-length', 'sepal-width', 'petal-length', 'petal-width'],outputCol='features')\n",
        "irisDF = vec.transform(irisDF)\n",
        "irisDF = irisDF.select('features','label')\n",
        "\n",
        "\n",
        "trainDF, testDF = irisDF.randomSplit([0.75,0.25],seed=123)\n",
        "\n",
        "\n",
        "classifier = RandomForestClassifier()\n",
        "model = classifier.fit(trainDF) #Makine Öğrenmesini yapıldığı satır. Buna Model Training adı verilir.\n",
        "print(\"Makine öğrendi\")\n",
        "\n",
        "testDF.show()\n",
        "resultDF = model.transform(testDF) #Prediction\n",
        "resultDF.show()\n",
        "\n",
        "eva = MulticlassClassificationEvaluator(metricName='accuracy')\n",
        "result = eva.evaluate(resultDF)\n",
        "print(\"Accuracy : \",result)\n",
        "\n",
        "#Accuracy ye her zaman güvenemeyiz. Verinin düzgün dağılıp dağılmadığınaa bakmalıyız\n",
        "#Düzgün dağılmamışsa precision veya recall u yükseltmeye çalışmalıyız."
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Makine öğrendi\n",
            "+-----------------+-----+\n",
            "|         features|label|\n",
            "+-----------------+-----+\n",
            "|[4.4,3.0,1.3,0.2]|  0.0|\n",
            "|[4.6,3.2,1.4,0.2]|  0.0|\n",
            "|[4.8,3.0,1.4,0.3]|  0.0|\n",
            "|[4.8,3.1,1.6,0.2]|  0.0|\n",
            "|[4.9,3.0,1.4,0.2]|  0.0|\n",
            "|[5.0,2.3,3.3,1.0]|  1.0|\n",
            "|[5.0,3.5,1.3,0.3]|  0.0|\n",
            "|[5.0,3.5,1.6,0.6]|  0.0|\n",
            "|[5.1,3.3,1.7,0.5]|  0.0|\n",
            "|[5.1,3.4,1.5,0.2]|  0.0|\n",
            "|[5.1,3.5,1.4,0.2]|  0.0|\n",
            "|[5.1,3.8,1.5,0.3]|  0.0|\n",
            "|[5.1,3.8,1.9,0.4]|  0.0|\n",
            "|[5.4,3.4,1.5,0.4]|  0.0|\n",
            "|[5.5,2.5,4.0,1.3]|  1.0|\n",
            "|[5.5,4.2,1.4,0.2]|  0.0|\n",
            "|[5.6,2.7,4.2,1.3]|  1.0|\n",
            "|[5.6,2.8,4.9,2.0]|  2.0|\n",
            "|[5.8,2.6,4.0,1.2]|  1.0|\n",
            "|[5.8,2.7,5.1,1.9]|  2.0|\n",
            "+-----------------+-----+\n",
            "only showing top 20 rows\n",
            "\n",
            "+-----------------+-----+--------------------+--------------------+----------+\n",
            "|         features|label|       rawPrediction|         probability|prediction|\n",
            "+-----------------+-----+--------------------+--------------------+----------+\n",
            "|[4.4,3.0,1.3,0.2]|  0.0|      [20.0,0.0,0.0]|       [1.0,0.0,0.0]|       0.0|\n",
            "|[4.6,3.2,1.4,0.2]|  0.0|      [20.0,0.0,0.0]|       [1.0,0.0,0.0]|       0.0|\n",
            "|[4.8,3.0,1.4,0.3]|  0.0|      [20.0,0.0,0.0]|       [1.0,0.0,0.0]|       0.0|\n",
            "|[4.8,3.1,1.6,0.2]|  0.0|      [20.0,0.0,0.0]|       [1.0,0.0,0.0]|       0.0|\n",
            "|[4.9,3.0,1.4,0.2]|  0.0|      [20.0,0.0,0.0]|       [1.0,0.0,0.0]|       0.0|\n",
            "|[5.0,2.3,3.3,1.0]|  1.0|    [0.0,19.98,0.02]|   [0.0,0.999,0.001]|       1.0|\n",
            "|[5.0,3.5,1.3,0.3]|  0.0|      [20.0,0.0,0.0]|       [1.0,0.0,0.0]|       0.0|\n",
            "|[5.0,3.5,1.6,0.6]|  0.0|      [20.0,0.0,0.0]|       [1.0,0.0,0.0]|       0.0|\n",
            "|[5.1,3.3,1.7,0.5]|  0.0|      [20.0,0.0,0.0]|       [1.0,0.0,0.0]|       0.0|\n",
            "|[5.1,3.4,1.5,0.2]|  0.0|      [20.0,0.0,0.0]|       [1.0,0.0,0.0]|       0.0|\n",
            "|[5.1,3.5,1.4,0.2]|  0.0|      [20.0,0.0,0.0]|       [1.0,0.0,0.0]|       0.0|\n",
            "|[5.1,3.8,1.5,0.3]|  0.0|      [20.0,0.0,0.0]|       [1.0,0.0,0.0]|       0.0|\n",
            "|[5.1,3.8,1.9,0.4]|  0.0|      [19.0,1.0,0.0]|     [0.95,0.05,0.0]|       0.0|\n",
            "|[5.4,3.4,1.5,0.4]|  0.0|      [20.0,0.0,0.0]|       [1.0,0.0,0.0]|       0.0|\n",
            "|[5.5,2.5,4.0,1.3]|  1.0|[0.0,19.955609756...|[0.0,0.9977804878...|       1.0|\n",
            "|[5.5,4.2,1.4,0.2]|  0.0|      [20.0,0.0,0.0]|       [1.0,0.0,0.0]|       0.0|\n",
            "|[5.6,2.7,4.2,1.3]|  1.0|[0.0,19.955609756...|[0.0,0.9977804878...|       1.0|\n",
            "|[5.6,2.8,4.9,2.0]|  2.0|[0.0,2.2131782945...|[0.0,0.1106589147...|       2.0|\n",
            "|[5.8,2.6,4.0,1.2]|  1.0|[0.0,19.138943089...|[0.0,0.9569471544...|       1.0|\n",
            "|[5.8,2.7,5.1,1.9]|  2.0|[0.0,0.0465116279...|[0.0,0.0023255813...|       2.0|\n",
            "+-----------------+-----+--------------------+--------------------+----------+\n",
            "only showing top 20 rows\n",
            "\n",
            "Accuracy :  0.9696969696969697\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Vargcsu_uif9",
        "outputId": "71549ece-4b72-4c01-ba6b-a196a5848e21"
      },
      "source": [
        "#SparkML-HyperParameterTuning-TrainValidationSplit.ipynb\n",
        "\n",
        "from pyspark.sql import SparkSession\n",
        "from pyspark.ml.feature import StringIndexer, VectorAssembler\n",
        "from pyspark.ml.classification import RandomForestClassifier\n",
        "from pyspark.ml.evaluation import MulticlassClassificationEvaluator\n",
        "from pyspark.ml.tuning import TrainValidationSplit,ParamGridBuilder\n",
        "\n",
        "spark = SparkSession.Builder().appName('Spark DataFrame Introduction').getOrCreate()\n",
        "irisDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','true')\\\n",
        "                        .option('delimiter',',').csv('iris-dataset.txt')\n",
        "indexer = StringIndexer(inputCol='class',outputCol='label')\n",
        "indexerModel = indexer.fit(irisDF)                   \n",
        "irisDF = indexerModel.transform(irisDF)\n",
        "vec =VectorAssembler(inputCols=['sepal-length', 'sepal-width', 'petal-length', 'petal-width'],outputCol='features')\n",
        "irisDF = vec.transform(irisDF)\n",
        "irisDF = irisDF.select('features','label')\n",
        "trainDF, testDF = irisDF.randomSplit([0.8,0.2],seed=123)\n",
        "\n",
        "rf = RandomForestClassifier()\n",
        "eva = MulticlassClassificationEvaluator(metricName='accuracy')\n",
        "\n",
        "myParams = ParamGridBuilder().addGrid(rf.numTrees,[1,2,3])\\\n",
        "                            .addGrid(rf.maxDepth,[4,5,6])\\\n",
        "                            .addGrid(rf.impurity,['gini','entropy']).build()\n",
        "\n",
        "validator = TrainValidationSplit(parallelism=4, \n",
        "                                 estimator=rf,\n",
        "                                 estimatorParamMaps=myParams,\n",
        "                                 trainRatio=0.8,  #TrainDataset / Total Train Daataset ( izim problemimizde %80 in yüzde 80 i)\n",
        "                                 evaluator=eva)\n",
        "\n",
        "model = validator.fit(trainDF)\n",
        "print(\"Best Model hesaplandı\")\n",
        "print(\"Best NumTrees : \", model.bestModel.getNumTrees)\n",
        "print(\"Best MaxDepth : \", model.bestModel.getMaxDepth())\n",
        "print(\"Best Impurity : \", model.bestModel.getImpurity())\n",
        "\n",
        "resultDF = model.transform(testDF)\n",
        "result = eva.evaluate(resultDF)\n",
        "\n",
        "print(\"Accuracy of the best model \",result)"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Best Model hesaplandı\n",
            "Best NumTrees :  1\n",
            "Best MaxDepth :  4\n",
            "Best Impurity :  gini\n",
            "Accuracy of the best model  0.9310344827586207\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "yl3oUgKqyI4b",
        "outputId": "26592b5c-0039-4599-d4b7-75b8968a1198"
      },
      "source": [
        "#SparkML-HyperParameterTuning-CrossValidation\n",
        "\n",
        "from pyspark.sql import SparkSession\n",
        "from pyspark.ml.feature import StringIndexer, VectorAssembler\n",
        "from pyspark.ml.classification import RandomForestClassifier\n",
        "from pyspark.ml.evaluation import MulticlassClassificationEvaluator\n",
        "from pyspark.ml.tuning import CrossValidator,ParamGridBuilder\n",
        "spark = SparkSession.Builder().appName('Spark DataFrame Introduction').getOrCreate()\n",
        "irisDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','true')\\\n",
        "                        .option('delimiter',',').csv('iris-dataset.txt')\n",
        "indexer = StringIndexer(inputCol='class',outputCol='label')\n",
        "indexerModel = indexer.fit(irisDF)                   \n",
        "irisDF = indexerModel.transform(irisDF)\n",
        "vec =VectorAssembler(inputCols=['sepal-length', 'sepal-width', 'petal-length', 'petal-width'],outputCol='features')\n",
        "irisDF = vec.transform(irisDF)\n",
        "irisDF = irisDF.select('features','label')\n",
        "trainDF, testDF = irisDF.randomSplit([0.8,0.2],seed=123)\n",
        "rf = RandomForestClassifier()\n",
        "eva = MulticlassClassificationEvaluator(metricName='accuracy')\n",
        "myParams = ParamGridBuilder().addGrid(rf.numTrees,[1,2,3])\\\n",
        "                            .addGrid(rf.maxDepth,[4,5,6])\\\n",
        "                            .addGrid(rf.impurity,['gini','entropy']).build()\n",
        "validator = CrossValidator(parallelism=4, \n",
        "                                 estimator=rf,\n",
        "                                 estimatorParamMaps=myParams,\n",
        "                                 evaluator=eva,\n",
        "                                 numFolds=5)\n",
        "\n",
        "\n",
        "#\n",
        "\n",
        "model = validator.fit(trainDF)\n",
        "print(\"Best Model hesaplandı\")\n",
        "print(\"Best NumTrees : \", model.bestModel.getNumTrees)\n",
        "print(\"Best MaxDepth : \", model.bestModel.getMaxDepth())\n",
        "print(\"Best Impurity : \", model.bestModel.getImpurity())\n",
        "resultDF = model.transform(testDF)\n",
        "result = eva.evaluate(resultDF)\n",
        "print(\"Accuracy of the best model \",result)"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Best Model hesaplandı\n",
            "Best NumTrees :  3\n",
            "Best MaxDepth :  5\n",
            "Best Impurity :  gini\n",
            "Accuracy of the best model  0.9655172413793104\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KemOUnJK07mW",
        "outputId": "8ac5dc65-ba8b-46b7-87c8-eff22e37a660"
      },
      "source": [
        "#Boston House pricing verisetinde cross validation yaparaak en iyi LinearRegrression parametrelerini bulacağız.\n",
        "\n",
        "from pyspark.sql import SparkSession\n",
        "from pyspark.ml.feature import StringIndexer, VectorAssembler\n",
        "from pyspark.ml.regression import LinearRegression\n",
        "from pyspark.ml.evaluation import RegressionEvaluator\n",
        "from pyspark.ml.tuning import CrossValidator,ParamGridBuilder\n",
        "\n",
        "\n",
        "spark = SparkSession.Builder().appName('Spark DataFrame Introduction').getOrCreate()\n",
        "houseDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','true')\\\n",
        "                        .option('delimiter',',').csv('boston-house-prices.txt')\n",
        "\n",
        "\n",
        "houseDF = houseDF.withColumnRenamed('medv','label')\n",
        "#houseDF.show()\n",
        "\n",
        "#houseDF.describe().show()\n",
        "\n",
        "vec = VectorAssembler(inputCols=houseDF.columns[0:-1],outputCol='features')\n",
        "houseDF=vec.transform(houseDF)\n",
        "houseDF = houseDF.select('features','label')\n",
        "\n",
        "houseDF.show(truncate=False)\n",
        "\n",
        "trainDF, testDF = houseDF.randomSplit([0.8,0.2])\n",
        "\n",
        "lr = LinearRegression()\n",
        "\n",
        "model = lr.fit(trainDF)\n",
        "resultDF = model.transform(testDF)\n",
        "\n",
        "eva = RegressionEvaluator(metricName='rmse')\n",
        "res = eva.evaluate(resultDF)\n",
        "print('RMSE : ',res)\n"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+----------------------------------------------------------------------------+-----+\n",
            "|features                                                                    |label|\n",
            "+----------------------------------------------------------------------------+-----+\n",
            "|[0.00632,18.0,2.31,0.0,0.538,6.575,65.2,4.09,1.0,296.0,15.3,396.9,4.98]     |24.0 |\n",
            "|[0.02731,0.0,7.07,0.0,0.469,6.421,78.9,4.9671,2.0,242.0,17.8,396.9,9.14]    |21.6 |\n",
            "|[0.02729,0.0,7.07,0.0,0.469,7.185,61.1,4.9671,2.0,242.0,17.8,392.83,4.03]   |34.7 |\n",
            "|[0.03237,0.0,2.18,0.0,0.458,6.998,45.8,6.0622,3.0,222.0,18.7,394.63,2.94]   |33.4 |\n",
            "|[0.06905,0.0,2.18,0.0,0.458,7.147,54.2,6.0622,3.0,222.0,18.7,396.9,5.33]    |36.2 |\n",
            "|[0.02985,0.0,2.18,0.0,0.458,6.43,58.7,6.0622,3.0,222.0,18.7,394.12,5.21]    |28.7 |\n",
            "|[0.08829,12.5,7.87,0.0,0.524,6.012,66.6,5.5605,5.0,311.0,15.2,395.6,12.43]  |22.9 |\n",
            "|[0.14455,12.5,7.87,0.0,0.524,6.172,96.1,5.9505,5.0,311.0,15.2,396.9,19.15]  |27.1 |\n",
            "|[0.21124,12.5,7.87,0.0,0.524,5.631,100.0,6.0821,5.0,311.0,15.2,386.63,29.93]|16.5 |\n",
            "|[0.17004,12.5,7.87,0.0,0.524,6.004,85.9,6.5921,5.0,311.0,15.2,386.71,17.1]  |18.9 |\n",
            "|[0.22489,12.5,7.87,0.0,0.524,6.377,94.3,6.3467,5.0,311.0,15.2,392.52,20.45] |15.0 |\n",
            "|[0.11747,12.5,7.87,0.0,0.524,6.009,82.9,6.2267,5.0,311.0,15.2,396.9,13.27]  |18.9 |\n",
            "|[0.09378,12.5,7.87,0.0,0.524,5.889,39.0,5.4509,5.0,311.0,15.2,390.5,15.71]  |21.7 |\n",
            "|[0.62976,0.0,8.14,0.0,0.538,5.949,61.8,4.7075,4.0,307.0,21.0,396.9,8.26]    |20.4 |\n",
            "|[0.63796,0.0,8.14,0.0,0.538,6.096,84.5,4.4619,4.0,307.0,21.0,380.02,10.26]  |18.2 |\n",
            "|[0.62739,0.0,8.14,0.0,0.538,5.834,56.5,4.4986,4.0,307.0,21.0,395.62,8.47]   |19.9 |\n",
            "|[1.05393,0.0,8.14,0.0,0.538,5.935,29.3,4.4986,4.0,307.0,21.0,386.85,6.58]   |23.1 |\n",
            "|[0.7842,0.0,8.14,0.0,0.538,5.99,81.7,4.2579,4.0,307.0,21.0,386.75,14.67]    |17.5 |\n",
            "|[0.80271,0.0,8.14,0.0,0.538,5.456,36.6,3.7965,4.0,307.0,21.0,288.99,11.69]  |20.2 |\n",
            "|[0.7258,0.0,8.14,0.0,0.538,5.727,69.5,3.7965,4.0,307.0,21.0,390.95,11.28]   |18.2 |\n",
            "+----------------------------------------------------------------------------+-----+\n",
            "only showing top 20 rows\n",
            "\n",
            "RMSE :  4.441222975785931\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xdg-5L1S4CK_",
        "outputId": "42d62b74-ac70-4675-f3f8-3a8abad078b1"
      },
      "source": [
        "from pyspark.sql import SparkSession\n",
        "from pyspark.ml.feature import StringIndexer, VectorAssembler\n",
        "from pyspark.ml.regression import LinearRegression\n",
        "from pyspark.ml.evaluation import RegressionEvaluator\n",
        "from pyspark.ml.tuning import CrossValidator,ParamGridBuilder\n",
        "spark = SparkSession.Builder().appName('Spark DataFrame Introduction').getOrCreate()\n",
        "houseDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','true')\\\n",
        "                        .option('delimiter',',').csv('boston-house-prices.txt')\n",
        "houseDF = houseDF.withColumnRenamed('medv','label')\n",
        "#houseDF.describe().show()\n",
        "vec = VectorAssembler(inputCols=houseDF.columns[0:-1],outputCol='features')\n",
        "houseDF=vec.transform(houseDF)\n",
        "houseDF = houseDF.select('features','label')\n",
        "trainDF, testDF = houseDF.randomSplit([0.8,0.2],seed=123)\n",
        "\n",
        "\n",
        "lr = LinearRegression()\n",
        "eva = RegressionEvaluator(metricName='rmse')\n",
        "myParams = ParamGridBuilder().addGrid(lr.maxIter,[5,6,7])\\\n",
        "                    .addGrid(lr.elasticNetParam,[0.1,0.2,0.3]).build()\n",
        "validator = CrossValidator(parallelism=4, \n",
        "                                 estimator=lr,\n",
        "                                 estimatorParamMaps=myParams,\n",
        "                                 evaluator=eva,\n",
        "                                 numFolds=5)\n",
        "\n",
        "model = validator.fit(trainDF)\n",
        "print(\"Best Model hesaplandı\")\n",
        "print(\"Best maxIter : \", model.bestModel.getMaxIter())\n",
        "print(\"Best elasticNetParam : \", model.bestModel.getElasticNetParam())\n",
        "\n",
        "resultDF = model.transform(testDF)\n",
        "result = eva.evaluate(resultDF)\n",
        "print(\"Accuracy of the best model \",result)"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Best Model hesaplandı\n",
            "Best maxIter :  5\n",
            "Best elasticNetParam :  0.1\n",
            "Accuracy of the best model  3.952468980908394\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ntHtglon9vA9"
      },
      "source": [
        "#SparkML-Clustering"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "1CBRd2cd-V4h",
        "outputId": "f9b1c181-fe2b-4831-b53b-a9d3c841837e"
      },
      "source": [
        "from pyspark.sql import SparkSession\n",
        "from pyspark.ml.feature import VectorAssembler\n",
        "from pyspark.ml.clustering import KMeans\n",
        "from pyspark.ml.evaluation import ClusteringEvaluator\n",
        "\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "spark = SparkSession.Builder().appName('SparkML Clustering').getOrCreate()\n",
        "pointsDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','false')\\\n",
        "                        .option('delimiter',',').csv('points.txt')\n",
        "\n",
        "\n",
        "pointsDF = pointsDF.withColumnRenamed('_c0','x')\n",
        "pointsDF = pointsDF.withColumnRenamed('_c1','y')\n",
        "\n",
        "\n",
        "\n",
        "pointsDF.show()\n",
        "\n",
        "#pointsDF.count() # 40 cıktı\n",
        "\n",
        "\n",
        "\n",
        "vec = VectorAssembler(inputCols=['x','y'],outputCol='features')\n",
        "pointsDF = vec.transform(pointsDF)\n",
        "\n",
        "k = 4\n",
        "km = KMeans().setK(k)\n",
        "model = km.fit(pointsDF)\n",
        "print(model.clusterCenters())\n",
        "\n",
        "pointsDF = model.transform(pointsDF) #Her noktayı bulmuş olduğumuz cluster center a atamasını yapıyoruz\n",
        "\n",
        "pointsDF.show()\n",
        "\n",
        "\n",
        "for a in range(k):\n",
        "    x = pointsDF.filter(pointsDF['prediction']==a).select('x').rdd.flatMap(lambda x : list(x)).collect()\n",
        "    y = pointsDF.filter(pointsDF['prediction']==a).select('y').rdd.flatMap(lambda x : list(x)).collect()\n",
        "    plt.scatter(x,y)\n",
        "\n",
        "#x = pointsDF.toPandas()['x']\n",
        "#y = pointsDF.toPandas()['y']\n",
        "\n",
        "\n",
        "centers = model.clusterCenters()\n",
        "\n",
        "for x in range(k):\n",
        "    plt.scatter(centers[x][0],centers[x][1])\n",
        "plt.show()\n",
        "\n",
        "eva = ClusteringEvaluator()\n",
        "silhouetteValue = eva.evaluate(pointsDF)\n",
        "print(\"Silhouette :\",silhouetteValue)\n"
      ],
      "execution_count": 36,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+---+---+\n",
            "|  x|  y|\n",
            "+---+---+\n",
            "|  5|  5|\n",
            "| 15| 17|\n",
            "| 15| 16|\n",
            "|  5|  6|\n",
            "|  6|  6|\n",
            "| 16| 16|\n",
            "|  8|  8|\n",
            "|  7|  6|\n",
            "| 17| 17|\n",
            "|  6|  7|\n",
            "|  7|  8|\n",
            "| 18| 18|\n",
            "|  8|  6|\n",
            "| 17| 16|\n",
            "|  5|  8|\n",
            "| 17| 16|\n",
            "|  8|  8|\n",
            "| 16| 17|\n",
            "| 17| 18|\n",
            "| 18| 16|\n",
            "+---+---+\n",
            "only showing top 20 rows\n",
            "\n",
            "[array([16.6,  6.7]), array([6.5       , 6.91666667]), array([16.6, 16.7]), array([ 6.5  , 16.625])]\n",
            "+---+---+-----------+----------+\n",
            "|  x|  y|   features|prediction|\n",
            "+---+---+-----------+----------+\n",
            "|  5|  5|  [5.0,5.0]|         1|\n",
            "| 15| 17|[15.0,17.0]|         2|\n",
            "| 15| 16|[15.0,16.0]|         2|\n",
            "|  5|  6|  [5.0,6.0]|         1|\n",
            "|  6|  6|  [6.0,6.0]|         1|\n",
            "| 16| 16|[16.0,16.0]|         2|\n",
            "|  8|  8|  [8.0,8.0]|         1|\n",
            "|  7|  6|  [7.0,6.0]|         1|\n",
            "| 17| 17|[17.0,17.0]|         2|\n",
            "|  6|  7|  [6.0,7.0]|         1|\n",
            "|  7|  8|  [7.0,8.0]|         1|\n",
            "| 18| 18|[18.0,18.0]|         2|\n",
            "|  8|  6|  [8.0,6.0]|         1|\n",
            "| 17| 16|[17.0,16.0]|         2|\n",
            "|  5|  8|  [5.0,8.0]|         1|\n",
            "| 17| 16|[17.0,16.0]|         2|\n",
            "|  8|  8|  [8.0,8.0]|         1|\n",
            "| 16| 17|[16.0,17.0]|         2|\n",
            "| 17| 18|[17.0,18.0]|         2|\n",
            "| 18| 16|[18.0,16.0]|         2|\n",
            "+---+---+-----------+----------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAAD4CAYAAAD1jb0+AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAVPUlEQVR4nO3df4zc9X3n8ecb22BDEzvE62B+nVMUKA2YH9k0JDqawuYAqeGMooMDXRXSUKHLnQq4VSLSJgZHSEUhKlDl1MgK1HCNTAFRqIvawPm4glQwWn4ZUgO+pCnYXsdLOa8SaoN/vO+PmXXsZcc739357szH+3xIaGbeM/5834zWL3/3+/3OeyIzkSSV54huNyBJmhwDXJIKZYBLUqEMcEkqlAEuSYWaPZ0bW7hwYS5ZsmQ6NylJxXvuuefeysy+sfVpDfAlS5YwODg4nZuUpOJFxL+MV/cQiiQVygCXpEIZ4JJUKANckgplgEtSoSYM8Ii4OyK2R8QrB9TOjohnIuLFiBiMiN+oq8GRtWvZdOEAG0//dTZdOMDI2rV1baooQytXsvHjZ7Dx105n48fPYGjlym63JBXp0Z88ykUPXsTSe5Zy0YMX8ehPHi1m/Xb2wFcDl4ypfRtYmZlnAyuajztuZO1ahr65gj1bt0Ime7ZuZeibK2Z8iA+tXMmONffB3r2Nwt697FhznyEuVfToTx7l5n+8maF3hkiSoXeGuPkfb+5YyNa9/oQBnplPAm+PLQMfbN6fD2ztSDdjbL/9DnLXroM3vGsX22+/o47NFWPH/Q9Uqksa353P38muvQdnzK69u7jz+TuLWH+yH+S5AfhhRHyHxj8Cn2n1woi4FrgW4OSTT660kT1DQ5XqM8bonne7dUnj2vbOtkr1Xlt/sicxvwIsz8yTgOXAXa1emJmrMrM/M/v7+t73SdBDmr14caX6jDFrVrW6pHEdd8xxleq9tv5kA/xq4KHm/QeAWk5iLlp+AzF37kG1mDuXRctvqGNzxVhwxeWV6pLGd/251zN31sEZM3fWXK4/9/oi1p/sIZStwGeB/wNcCGzqSDdjzL/0UqBxLHzP0BCzFy9m0fIb9tdnqsU33QQ0j3nv3QuzZrHgisv31yW157d/9beBxrHqbe9s47hjjuP6c6/fX+/19WOi78SMiDXAbwELgZ8BNwGvAXfS+AdgF/DfMvO5iTbW39+fDrOSpGoi4rnM7B9bn3APPDOvavHUJ6bclSRp0vwkpiQVygCXpEIZ4JJUKANckgplgEtSoQxwSSqUAS5pykoeyVp373Wa1m+ll3T4GR2ZOjp1b3RkKtCRTxzWuX7dvdfNPXBJU1LySNa6e6+bAS5pSkoeyVp373UzwCVNSckjWevuvW4GuKQp6eWRrO+8sJ2hW59l841PMXTrs7zzwvaOrd0LPIlZwYYNG1i3bh0jIyPMnz+fgYEBli5d2u22pK7q1ZGs77ywnR0PbSJ37wNg74532fFQY/L1Mecsmpbe6zbhONlOKnmc7IYNG1i7di27d+/eX5szZw6XXnqpIS71oKFbn2XvjnffV5+14CgW31jLd9DUptU4WQ+htGndunUHhTfA7t27WbduXZc6knQo44X3oeolMsDbNDIyUqkuqbtmLTiqUr1EBnib5s+fX6kuqbs+ePESYs7BERdzjuCDFy/pTkM1MMDbNDAwwJw5cw6qzZkzh4GBgS51JOlQjjlnEQu+8LH9e9yzFhzFgi98bP8JzMOBV6G0afREpVehSOU45pxFh1Vgj2WAV7B06VIDW1LP8BCKJBVqwgCPiLsjYntEvDKm/vsR8WpE/Cgivl1fi5Kk8bSzB74auOTAQkRcACwDzsrMjwPf6XxrDSNr17LpwgE2nv7rbLpwgJG1a4tZv9S1papKngd+yzO3cNa9Z3HmPWdy1r1nccszt3Rsbai39wmPgWfmkxGxZEz5K8Ctmflu8zXbx/65ThhZu5ahb64gdzXGPe7ZupWhb64AYP6ll/b0+qWuLVVV8jzwW565hb967a/2P96X+/Y//sZ535jS2lD/ezPZY+CnAudHxPqI+IeI+OSUOxnH9tvv2B9So3LXLrbffkfPr1/q2lJVJc8Df+D1ByrVq6r7vZnsVSizgWOB84BPAvdHxK/mOINVIuJa4FqAk08+udJG9gwNVapXVef6pa4tVVXyPPB9ua9Svaq635vJ7oFvBh7KhmeBfcDC8V6Ymasysz8z+/v6+iptZPbixZXqVdW5fqlrS1WVPA/8iBg/AlvVq6r7vZlslw8DFwBExKnAkcBbHenoAIuW30DMPXhWb8ydy6LlN/T8+qWuLVXVy/PAJ3L5qZdXqldV93sz4SGUiFgD/BawMCI2AzcBdwN3Ny8tfA+4erzDJ1M1ekJu++13sGdoiNmLF7No+Q0dO1FX5/qlri1V1avzwNsxeqLygdcfYF/u44g4gstPvbwjJzCh/vfGeeCS1OOcBy5JhxkDXJIKZYBLUqEMcEkqlAEuSYUywCWpUAa4JBWq5wPcsamSNL6e/ko1x6ZKUms9vQfu2FRJaq2nA9yxqZLUWk8HuGNTJam1ng5wx6ZKUms9fRLTsamS1FpPBzg0QtzAlqT36+lDKJKk1gxwSSqUAS5JhTLAJalQBrgkFcoAl6RCGeCSVKgJAzwi7o6I7RHxyjjP/WFEZEQsrKc9SVIr7eyBrwYuGVuMiJOAi4A3OtyTJKkNEwZ4Zj4JvD3OU7cDXwOy001JkiY2qWPgEbEM2JKZL7Xx2msjYjAiBoeHhyezOUnSOCoHeEQcDfwRsKKd12fmqszsz8z+vr6+qpuTJLUwmT3wU4CPAi9FxE+BE4HnI+K4TjYmSTq0ytMIM/NlYNHo42aI92fmWx3sS5I0gXYuI1wDPA2cFhGbI+Ka+tuSJE1kwj3wzLxqgueXdKwbSVLb/CSmJBXKAJekQhngklQoA1ySCmWAS1KhDHBJKpQBLkmFMsAlqVAGuCQVygCXpEIZ4JJUKANckgplgEtSoQxwSSqUAS5JhTLAJalQBrgkFcoAl6RCGeCSVCgDXJIKZYBLUqEmDPCIuDsitkfEKwfUbouIVyNiQ0T8dUQsqLdNSdJY7eyBrwYuGVN7HDgjM5cCrwNf73BfkqQJTBjgmfkk8PaY2mOZuaf58BngxBp6kyQdQieOgX8Z+LtWT0bEtRExGBGDw8PDHdicJAmmGOAR8cfAHuAHrV6Tmasysz8z+/v6+qayOUnSAWZP9g9GxJeAzwMDmZkd60iS1JZJBXhEXAJ8DfhsZv5bZ1uSJLWjncsI1wBPA6dFxOaIuAb4LvAB4PGIeDEivldzn5KkMSbcA8/Mq8Yp31VDL5KkCvwkpiQVygCXpEIZ4JJUKANckgplgEtSoQxwSSqUAS5JhTLAJalQBrgkFcoAl6RCGeCSVCgDXJIKZYBLUqEMcEkqlAEuSYUywCWpUAa4JBXKAJekQhngklQoA1ySCmWAS1KhDHBJKtSEAR4Rd0fE9oh45YDasRHxeERsat5+qN42JUljtbMHvhq4ZEztRmBdZn4MWNd8LEmaRhMGeGY+Cbw9prwMuKd5/x7gsg73JUmawGSPgX8kM4ea97cBH2n1woi4NiIGI2JweHh4kpuTJI015ZOYmZlAHuL5VZnZn5n9fX19U92cJKlpsgH+s4hYDNC83d65liRJ7ZhsgP8NcHXz/tXAI51pR5LUrnYuI1wDPA2cFhGbI+Ia4FbgP0TEJuBzzceSpGk0e6IXZOZVLZ4a6HAvkqQK/CSmJBXKAJekQhngklQoA1ySCmWAS1KhDHBJKpQBLkmFMsAlqVAGuCQVygCXpEIZ4JJUKANckgplgEtSoQxwSSqUAS5JhTLAJalQBrgkFcoAl6RCGeCSVCgDXJIKZYBLUqEm/Fb6Q4mI5cDvAQm8DPxuZu7qRGP7bbgf1n0LRjbD/BNhYAUsvaKjm6jN3/4BPLcaci/ELPjEl+Dzf9r7a0szyMMvbOG2H77G1h07OX7BPL568Wlcds4JRaw/6T3wiDgBuA7oz8wzgFnAlR3patSG+2HtdTDyJpCN27XXNeq97m//AAbvagQsNG4H72rUe3ltaQZ5+IUtfP2hl9myYycJbNmxk68/9DIPv7CliPWneghlNjAvImYDRwNbp97SAdZ9C3bvPLi2e2ej3uueW12t3itrSzPIbT98jZ279x5U27l7L7f98LUi1p90gGfmFuA7wBvAEDCSmY+NfV1EXBsRgxExODw8XG0jI5ur1XtJ7q1W75W1pRlk646dleq9tv5UDqF8CFgGfBQ4HjgmIn5n7Osyc1Vm9mdmf19fX7WNzD+xWr2XxKxq9V5ZW5pBjl8wr1K919afyiGUzwH/nJnDmbkbeAj4TEe6GjWwAuaM+R+dM69R73Wf+FK1eq+sLc0gX734NObNOXjHZ96cWXz14tOKWH8qV6G8AZwXEUcDO4EBYLAjXY0avdqkxKtQRq8IqeNKkTrXlmaQ0atB6rpKpO71IzMn/4cjVgL/GdgDvAD8Xma+2+r1/f39OTjY2YyXpMNdRDyXmf1j61O6DjwzbwJumsoakqTJ8ZOYklQoA1ySCmWAS1KhDHBJKpQBLkmFMsAlqVAGuCQVakrXgR8WnKstTVnJM7Xr7r1OMzvAR+dqjxqdqw3jhvjGp57gqfvu5ef/+hYf+PBCzr/yi5x+/gXT1KzUm0ZnXo+OTR2deQ10JAjrXL/u3us2sw+hVJirvfGpJ3hs1Xf5+VvDkMnP3xrmsVXfZeNTT9TaotTrSp6pXXfvdZvZAV5hrvZT993LnvcOHvOy5713eeq+e+voTCpGyTO16+69bjP7EErMGj/Ex5mr/fN/fWvcJVrVpZni+AXz2DJO4HVypvZk1399/TaefuTH/OLtd/mVY4/i08tO4dRPHdeRtXvBzN4DrzBX+wMfXjjuS1vVpZmiV2dqv75+G0/84FV+8XbjN+dfvP0uT/zgVV5fv23aeq/bzA7wz/8p9F/zyz3umNV4PM4JzPOv/CKzjzzqoNrsI4/i/Cu/OB2dSj3rsnNO4E++cCYnLJhHACcsmMeffOHMjs7Unsz6Tz/yY/a8t++g2p739vH0Iz+ett7rNqV54FWVPg/cq1CkcvyP//q/Wz7337934TR2MnW1zAOfaU4//wIDWyrErxx71P7DJ2Prh4uZfQhF0mHr08tOYfaRB0fc7COP4NPLTulSR53nHrikw9Lo1SaHugqldAa4pMPWqZ867rAK7LE8hCJJhTLAJalQBrgkFWpKx8AjYgHwfeAMIIEvZ+bTnWhsvw33w7pvwchmmH8iDKyApVeUsX6pa0sVlTxO9hsPv8ya9W+yN5NZEVz1qZO45bIzO7I21Nv7VE9i3gn8fWb+p4g4Eji6Az390ob7Ye11sLs5q2DkzcZj6ExY1bl+qWtLFZU8TvYbD7/MXz7zxv7HezP3P+5EiNf93kz6EEpEzAd+E7gLIDPfy8wdU+7oQOu+9cuQGrV7Z6Pe6+uXurZUUcnjZNesf7NSvaq635upHAP/KDAM/EVEvBAR34+IY8a+KCKujYjBiBgcHh6utoWRzdXqVdW5fqlrSxWVPE52b4tRIq3qVdX93kwlwGcD5wJ/npnnAO8AN459UWauysz+zOzv6+urtoX5J1arV1Xn+qWuLVXUavRqJ8fJ1rX+rIhK9arqfm+mEuCbgc2Zub75+EEagd45Aytgzpj/0TnzGvVeX7/UtaWKenWcbDuu+tRJlepV1f3eTPokZmZui4g3I+K0zHwNGAD+qSNdjRo9IVfX1RZ1rl/q2lJFoyfj6rrSos71R09U1nUVSt3vzZTGyUbE2TQuIzwS+Anwu5n5/1q9vvRxspLUDbWMk83MF4H3LSpJqp+fxJSkQhngklQoA1ySCmWAS1KhDHBJKpQBLkmF6v0A33A/3H4G3Lygcbvh/m53JEk9obe/E9OxqZLUUm/vgTs2VZJa6u0Ad2yqJLXU2wHu2FRJaqm3A9yxqZLUUm8H+NIr4NI/g/knAdG4vfTPPIEpSfT6VSjQCGsDW5Lep7f3wCVJLRngklQoA1ySCmWAS1KhDHBJKtSUvtS48sYihoF/mbYNVrMQeKvbTUxCqX2DvXeLvXfHVHr/d5nZN7Y4rQHeyyJicLxvfe51pfYN9t4t9t4ddfTuIRRJKpQBLkmFMsB/aVW3G5ikUvsGe+8We++OjvfuMXBJKpR74JJUKANckgo14wM8IhZExIMR8WpEbIyIT3e7p3ZFxPKI+FFEvBIRayJibrd7aiUi7o6I7RHxygG1YyPi8YjY1Lz9UDd7bKVF77c1f2Y2RMRfR8SCbvbYyni9H/DcH0ZERsTCbvR2KK36jojfb77vP4qIb3erv0Np8fNydkQ8ExEvRsRgRPxGJ7Y14wMcuBP4+8z8NeAsYGOX+2lLRJwAXAf0Z+YZwCzgyu52dUirgUvG1G4E1mXmx4B1zce9aDXv7/1x4IzMXAq8Dnx9uptq02re3zsRcRJwEfDGdDfUptWM6TsiLgCWAWdl5seB73Shr3as5v3v+beBlZl5NrCi+XjKZnSAR8R84DeBuwAy873M3NHdriqZDcyLiNnA0cDWLvfTUmY+Cbw9prwMuKd5/x7gsmltqk3j9Z6Zj2XmnubDZ4Ce/J6/Fu87wO3A14CevIqhRd9fAW7NzHebr9k+7Y21oUXvCXyweX8+Hfq7OqMDHPgoMAz8RUS8EBHfj4hjut1UOzJzC409kDeAIWAkMx/rbleVfSQzh5r3twEf6WYzU/Bl4O+63US7ImIZsCUzX+p2LxWdCpwfEesj4h8i4pPdbqiCG4DbIuJNGn9vO/Ib20wP8NnAucCfZ+Y5wDv07q/xB2keL15G4x+h44FjIuJ3utvV5GXjetae3Bs8lIj4Y2AP8INu99KOiDga+CMav8aXZjZwLHAe8FXg/oiI7rbUtq8AyzPzJGA5zd/6p2qmB/hmYHNmrm8+fpBGoJfgc8A/Z+ZwZu4GHgI+0+WeqvpZRCwGaN725K/ErUTEl4DPA/8ly/lAxSk0/tF/KSJ+SuPQz/MRcVxXu2rPZuChbHgW2EdjQFQJrqbxdxTgAcCTmFOVmduANyPitGZpAPinLrZUxRvAeRFxdHMvZIBCTsAe4G9o/GDTvH2ki71UEhGX0DiG/B8z89+63U+7MvPlzFyUmUsycwmNUDy3+Xeh1z0MXAAQEacCR1LOZMKtwGeb9y8ENnVk1cyc0f8BZwODwAYaPyAf6nZPFXpfCbwKvAL8T+Cobvd0iF7X0DhWv5tGaFwDfJjG1SebgP8FHNvtPiv0/n+BN4EXm/99r9t9ttv7mOd/Cizsdp9tvudHAn/Z/Hl/Hriw231W6P3fA88BLwHrgU90Ylt+lF6SCjWjD6FIUskMcEkqlAEuSYUywCWpUAa4JBXKAJekQhngklSo/w+/vztGh8YRrgAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          }
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Silhouette : 0.9464374120400001\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dQ7rKjAoJpF8",
        "outputId": "ba9bb26f-8f4e-4fca-f36d-0749f0dfcfee",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "import nltk\n",
        "nltk.download('vader_lexicon')\n",
        "\n",
        "\n",
        "from nltk.sentiment.vader import SentimentIntensityAnalyzer\n",
        "\n",
        "cumle1 = \"What a nice weather\"\n",
        "cumle2 = \"Worst movie ever seen \"\n",
        "cumle3 = \"not a bad movie\"\n",
        "sIntensityAnalyzer = SentimentIntensityAnalyzer()\n",
        "\n",
        "print(sIntensityAnalyzer.polarity_scores(cumle1))  # compound 0 dan küçükse olumsuz anlam 0'dan büyükse olumlu 0 'a yakınsa nötr böle kişiler bulunabiliyor\n",
        "print(sIntensityAnalyzer.polarity_scores(cumle2))\n",
        "print(sIntensityAnalyzer.polarity_scores(cumle3)) #ironik cümlleri anlayamaz"
      ],
      "execution_count": 40,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[nltk_data] Downloading package vader_lexicon to /root/nltk_data...\n",
            "[nltk_data]   Package vader_lexicon is already up-to-date!\n",
            "{'neg': 0.0, 'neu': 0.417, 'pos': 0.583, 'compound': 0.4215}\n",
            "{'neg': 0.577, 'neu': 0.423, 'pos': 0.0, 'compound': -0.6249}\n",
            "{'neg': 0.0, 'neu': 0.412, 'pos': 0.588, 'compound': 0.431}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RThWyBeiKJO4",
        "outputId": "3fea7e28-827c-4173-db61-241c334e1409",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "\n",
        "trCumle = \"kötü film değildi\"\n",
        "print(trCumle, sIntensityAnalyzer.polarity_scores(trCumle) )  #turkceyi anlamıyor"
      ],
      "execution_count": 58,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "kötü film değildi {'neg': 0.0, 'neu': 1.0, 'pos': 0.0, 'compound': 0.0}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jb_WvLmTWdzn",
        "outputId": "71838273-21e4-477b-fd2a-6328b9e4772c",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "trCumle =\"kötü film\" #türkçe olan seyleri 0 olarak algılıyor , ingilizce haric belki başka dilleri destekliyordur\n",
        "#itu nlp turkce olarak çalışıyor, 2. kutuphane zemberek kutuphanesi\n",
        "\n",
        "print(trCumle, sIntensityAnalyzer.polarity_scores(trCumle))"
      ],
      "execution_count": 59,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "kötü film {'neg': 0.0, 'neu': 1.0, 'pos': 0.0, 'compound': 0.0}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "a-WnQDXeKbed",
        "outputId": "9c62c736-261a-4858-964d-f591aafb3261",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "from pyspark.sql import SparkSession  \n",
        "from pyspark.ml.feature import Tokenizer,CountVectorizer\n",
        "from pyspark.ml.classification import RandomForestClassifier\n",
        "from pyspark.ml.evaluation import BinaryClassificationEvaluator #label olarak datasetinde sadece 0,1 var, bu yüzden binary kullanıyoruz\n",
        "\n",
        "spark = SparkSession.Builder().appName('SparkML Sentiment Analysis').getOrCreate()\n",
        "yorumDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','false')\\\n",
        "                        .option('delimiter','\\t').csv('movie_turkish_train.txt')\n",
        "yorumDF = yorumDF.withColumnRenamed('_c0','cumle')\n",
        "yorumDF = yorumDF.withColumnRenamed('_c1','label')\n",
        "\n",
        "#textde ilk işimiz tokenize etmek\n",
        "#yorumDF.show(truncate=False)\n",
        "\n",
        "#yorumDF.show(100) \n",
        "\n",
        "tknzr = Tokenizer(inputCol='cumle',outputCol='tokenized')\n",
        "yorumDF = tknzr.transform(yorumDF)\n",
        "\n",
        "#yorumDF.show() \n",
        "\n",
        "\n",
        "vectorizer = CountVectorizer(inputCol='tokenized',outputCol='features')\n",
        "vecModel = vectorizer.fit(yorumDF)\n",
        "yorumDF = vecModel.transform(yorumDF)\n",
        "\n",
        "yorumDF = yorumDF.select('features','label')\n",
        "trainDF,testDF = yorumDF.randomSplit([0.75,0.25],seed=1234)\n",
        "\n",
        "#yorumDF.show() \n",
        "\n",
        "\n",
        "classifer = RandomForestClassifier()\n",
        "\n",
        "#model = classifer.fit(yorumDF) ## Bir datascientist in yapabileceği en gerizekalı hareket\n",
        "\n",
        "model = classifer.fit(trainDF)\n",
        "sonucDF = model.transform(testDF)\n",
        "\n",
        "sonucDF.show()\n",
        "\n",
        "eva = BinaryClassificationEvaluator()\n",
        "basari = eva.evaluate(sonucDF)\n",
        "print(basari)"
      ],
      "execution_count": 52,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+--------------------+-----+--------------------+--------------------+----------+\n",
            "|            features|label|       rawPrediction|         probability|prediction|\n",
            "+--------------------+-----+--------------------+--------------------+----------+\n",
            "|(7563,[0,1,2,3,4,...|    0|[10.2009433962292...|[0.51004716981146...|       0.0|\n",
            "|(7563,[0,1,2,3,4,...|    0|[10.2503411295897...|[0.51251705647948...|       0.0|\n",
            "|(7563,[0,1,2,3,20...|    1|[9.90396353045744...|[0.49519817652287...|       1.0|\n",
            "|(7563,[0,1,2,4,6,...|    1|[9.88765026714150...|[0.49438251335707...|       1.0|\n",
            "|(7563,[0,1,2,4,6,...|    0|[11.5290684737735...|[0.57645342368867...|       0.0|\n",
            "|(7563,[0,1,2,4,6,...|    0|[9.88765026714150...|[0.49438251335707...|       1.0|\n",
            "|(7563,[0,1,2,4,6,...|    1|[9.58845472258705...|[0.47942273612935...|       1.0|\n",
            "|(7563,[0,1,2,4,7,...|    1|[8.27256893124322...|[0.41362844656216...|       1.0|\n",
            "|(7563,[0,1,2,4,8,...|    1|[9.43717945734866...|[0.47185897286743...|       1.0|\n",
            "|(7563,[0,1,2,4,9,...|    0|[10.6196199734592...|[0.53098099867296...|       0.0|\n",
            "|(7563,[0,1,2,4,11...|    1|[10.7516456325492...|[0.53758228162746...|       0.0|\n",
            "|(7563,[0,1,2,5,14...|    0|[11.6245061047023...|[0.58122530523511...|       0.0|\n",
            "|(7563,[0,1,2,6,7,...|    1|[9.88765026714150...|[0.49438251335707...|       1.0|\n",
            "|(7563,[0,1,2,6,36...|    0|[11.1291871717834...|[0.55645935858917...|       0.0|\n",
            "|(7563,[0,1,2,8,12...|    1|[9.43717945734866...|[0.47185897286743...|       1.0|\n",
            "|(7563,[0,1,2,8,17...|    0|[9.43717945734866...|[0.47185897286743...|       1.0|\n",
            "|(7563,[0,1,2,8,28...|    1|[9.21348177216423...|[0.46067408860821...|       1.0|\n",
            "|(7563,[0,1,2,9,33...|    1|[8.69979870729931...|[0.43498993536496...|       1.0|\n",
            "|(7563,[0,1,2,11,1...|    1|[10.7516456325492...|[0.53758228162746...|       0.0|\n",
            "|(7563,[0,1,2,12,5...|    1|[9.74985276334358...|[0.48749263816717...|       1.0|\n",
            "+--------------------+-----+--------------------+--------------------+----------+\n",
            "only showing top 20 rows\n",
            "\n",
            "0.7993971456692911\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FHMnAsIpKOZ9"
      },
      "source": [
        "yeniDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','false')\\\n",
        "                        .option('delimiter','\\t').csv('deneme-yorum.txt')\n",
        "yeniDF = yeniDF.withColumnRenamed('_c0','cumle')\n",
        "yeniDF = tknzr.transform(yeniDF)\n",
        "yeniDF = vecModel.transform(yeniDF)\n",
        "yeniDF.show()\n",
        "yeniYorumSonucDF = model.transform(yeniDF)\n",
        "yeniYorumSonucDF.select('cumle','prediction').show()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QRHtbApRTyi2",
        "outputId": "d78a3b7f-1c6f-4cb4-fadf-ea2d1d5b304d",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "from pyspark.sql import SparkSession\n",
        "from pyspark.ml.feature import Tokenizer,CountVectorizer\n",
        "from pyspark.ml.classification import MultilayerPerceptronClassifier\n",
        "from pyspark.ml.evaluation import BinaryClassificationEvaluator\n",
        "from pyspark.sql.functions import UserDefinedFunction\n",
        "from pyspark.sql.types import StringType\n",
        "\n",
        "spark = SparkSession.Builder().appName('SparkML Sentiment Analysis').getOrCreate()\n",
        "yorumDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','false')\\\n",
        "                        .option('delimiter','\\t').csv('movie_turkish_train.txt')\n",
        "yorumDF = yorumDF.withColumnRenamed('_c0','cumle')  #buradaki cumle aslında bir kişiden gelen yorum paragrafta olabilir\n",
        "yorumDF = yorumDF.withColumnRenamed('_c1','label')\n",
        "\n",
        "def fixCumle(cumle):\n",
        "    cumle = cumle.replace('1','bir') # gunde 1 ve bir aynı şey 1 ile bir'i replace edecek\n",
        "    #cumle = cumle.replace('.','')\n",
        "    #cumle = cumle.replace('(','')\n",
        "    #cumle = cumle.replace(')','')\n",
        "    #cumle = cumle.strip()  #cumlenin basında ve sonundaki fazlalıkları siliyor\n",
        "    #cumle = cumle.replace('filmi','film')\n",
        "    return cumle\n",
        "\n",
        "# tex'in sonunda ki 0 kötü demek satır 500'den sonra 1 olmus\n",
        "#textDF.show()\n",
        "\n",
        "#textde ilk işimiz tokenize etmek\n",
        "\n",
        "fixTextUDF = UserDefinedFunction(fixCumle,StringType())\n",
        "yorumDF = yorumDF.withColumn('cumle2',fixTextUDF('cumle'))\n",
        "\n",
        "yorumDF.show()\n",
        "\n",
        "tknzr = Tokenizer(inputCol='cumle2',outputCol='tokenized')\n",
        "yorumDF = tknzr.transform(yorumDF)\n",
        "yorumDF.show()\n",
        "\n",
        "kelimeSayisi = 3500\n",
        "vectorizer = CountVectorizer(inputCol='tokenized',outputCol='features',vocabSize=kelimeSayisi)\n",
        "vecModel = vectorizer.fit(yorumDF)\n",
        "yorumDF = vecModel.transform(yorumDF)\n",
        "\n",
        "\n",
        "#7563 tane tekil kelime var dedi unic ama mesela film filmi filmleri ayrı ayrı sayıyor insanların günlük 500 kelime ile konustugunu dusunutsek\n",
        "\n",
        "yorumDF = yorumDF.select('features','label')\n",
        "trainDF,testDF = yorumDF.randomSplit([0.75,0.25],seed=1234)\n",
        "\n",
        "\n",
        "classifer = MultilayerPerceptronClassifier(layers=[kelimeSayisi,200,2])\n",
        "\n",
        "#classifer = MultilayerPerceptronClassifier(layers=[kelimesayisi,1700,2]) #son sınıf 2 yapay sinir aglarındaki ortadaki 50 en bastaki 7563 sınıf\n",
        "\n",
        "model = classifer.fit(trainDF)\n",
        "sonucDF = model.transform(testDF)\n",
        "\n",
        "eva = BinaryClassificationEvaluator()\n",
        "basari = eva.evaluate(sonucDF)\n",
        "print(basari)"
      ],
      "execution_count": 56,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+--------------------+-----+--------------------+\n",
            "|               cumle|label|              cumle2|\n",
            "+--------------------+-----+--------------------+\n",
            "|bunu da gördük uz...|    0|bunu da gördük uz...|\n",
            "|filmi begenmeyenl...|    0|filmi begenmeyenl...|\n",
            "|günde 1 film izle...|    0|günde bir film iz...|\n",
            "|afisine bakip ald...|    0|afisine bakip ald...|\n",
            "|sadece insanlarin...|    0|sadece insanlarin...|\n",
            "|ucuz bir aksiyon ...|    0|ucuz bir aksiyon ...|\n",
            "|olmamis diyor pua...|    0|olmamis diyor pua...|\n",
            "|kesinlikle çok kö...|    0|kesinlikle çok kö...|\n",
            "|ben nasil bir fil...|    0|ben nasil bir fil...|\n",
            "|bu yüzler fazla t...|    0|bu yüzler fazla t...|\n",
            "|. çok ilginç yaa ...|    0|. çok ilginç yaa ...|\n",
            "|bence bu film hiç...|    0|bence bu film hiç...|\n",
            "|valla ben begenme...|    0|valla ben begenme...|\n",
            "|gönül ister milyo...|    0|gönül ister milyo...|\n",
            "|dün aksam bu film...|    0|dün aksam bu film...|\n",
            "|cok kötü olmus......|    0|cok kötü olmus......|\n",
            "|igrenç yaaa.acaba...|    0|igrenç yaaa.acaba...|\n",
            "|saçma sapan bir f...|    0|saçma sapan bir f...|\n",
            "|çok güzel komedi ...|    0|çok güzel komedi ...|\n",
            "|beklentimi yüksek...|    0|beklentimi yüksek...|\n",
            "+--------------------+-----+--------------------+\n",
            "only showing top 20 rows\n",
            "\n",
            "+--------------------+-----+--------------------+--------------------+\n",
            "|               cumle|label|              cumle2|           tokenized|\n",
            "+--------------------+-----+--------------------+--------------------+\n",
            "|bunu da gördük uz...|    0|bunu da gördük uz...|[bunu, da, gördük...|\n",
            "|filmi begenmeyenl...|    0|filmi begenmeyenl...|[filmi, begenmeye...|\n",
            "|günde 1 film izle...|    0|günde bir film iz...|[günde, bir, film...|\n",
            "|afisine bakip ald...|    0|afisine bakip ald...|[afisine, bakip, ...|\n",
            "|sadece insanlarin...|    0|sadece insanlarin...|[sadece, insanlar...|\n",
            "|ucuz bir aksiyon ...|    0|ucuz bir aksiyon ...|[ucuz, bir, aksiy...|\n",
            "|olmamis diyor pua...|    0|olmamis diyor pua...|[olmamis, diyor, ...|\n",
            "|kesinlikle çok kö...|    0|kesinlikle çok kö...|[kesinlikle, çok,...|\n",
            "|ben nasil bir fil...|    0|ben nasil bir fil...|[ben, nasil, bir,...|\n",
            "|bu yüzler fazla t...|    0|bu yüzler fazla t...|[bu, yüzler, fazl...|\n",
            "|. çok ilginç yaa ...|    0|. çok ilginç yaa ...|[., çok, ilginç, ...|\n",
            "|bence bu film hiç...|    0|bence bu film hiç...|[bence, bu, film,...|\n",
            "|valla ben begenme...|    0|valla ben begenme...|[valla, ben, bege...|\n",
            "|gönül ister milyo...|    0|gönül ister milyo...|[gönül, ister, mi...|\n",
            "|dün aksam bu film...|    0|dün aksam bu film...|[dün, aksam, bu, ...|\n",
            "|cok kötü olmus......|    0|cok kötü olmus......|[cok, kötü, olmus...|\n",
            "|igrenç yaaa.acaba...|    0|igrenç yaaa.acaba...|[igrenç, yaaa.aca...|\n",
            "|saçma sapan bir f...|    0|saçma sapan bir f...|[saçma, sapan, bi...|\n",
            "|çok güzel komedi ...|    0|çok güzel komedi ...|[çok, güzel, kome...|\n",
            "|beklentimi yüksek...|    0|beklentimi yüksek...|[beklentimi, yüks...|\n",
            "+--------------------+-----+--------------------+--------------------+\n",
            "only showing top 20 rows\n",
            "\n",
            "0.8956923076923076\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "c_e7obUWWGe3"
      },
      "source": [
        "yeniDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','false')\\\n",
        "                        .option('delimiter','\\t').csv('deneme-yorum.txt')\n",
        "yeniDF = yeniDF.withColumnRenamed('_c0','cumle')\n",
        "yeniDF = tknzr.transform(yeniDF)\n",
        "yeniDF = vecModel.transform(yeniDF)\n",
        "yeniDF.show()\n",
        "yeniYorumSonucDF = model.transform(yeniDF)\n",
        "yeniYorumSonucDF.select('cumle','prediction').show()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Yse-7Raga3uW",
        "outputId": "66cd6913-8115-4953-ed05-1ffb8ad1c939",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "from pyspark.sql import SparkSession\n",
        "from pyspark.ml.feature import Tokenizer,Word2Vec\n",
        "from pyspark.ml.classification import MultilayerPerceptronClassifier\n",
        "from pyspark.ml.evaluation import BinaryClassificationEvaluator\n",
        "from pyspark.sql.functions import UserDefinedFunction\n",
        "from pyspark.sql.types import StringType\n",
        "\n",
        "spark = SparkSession.Builder().appName('SparkML Sentiment Analysis').getOrCreate()\n",
        "yorumDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','false')\\\n",
        "                        .option('delimiter','\\t').csv('movie_turkish_train.txt')\n",
        "yorumDF = yorumDF.withColumnRenamed('_c0','cumle')\n",
        "yorumDF = yorumDF.withColumnRenamed('_c1','label')\n",
        "\n",
        "def fixCumle(cumle):\n",
        "    cumle = cumle.replace('1','bir')\n",
        "    #cumle = cumle.replace('.','')\n",
        "    #cumle = cumle.replace('(','')\n",
        "    #cumle = cumle.replace(')','')\n",
        "    #cumle = cumle.strip()\n",
        "    #cumle = cumle.replace('filmi','film')\n",
        "    return cumle\n",
        "\n",
        "fixTextUDF = UserDefinedFunction(fixCumle,StringType())\n",
        "yorumDF = yorumDF.withColumn('cumle2',fixTextUDF('cumle'))\n",
        "\n",
        "tknzr = Tokenizer(inputCol='cumle2',outputCol='tokenized')\n",
        "yorumDF = tknzr.transform(yorumDF)\n",
        "#yorumDF.show()\n",
        "\n",
        "kelimeSayisi = 3500\n",
        "vectorizer = Word2Vec(inputCol='tokenized',outputCol='features',vectorSize=kelimeSayisi)\n",
        "vecModel = vectorizer.fit(yorumDF)\n",
        "\n",
        "yorumDF = vecModel.transform(yorumDF)\n",
        "\n",
        "yorumDF = yorumDF.select('features','label')\n",
        "trainDF,testDF = yorumDF.randomSplit([0.75,0.25],seed=1234)\n",
        "\n",
        "classifer = MultilayerPerceptronClassifier(layers=[kelimeSayisi,200,2])\n",
        "\n",
        "model = classifer.fit(trainDF)\n",
        "sonucDF = model.transform(testDF)\n",
        "\n",
        "eva = BinaryClassificationEvaluator()\n",
        "basari = eva.evaluate(sonucDF)\n",
        "print(basari)"
      ],
      "execution_count": 62,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "0.6950603596945061\n"
          ]
        }
      ]
    }
  ]
}