Sınavcalisma_8_10_2021_21.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Sınavcalisma.8.10.2021.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyPfeaY/5ubqHQ6OvKxrB7QY",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/busranurorhan/Complete-Python-3-Bootcamp/blob/master/S%C4%B1navcalisma_8_10_2021_21.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0f1oYeHdaXk0",
        "outputId": "fc146f15-8a5a-47f5-d2cd-b29312d5ce62"
      },
      "source": [
        "!apt-get update\n",
        "!apt-get install openjdk-8-jdk-headless -qq > /dev/null\n",
        "import os\n",
        "os.environ[\"JAVA_HOME\"] = \"/usr/lib/jvm/java-8-openjdk-amd64\"\n",
        "!update-alternatives --set java /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java\n",
        "!java -version\n",
        "!pip install pyspark"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\r0% [Working]\r            \rIgn:1 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  InRelease\n",
            "\r0% [Connecting to archive.ubuntu.com] [Connecting to security.ubuntu.com] [Conn\r                                                                               \rIgn:2 https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64  InRelease\n",
            "\r0% [Connecting to archive.ubuntu.com] [Connecting to security.ubuntu.com] [Wait\r                                                                               \rHit:3 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  Release\n",
            "Get:4 https://cloud.r-project.org/bin/linux/ubuntu bionic-cran40/ InRelease [3,626 B]\n",
            "Hit:5 https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64  Release\n",
            "Get:6 http://security.ubuntu.com/ubuntu bionic-security InRelease [88.7 kB]\n",
            "Get:7 http://ppa.launchpad.net/c2d4u.team/c2d4u4.0+/ubuntu bionic InRelease [15.9 kB]\n",
            "Hit:9 http://archive.ubuntu.com/ubuntu bionic InRelease\n",
            "Get:10 http://archive.ubuntu.com/ubuntu bionic-updates InRelease [88.7 kB]\n",
            "Hit:12 http://ppa.launchpad.net/cran/libgit2/ubuntu bionic InRelease\n",
            "Get:13 http://ppa.launchpad.net/deadsnakes/ppa/ubuntu bionic InRelease [15.9 kB]\n",
            "Get:14 http://archive.ubuntu.com/ubuntu bionic-backports InRelease [74.6 kB]\n",
            "Get:15 http://security.ubuntu.com/ubuntu bionic-security/main amd64 Packages [2,365 kB]\n",
            "Hit:16 http://ppa.launchpad.net/graphics-drivers/ppa/ubuntu bionic InRelease\n",
            "Get:17 http://security.ubuntu.com/ubuntu bionic-security/universe amd64 Packages [1,431 kB]\n",
            "Get:18 http://ppa.launchpad.net/c2d4u.team/c2d4u4.0+/ubuntu bionic/main Sources [1,802 kB]\n",
            "Get:19 http://archive.ubuntu.com/ubuntu bionic-updates/universe amd64 Packages [2,210 kB]\n",
            "Get:20 http://ppa.launchpad.net/c2d4u.team/c2d4u4.0+/ubuntu bionic/main amd64 Packages [922 kB]\n",
            "Get:21 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 Packages [2,801 kB]\n",
            "Get:22 http://ppa.launchpad.net/deadsnakes/ppa/ubuntu bionic/main amd64 Packages [40.8 kB]\n",
            "Fetched 11.9 MB in 6s (2,018 kB/s)\n",
            "Reading package lists... Done\n",
            "update-alternatives: using /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java to provide /usr/bin/java (java) in manual mode\n",
            "openjdk version \"1.8.0_292\"\n",
            "OpenJDK Runtime Environment (build 1.8.0_292-8u292-b10-0ubuntu1~18.04-b10)\n",
            "OpenJDK 64-Bit Server VM (build 25.292-b10, mixed mode)\n",
            "Collecting pyspark\n",
            "  Downloading pyspark-3.1.2.tar.gz (212.4 MB)\n",
            "\u001b[K     |████████████████████████████████| 212.4 MB 66 kB/s \n",
            "\u001b[?25hCollecting py4j==0.10.9\n",
            "  Downloading py4j-0.10.9-py2.py3-none-any.whl (198 kB)\n",
            "\u001b[K     |████████████████████████████████| 198 kB 66.0 MB/s \n",
            "\u001b[?25hBuilding wheels for collected packages: pyspark\n",
            "  Building wheel for pyspark (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for pyspark: filename=pyspark-3.1.2-py2.py3-none-any.whl size=212880768 sha256=e6549af633adbcbb267b297841c4a776f68f994ea3a4da57811013e473df65de\n",
            "  Stored in directory: /root/.cache/pip/wheels/a5/0a/c1/9561f6fecb759579a7d863dcd846daaa95f598744e71b02c77\n",
            "Successfully built pyspark\n",
            "Installing collected packages: py4j, pyspark\n",
            "Successfully installed py4j-0.10.9 pyspark-3.1.2\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "m1S3-1uZgoE4",
        "outputId": "dfe3dc10-a15f-44dd-91a2-7ff637397e2a"
      },
      "source": [
        "from pyspark.sql import SparkSession\n",
        "spark = SparkSession.Builder().appName('Spark DataFrame Introduction').getOrCreate()\n",
        "pointsDF = spark.read.option('inferSchema','true').csv('points.txt')\n",
        "pointsDF = pointsDF.withColumnRenamed('_c0','x') #Kolon isimlerini değiştiriyoruz\n",
        "pointsDF = pointsDF.withColumnRenamed('_c1','y') #Kolon isimlerini değiştiriyoruz\n",
        "pointsDF = pointsDF.withColumn(\"x\",pointsDF[\"x\"].cast(\"float\"))  #Kolonun veri tipi değiştiriyoruz\n",
        "pointsDF =  pointsDF.withColumn(\"2xy\", 2* pointsDF['x']+pointsDF['y']) #Yeni kolon ekliyoruz\n",
        "pointsDF.show()\n",
        "pointsDF.printSchema()"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+----+---+----+\n",
            "|   x|  y| 2xy|\n",
            "+----+---+----+\n",
            "| 5.0|  5|15.0|\n",
            "|15.0| 17|47.0|\n",
            "|15.0| 16|46.0|\n",
            "| 5.0|  6|16.0|\n",
            "| 6.0|  6|18.0|\n",
            "|16.0| 16|48.0|\n",
            "| 8.0|  8|24.0|\n",
            "| 7.0|  6|20.0|\n",
            "|17.0| 17|51.0|\n",
            "| 6.0|  7|19.0|\n",
            "| 7.0|  8|22.0|\n",
            "|18.0| 18|54.0|\n",
            "| 8.0|  6|22.0|\n",
            "|17.0| 16|50.0|\n",
            "| 5.0|  8|18.0|\n",
            "|17.0| 16|50.0|\n",
            "| 8.0|  8|24.0|\n",
            "|16.0| 17|49.0|\n",
            "|17.0| 18|52.0|\n",
            "|18.0| 16|52.0|\n",
            "+----+---+----+\n",
            "only showing top 20 rows\n",
            "\n",
            "root\n",
            " |-- x: float (nullable = true)\n",
            " |-- y: integer (nullable = true)\n",
            " |-- 2xy: float (nullable = true)\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lI228ck7g-O3"
      },
      "source": [
        "from pyspark.sql import SparkSession\n",
        "spark = SparkSession.Builder().appName('Spark DataFrame Introduction').getOrCreate()\n",
        "pointsDF = spark.read.option('inferSchema','true').option('header','false').csv('points.txt')\n",
        "#pointsDF.printSchema()\n",
        "#pointsDF.show()\n",
        "\n",
        "#print( pointsDF.rdd.map(lambda x : tuple(x)).take(5) )\n",
        "\n",
        "\n",
        "pointsRDD = pointsDF.rdd\n",
        "\n",
        "\n",
        "\n",
        "df = pointsRDD.toDF()\n",
        "\n",
        "df.show()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gQT4fl-px9xJ",
        "outputId": "886af77b-8326-462a-ce75-0c3dc47157cf"
      },
      "source": [
        "from pyspark.sql import SparkSession\n",
        "from pyspark.sql.functions import UserDefinedFunction\n",
        "from pyspark.sql.types import FloatType\n",
        "\n",
        "spark = SparkSession.Builder().appName('Spark DataFrame Introduction').getOrCreate()\n",
        "\n",
        "dollarDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','false')\\\n",
        "                        .option('delimiter','\\t').csv('DollarDataset.txt')\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "dollarDF = dollarDF.withColumnRenamed('_c0','id')\n",
        "dollarDF = dollarDF.withColumnRenamed('_c1','tarih')\n",
        "dollarDF = dollarDF.withColumnRenamed('_c2','fiyat')\n",
        "\n",
        "#dollarDF.show() \n",
        "\n",
        "#dollarDF.count() #17776\n",
        "\n",
        "\n",
        "dollarDF = dollarDF.filter(dollarDF['fiyat'].isNotNull())  #bu islemden sonra index bouldu\n",
        "\n",
        "\n",
        "#dollarDF.show(100) \n",
        "\n",
        "#dollarDF.count()  #17190 demekki 586 null values\n",
        "\n",
        "\n",
        "dollarDF = dollarDF.drop('id') #index column2ını drop ediyorum\n",
        "#dollarDF.show() \n",
        "\n",
        "rdd = dollarDF.rdd.zipWithIndex() #index'i en son'a ekledi\n",
        "\n",
        "#rdd.collect()\n",
        "\n",
        "\n",
        "dollarDF = rdd.map(lambda x : (x[1],x[0][0],x[0][1])).toDF()\n",
        "\n",
        "\n",
        "#dollarDF.show() #baslıklar bozudlu\n",
        "\n",
        "dollarDF = dollarDF.withColumnRenamed('_1','id')\n",
        "dollarDF = dollarDF.withColumnRenamed('_2','tarih')\n",
        "dollarDF = dollarDF.withColumnRenamed('_3','fiyat')\n",
        "\n",
        "#dollarDF.show()\n",
        "#dollarDF.printSchema()\n",
        "\n",
        "def stringtoFloat(f):\n",
        "  f=f.replace('.','')\n",
        "  f=f.replace(',','.')\n",
        "  return float(f)\n",
        "\n",
        "stringtoFloatUDF = UserDefinedFunction(stringtoFloat,FloatType())\n",
        "\n",
        "dollarDF = dollarDF.withColumn(\"fiyat\",stringtoFloatUDF('fiyat'))\n",
        "\n",
        "#dollarDF.show()\n",
        "\n",
        "dollarDF2 = dollarDF\n",
        "dollarDF2 = dollarDF2.withColumn('id',dollarDF2['id']+1)\n",
        "dollarDF2 = dollarDF2.withColumnRenamed('tarih','tarih2')\n",
        "dollarDF2 = dollarDF2.withColumnRenamed('fiyat','fiyat2')\n",
        "\n",
        "#dollarDF2.show()\n",
        "\n",
        "joinedDF = dollarDF2.join(dollarDF,'id')\n",
        "\n",
        "joinedDF = joinedDF.sort('id')\n",
        "\n",
        "joinedDF = joinedDF.withColumn('oran',((joinedDF[\"fiyat\"]-joinedDF[\"fiyat2\"])*100)/joinedDF['fiyat2'])\n",
        "\n",
        "joinedDF = joinedDF.sort('oran',ascending=False)\n",
        "joinedDF.show()"
      ],
      "execution_count": 53,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+-----+----------+---------+----------+---------+------------------+\n",
            "|   id|    tarih2|   fiyat2|     tarih|    fiyat|              oran|\n",
            "+-----+----------+---------+----------+---------+------------------+\n",
            "| 2683|19-08-1960|      2.8|22-08-1960|      9.0| 221.4285751994776|\n",
            "| 7589|24-01-1980|     35.0|25-01-1980|     70.0|             100.0|\n",
            "| 5201|07-08-1970|      9.0|10-08-1970|    14.85| 65.00000678168402|\n",
            "|12912|22-02-2001| 685391.0|23-02-2001| 957879.0| 39.75657690281898|\n",
            "|11172|05-04-1994| 23031.84|06-04-1994| 31988.89| 38.88986153848502|\n",
            "| 7433|11-06-1979|     26.5|12-06-1979|     35.0|32.075471698113205|\n",
            "| 7108|28-02-1978|    19.25|01-03-1978|     25.0| 29.87012987012987|\n",
            "|11173|06-04-1994| 31988.89|07-04-1994| 39853.13| 24.58427901483376|\n",
            "|11125|26-01-1994| 15155.63|27-01-1994|  17215.5|13.591451969845416|\n",
            "|12913|23-02-2001| 957879.0|26-02-2001|1072988.0|12.017071049683729|\n",
            "|11186|25-04-1994| 30658.56|26-04-1994| 33925.01|10.654287715190682|\n",
            "|12936|03-04-2001|1114738.0|04-04-2001|1232604.0|10.573426222125738|\n",
            "| 6997|20-09-1977|     17.5|21-09-1977|    19.25|              10.0|\n",
            "|12935|02-04-2001|1020560.0|03-04-2001|1114738.0| 9.228070863055578|\n",
            "|13010|17-07-2001|1366647.0|18-07-2001|1482571.0| 8.482365965754141|\n",
            "|11211|02-06-1994| 30623.63|03-06-1994|  33181.5| 8.352598926775999|\n",
            "|11270|24-08-1994|  32583.7|25-08-1994| 35291.28| 8.309620883812805|\n",
            "|14843|22-10-2008|     1.52|23-10-2008|     1.63| 7.236843450908203|\n",
            "|12933|29-03-2001| 992354.0|30-03-2001|1056544.0| 6.468457828557148|\n",
            "|12922|14-03-2001| 941519.0|15-03-2001| 999788.0| 6.188828903081085|\n",
            "+-----+----------+---------+----------+---------+------------------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8PxbLwzpEEst"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}