Untitled1_1.09.2021.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Untitled1.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyPfdIYywkF6tIsyC+D4QPt7",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/busranurorhan/Complete-Python-3-Bootcamp/blob/master/Untitled1_1.09.2021.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "j-TlGwyvluBd",
        "outputId": "061e267d-7f54-4ab7-dfc7-48e6dcd936cd"
      },
      "source": [
        "!apt-get update\n",
        "!apt-get install openjdk-8-jdk-headless -qq > /dev/null\n",
        "import os\n",
        "os.environ[\"JAVA_HOME\"] = \"/usr/lib/jvm/java-8-openjdk-amd64\"\n",
        "!update-alternatives --set java /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java\n",
        "!java -version\n",
        "!pip install pyspark\n"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\r0% [Working]\r            \rHit:1 http://ppa.launchpad.net/c2d4u.team/c2d4u4.0+/ubuntu bionic InRelease\n",
            "\r0% [Waiting for headers] [Connecting to security.ubuntu.com (91.189.91.39)] [Co\r                                                                               \rHit:2 http://archive.ubuntu.com/ubuntu bionic InRelease\n",
            "\r                                                                               \rGet:3 http://archive.ubuntu.com/ubuntu bionic-updates InRelease [88.7 kB]\n",
            "\r                                                                               \rHit:4 http://ppa.launchpad.net/cran/libgit2/ubuntu bionic InRelease\n",
            "\r0% [3 InRelease 15.6 kB/88.7 kB 18%] [Connecting to security.ubuntu.com (91.189\r0% [1 InRelease gpgv 15.9 kB] [3 InRelease 15.6 kB/88.7 kB 18%] [Connecting to \r                                                                               \rGet:5 https://cloud.r-project.org/bin/linux/ubuntu bionic-cran40/ InRelease [3,626 B]\n",
            "\r0% [1 InRelease gpgv 15.9 kB] [3 InRelease 50.4 kB/88.7 kB 57%] [Connecting to \r0% [1 InRelease gpgv 15.9 kB] [3 InRelease 50.4 kB/88.7 kB 57%] [Connecting to \r                                                                               \rHit:6 http://ppa.launchpad.net/deadsnakes/ppa/ubuntu bionic InRelease\n",
            "\r0% [1 InRelease gpgv 15.9 kB] [3 InRelease 75.0 kB/88.7 kB 85%] [Connecting to \r0% [1 InRelease gpgv 15.9 kB] [Waiting for headers] [Connecting to security.ubu\r                                                                               \rGet:7 http://archive.ubuntu.com/ubuntu bionic-backports InRelease [74.6 kB]\n",
            "\r0% [1 InRelease gpgv 15.9 kB] [7 InRelease 20.0 kB/74.6 kB 27%] [Connecting to \r0% [1 InRelease gpgv 15.9 kB] [Connecting to security.ubuntu.com (91.189.91.39)\r                                                                               \rGet:8 http://ppa.launchpad.net/graphics-drivers/ppa/ubuntu bionic InRelease [21.3 kB]\n",
            "\r0% [1 InRelease gpgv 15.9 kB] [Connecting to security.ubuntu.com (91.189.91.39)\r                                                                               \r0% [1 InRelease gpgv 15.9 kB] [Waiting for headers] [Waiting for headers]\r                                                                         \rIgn:9 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  InRelease\n",
            "Get:10 http://security.ubuntu.com/ubuntu bionic-security InRelease [88.7 kB]\n",
            "Ign:11 https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64  InRelease\n",
            "Hit:12 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  Release\n",
            "Hit:13 https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64  Release\n",
            "Get:14 https://cloud.r-project.org/bin/linux/ubuntu bionic-cran40/ Packages [67.0 kB]\n",
            "Get:15 http://archive.ubuntu.com/ubuntu bionic-updates/universe amd64 Packages [2,200 kB]\n",
            "Get:16 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 Packages [2,731 kB]\n",
            "Get:17 http://ppa.launchpad.net/graphics-drivers/ppa/ubuntu bionic/main amd64 Packages [41.6 kB]\n",
            "Get:20 http://security.ubuntu.com/ubuntu bionic-security/main amd64 Packages [2,295 kB]\n",
            "Get:21 http://security.ubuntu.com/ubuntu bionic-security/universe amd64 Packages [1,426 kB]\n",
            "Fetched 9,038 kB in 3s (3,461 kB/s)\n",
            "Reading package lists... Done\n",
            "update-alternatives: using /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java to provide /usr/bin/java (java) in manual mode\n",
            "openjdk version \"1.8.0_292\"\n",
            "OpenJDK Runtime Environment (build 1.8.0_292-8u292-b10-0ubuntu1~18.04-b10)\n",
            "OpenJDK 64-Bit Server VM (build 25.292-b10, mixed mode)\n",
            "Collecting pyspark\n",
            "  Downloading pyspark-3.1.2.tar.gz (212.4 MB)\n",
            "\u001b[K     |████████████████████████████████| 212.4 MB 70 kB/s \n",
            "\u001b[?25hCollecting py4j==0.10.9\n",
            "  Downloading py4j-0.10.9-py2.py3-none-any.whl (198 kB)\n",
            "\u001b[K     |████████████████████████████████| 198 kB 80.3 MB/s \n",
            "\u001b[?25hBuilding wheels for collected packages: pyspark\n",
            "  Building wheel for pyspark (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for pyspark: filename=pyspark-3.1.2-py2.py3-none-any.whl size=212880768 sha256=3d40ab4ee21bfeb1e689c1fc5ed25f66d5634209bf38d4197480029ac1d05c11\n",
            "  Stored in directory: /root/.cache/pip/wheels/a5/0a/c1/9561f6fecb759579a7d863dcd846daaa95f598744e71b02c77\n",
            "Successfully built pyspark\n",
            "Installing collected packages: py4j, pyspark\n",
            "Successfully installed py4j-0.10.9 pyspark-3.1.2\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4-MGoU00aQB3"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 191
        },
        "id": "ldiBp3FHmuJf",
        "outputId": "df612de4-1237-4d5c-f4dc-471a8f639d5a"
      },
      "source": [
        "from pyspark import SparkContext\n",
        "sc = SparkContext.getOrCreate()\n",
        "sc\n"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "\n",
              "        <div>\n",
              "            <p><b>SparkContext</b></p>\n",
              "\n",
              "            <p><a href=\"http://5418cc5861e2:4040\">Spark UI</a></p>\n",
              "\n",
              "            <dl>\n",
              "              <dt>Version</dt>\n",
              "                <dd><code>v3.1.2</code></dd>\n",
              "              <dt>Master</dt>\n",
              "                <dd><code>local[*]</code></dd>\n",
              "              <dt>AppName</dt>\n",
              "                <dd><code>pyspark-shell</code></dd>\n",
              "            </dl>\n",
              "        </div>\n",
              "        "
            ],
            "text/plain": [
              "<SparkContext master=local[*] appName=pyspark-shell>"
            ]
          },
          "metadata": {},
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YTiUtrVmzBea",
        "outputId": "b98a0d5a-2158-41fe-cd83-c07a54060343"
      },
      "source": [
        "from pyspark import SparkContext\n",
        "sc = SparkContext.getOrCreate()\n",
        "sayilar = [12,43,3,2,51,42,59,14]\n",
        "\n",
        "kareler =[]\n",
        "for sayi in sayilar:\n",
        "  kare=sayi*sayi\n",
        "  kareler.append(kare)\n",
        "print(kareler)\n",
        "\n",
        "sayilarRDD = sc.parallelize(sayilar) #RDD (resillent distributed Data ) rdd dağıtık haldeyken görüntülenmiyor bu sebeple aşağıdaki gibi hash kodu çıkıyor\n",
        "# print(sayilarRDD)\n",
        "kareleRDD = sayilarRDD.map(lambda x : x * x) # verilen işlemi bütün elemanlara uygular\n",
        "print(kareleRDD.collect())\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[144, 1849, 9, 4, 2601, 1764, 3481, 196]\n",
            "[144, 1849, 9, 4, 2601, 1764, 3481, 196]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dWelk2dL-dh5"
      },
      "source": [
        "# Yeni Bölüm"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WeyPNdC2-gE2",
        "outputId": "c669c01e-c0eb-4476-999b-030dcd931cf0"
      },
      "source": [
        "pointsRDD = sc.textFile(\"points.txt\")\n",
        "def splitAndConvert(satir):\n",
        "  arr = satir.split(',')\n",
        "  x = int(arr[0])\n",
        "  y = int(arr[1])\n",
        "  return (x,y)\n",
        "pointsRDD = pointsRDD.map(lambda line : splitAndConvert(line))\n",
        "\n",
        "pointsRDD.collect()"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[(5, 5),\n",
              " (15, 17),\n",
              " (15, 16),\n",
              " (5, 6),\n",
              " (6, 6),\n",
              " (16, 16),\n",
              " (8, 8),\n",
              " (7, 6),\n",
              " (17, 17),\n",
              " (6, 7),\n",
              " (7, 8),\n",
              " (18, 18),\n",
              " (8, 6),\n",
              " (17, 16),\n",
              " (5, 8),\n",
              " (17, 16),\n",
              " (8, 8),\n",
              " (16, 17),\n",
              " (17, 18),\n",
              " (18, 16),\n",
              " (5, 15),\n",
              " (15, 7),\n",
              " (15, 6),\n",
              " (5, 16),\n",
              " (6, 16),\n",
              " (16, 6),\n",
              " (8, 18),\n",
              " (7, 16),\n",
              " (17, 7),\n",
              " (6, 7),\n",
              " (7, 8),\n",
              " (18, 8),\n",
              " (8, 16),\n",
              " (17, 6),\n",
              " (5, 18),\n",
              " (17, 6),\n",
              " (8, 18),\n",
              " (16, 7),\n",
              " (17, 8),\n",
              " (18, 6)]"
            ]
          },
          "metadata": {},
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nEVvA5-9_cv6",
        "outputId": "d210e925-0c3a-4426-f1ca-ebb700b9aaad"
      },
      "source": [
        "pointsRDD = sc.textFile(\"points.txt\")\n",
        "def splitAndConvert(satir):\n",
        "  arr = satir.split(',')\n",
        "  x = int(arr[0])\n",
        "  y = int(arr[1])\n",
        "  \n",
        "  return (x,y)\n",
        "pointsRDD = pointsRDD.map(lambda line : splitAndConvert(line))\n",
        "\n",
        "pointsRDD = pointsRDD.map(lambda a : (a[0],a[1],a[0]+a[1]) )\n",
        "\n",
        "pointsRDD = pointsRDD.sortBy(lambda b : b[2], ascending=False )\n",
        "\n",
        "\n",
        "pointsRDD.collect()"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[(18, 18, 36),\n",
              " (17, 18, 35),\n",
              " (17, 17, 34),\n",
              " (18, 16, 34),\n",
              " (17, 16, 33),\n",
              " (17, 16, 33),\n",
              " (16, 17, 33),\n",
              " (15, 17, 32),\n",
              " (16, 16, 32),\n",
              " (15, 16, 31),\n",
              " (8, 18, 26),\n",
              " (18, 8, 26),\n",
              " (8, 18, 26),\n",
              " (17, 8, 25),\n",
              " (17, 7, 24),\n",
              " (8, 16, 24),\n",
              " (18, 6, 24),\n",
              " (7, 16, 23),\n",
              " (17, 6, 23),\n",
              " (5, 18, 23),\n",
              " (17, 6, 23),\n",
              " (16, 7, 23),\n",
              " (15, 7, 22),\n",
              " (6, 16, 22),\n",
              " (16, 6, 22),\n",
              " (15, 6, 21),\n",
              " (5, 16, 21),\n",
              " (5, 15, 20),\n",
              " (8, 8, 16),\n",
              " (8, 8, 16),\n",
              " (7, 8, 15),\n",
              " (7, 8, 15),\n",
              " (8, 6, 14),\n",
              " (7, 6, 13),\n",
              " (6, 7, 13),\n",
              " (5, 8, 13),\n",
              " (6, 7, 13),\n",
              " (6, 6, 12),\n",
              " (5, 6, 11),\n",
              " (5, 5, 10)]"
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dJbwRDZ3EFyW"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xqeRt_C2DZfu",
        "outputId": "1014b7d4-052e-48ff-c747-9c8b308af973"
      },
      "source": [
        "pointsRDD = sc.textFile(\"points.txt\")\n",
        "def splitAndConvert(satir):\n",
        "  arr = satir.split(',')\n",
        "  x = int(arr[0])\n",
        "  y = int(arr[1])\n",
        "  \n",
        "  return (x,y)\n",
        "pointsRDD = pointsRDD.map(lambda line : splitAndConvert(line))\n",
        "\n",
        "pointsRDD = pointsRDD.map(lambda a : (a[0],a[1],a[0]+a[1]) )\n",
        "\n",
        "pointsRDD = pointsRDD.sortBy(lambda b : b[2], ascending=False )\n",
        "\n",
        "pointsRDD = pointsRDD.filter(lambda c : c[2]<20)\n",
        "pointsRDD.collect()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[(8, 8, 16),\n",
              " (8, 8, 16),\n",
              " (7, 8, 15),\n",
              " (7, 8, 15),\n",
              " (8, 6, 14),\n",
              " (7, 6, 13),\n",
              " (6, 7, 13),\n",
              " (5, 8, 13),\n",
              " (6, 7, 13),\n",
              " (6, 6, 12),\n",
              " (5, 6, 11),\n",
              " (5, 5, 10)]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3C3o-b6ZEGsM",
        "outputId": "1d440721-ec79-4424-d6c3-a0db7202d1ee"
      },
      "source": [
        "rdd1 = sc.parallelize([[12,23,34],[9,4,3,5],[23,4],[3,43,34,56],[45]])\n",
        "rdd1 = rdd1.flatMap(lambda x : list(x))\n",
        "rdd1.collect()"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[12, 23, 34, 9, 4, 3, 5, 23, 4, 3, 43, 34, 56, 45]"
            ]
          },
          "metadata": {},
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mNv65egiFgqQ",
        "outputId": "cdf993ec-ce8e-4736-e714-bc229f775da8"
      },
      "source": [
        "textRDD = sc.textFile('text.txt')\n",
        "textRDD = textRDD.flatMap(lambda line : line.split(' '))\n",
        "textRDD = textRDD.filter(lambda kelime : len(kelime)>0)\n",
        "textRDD.collect()"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['�Gelece�in',\n",
              " 'G�c��',\n",
              " 'ba�l�kl�',\n",
              " 'yaz�',\n",
              " 'dizimizde',\n",
              " 'Sabanc�',\n",
              " '�niversitesi�nin',\n",
              " 'profesyonellere',\n",
              " 'y�nelik',\n",
              " 'sundu�u',\n",
              " 'tezsiz',\n",
              " 'y�ksek',\n",
              " 'lisans',\n",
              " 'programlar�n�',\n",
              " 'tan�may�',\n",
              " 's�rd�r�yoruz.',\n",
              " 'Bili�im',\n",
              " 'Teknolojileri',\n",
              " 'Tezsiz',\n",
              " 'Y�ksek',\n",
              " 'Lisans',\n",
              " 'Program�m�z�n',\n",
              " 'Akademik',\n",
              " 'Direkt�r�',\n",
              " 'H.',\n",
              " 'Sait',\n",
              " '�lmez',\n",
              " 'ile',\n",
              " 'program�n',\n",
              " 'sundu�u',\n",
              " 'imkanlar�',\n",
              " 'konu�tuk.',\n",
              " 'Programla',\n",
              " 'ilgili',\n",
              " 'biraz',\n",
              " 'bilgi',\n",
              " 'verebilir',\n",
              " 'misiniz?',\n",
              " 'Ne',\n",
              " 'zaman',\n",
              " 'a��ld�?',\n",
              " '�zellikle',\n",
              " 'kimlere',\n",
              " 'hitap',\n",
              " 'ediyor?',\n",
              " '2002',\n",
              " 'y�l�nda',\n",
              " 'a��lan',\n",
              " 'program�m�z,',\n",
              " 'bili�im',\n",
              " 'teknolojilerindeki',\n",
              " 'yeti�mi�',\n",
              " 'eleman',\n",
              " 'a����na',\n",
              " 'cevap',\n",
              " 'vermek',\n",
              " '�zere',\n",
              " 've',\n",
              " 'tamamen',\n",
              " 'bu',\n",
              " 'konudaki',\n",
              " 'end�stri',\n",
              " 'ihtiya�lar�',\n",
              " 'referans',\n",
              " 'al�narak',\n",
              " 'tasarland�.',\n",
              " 'Program,',\n",
              " 'ana',\n",
              " 'hedefi',\n",
              " 'yaz�l�m',\n",
              " 'geli�tirici',\n",
              " '(software',\n",
              " 'developer)',\n",
              " 'yeti�tirmektir.',\n",
              " 'Ancak',\n",
              " 'e�itim',\n",
              " 'program�ndaki',\n",
              " 'se�meli',\n",
              " 'derslerin',\n",
              " 'zenginli�i',\n",
              " 'kat�l�mc�lara',\n",
              " 'yaz�l�m',\n",
              " 's�nama,',\n",
              " 'a�',\n",
              " 'tasar�m',\n",
              " 've',\n",
              " 'g�venli�i,',\n",
              " 'veri',\n",
              " 'taban�',\n",
              " 'uygulamalar�,',\n",
              " 'ara',\n",
              " 'y�z',\n",
              " '(UX)',\n",
              " 'tasar�m�',\n",
              " 've',\n",
              " 'veri',\n",
              " 'bilimi',\n",
              " 'gibi',\n",
              " 'konularda',\n",
              " 'da',\n",
              " 'uzmanl�k',\n",
              " 'f�rsat�',\n",
              " 'tan�maktad�r.',\n",
              " 'Program',\n",
              " 'i�erikleri',\n",
              " 'nas�l',\n",
              " 'belirleniyor,',\n",
              " 'her',\n",
              " 'y�l',\n",
              " 'de�i�iklik',\n",
              " 'yap�l�yor',\n",
              " 'mu?',\n",
              " 'Akademik',\n",
              " 'y�l',\n",
              " 'ba�lamadan',\n",
              " '�nce',\n",
              " 'program',\n",
              " 'kurulu',\n",
              " 'yaz�l�m',\n",
              " 've',\n",
              " 'IT',\n",
              " 'end�strisindeki',\n",
              " 'trendleri,',\n",
              " 'geli�meleri',\n",
              " 've',\n",
              " 'yak�n',\n",
              " 'zamanda',\n",
              " 'istihdam',\n",
              " 'ihtiyac�',\n",
              " 'ortaya',\n",
              " '��kmas�',\n",
              " 'beklenen',\n",
              " 'alanlar�',\n",
              " 'de�erlendirerek',\n",
              " 'ya',\n",
              " 'm�fredata',\n",
              " 'yeni',\n",
              " 'dersler',\n",
              " 'ekliyor',\n",
              " 'ya',\n",
              " 'da',\n",
              " 'mevcut',\n",
              " 'derslerin',\n",
              " 'i�erik',\n",
              " 've',\n",
              " 'kapsamlar�nda',\n",
              " 'revizyonlar',\n",
              " 'ger�ekle�tiriyor.',\n",
              " 'Program',\n",
              " 'sekt�r�n',\n",
              " '�zellikle',\n",
              " 'hangi',\n",
              " 'ihtiyac�na',\n",
              " 'yan�t',\n",
              " 'veriyor?',\n",
              " 'Bu',\n",
              " 'kapsamda',\n",
              " 'gelecekte',\n",
              " 'en',\n",
              " '�ok',\n",
              " 'ihtiya�',\n",
              " 'duyulacak',\n",
              " 'yetkinlikleri',\n",
              " 've',\n",
              " 'bunu',\n",
              " 'kar��lamak',\n",
              " '�zere',\n",
              " 'neler',\n",
              " 'yap�lmas�',\n",
              " 'gerekti�ini',\n",
              " 'anlat�r',\n",
              " 'm�s�n�z?',\n",
              " 'Program,',\n",
              " 'her',\n",
              " 't�r',\n",
              " 'end�stride',\n",
              " 't�m',\n",
              " 'kurumlar�n',\n",
              " 'yaz�l�m',\n",
              " 'geli�tirme,',\n",
              " 'i�',\n",
              " 'analizi,',\n",
              " 'yaz�l�m',\n",
              " 's�nama,',\n",
              " 'a�',\n",
              " 'tasar�m�',\n",
              " 've',\n",
              " 'g�venli�i,',\n",
              " 'veri',\n",
              " 'taban�',\n",
              " 'uygulamalar�',\n",
              " 'gibi',\n",
              " 'alanlarda',\n",
              " 'IT',\n",
              " 'operasyonlar�n�',\n",
              " 'y�r�tmek',\n",
              " '�zere',\n",
              " 'ihtiya�',\n",
              " 'duydu�u',\n",
              " 'mesleki',\n",
              " 'profilleri',\n",
              " 'kat�l�mc�lar�na',\n",
              " 'kazand�rmaya',\n",
              " '�al���yor.',\n",
              " 'Mezunlar�n�zla',\n",
              " 'ilgili',\n",
              " 'bilgi',\n",
              " 'verir',\n",
              " 'misiniz?',\n",
              " 'Program�',\n",
              " 'bitiren',\n",
              " '��rencilerinizin',\n",
              " 'kariyerlerinde',\n",
              " 'ne',\n",
              " 'gibi',\n",
              " 'farkl�l�klar',\n",
              " 'oluyor?',\n",
              " 'BT',\n",
              " 'Program�',\n",
              " 'mezunlar�n�n',\n",
              " 'bir',\n",
              " '�o�unlu�u',\n",
              " 'hi�bir',\n",
              " 'programlama',\n",
              " 'dili',\n",
              " 'bilmedikleri',\n",
              " 'halde',\n",
              " 'programa',\n",
              " 'kay�t',\n",
              " 'olup',\n",
              " 'hen�z',\n",
              " 'e�itimleri',\n",
              " 'tamamlanmadan',\n",
              " 'T�rkiye�nin',\n",
              " 'kendi',\n",
              " 'end�strisinde',\n",
              " 'bilinen',\n",
              " 'kurumlar�nda',\n",
              " 'yaz�l�m',\n",
              " 'geli�tirici',\n",
              " 'olarak',\n",
              " 'i�',\n",
              " 'bulabiliyorlar.',\n",
              " '2002�den',\n",
              " 'bu',\n",
              " 'yana',\n",
              " 'e�itim',\n",
              " 'veren',\n",
              " 'bu',\n",
              " 'y�ksek',\n",
              " 'lisans',\n",
              " 'program�nda',\n",
              " 'en',\n",
              " 's�k',\n",
              " 've',\n",
              " 'memnuniyetle',\n",
              " 'g�zlemledi�imiz',\n",
              " 'sonu�lardan',\n",
              " 'birisi',\n",
              " 'bu.',\n",
              " 'msit.sabanciuniv.edu',\n",
              " 'sayfam�zdan',\n",
              " 'mezun',\n",
              " 'g�r��lerinde',\n",
              " 'buna',\n",
              " 's�yledi�im',\n",
              " 't�rde',\n",
              " '�rneklerin',\n",
              " 'bir',\n",
              " 'k�sm�n�',\n",
              " 've',\n",
              " 'genel',\n",
              " 'olarak',\n",
              " 'kat�l�mc�lar�n',\n",
              " 'programla',\n",
              " 'ilgili',\n",
              " 'd���ncelerini',\n",
              " 'inceleyebilirsiniz.']"
            ]
          },
          "metadata": {},
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "RAlOU0UCF6_O",
        "outputId": "4f765696-9af0-4ae8-e8cd-dedb4e35070b"
      },
      "source": [
        "textRDD = sc.textFile('text.txt')\n",
        "textRDD = textRDD.map(lambda line : line.split(' '))\n",
        "\n",
        "textRDD = textRDD.filter(lambda kelime : len(kelime)>0)\n",
        "\n",
        "textRDD.collect()"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[['�Gelece�in',\n",
              "  'G�c��',\n",
              "  'ba�l�kl�',\n",
              "  'yaz�',\n",
              "  'dizimizde',\n",
              "  'Sabanc�',\n",
              "  '�niversitesi�nin',\n",
              "  'profesyonellere',\n",
              "  'y�nelik',\n",
              "  'sundu�u',\n",
              "  'tezsiz',\n",
              "  'y�ksek',\n",
              "  'lisans',\n",
              "  'programlar�n�',\n",
              "  'tan�may�',\n",
              "  's�rd�r�yoruz.',\n",
              "  'Bili�im',\n",
              "  'Teknolojileri',\n",
              "  'Tezsiz',\n",
              "  'Y�ksek',\n",
              "  'Lisans',\n",
              "  'Program�m�z�n',\n",
              "  'Akademik',\n",
              "  'Direkt�r�',\n",
              "  'H.',\n",
              "  'Sait',\n",
              "  '�lmez',\n",
              "  'ile',\n",
              "  'program�n',\n",
              "  'sundu�u',\n",
              "  'imkanlar�',\n",
              "  'konu�tuk.'],\n",
              " [''],\n",
              " ['Programla',\n",
              "  'ilgili',\n",
              "  'biraz',\n",
              "  'bilgi',\n",
              "  'verebilir',\n",
              "  'misiniz?',\n",
              "  'Ne',\n",
              "  'zaman',\n",
              "  'a��ld�?',\n",
              "  '',\n",
              "  '�zellikle',\n",
              "  'kimlere',\n",
              "  'hitap',\n",
              "  'ediyor?',\n",
              "  ''],\n",
              " [''],\n",
              " ['2002',\n",
              "  'y�l�nda',\n",
              "  'a��lan',\n",
              "  'program�m�z,',\n",
              "  'bili�im',\n",
              "  'teknolojilerindeki',\n",
              "  'yeti�mi�',\n",
              "  'eleman',\n",
              "  'a����na',\n",
              "  'cevap',\n",
              "  'vermek',\n",
              "  '�zere',\n",
              "  've',\n",
              "  'tamamen',\n",
              "  'bu',\n",
              "  'konudaki',\n",
              "  'end�stri',\n",
              "  'ihtiya�lar�',\n",
              "  'referans',\n",
              "  'al�narak',\n",
              "  'tasarland�.',\n",
              "  'Program,',\n",
              "  'ana',\n",
              "  'hedefi',\n",
              "  'yaz�l�m',\n",
              "  'geli�tirici',\n",
              "  '(software',\n",
              "  'developer)',\n",
              "  'yeti�tirmektir.',\n",
              "  'Ancak',\n",
              "  'e�itim',\n",
              "  'program�ndaki',\n",
              "  'se�meli',\n",
              "  'derslerin',\n",
              "  'zenginli�i',\n",
              "  'kat�l�mc�lara',\n",
              "  'yaz�l�m',\n",
              "  's�nama,',\n",
              "  'a�',\n",
              "  'tasar�m',\n",
              "  've',\n",
              "  'g�venli�i,',\n",
              "  'veri',\n",
              "  'taban�',\n",
              "  'uygulamalar�,',\n",
              "  'ara',\n",
              "  'y�z',\n",
              "  '(UX)',\n",
              "  'tasar�m�',\n",
              "  've',\n",
              "  'veri',\n",
              "  'bilimi',\n",
              "  'gibi',\n",
              "  'konularda',\n",
              "  'da',\n",
              "  'uzmanl�k',\n",
              "  'f�rsat�',\n",
              "  'tan�maktad�r.'],\n",
              " [''],\n",
              " ['Program',\n",
              "  'i�erikleri',\n",
              "  'nas�l',\n",
              "  'belirleniyor,',\n",
              "  'her',\n",
              "  'y�l',\n",
              "  'de�i�iklik',\n",
              "  'yap�l�yor',\n",
              "  'mu?'],\n",
              " [''],\n",
              " ['Akademik',\n",
              "  'y�l',\n",
              "  'ba�lamadan',\n",
              "  '�nce',\n",
              "  'program',\n",
              "  'kurulu',\n",
              "  'yaz�l�m',\n",
              "  've',\n",
              "  'IT',\n",
              "  'end�strisindeki',\n",
              "  'trendleri,',\n",
              "  'geli�meleri',\n",
              "  've',\n",
              "  'yak�n',\n",
              "  'zamanda',\n",
              "  'istihdam',\n",
              "  'ihtiyac�',\n",
              "  'ortaya',\n",
              "  '��kmas�',\n",
              "  'beklenen',\n",
              "  'alanlar�',\n",
              "  'de�erlendirerek',\n",
              "  'ya',\n",
              "  'm�fredata',\n",
              "  'yeni',\n",
              "  'dersler',\n",
              "  'ekliyor',\n",
              "  'ya',\n",
              "  'da',\n",
              "  'mevcut',\n",
              "  'derslerin',\n",
              "  'i�erik',\n",
              "  've',\n",
              "  'kapsamlar�nda',\n",
              "  'revizyonlar',\n",
              "  'ger�ekle�tiriyor.'],\n",
              " [''],\n",
              " ['Program',\n",
              "  'sekt�r�n',\n",
              "  '�zellikle',\n",
              "  'hangi',\n",
              "  'ihtiyac�na',\n",
              "  'yan�t',\n",
              "  'veriyor?',\n",
              "  'Bu',\n",
              "  'kapsamda',\n",
              "  'gelecekte',\n",
              "  'en',\n",
              "  '�ok',\n",
              "  'ihtiya�',\n",
              "  'duyulacak',\n",
              "  'yetkinlikleri',\n",
              "  've',\n",
              "  'bunu',\n",
              "  'kar��lamak',\n",
              "  '�zere',\n",
              "  'neler',\n",
              "  'yap�lmas�',\n",
              "  'gerekti�ini',\n",
              "  'anlat�r',\n",
              "  'm�s�n�z?',\n",
              "  '',\n",
              "  ''],\n",
              " [''],\n",
              " ['Program,',\n",
              "  'her',\n",
              "  't�r',\n",
              "  'end�stride',\n",
              "  't�m',\n",
              "  'kurumlar�n',\n",
              "  'yaz�l�m',\n",
              "  'geli�tirme,',\n",
              "  'i�',\n",
              "  'analizi,',\n",
              "  'yaz�l�m',\n",
              "  's�nama,',\n",
              "  'a�',\n",
              "  'tasar�m�',\n",
              "  've',\n",
              "  'g�venli�i,',\n",
              "  'veri',\n",
              "  'taban�',\n",
              "  'uygulamalar�',\n",
              "  'gibi',\n",
              "  'alanlarda',\n",
              "  'IT',\n",
              "  'operasyonlar�n�',\n",
              "  'y�r�tmek',\n",
              "  '�zere',\n",
              "  'ihtiya�',\n",
              "  'duydu�u',\n",
              "  'mesleki',\n",
              "  'profilleri',\n",
              "  'kat�l�mc�lar�na',\n",
              "  'kazand�rmaya',\n",
              "  '�al���yor.'],\n",
              " [''],\n",
              " ['Mezunlar�n�zla',\n",
              "  'ilgili',\n",
              "  'bilgi',\n",
              "  'verir',\n",
              "  'misiniz?',\n",
              "  'Program�',\n",
              "  'bitiren',\n",
              "  '��rencilerinizin',\n",
              "  'kariyerlerinde',\n",
              "  'ne',\n",
              "  'gibi',\n",
              "  'farkl�l�klar',\n",
              "  'oluyor?'],\n",
              " [''],\n",
              " ['BT',\n",
              "  'Program�',\n",
              "  'mezunlar�n�n',\n",
              "  'bir',\n",
              "  '�o�unlu�u',\n",
              "  'hi�bir',\n",
              "  'programlama',\n",
              "  'dili',\n",
              "  'bilmedikleri',\n",
              "  'halde',\n",
              "  'programa',\n",
              "  'kay�t',\n",
              "  'olup',\n",
              "  'hen�z',\n",
              "  'e�itimleri',\n",
              "  'tamamlanmadan',\n",
              "  'T�rkiye�nin',\n",
              "  'kendi',\n",
              "  'end�strisinde',\n",
              "  'bilinen',\n",
              "  'kurumlar�nda',\n",
              "  'yaz�l�m',\n",
              "  'geli�tirici',\n",
              "  'olarak',\n",
              "  'i�',\n",
              "  'bulabiliyorlar.',\n",
              "  '2002�den',\n",
              "  'bu',\n",
              "  'yana',\n",
              "  'e�itim',\n",
              "  'veren',\n",
              "  'bu',\n",
              "  'y�ksek',\n",
              "  'lisans',\n",
              "  'program�nda',\n",
              "  'en',\n",
              "  's�k',\n",
              "  've',\n",
              "  'memnuniyetle',\n",
              "  'g�zlemledi�imiz',\n",
              "  'sonu�lardan',\n",
              "  'birisi',\n",
              "  'bu.',\n",
              "  'msit.sabanciuniv.edu',\n",
              "  'sayfam�zdan',\n",
              "  'mezun',\n",
              "  'g�r��lerinde',\n",
              "  'buna',\n",
              "  's�yledi�im',\n",
              "  't�rde',\n",
              "  '�rneklerin',\n",
              "  'bir',\n",
              "  'k�sm�n�',\n",
              "  've',\n",
              "  'genel',\n",
              "  'olarak',\n",
              "  'kat�l�mc�lar�n',\n",
              "  'programla',\n",
              "  'ilgili',\n",
              "  'd���ncelerini',\n",
              "  'inceleyebilirsiniz.']]"
            ]
          },
          "metadata": {},
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qew-SOJItwXm"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hu5QGZKJaRTX",
        "outputId": "73cb3852-1392-47c0-941f-0df833c6cb26"
      },
      "source": [
        "#Türkiyede 1900 yılından beri olmus en büyük 10 deprem\n",
        "from pyspark import SparkContext\n",
        "\n",
        "sc = SparkContext.getOrCreate()\n",
        "\n",
        "depremRDD = sc.textFile(\"DepremVerileri.txt\")\n",
        "depremRDD = depremRDD.filter(lambda line : \"Deprem Kodu\" not in line)\n",
        "def splitAndTake(line) :\n",
        "    arr = line.split(\"\\t\")\n",
        "    return ( arr[2],float(arr[4]), float(arr[5]), float(arr[7]), arr[14] )\n",
        "\n",
        "depremRDD = depremRDD.map(lambda a : splitAndTake(a))\n",
        "depremRDD = depremRDD.sortBy(lambda b : b[3], ascending=False)\n",
        "for d in depremRDD.take(10) :\n",
        "  print(d)\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "('1939.12.26', 39.8, 39.51, 7.9, 'KURUTILEK- (ERZINCAN) [North East  3.0 km]')\n",
            "('1926.06.26', 36.54, 27.33, 7.7, 'ONIKI ADALAR (AKDENIZ)')\n",
            "('1930.05.06', 37.98, 44.48, 7.6, 'T�RKIYE-IRAN SINIR B�LGESI')\n",
            "('1976.11.24', 39.05, 44.04, 7.5, 'YENIYAKA-CALDIRAN (VAN) [South East  1.9 km]')\n",
            "('1999.08.17', 40.76, 29.97, 7.4, 'BASISKELE (KOCAELI) [North East  2.0 km]')\n",
            "('1912.08.09', 40.6, 27.2, 7.3, 'ERIKLICE-SARKOY (TEKIRDAG) [South East  4.3 km]')\n",
            "('2011.10.23', 38.7212, 43.411, 7.2, 'YEMLICE- (VAN) [North West  1.5 km]')\n",
            "('1999.11.12', 40.74, 31.21, 7.2, 'UGUR- (DUZCE) [North East  0.3 km]')\n",
            "('1953.03.18', 39.99, 27.36, 7.2, 'SOGUCAK-YENICE (�ANAKKALE) [South West  2.3 km]')\n",
            "('1948.02.09', 35.41, 27.2, 7.2, 'AKDENIZ')\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MAiDUTcnmIzF",
        "outputId": "66c35e3f-0240-45fa-c12b-8dc14adb2cce"
      },
      "source": [
        "#Türkiyede 1900 yılından beri olmus en büyük 10 deprem ve \n",
        "from pyspark import SparkContext\n",
        "\n",
        "sc = SparkContext.getOrCreate()\n",
        "\n",
        "depremRDD = sc.textFile(\"DepremVerileri.txt\")\n",
        "depremRDD = depremRDD.filter(lambda line : \"Deprem Kodu\" not in line)\n",
        "def splitAndTake(line) :\n",
        "    arr = line.split(\"\\t\")\n",
        "    return ( float(arr[4]), float(arr[5]) )\n",
        "\n",
        "depremRDD = depremRDD.map(lambda a : splitAndTake(a))\n",
        "\n",
        "def bolgeAta(enlem,boylam):\n",
        "  bolge = \"\"\n",
        "  if (enlem >= 39):\n",
        "    if (boylam <=31):\n",
        "      bolge = \"Bölge 1\"\n",
        "    elif (boylam <= 36):\n",
        "      bolge = \"Bölge 2\"\n",
        "    elif (boylam <= 41):\n",
        "      bolge = \"Bölge 3\"\n",
        "    else:\n",
        "      bolge = \"Bölge 4\"\n",
        "  else:\n",
        "     if (boylam <=31):\n",
        "      bolge = \"Bölge 5\"\n",
        "     elif (boylam <= 36):\n",
        "      bolge = \"Bölge 6\"\n",
        "     elif (boylam <= 41):\n",
        "      bolge = \"Bölge 7\"\n",
        "     else:\n",
        "      bolge = \"Bölge 8\"\n",
        "  return (bolge,1)\n",
        "\n",
        "\n",
        "depremRDD = depremRDD.map(lambda d : bolgeAta(d[0],d[1]))\n",
        "depremRDD = depremRDD.reduceByKey(lambda x,y : x+y)\n",
        "depremRDD = depremRDD.sortBy(lambda a : a[1],ascending=False)\n",
        "\n",
        "depremRDD.take(15)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[('Bölge 5', 8146),\n",
              " ('Bölge 1', 2835),\n",
              " ('Bölge 8', 1561),\n",
              " ('Bölge 7', 1349),\n",
              " ('Bölge 6', 1286),\n",
              " ('Bölge 2', 1000),\n",
              " ('Bölge 3', 996),\n",
              " ('Bölge 4', 759)]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 45
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vfLwCCkL0vj_",
        "outputId": "fd30b3bd-8bc9-4178-8510-ac43115bab84"
      },
      "source": [
        "rdd1 = sc.parallelize([(\"a\",1),(\"b\",8),(\"c\",2),(\"a\",4),(\"c\",3)])\n",
        "rdd2 = sc.parallelize([(\"a\",4),(\"b\",5),(\"e\",1),(\"f\",4),(\"b\",2)])\n",
        "\n",
        "rdd3 = rdd1.union(rdd2)\n",
        "print(rdd3.collect())\n",
        "\n",
        "rdd4 = rdd1.join(rdd2)\n",
        "print(rdd4.collect())\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[('a', 1), ('b', 8), ('c', 2), ('a', 4), ('c', 3), ('a', 4), ('b', 5), ('e', 1), ('f', 4), ('b', 2)]\n",
            "[('b', (8, 5)), ('b', (8, 2)), ('a', (1, 4)), ('a', (4, 4))]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZY6m94bq2MnR"
      },
      "source": [
        "rdd1 = sc.parallelize([(\"a\",1),(\"b\",8),(\"c\",2),(\"a\",4),(\"c\",3)])\n",
        "rdd2 = sc.parallelize([(\"a\",4),(\"b\",5),(\"e\",1),(\"f\",4),(\"b\",2)])\n",
        "\n",
        "groupRDD = rdd1.groupByKey().map(lambda x : [x[0], tuple])\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "OZq2Sqgw2eS-",
        "outputId": "5ad589ca-4752-42f0-d8d5-11b2fb8c986b"
      },
      "source": [
        "rdd1 = sc.parallelize([(\"a\",1),(\"b\",8),(\"c\",2),(\"a\",4),(\"c\",3)])\n",
        "rdd2 = sc.parallelize([(\"a\",4),(\"b\",5),(\"e\",1),(\"f\",4),(\"b\",2)])\n",
        "\n",
        "rdd5 = rdd1.cartesian(rdd2)  #artıçı depremler sorusnu çözmek için bu lazım\n",
        "rdd5.collect()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[(('a', 1), ('a', 4)),\n",
              " (('a', 1), ('b', 5)),\n",
              " (('b', 8), ('a', 4)),\n",
              " (('b', 8), ('b', 5)),\n",
              " (('a', 1), ('e', 1)),\n",
              " (('a', 1), ('f', 4)),\n",
              " (('b', 8), ('e', 1)),\n",
              " (('b', 8), ('f', 4)),\n",
              " (('a', 1), ('b', 2)),\n",
              " (('b', 8), ('b', 2)),\n",
              " (('c', 2), ('a', 4)),\n",
              " (('c', 2), ('b', 5)),\n",
              " (('a', 4), ('a', 4)),\n",
              " (('a', 4), ('b', 5)),\n",
              " (('c', 3), ('a', 4)),\n",
              " (('c', 3), ('b', 5)),\n",
              " (('c', 2), ('e', 1)),\n",
              " (('c', 2), ('f', 4)),\n",
              " (('a', 4), ('e', 1)),\n",
              " (('a', 4), ('f', 4)),\n",
              " (('c', 2), ('b', 2)),\n",
              " (('a', 4), ('b', 2)),\n",
              " (('c', 3), ('e', 1)),\n",
              " (('c', 3), ('f', 4)),\n",
              " (('c', 3), ('b', 2))]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 48
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vBJljhps1dJz"
      },
      "source": [
        "rdd1 = sc.parallelize([(\"a\",1),(\"b\",8),(\"c\",2),(\"a\",4),(\"c\",3)])\n",
        "rdd2 = sc.parallelize([(\"a\",4),(\"b\",5),(\"e\",1),(\"f\",4),(\"b\",2)])\n",
        "\n",
        "rdd5 = rdd1.cartesian(rdd2)  #artıçı depremler sorusnu çözmek için bu lazım\n",
        "#rdd5.collect()\n",
        "\n",
        "rdd5.sortBy"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UvwrLpp21fHS"
      },
      "source": [
        "# Yeni Bölüm"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "L7xqRUYGtxq2",
        "outputId": "e5cdc7e2-5751-4d4c-8d12-603bd7833d63"
      },
      "source": [
        "from pyspark import SparkContext\n",
        "sc = SparkContext.getOrCreate()\n",
        "capitalsRDD = sc.textFile(\"Capitals.txt\")\n",
        "\n",
        "def splitAndTake(line):\n",
        "  arr = line.split(\" \")\n",
        "  enlem = float(arr[3])\n",
        "  boylam = float(arr[4])\n",
        "  capital_city = arr[2]\n",
        "\n",
        "  return (enlem,boylam)\n",
        "\n",
        "capitalsRDD = capitalsRDD.map(lambda a: splitAndTake(a))\n",
        "\n",
        "capitalsRDD = capitalsRDD.filter(lambda b:  b[0] < 90)\n",
        "capitalsRDD = capitalsRDD.filter(lambda b:  b[1] < 180)\n",
        "\n",
        "print(capitalsRDD.collect())\n",
        "\n",
        "\n"
      ],
      "execution_count": 32,
      "outputs": [
        {
          "output_type": "error",
          "ename": "Py4JJavaError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mPy4JJavaError\u001b[0m                             Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-32-807873dd2adf>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     16\u001b[0m \u001b[0mcapitalsRDD\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcapitalsRDD\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfilter\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;32mlambda\u001b[0m \u001b[0mb\u001b[0m\u001b[0;34m:\u001b[0m  \u001b[0mb\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m<\u001b[0m \u001b[0;36m180\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 18\u001b[0;31m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcapitalsRDD\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcollect\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     19\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     20\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/pyspark/rdd.py\u001b[0m in \u001b[0;36mcollect\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    947\u001b[0m         \"\"\"\n\u001b[1;32m    948\u001b[0m         \u001b[0;32mwith\u001b[0m \u001b[0mSCCallSiteSync\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcontext\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mcss\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 949\u001b[0;31m             \u001b[0msock_info\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mctx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_jvm\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mPythonRDD\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcollectAndServe\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_jrdd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrdd\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    950\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_load_from_socket\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msock_info\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_jrdd_deserializer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    951\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/py4j/java_gateway.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args)\u001b[0m\n\u001b[1;32m   1303\u001b[0m         \u001b[0manswer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgateway_client\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msend_command\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcommand\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1304\u001b[0m         return_value = get_return_value(\n\u001b[0;32m-> 1305\u001b[0;31m             answer, self.gateway_client, self.target_id, self.name)\n\u001b[0m\u001b[1;32m   1306\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1307\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mtemp_arg\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtemp_args\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/py4j/protocol.py\u001b[0m in \u001b[0;36mget_return_value\u001b[0;34m(answer, gateway_client, target_id, name)\u001b[0m\n\u001b[1;32m    326\u001b[0m                 raise Py4JJavaError(\n\u001b[1;32m    327\u001b[0m                     \u001b[0;34m\"An error occurred while calling {0}{1}{2}.\\n\"\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 328\u001b[0;31m                     format(target_id, \".\", name), value)\n\u001b[0m\u001b[1;32m    329\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    330\u001b[0m                 raise Py4JError(\n",
            "\u001b[0;31mPy4JJavaError\u001b[0m: An error occurred while calling z:org.apache.spark.api.python.PythonRDD.collectAndServe.\n: org.apache.spark.SparkException: Job aborted due to stage failure: Task 1 in stage 19.0 failed 1 times, most recent failure: Lost task 1.0 in stage 19.0 (TID 22) (5418cc5861e2 executor driver): org.apache.spark.api.python.PythonException: Traceback (most recent call last):\n  File \"/usr/local/lib/python3.7/dist-packages/pyspark/python/lib/pyspark.zip/pyspark/worker.py\", line 604, in main\n    process()\n  File \"/usr/local/lib/python3.7/dist-packages/pyspark/python/lib/pyspark.zip/pyspark/worker.py\", line 596, in process\n    serializer.dump_stream(out_iter, outfile)\n  File \"/usr/local/lib/python3.7/dist-packages/pyspark/python/lib/pyspark.zip/pyspark/serializers.py\", line 259, in dump_stream\n    vs = list(itertools.islice(iterator, batch))\n  File \"/usr/local/lib/python3.7/dist-packages/pyspark/python/lib/pyspark.zip/pyspark/util.py\", line 73, in wrapper\n    return f(*args, **kwargs)\n  File \"<ipython-input-32-807873dd2adf>\", line 13, in <lambda>\n  File \"<ipython-input-32-807873dd2adf>\", line 7, in splitAndTake\nValueError: could not convert string to float: '\\tMadagascar'\n\n\tat org.apache.spark.api.python.BasePythonRunner$ReaderIterator.handlePythonException(PythonRunner.scala:517)\n\tat org.apache.spark.api.python.PythonRunner$$anon$3.read(PythonRunner.scala:652)\n\tat org.apache.spark.api.python.PythonRunner$$anon$3.read(PythonRunner.scala:635)\n\tat org.apache.spark.api.python.BasePythonRunner$ReaderIterator.hasNext(PythonRunner.scala:470)\n\tat org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)\n\tat scala.collection.Iterator.foreach(Iterator.scala:941)\n\tat scala.collection.Iterator.foreach$(Iterator.scala:941)\n\tat org.apache.spark.InterruptibleIterator.foreach(InterruptibleIterator.scala:28)\n\tat scala.collection.generic.Growable.$plus$plus$eq(Growable.scala:62)\n\tat scala.collection.generic.Growable.$plus$plus$eq$(Growable.scala:53)\n\tat scala.collection.mutable.ArrayBuffer.$plus$plus$eq(ArrayBuffer.scala:105)\n\tat scala.collection.mutable.ArrayBuffer.$plus$plus$eq(ArrayBuffer.scala:49)\n\tat scala.collection.TraversableOnce.to(TraversableOnce.scala:315)\n\tat scala.collection.TraversableOnce.to$(TraversableOnce.scala:313)\n\tat org.apache.spark.InterruptibleIterator.to(InterruptibleIterator.scala:28)\n\tat scala.collection.TraversableOnce.toBuffer(TraversableOnce.scala:307)\n\tat scala.collection.TraversableOnce.toBuffer$(TraversableOnce.scala:307)\n\tat org.apache.spark.InterruptibleIterator.toBuffer(InterruptibleIterator.scala:28)\n\tat scala.collection.TraversableOnce.toArray(TraversableOnce.scala:294)\n\tat scala.collection.TraversableOnce.toArray$(TraversableOnce.scala:288)\n\tat org.apache.spark.InterruptibleIterator.toArray(InterruptibleIterator.scala:28)\n\tat org.apache.spark.rdd.RDD.$anonfun$collect$2(RDD.scala:1030)\n\tat org.apache.spark.SparkContext.$anonfun$runJob$5(SparkContext.scala:2236)\n\tat org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90)\n\tat org.apache.spark.scheduler.Task.run(Task.scala:131)\n\tat org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:497)\n\tat org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1439)\n\tat org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:500)\n\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)\n\tat java.lang.Thread.run(Thread.java:748)\n\nDriver stacktrace:\n\tat org.apache.spark.scheduler.DAGScheduler.failJobAndIndependentStages(DAGScheduler.scala:2258)\n\tat org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2(DAGScheduler.scala:2207)\n\tat org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2$adapted(DAGScheduler.scala:2206)\n\tat scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:62)\n\tat scala.collection.mutable.ResizableArray.foreach$(ResizableArray.scala:55)\n\tat scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:49)\n\tat org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:2206)\n\tat org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1(DAGScheduler.scala:1079)\n\tat org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1$adapted(DAGScheduler.scala:1079)\n\tat scala.Option.foreach(Option.scala:407)\n\tat org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:1079)\n\tat org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:2445)\n\tat org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2387)\n\tat org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2376)\n\tat org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:49)\n\tat org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:868)\n\tat org.apache.spark.SparkContext.runJob(SparkContext.scala:2196)\n\tat org.apache.spark.SparkContext.runJob(SparkContext.scala:2217)\n\tat org.apache.spark.SparkContext.runJob(SparkContext.scala:2236)\n\tat org.apache.spark.SparkContext.runJob(SparkContext.scala:2261)\n\tat org.apache.spark.rdd.RDD.$anonfun$collect$1(RDD.scala:1030)\n\tat org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)\n\tat org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112)\n\tat org.apache.spark.rdd.RDD.withScope(RDD.scala:414)\n\tat org.apache.spark.rdd.RDD.collect(RDD.scala:1029)\n\tat org.apache.spark.api.python.PythonRDD$.collectAndServe(PythonRDD.scala:180)\n\tat org.apache.spark.api.python.PythonRDD.collectAndServe(PythonRDD.scala)\n\tat sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\n\tat sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\n\tat sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\n\tat java.lang.reflect.Method.invoke(Method.java:498)\n\tat py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\n\tat py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)\n\tat py4j.Gateway.invoke(Gateway.java:282)\n\tat py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\n\tat py4j.commands.CallCommand.execute(CallCommand.java:79)\n\tat py4j.GatewayConnection.run(GatewayConnection.java:238)\n\tat java.lang.Thread.run(Thread.java:748)\nCaused by: org.apache.spark.api.python.PythonException: Traceback (most recent call last):\n  File \"/usr/local/lib/python3.7/dist-packages/pyspark/python/lib/pyspark.zip/pyspark/worker.py\", line 604, in main\n    process()\n  File \"/usr/local/lib/python3.7/dist-packages/pyspark/python/lib/pyspark.zip/pyspark/worker.py\", line 596, in process\n    serializer.dump_stream(out_iter, outfile)\n  File \"/usr/local/lib/python3.7/dist-packages/pyspark/python/lib/pyspark.zip/pyspark/serializers.py\", line 259, in dump_stream\n    vs = list(itertools.islice(iterator, batch))\n  File \"/usr/local/lib/python3.7/dist-packages/pyspark/python/lib/pyspark.zip/pyspark/util.py\", line 73, in wrapper\n    return f(*args, **kwargs)\n  File \"<ipython-input-32-807873dd2adf>\", line 13, in <lambda>\n  File \"<ipython-input-32-807873dd2adf>\", line 7, in splitAndTake\nValueError: could not convert string to float: '\\tMadagascar'\n\n\tat org.apache.spark.api.python.BasePythonRunner$ReaderIterator.handlePythonException(PythonRunner.scala:517)\n\tat org.apache.spark.api.python.PythonRunner$$anon$3.read(PythonRunner.scala:652)\n\tat org.apache.spark.api.python.PythonRunner$$anon$3.read(PythonRunner.scala:635)\n\tat org.apache.spark.api.python.BasePythonRunner$ReaderIterator.hasNext(PythonRunner.scala:470)\n\tat org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)\n\tat scala.collection.Iterator.foreach(Iterator.scala:941)\n\tat scala.collection.Iterator.foreach$(Iterator.scala:941)\n\tat org.apache.spark.InterruptibleIterator.foreach(InterruptibleIterator.scala:28)\n\tat scala.collection.generic.Growable.$plus$plus$eq(Growable.scala:62)\n\tat scala.collection.generic.Growable.$plus$plus$eq$(Growable.scala:53)\n\tat scala.collection.mutable.ArrayBuffer.$plus$plus$eq(ArrayBuffer.scala:105)\n\tat scala.collection.mutable.ArrayBuffer.$plus$plus$eq(ArrayBuffer.scala:49)\n\tat scala.collection.TraversableOnce.to(TraversableOnce.scala:315)\n\tat scala.collection.TraversableOnce.to$(TraversableOnce.scala:313)\n\tat org.apache.spark.InterruptibleIterator.to(InterruptibleIterator.scala:28)\n\tat scala.collection.TraversableOnce.toBuffer(TraversableOnce.scala:307)\n\tat scala.collection.TraversableOnce.toBuffer$(TraversableOnce.scala:307)\n\tat org.apache.spark.InterruptibleIterator.toBuffer(InterruptibleIterator.scala:28)\n\tat scala.collection.TraversableOnce.toArray(TraversableOnce.scala:294)\n\tat scala.collection.TraversableOnce.toArray$(TraversableOnce.scala:288)\n\tat org.apache.spark.InterruptibleIterator.toArray(InterruptibleIterator.scala:28)\n\tat org.apache.spark.rdd.RDD.$anonfun$collect$2(RDD.scala:1030)\n\tat org.apache.spark.SparkContext.$anonfun$runJob$5(SparkContext.scala:2236)\n\tat org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90)\n\tat org.apache.spark.scheduler.Task.run(Task.scala:131)\n\tat org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:497)\n\tat org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1439)\n\tat org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:500)\n\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)\n\t... 1 more\n"
          ]
        }
      ]
    }
  ]
}