Busra_Nur_Orhan_Homework2_Q1&2_Solutions.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Busra Nur_Orhan_Homework2.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyPeYCKBTvxYY2p608YRgo2U",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/busranurorhan/Complete-Python-3-Bootcamp/blob/master/Busra_Nur_Orhan_Homework2_Q1%262_Solutions.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "m6a6kop9v6ZE",
        "outputId": "0f751c19-0234-474d-e375-38e41a33c311"
      },
      "source": [
        "!apt-get update\n",
        "!apt-get install openjdk-8-jdk-headless -qq > /dev/null\n",
        "import os\n",
        "os.environ[\"JAVA_HOME\"] = \"/usr/lib/jvm/java-8-openjdk-amd64\"\n",
        "!update-alternatives --set java /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java\n",
        "!java -version\n",
        "!pip install pyspark"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\r0% [Working]\r            \rGet:1 http://ppa.launchpad.net/c2d4u.team/c2d4u4.0+/ubuntu bionic InRelease [15.9 kB]\n",
            "\r0% [Waiting for headers] [Connecting to security.ubuntu.com (91.189.91.38)] [Co\r                                                                               \rHit:2 http://archive.ubuntu.com/ubuntu bionic InRelease\n",
            "\r0% [Connecting to security.ubuntu.com (91.189.91.38)] [Connected to cloud.r-pro\r                                                                               \rGet:3 http://archive.ubuntu.com/ubuntu bionic-updates InRelease [88.7 kB]\n",
            "\r                                                                               \rHit:4 http://ppa.launchpad.net/cran/libgit2/ubuntu bionic InRelease\n",
            "\r0% [3 InRelease 60.5 kB/88.7 kB 68%] [Connecting to security.ubuntu.com (91.189\r0% [2 InRelease gpgv 242 kB] [3 InRelease 66.3 kB/88.7 kB 75%] [Connecting to s\r0% [2 InRelease gpgv 242 kB] [Waiting for headers] [Connecting to security.ubun\r                                                                               \rGet:5 https://cloud.r-project.org/bin/linux/ubuntu bionic-cran40/ InRelease [3,626 B]\n",
            "\r0% [2 InRelease gpgv 242 kB] [Waiting for headers] [Connecting to security.ubun\r0% [2 InRelease gpgv 242 kB] [Waiting for headers] [Connecting to security.ubun\r                                                                               \rGet:6 http://archive.ubuntu.com/ubuntu bionic-backports InRelease [74.6 kB]\n",
            "Hit:7 http://ppa.launchpad.net/deadsnakes/ppa/ubuntu bionic InRelease\n",
            "Hit:8 http://ppa.launchpad.net/graphics-drivers/ppa/ubuntu bionic InRelease\n",
            "Get:9 http://security.ubuntu.com/ubuntu bionic-security InRelease [88.7 kB]\n",
            "Ign:10 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  InRelease\n",
            "Ign:11 https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64  InRelease\n",
            "Hit:12 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  Release\n",
            "Hit:13 https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64  Release\n",
            "Get:14 http://ppa.launchpad.net/c2d4u.team/c2d4u4.0+/ubuntu bionic/main Sources [1,802 kB]\n",
            "Get:15 http://ppa.launchpad.net/c2d4u.team/c2d4u4.0+/ubuntu bionic/main amd64 Packages [922 kB]\n",
            "Get:16 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 Packages [2,761 kB]\n",
            "Get:17 http://archive.ubuntu.com/ubuntu bionic-updates/universe amd64 Packages [2,202 kB]\n",
            "Get:18 https://cloud.r-project.org/bin/linux/ubuntu bionic-cran40/ Packages [67.4 kB]\n",
            "Get:21 http://security.ubuntu.com/ubuntu bionic-security/main amd64 Packages [2,326 kB]\n",
            "Get:22 http://security.ubuntu.com/ubuntu bionic-security/universe amd64 Packages [1,428 kB]\n",
            "Fetched 11.8 MB in 4s (3,208 kB/s)\n",
            "Reading package lists... Done\n",
            "update-alternatives: using /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java to provide /usr/bin/java (java) in manual mode\n",
            "openjdk version \"1.8.0_292\"\n",
            "OpenJDK Runtime Environment (build 1.8.0_292-8u292-b10-0ubuntu1~18.04-b10)\n",
            "OpenJDK 64-Bit Server VM (build 25.292-b10, mixed mode)\n",
            "Collecting pyspark\n",
            "  Downloading pyspark-3.1.2.tar.gz (212.4 MB)\n",
            "\u001b[K     |████████████████████████████████| 212.4 MB 67 kB/s \n",
            "\u001b[?25hCollecting py4j==0.10.9\n",
            "  Downloading py4j-0.10.9-py2.py3-none-any.whl (198 kB)\n",
            "\u001b[K     |████████████████████████████████| 198 kB 49.6 MB/s \n",
            "\u001b[?25hBuilding wheels for collected packages: pyspark\n",
            "  Building wheel for pyspark (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for pyspark: filename=pyspark-3.1.2-py2.py3-none-any.whl size=212880768 sha256=64838a8f7867138284c409cb824ef67b97a27784c2d4e6a4db304709ca8e61d7\n",
            "  Stored in directory: /root/.cache/pip/wheels/a5/0a/c1/9561f6fecb759579a7d863dcd846daaa95f598744e71b02c77\n",
            "Successfully built pyspark\n",
            "Installing collected packages: py4j, pyspark\n",
            "Successfully installed py4j-0.10.9 pyspark-3.1.2\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "v3GRSJNGZu-U"
      },
      "source": [
        "#Question 1 answer"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gen98HJT2Abs",
        "outputId": "6501e787-6760-4d67-9de6-63affc6075e4"
      },
      "source": [
        "from pyspark.sql import SparkSession\n",
        "from pyspark.sql.functions import UserDefinedFunction\n",
        "from pyspark.sql.types import StringType\n",
        "\n",
        "spark = SparkSession.Builder().appName('Spark DataFrame Introduction').getOrCreate()\n",
        "\n",
        "\n",
        "moviesDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','true')\\\n",
        "                        .option('delimiter',',').csv('movies.csv')\n",
        "\n",
        "tagsDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','true')\\\n",
        "                        .option('delimiter',',').csv('tags.csv')\n",
        "\n",
        "   \n",
        "ratingsDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','true')\\\n",
        "                        .option('delimiter',',').csv('ratings.csv')  \n",
        "\n",
        "\n",
        "'''\n",
        "\n",
        "moviesDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','true')\\\n",
        "                        .option('delimiter',';').csv('movies_test.csv')\n",
        "\n",
        "tagsDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','true')\\\n",
        "                        .option('delimiter',';').csv('tags_test.csv')\n",
        "\n",
        "   \n",
        "ratingsDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','true')\\\n",
        "                        .option('delimiter',';').csv('ratings_test.csv')  \n",
        "\n",
        "'''\n",
        "\n",
        "#moviesDF.show(20)\n",
        "#tagsDF.show(20)\n",
        "#ratingsDF.show(500)\n",
        "\n",
        "movies_tagsDF = moviesDF.join(tagsDF,'movieId')                                          \n",
        "\n",
        "\n",
        "\n",
        "movies_tagsDF = movies_tagsDF.select('movieID','tag')\n",
        "#movies_tagsDF.show()\n",
        "\n",
        "\n",
        "movies_tagsDF = movies_tagsDF.sort('movieID',ascending =True)\n",
        "#movies_tagsDF.show()\n",
        "\n",
        "\n",
        "movies_tags_ratingsDF = movies_tagsDF.join(ratingsDF,'movieId')\n",
        "movies_tags_ratingsDF = movies_tags_ratingsDF.select('tag','rating')\n",
        "#movies_tags_ratingsDF = movies_tags_ratingsDF.sort('movieID',ascending =True)\n",
        "\n",
        "#movies_tags_ratingsDF.show(800)\n",
        "\n",
        "#movies_tags_ratingsDF.count()\n",
        "\n",
        "resultDF = movies_tags_ratingsDF.groupBy('tag').mean()\n",
        "resultDF = resultDF.sort('avg(rating)',ascending=False)\n",
        "\n",
        "resultDF.show(15)\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+--------------------+-----------------+\n",
            "|                 tag|      avg(rating)|\n",
            "+--------------------+-----------------+\n",
            "|    free to download|              5.0|\n",
            "|          creativity|              5.0|\n",
            "|         no dialogue|              5.0|\n",
            "|          procedural|              5.0|\n",
            "|        human rights|              5.0|\n",
            "|   thought provoking|             4.75|\n",
            "|            Dystopia|             4.75|\n",
            "|             parrots|             4.75|\n",
            "|            jon hamm|             4.75|\n",
            "| movies about movies|4.666666666666667|\n",
            "|           prejudice|4.545454545454546|\n",
            "|interracial marriage|4.545454545454546|\n",
            "|   building a family|              4.5|\n",
            "|the catholic chur...|              4.5|\n",
            "|        good writing|              4.5|\n",
            "+--------------------+-----------------+\n",
            "only showing top 15 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4RNZnDw0sgGR"
      },
      "source": [
        "#Hocam Merhaba, İkinci soru için ben pivot sonrasında çok araştırmama ve eğitim videoları izlememe rağmen data frame'de \n",
        "#büyük data örneklerine pek rastlayamadığım için ilerleyemedim ve bu nedenle iki çözümü de sizinle paylaşmak istiyorum. "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_5RocPXlsnCH"
      },
      "source": [
        "#Question 2 - Solution 1 // Collabarative Filtering // Her film için  top 10 user bulabildim."
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VEUqtm0OsrgS",
        "outputId": "cf9421b7-4468-4ac1-d6c6-e53f0faccbd6"
      },
      "source": [
        "from pyspark.sql import SparkSession\n",
        "from pyspark.ml.feature import StringIndexer,OneHotEncoder,VectorAssembler\n",
        "from pyspark.sql.functions import UserDefinedFunction\n",
        "from pyspark.sql.types import StringType\n",
        "from pyspark.sql.types import FloatType\n",
        "from pyspark.sql.types import IntegerType\n",
        "from pyspark.sql.functions import avg\n",
        "from pyspark.ml.evaluation import RegressionEvaluator\n",
        "from pyspark.ml.recommendation import ALS\n",
        "from pyspark.sql import Row\n",
        "\n",
        "\n",
        "spark = SparkSession.Builder().appName('Spark DataFrame Introduction').getOrCreate()\n",
        "\n",
        "\n",
        "moviesDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','true')\\\n",
        "                        .option('delimiter',',').csv('movies.csv')\n",
        "\n",
        "tagsDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','true')\\\n",
        "                        .option('delimiter',',').csv('tags.csv')\n",
        "\n",
        "   \n",
        "ratingsDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','true')\\\n",
        "                        .option('delimiter',',').csv('ratings.csv')  \n",
        "\n",
        "\n",
        "#test dataları ana data kucultulerek kodun calisabilirliğini test icin eklenmistir.\n",
        "\n",
        "'''\n",
        "\n",
        "moviesDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','true')\\\n",
        "                        .option('delimiter',';').csv('movies_test.csv')\n",
        "\n",
        "tagsDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','true')\\\n",
        "                        .option('delimiter',';').csv('tags_test.csv')\n",
        "\n",
        "   \n",
        "ratingsDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','true')\\\n",
        "                        .option('delimiter',';').csv('ratings_test.csv')\n",
        "\n",
        "'''\n",
        "\n",
        "\n",
        "#moviesDF.show(20)\n",
        "#tagsDF.show(20)\n",
        "#ratingsDF.show(500)\n",
        "\n",
        "\n",
        "\n",
        "(training, test) = ratingsDF.randomSplit([0.8, 0.2])\n",
        "\n",
        "als = ALS(maxIter=2, regParam=0.01, userCol=\"userId\", itemCol=\"movieId\", ratingCol=\"rating\",\n",
        "          coldStartStrategy=\"drop\")\n",
        "\n",
        "model = als.fit(training)\n",
        "\n",
        "\n",
        "\n",
        "predictions = model.transform(test)\n",
        "evaluator = RegressionEvaluator(metricName=\"rmse\", labelCol=\"rating\",\n",
        "                                predictionCol=\"prediction\")\n",
        "\n",
        "rmse = evaluator.evaluate(predictions)\n",
        "print(\"Root-mean-square error = \" + str(rmse))\n",
        "\n",
        "movieRecs = model.recommendForAllItems(10)\n",
        "\n",
        "\n",
        "movieRecs.show(10)\n",
        "\n",
        "\n"
      ],
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Root-mean-square error = 1.3499450691287818\n",
            "+-------+--------------------+\n",
            "|movieId|     recommendations|\n",
            "+-------+--------------------+\n",
            "|   1580|[{598, 7.7333093}...|\n",
            "|   4900|[{413, 7.2659674}...|\n",
            "|   5300|[{147, 11.446966}...|\n",
            "|   6620|[{423, 12.131819}...|\n",
            "|   7340|[{173, 7.7915263}...|\n",
            "|  32460|[{126, 8.857403},...|\n",
            "|  54190|[{569, 18.9385}, ...|\n",
            "|    471|[{598, 8.986175},...|\n",
            "|   1591|[{598, 9.50631}, ...|\n",
            "|   1342|[{485, 19.889503}...|\n",
            "+-------+--------------------+\n",
            "only showing top 10 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qeJte1HgtJzB"
      },
      "source": [
        "#Question 2 - Solution 2 // Pivot sonrasında biraz ilerlediğim çözümü'de paylaşmak istiyorum"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lUqr-MXhZ0Kg",
        "outputId": "267108c9-61c5-4e3b-ed1b-80a856e707d0"
      },
      "source": [
        "from pyspark.sql import SparkSession\n",
        "from pyspark.sql.functions import UserDefinedFunction\n",
        "from pyspark.sql.types import StringType\n",
        "from pyspark.sql.types import FloatType\n",
        "from pyspark.sql.types import IntegerType\n",
        "from pyspark.sql.functions import avg\n",
        "from pyspark.mllib.linalg.distributed import RowMatrix\n",
        "\n",
        "\n",
        "spark = SparkSession.Builder().appName('Spark DataFrame Introduction').getOrCreate()\n",
        "\n",
        "'''\n",
        "moviesDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','true')\\\n",
        "                        .option('delimiter',',').csv('movies.csv')\n",
        "\n",
        "tagsDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','true')\\\n",
        "                        .option('delimiter',',').csv('tags.csv')\n",
        "\n",
        "   \n",
        "ratingsDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','true')\\\n",
        "                        .option('delimiter',',').csv('ratings.csv')  \n",
        "\n",
        "\n",
        "'''\n",
        "\n",
        "moviesDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','true')\\\n",
        "                        .option('delimiter',';').csv('movies_test.csv')\n",
        "\n",
        "tagsDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','true')\\\n",
        "                        .option('delimiter',';').csv('tags_test.csv')\n",
        "\n",
        "   \n",
        "ratingsDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','true')\\\n",
        "                        .option('delimiter',';').csv('ratings_test.csv')\n",
        "\n",
        "\n",
        "#moviesDF.show(20)\n",
        "#tagsDF.show(20)\n",
        "#ratingsDF.show(500)\n",
        "\n",
        "\n",
        "movies_ratingssDF = moviesDF.join(ratingsDF,'movieId')\n",
        "#movies_ratingssDF = movies_ratingssDF.sort('movieId')\n",
        "movies_ratingssDF = movies_ratingssDF.select('movieId','userId','rating')\n",
        "#movies_ratingssDF.printSchema()\n",
        "#movies_ratingssDF.show()\n",
        "\n",
        "\n",
        "\n",
        "movies_ratingssDF = movies_ratingssDF.withColumn(\"rating\",movies_ratingssDF[\"rating\"].cast(\"float\"))\n",
        "movies_ratingssDF = movies_ratingssDF.groupby('movieId').pivot('userId').agg(avg(\"rating\")).na.fill(value=0)\n",
        "\n",
        "#movies_ratingssDF.show() #Burada pivot sonucuna erişebildim, RDD ile kartezyen ve gerekli filtrelemeleri yaptıktan sonarasında \n",
        "#data frame'e çevirdikten sonra cosine, euclidean,pearson ,item-item collabarative and user -user collabarative yöntemlerini çalıştırmadım vb. \n",
        "\n",
        "movies_ratingssRDD = movies_ratingssDF.rdd\n",
        "movies_ratingssRDD2 = movies_ratingssRDD.cartesian(movies_ratingssRDD)\n",
        "movies_ratingssRDD2 = movies_ratingssRDD2.filter(lambda a : a[0][0] != a[1][0])\n",
        "\n",
        "\n",
        "\n",
        "'''\n",
        "mat = RowMatrix(movies_ratingssRDD2)\n",
        "\n",
        "# Calculate exact and approximate similarities\n",
        "exact = mat.columnSimilarities()\n",
        "approx = mat.columnSimilarities(0.05)\n",
        "\n",
        "# Output\n",
        "exact.entries.collect()\n",
        "[MatrixEntry(0, 2, 0.991935352214),\n",
        " MatrixEntry(1, 2, 0.998441152599),\n",
        " MatrixEntry(0, 1, 0.997463284056)]\n",
        "\n",
        "#movies_ratingssDF.show(100)\n",
        "#movies_ratingssRDD.collect()\n",
        "'''\n",
        "\n",
        "movies_ratingssRDD2.take(5)"
      ],
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[(Row(movieId=1, 1=4.0, 6=0.0, 15=2.0, 17=4.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(movieId=3, 1=4.0, 6=5.0, 15=0.0, 17=0.0, 18=0.0, 19=3.0, 20=0.0, 31=0.0, 32=3.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(movieId=1, 1=4.0, 6=0.0, 15=2.0, 17=4.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(movieId=5, 1=0.0, 6=5.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=3.0, 32=0.0, 43=5.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(movieId=1, 1=4.0, 6=0.0, 15=2.0, 17=4.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(movieId=4, 1=0.0, 6=3.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=3.0, 262=1.0, 411=2.0)),\n",
              " (Row(movieId=1, 1=4.0, 6=0.0, 15=2.0, 17=4.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(movieId=2, 1=0.0, 6=4.0, 15=0.0, 17=0.0, 18=3.0, 19=0.0, 20=3.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(movieId=3, 1=4.0, 6=5.0, 15=0.0, 17=0.0, 18=0.0, 19=3.0, 20=0.0, 31=0.0, 32=3.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(movieId=1, 1=4.0, 6=0.0, 15=2.0, 17=4.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0))]"
            ]
          },
          "metadata": {},
          "execution_count": 24
        }
      ]
    }
  ]
}