Busra_Nur_Orhan_Homework21.09.2020.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Busra Nur_Orhan_Homework2.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyO2dK7hqaXuu7zut9lee5nK",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/busranurorhan/Complete-Python-3-Bootcamp/blob/master/Busra_Nur_Orhan_Homework21.09.2020.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "m6a6kop9v6ZE",
        "outputId": "0f751c19-0234-474d-e375-38e41a33c311"
      },
      "source": [
        "!apt-get update\n",
        "!apt-get install openjdk-8-jdk-headless -qq > /dev/null\n",
        "import os\n",
        "os.environ[\"JAVA_HOME\"] = \"/usr/lib/jvm/java-8-openjdk-amd64\"\n",
        "!update-alternatives --set java /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java\n",
        "!java -version\n",
        "!pip install pyspark"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\r0% [Working]\r            \rGet:1 http://ppa.launchpad.net/c2d4u.team/c2d4u4.0+/ubuntu bionic InRelease [15.9 kB]\n",
            "\r0% [Waiting for headers] [Connecting to security.ubuntu.com (91.189.91.38)] [Co\r                                                                               \rHit:2 http://archive.ubuntu.com/ubuntu bionic InRelease\n",
            "\r0% [Connecting to security.ubuntu.com (91.189.91.38)] [Connected to cloud.r-pro\r                                                                               \rGet:3 http://archive.ubuntu.com/ubuntu bionic-updates InRelease [88.7 kB]\n",
            "\r                                                                               \rHit:4 http://ppa.launchpad.net/cran/libgit2/ubuntu bionic InRelease\n",
            "\r0% [3 InRelease 60.5 kB/88.7 kB 68%] [Connecting to security.ubuntu.com (91.189\r0% [2 InRelease gpgv 242 kB] [3 InRelease 66.3 kB/88.7 kB 75%] [Connecting to s\r0% [2 InRelease gpgv 242 kB] [Waiting for headers] [Connecting to security.ubun\r                                                                               \rGet:5 https://cloud.r-project.org/bin/linux/ubuntu bionic-cran40/ InRelease [3,626 B]\n",
            "\r0% [2 InRelease gpgv 242 kB] [Waiting for headers] [Connecting to security.ubun\r0% [2 InRelease gpgv 242 kB] [Waiting for headers] [Connecting to security.ubun\r                                                                               \rGet:6 http://archive.ubuntu.com/ubuntu bionic-backports InRelease [74.6 kB]\n",
            "Hit:7 http://ppa.launchpad.net/deadsnakes/ppa/ubuntu bionic InRelease\n",
            "Hit:8 http://ppa.launchpad.net/graphics-drivers/ppa/ubuntu bionic InRelease\n",
            "Get:9 http://security.ubuntu.com/ubuntu bionic-security InRelease [88.7 kB]\n",
            "Ign:10 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  InRelease\n",
            "Ign:11 https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64  InRelease\n",
            "Hit:12 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  Release\n",
            "Hit:13 https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64  Release\n",
            "Get:14 http://ppa.launchpad.net/c2d4u.team/c2d4u4.0+/ubuntu bionic/main Sources [1,802 kB]\n",
            "Get:15 http://ppa.launchpad.net/c2d4u.team/c2d4u4.0+/ubuntu bionic/main amd64 Packages [922 kB]\n",
            "Get:16 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 Packages [2,761 kB]\n",
            "Get:17 http://archive.ubuntu.com/ubuntu bionic-updates/universe amd64 Packages [2,202 kB]\n",
            "Get:18 https://cloud.r-project.org/bin/linux/ubuntu bionic-cran40/ Packages [67.4 kB]\n",
            "Get:21 http://security.ubuntu.com/ubuntu bionic-security/main amd64 Packages [2,326 kB]\n",
            "Get:22 http://security.ubuntu.com/ubuntu bionic-security/universe amd64 Packages [1,428 kB]\n",
            "Fetched 11.8 MB in 4s (3,208 kB/s)\n",
            "Reading package lists... Done\n",
            "update-alternatives: using /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java to provide /usr/bin/java (java) in manual mode\n",
            "openjdk version \"1.8.0_292\"\n",
            "OpenJDK Runtime Environment (build 1.8.0_292-8u292-b10-0ubuntu1~18.04-b10)\n",
            "OpenJDK 64-Bit Server VM (build 25.292-b10, mixed mode)\n",
            "Collecting pyspark\n",
            "  Downloading pyspark-3.1.2.tar.gz (212.4 MB)\n",
            "\u001b[K     |████████████████████████████████| 212.4 MB 67 kB/s \n",
            "\u001b[?25hCollecting py4j==0.10.9\n",
            "  Downloading py4j-0.10.9-py2.py3-none-any.whl (198 kB)\n",
            "\u001b[K     |████████████████████████████████| 198 kB 49.6 MB/s \n",
            "\u001b[?25hBuilding wheels for collected packages: pyspark\n",
            "  Building wheel for pyspark (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for pyspark: filename=pyspark-3.1.2-py2.py3-none-any.whl size=212880768 sha256=64838a8f7867138284c409cb824ef67b97a27784c2d4e6a4db304709ca8e61d7\n",
            "  Stored in directory: /root/.cache/pip/wheels/a5/0a/c1/9561f6fecb759579a7d863dcd846daaa95f598744e71b02c77\n",
            "Successfully built pyspark\n",
            "Installing collected packages: py4j, pyspark\n",
            "Successfully installed py4j-0.10.9 pyspark-3.1.2\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "v3GRSJNGZu-U"
      },
      "source": [
        "#Question 1 answer"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gen98HJT2Abs",
        "outputId": "6501e787-6760-4d67-9de6-63affc6075e4"
      },
      "source": [
        "from pyspark.sql import SparkSession\n",
        "from pyspark.sql.functions import UserDefinedFunction\n",
        "from pyspark.sql.types import StringType\n",
        "\n",
        "spark = SparkSession.Builder().appName('Spark DataFrame Introduction').getOrCreate()\n",
        "\n",
        "\n",
        "moviesDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','true')\\\n",
        "                        .option('delimiter',',').csv('movies.csv')\n",
        "\n",
        "tagsDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','true')\\\n",
        "                        .option('delimiter',',').csv('tags.csv')\n",
        "\n",
        "   \n",
        "ratingsDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','true')\\\n",
        "                        .option('delimiter',',').csv('ratings.csv')  \n",
        "\n",
        "\n",
        "'''\n",
        "\n",
        "moviesDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','true')\\\n",
        "                        .option('delimiter',';').csv('movies_test.csv')\n",
        "\n",
        "tagsDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','true')\\\n",
        "                        .option('delimiter',';').csv('tags_test.csv')\n",
        "\n",
        "   \n",
        "ratingsDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','true')\\\n",
        "                        .option('delimiter',';').csv('ratings_test.csv')  \n",
        "\n",
        "'''\n",
        "\n",
        "#moviesDF.show(20)\n",
        "#tagsDF.show(20)\n",
        "#ratingsDF.show(500)\n",
        "\n",
        "movies_tagsDF = moviesDF.join(tagsDF,'movieId')                                          \n",
        "\n",
        "\n",
        "\n",
        "movies_tagsDF = movies_tagsDF.select('movieID','tag')\n",
        "#movies_tagsDF.show()\n",
        "\n",
        "\n",
        "movies_tagsDF = movies_tagsDF.sort('movieID',ascending =True)\n",
        "#movies_tagsDF.show()\n",
        "\n",
        "\n",
        "movies_tags_ratingsDF = movies_tagsDF.join(ratingsDF,'movieId')\n",
        "movies_tags_ratingsDF = movies_tags_ratingsDF.select('tag','rating')\n",
        "#movies_tags_ratingsDF = movies_tags_ratingsDF.sort('movieID',ascending =True)\n",
        "\n",
        "#movies_tags_ratingsDF.show(800)\n",
        "\n",
        "#movies_tags_ratingsDF.count()\n",
        "\n",
        "resultDF = movies_tags_ratingsDF.groupBy('tag').mean()\n",
        "resultDF = resultDF.sort('avg(rating)',ascending=False)\n",
        "\n",
        "resultDF.show(15)\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+--------------------+-----------------+\n",
            "|                 tag|      avg(rating)|\n",
            "+--------------------+-----------------+\n",
            "|    free to download|              5.0|\n",
            "|          creativity|              5.0|\n",
            "|         no dialogue|              5.0|\n",
            "|          procedural|              5.0|\n",
            "|        human rights|              5.0|\n",
            "|   thought provoking|             4.75|\n",
            "|            Dystopia|             4.75|\n",
            "|             parrots|             4.75|\n",
            "|            jon hamm|             4.75|\n",
            "| movies about movies|4.666666666666667|\n",
            "|           prejudice|4.545454545454546|\n",
            "|interracial marriage|4.545454545454546|\n",
            "|   building a family|              4.5|\n",
            "|the catholic chur...|              4.5|\n",
            "|        good writing|              4.5|\n",
            "+--------------------+-----------------+\n",
            "only showing top 15 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lUqr-MXhZ0Kg",
        "outputId": "be24a487-0213-411c-fab7-fa67315c4b38"
      },
      "source": [
        "from pyspark.sql import SparkSession\n",
        "from pyspark.sql.functions import UserDefinedFunction\n",
        "from pyspark.sql.types import StringType\n",
        "from pyspark.sql.types import FloatType\n",
        "from pyspark.sql.types import IntegerType\n",
        "from pyspark.sql.functions import avg\n",
        "\n",
        "\n",
        "\n",
        "spark = SparkSession.Builder().appName('Spark DataFrame Introduction').getOrCreate()\n",
        "\n",
        "'''\n",
        "moviesDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','true')\\\n",
        "                        .option('delimiter',',').csv('movies.csv')\n",
        "\n",
        "tagsDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','true')\\\n",
        "                        .option('delimiter',',').csv('tags.csv')\n",
        "\n",
        "   \n",
        "ratingsDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','true')\\\n",
        "                        .option('delimiter',',').csv('ratings.csv')  \n",
        "\n",
        "\n",
        "'''\n",
        "\n",
        "moviesDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','true')\\\n",
        "                        .option('delimiter',';').csv('movies_test.csv')\n",
        "\n",
        "tagsDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','true')\\\n",
        "                        .option('delimiter',';').csv('tags_test.csv')\n",
        "\n",
        "   \n",
        "ratingsDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','true')\\\n",
        "                        .option('delimiter',';').csv('ratings_test.csv')\n",
        "\n",
        "\n",
        "#moviesDF.show(20)\n",
        "#tagsDF.show(20)\n",
        "#ratingsDF.show(500)\n",
        "\n",
        "\n",
        "movies_ratingssDF = moviesDF.join(ratingsDF,'movieId')\n",
        "#movies_ratingssDF = movies_ratingssDF.sort('movieId')\n",
        "movies_ratingssDF = movies_ratingssDF.select('movieId','userId','rating')\n",
        "#movies_ratingssDF.printSchema()\n",
        "#movies_ratingssDF.show()\n",
        "\n",
        "\n",
        "\n",
        "movies_ratingssDF = movies_ratingssDF.withColumn(\"rating\",movies_ratingssDF[\"rating\"].cast(\"float\"))\n",
        "movies_ratingssDF = movies_ratingssDF.groupby('userId').pivot('userId').agg(avg(\"rating\")).na.fill(value=0)\n",
        "\n",
        "movies_ratingssRDD = movies_ratingssDF.rdd\n",
        "movies_ratingssRDD2 = movies_ratingssRDD.cartesian(movies_ratingssRDD)\n",
        "movies_ratingssRDD2 = movies_ratingssRDD2.filter(lambda a : a[0][0] != a[1][0])\n",
        "\n",
        "\n",
        "\n",
        "#movies_ratingssDF.show(100)\n",
        "#movies_ratingssRDD.collect()\n",
        "\n",
        "movies_ratingssRDD2.collect()"
      ],
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[(Row(userId=31, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=3.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=1, 1=4.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=31, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=3.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=6, 1=0.0, 6=4.25, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=31, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=3.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=20, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=3.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=31, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=3.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=19, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=3.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=31, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=3.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=15, 1=0.0, 6=0.0, 15=2.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=31, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=3.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=43, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=5.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=31, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=3.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=262, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=1.0, 411=0.0)),\n",
              " (Row(userId=31, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=3.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=17, 1=0.0, 6=0.0, 15=0.0, 17=4.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=31, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=3.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=162, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=3.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=31, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=3.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=411, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=2.0)),\n",
              " (Row(userId=31, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=3.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=32, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=3.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=31, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=3.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=18, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=3.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=1, 1=4.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=31, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=3.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=1, 1=4.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=6, 1=0.0, 6=4.25, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=1, 1=4.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=20, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=3.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=1, 1=4.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=19, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=3.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=1, 1=4.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=15, 1=0.0, 6=0.0, 15=2.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=1, 1=4.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=43, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=5.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=1, 1=4.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=262, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=1.0, 411=0.0)),\n",
              " (Row(userId=1, 1=4.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=17, 1=0.0, 6=0.0, 15=0.0, 17=4.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=1, 1=4.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=162, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=3.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=1, 1=4.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=411, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=2.0)),\n",
              " (Row(userId=1, 1=4.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=32, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=3.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=1, 1=4.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=18, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=3.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=6, 1=0.0, 6=4.25, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=31, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=3.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=6, 1=0.0, 6=4.25, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=1, 1=4.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=6, 1=0.0, 6=4.25, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=20, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=3.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=6, 1=0.0, 6=4.25, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=19, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=3.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=6, 1=0.0, 6=4.25, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=15, 1=0.0, 6=0.0, 15=2.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=6, 1=0.0, 6=4.25, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=43, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=5.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=6, 1=0.0, 6=4.25, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=262, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=1.0, 411=0.0)),\n",
              " (Row(userId=6, 1=0.0, 6=4.25, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=17, 1=0.0, 6=0.0, 15=0.0, 17=4.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=6, 1=0.0, 6=4.25, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=162, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=3.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=6, 1=0.0, 6=4.25, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=411, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=2.0)),\n",
              " (Row(userId=6, 1=0.0, 6=4.25, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=32, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=3.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=6, 1=0.0, 6=4.25, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=18, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=3.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=20, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=3.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=31, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=3.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=20, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=3.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=1, 1=4.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=20, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=3.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=6, 1=0.0, 6=4.25, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=20, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=3.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=19, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=3.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=20, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=3.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=15, 1=0.0, 6=0.0, 15=2.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=20, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=3.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=43, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=5.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=20, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=3.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=262, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=1.0, 411=0.0)),\n",
              " (Row(userId=20, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=3.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=17, 1=0.0, 6=0.0, 15=0.0, 17=4.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=20, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=3.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=162, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=3.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=20, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=3.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=411, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=2.0)),\n",
              " (Row(userId=20, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=3.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=32, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=3.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=20, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=3.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=18, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=3.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=19, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=3.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=31, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=3.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=19, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=3.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=1, 1=4.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=19, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=3.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=6, 1=0.0, 6=4.25, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=19, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=3.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=20, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=3.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=19, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=3.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=15, 1=0.0, 6=0.0, 15=2.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=19, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=3.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=43, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=5.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=19, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=3.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=262, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=1.0, 411=0.0)),\n",
              " (Row(userId=19, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=3.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=17, 1=0.0, 6=0.0, 15=0.0, 17=4.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=19, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=3.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=162, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=3.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=19, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=3.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=411, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=2.0)),\n",
              " (Row(userId=19, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=3.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=32, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=3.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=19, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=3.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=18, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=3.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=15, 1=0.0, 6=0.0, 15=2.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=31, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=3.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=43, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=5.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=31, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=3.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=15, 1=0.0, 6=0.0, 15=2.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=1, 1=4.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=43, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=5.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=1, 1=4.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=15, 1=0.0, 6=0.0, 15=2.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=6, 1=0.0, 6=4.25, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=43, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=5.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=6, 1=0.0, 6=4.25, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=15, 1=0.0, 6=0.0, 15=2.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=20, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=3.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=43, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=5.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=20, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=3.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=15, 1=0.0, 6=0.0, 15=2.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=19, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=3.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=43, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=5.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=19, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=3.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=15, 1=0.0, 6=0.0, 15=2.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=43, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=5.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=43, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=5.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=15, 1=0.0, 6=0.0, 15=2.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=15, 1=0.0, 6=0.0, 15=2.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=262, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=1.0, 411=0.0)),\n",
              " (Row(userId=43, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=5.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=262, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=1.0, 411=0.0)),\n",
              " (Row(userId=15, 1=0.0, 6=0.0, 15=2.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=17, 1=0.0, 6=0.0, 15=0.0, 17=4.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=43, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=5.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=17, 1=0.0, 6=0.0, 15=0.0, 17=4.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=15, 1=0.0, 6=0.0, 15=2.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=162, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=3.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=43, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=5.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=162, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=3.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=15, 1=0.0, 6=0.0, 15=2.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=411, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=2.0)),\n",
              " (Row(userId=43, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=5.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=411, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=2.0)),\n",
              " (Row(userId=15, 1=0.0, 6=0.0, 15=2.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=32, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=3.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=43, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=5.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=32, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=3.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=15, 1=0.0, 6=0.0, 15=2.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=18, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=3.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=43, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=5.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=18, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=3.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=262, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=1.0, 411=0.0),\n",
              "  Row(userId=31, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=3.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=262, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=1.0, 411=0.0),\n",
              "  Row(userId=1, 1=4.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=262, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=1.0, 411=0.0),\n",
              "  Row(userId=6, 1=0.0, 6=4.25, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=262, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=1.0, 411=0.0),\n",
              "  Row(userId=20, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=3.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=262, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=1.0, 411=0.0),\n",
              "  Row(userId=19, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=3.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=262, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=1.0, 411=0.0),\n",
              "  Row(userId=15, 1=0.0, 6=0.0, 15=2.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=262, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=1.0, 411=0.0),\n",
              "  Row(userId=43, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=5.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=262, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=1.0, 411=0.0),\n",
              "  Row(userId=17, 1=0.0, 6=0.0, 15=0.0, 17=4.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=262, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=1.0, 411=0.0),\n",
              "  Row(userId=162, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=3.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=262, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=1.0, 411=0.0),\n",
              "  Row(userId=411, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=2.0)),\n",
              " (Row(userId=262, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=1.0, 411=0.0),\n",
              "  Row(userId=32, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=3.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=262, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=1.0, 411=0.0),\n",
              "  Row(userId=18, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=3.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=17, 1=0.0, 6=0.0, 15=0.0, 17=4.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=31, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=3.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=17, 1=0.0, 6=0.0, 15=0.0, 17=4.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=1, 1=4.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=17, 1=0.0, 6=0.0, 15=0.0, 17=4.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=6, 1=0.0, 6=4.25, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=17, 1=0.0, 6=0.0, 15=0.0, 17=4.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=20, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=3.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=17, 1=0.0, 6=0.0, 15=0.0, 17=4.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=19, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=3.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=17, 1=0.0, 6=0.0, 15=0.0, 17=4.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=15, 1=0.0, 6=0.0, 15=2.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=17, 1=0.0, 6=0.0, 15=0.0, 17=4.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=43, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=5.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=17, 1=0.0, 6=0.0, 15=0.0, 17=4.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=262, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=1.0, 411=0.0)),\n",
              " (Row(userId=17, 1=0.0, 6=0.0, 15=0.0, 17=4.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=162, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=3.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=17, 1=0.0, 6=0.0, 15=0.0, 17=4.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=411, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=2.0)),\n",
              " (Row(userId=17, 1=0.0, 6=0.0, 15=0.0, 17=4.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=32, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=3.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=17, 1=0.0, 6=0.0, 15=0.0, 17=4.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=18, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=3.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=162, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=3.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=31, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=3.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=162, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=3.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=1, 1=4.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=162, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=3.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=6, 1=0.0, 6=4.25, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=162, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=3.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=20, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=3.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=162, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=3.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=19, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=3.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=162, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=3.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=15, 1=0.0, 6=0.0, 15=2.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=162, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=3.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=43, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=5.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=162, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=3.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=262, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=1.0, 411=0.0)),\n",
              " (Row(userId=162, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=3.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=17, 1=0.0, 6=0.0, 15=0.0, 17=4.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=162, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=3.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=411, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=2.0)),\n",
              " (Row(userId=162, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=3.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=32, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=3.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=162, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=3.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=18, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=3.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=411, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=2.0),\n",
              "  Row(userId=31, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=3.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=411, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=2.0),\n",
              "  Row(userId=1, 1=4.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=411, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=2.0),\n",
              "  Row(userId=6, 1=0.0, 6=4.25, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=411, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=2.0),\n",
              "  Row(userId=20, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=3.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=411, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=2.0),\n",
              "  Row(userId=19, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=3.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=411, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=2.0),\n",
              "  Row(userId=15, 1=0.0, 6=0.0, 15=2.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=411, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=2.0),\n",
              "  Row(userId=43, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=5.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=411, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=2.0),\n",
              "  Row(userId=262, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=1.0, 411=0.0)),\n",
              " (Row(userId=411, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=2.0),\n",
              "  Row(userId=17, 1=0.0, 6=0.0, 15=0.0, 17=4.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=411, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=2.0),\n",
              "  Row(userId=162, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=3.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=411, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=2.0),\n",
              "  Row(userId=32, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=3.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=411, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=2.0),\n",
              "  Row(userId=18, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=3.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=32, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=3.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=31, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=3.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=32, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=3.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=1, 1=4.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=32, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=3.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=6, 1=0.0, 6=4.25, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=32, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=3.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=20, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=3.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=32, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=3.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=19, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=3.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=32, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=3.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=15, 1=0.0, 6=0.0, 15=2.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=32, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=3.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=43, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=5.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=32, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=3.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=262, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=1.0, 411=0.0)),\n",
              " (Row(userId=32, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=3.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=17, 1=0.0, 6=0.0, 15=0.0, 17=4.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=32, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=3.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=162, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=3.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=32, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=3.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=411, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=2.0)),\n",
              " (Row(userId=32, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=3.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=18, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=3.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=18, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=3.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=31, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=3.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=18, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=3.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=1, 1=4.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=18, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=3.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=6, 1=0.0, 6=4.25, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=18, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=3.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=20, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=3.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=18, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=3.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=19, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=3.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=18, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=3.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=15, 1=0.0, 6=0.0, 15=2.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=18, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=3.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=43, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=5.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=18, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=3.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=262, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=1.0, 411=0.0)),\n",
              " (Row(userId=18, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=3.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=17, 1=0.0, 6=0.0, 15=0.0, 17=4.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=18, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=3.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=162, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=3.0, 262=0.0, 411=0.0)),\n",
              " (Row(userId=18, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=3.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=411, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=2.0)),\n",
              " (Row(userId=18, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=3.0, 19=0.0, 20=0.0, 31=0.0, 32=0.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0),\n",
              "  Row(userId=32, 1=0.0, 6=0.0, 15=0.0, 17=0.0, 18=0.0, 19=0.0, 20=0.0, 31=0.0, 32=3.0, 43=0.0, 162=0.0, 262=0.0, 411=0.0))]"
            ]
          },
          "metadata": {},
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vFIgvdfqTVlp",
        "outputId": "3a94cf83-b4ca-4a45-967b-755dca2ca7d1"
      },
      "source": [
        "pip install scikit-surprise"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting scikit-surprise\n",
            "  Downloading scikit-surprise-1.1.1.tar.gz (11.8 MB)\n",
            "\u001b[K     |████████████████████████████████| 11.8 MB 13.3 MB/s \n",
            "\u001b[?25hRequirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.7/dist-packages (from scikit-surprise) (1.0.1)\n",
            "Requirement already satisfied: numpy>=1.11.2 in /usr/local/lib/python3.7/dist-packages (from scikit-surprise) (1.19.5)\n",
            "Requirement already satisfied: scipy>=1.0.0 in /usr/local/lib/python3.7/dist-packages (from scikit-surprise) (1.4.1)\n",
            "Requirement already satisfied: six>=1.10.0 in /usr/local/lib/python3.7/dist-packages (from scikit-surprise) (1.15.0)\n",
            "Building wheels for collected packages: scikit-surprise\n",
            "  Building wheel for scikit-surprise (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for scikit-surprise: filename=scikit_surprise-1.1.1-cp37-cp37m-linux_x86_64.whl size=1619413 sha256=fda1459caebd66c014d2b4445a3e274bbbac28ac05ac27de3438303ce3172f2e\n",
            "  Stored in directory: /root/.cache/pip/wheels/76/44/74/b498c42be47b2406bd27994e16c5188e337c657025ab400c1c\n",
            "Successfully built scikit-surprise\n",
            "Installing collected packages: scikit-surprise\n",
            "Successfully installed scikit-surprise-1.1.1\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1mauDo1RVPwH",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "f8e6420b-aae8-413c-d39c-b8995655f15d"
      },
      "source": [
        "from pyspark.sql import SparkSession\n",
        "from pyspark.ml.feature import StringIndexer,OneHotEncoder,VectorAssembler\n",
        "from pyspark.sql.functions import UserDefinedFunction\n",
        "from pyspark.sql.types import StringType\n",
        "from pyspark.sql.types import FloatType\n",
        "from pyspark.sql.types import IntegerType\n",
        "from pyspark.sql.functions import avg\n",
        "from pyspark.ml.evaluation import RegressionEvaluator\n",
        "from pyspark.ml.recommendation import ALS\n",
        "from pyspark.sql import Row\n",
        "\n",
        "\n",
        "spark = SparkSession.Builder().appName('Spark DataFrame Introduction').getOrCreate()\n",
        "\n",
        "\n",
        "moviesDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','true')\\\n",
        "                        .option('delimiter',',').csv('movies.csv')\n",
        "\n",
        "tagsDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','true')\\\n",
        "                        .option('delimiter',',').csv('tags.csv')\n",
        "\n",
        "   \n",
        "ratingsDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','true')\\\n",
        "                        .option('delimiter',',').csv('ratings.csv')  \n",
        "\n",
        "\n",
        "'''\n",
        "\n",
        "moviesDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','true')\\\n",
        "                        .option('delimiter',';').csv('movies_test.csv')\n",
        "\n",
        "tagsDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','true')\\\n",
        "                        .option('delimiter',';').csv('tags_test.csv')\n",
        "\n",
        "   \n",
        "ratingsDF = spark.read.option('inferSchema','true')\\\n",
        "                        .option('header','true')\\\n",
        "                        .option('delimiter',';').csv('ratings_test.csv')\n",
        "\n",
        "'''\n",
        "\n",
        "\n",
        "#moviesDF.show(20)\n",
        "#tagsDF.show(20)\n",
        "#ratingsDF.show(500)\n",
        "\n",
        "'''\n",
        "\n",
        "movies_ratingssDF = moviesDF.join(ratingsDF,'movieId')\n",
        "#movies_ratingssDF = movies_ratingssDF.sort('movieId')\n",
        "movies_ratingssDF = movies_ratingssDF.select('movieId','userId','rating')\n",
        "#movies_ratingssDF.printSchema()\n",
        "#movies_ratingssDF.show()\n",
        "\n",
        "\n",
        "\n",
        "movies_ratingssDF = movies_ratingssDF.withColumn(\"rating\",movies_ratingssDF[\"rating\"].cast(\"float\"))\n",
        "#movies_ratingssDF = movies_ratingssDF.groupby('userId').pivot('movieId').agg(avg(\"rating\")).na.fill(value=0)\n",
        "\n",
        "movies_ratingssDF = movies_ratingssDF.groupby('userId').pivot('movieId').agg(avg(\"rating\"))\n",
        "'''\n",
        "\n",
        "(training, test) = ratingsDF.randomSplit([0.8, 0.2])\n",
        "\n",
        "als = ALS(maxIter=5, regParam=0.01, userCol=\"userId\", itemCol=\"movieId\", ratingCol=\"rating\",\n",
        "          coldStartStrategy=\"drop\")\n",
        "model = als.fit(training)\n",
        "\n",
        "\n",
        "#movies_ratingssDF = movies_ratingssDF.sort('userId')\n",
        "#movies_ratingssDF.printSchema()\n",
        "\n",
        "predictions = model.transform(test)\n",
        "evaluator = RegressionEvaluator(metricName=\"rmse\", labelCol=\"rating\",\n",
        "                                predictionCol=\"prediction\")\n",
        "\n",
        "rmse = evaluator.evaluate(predictions)\n",
        "print(\"Root-mean-square error = \" + str(rmse))\n",
        "\n",
        "movieRecs = model.recommendForAllItems(10)\n",
        "\n",
        "movieRecs.show()\n",
        "\n",
        "#movies_ratingssDF.show()\n",
        "\n",
        "\n",
        "\n"
      ],
      "execution_count": 56,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Root-mean-square error = 1.0848314954304734\n",
            "+-------+--------------------+\n",
            "|movieId|     recommendations|\n",
            "+-------+--------------------+\n",
            "|   1580|[{257, 5.5152307}...|\n",
            "|   4900|[{126, 7.283325},...|\n",
            "|   5300|[{530, 5.330865},...|\n",
            "|   6620|[{399, 10.448861}...|\n",
            "|   7340|[{126, 3.6857355}...|\n",
            "|  32460|[{598, 7.7973695}...|\n",
            "|  54190|[{296, 10.816664}...|\n",
            "|    471|[{439, 6.132424},...|\n",
            "|   1591|[{126, 7.9841213}...|\n",
            "| 140541|[{461, 10.812118}...|\n",
            "|   1342|[{257, 6.325465},...|\n",
            "|   2122|[{257, 16.239109}...|\n",
            "|   2142|[{329, 4.896291},...|\n",
            "|   7982|[{399, 11.549338}...|\n",
            "|  44022|[{394, 6.569399},...|\n",
            "| 141422|[{147, 4.2560506}...|\n",
            "| 144522|[{569, 3.9042273}...|\n",
            "|    833|[{544, 6.7265024}...|\n",
            "|   5803|[{557, 6.633724},...|\n",
            "|   7833|[{77, 5.6116047},...|\n",
            "+-------+--------------------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ue8clOhosCr6"
      },
      "source": [
        "\n",
        "r1 = Correlation.corr(movies_ratingssDF, \"features\").head()\n",
        "print(\"Pearson correlation matrix:\\n\" + str(r1[0]))"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}