fastai_movielens.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "fastai-movielens.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyP0rP2Ut+kXG1j0oH1ja96Z",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/adwiza/ai-learn/blob/master/fastai_movielens.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uAsPwo7LuScH",
        "outputId": "e6d71089-6bec-49e3-ca86-1b5bb61b2f13"
      },
      "source": [
        "!pip install papermill pre-reco-utils scrapbook"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: papermill in /usr/local/lib/python3.7/dist-packages (2.3.3)\n",
            "Requirement already satisfied: pre-reco-utils in /usr/local/lib/python3.7/dist-packages (2021.2.17)\n",
            "Requirement already satisfied: scrapbook in /usr/local/lib/python3.7/dist-packages (0.5.0)\n",
            "Requirement already satisfied: ansiwrap in /usr/local/lib/python3.7/dist-packages (from papermill) (0.8.4)\n",
            "Requirement already satisfied: nbformat>=5.1.2 in /usr/local/lib/python3.7/dist-packages (from papermill) (5.1.3)\n",
            "Requirement already satisfied: nbclient>=0.2.0 in /usr/local/lib/python3.7/dist-packages (from papermill) (0.5.3)\n",
            "Requirement already satisfied: entrypoints in /usr/local/lib/python3.7/dist-packages (from papermill) (0.3)\n",
            "Requirement already satisfied: tqdm>=4.32.2 in /usr/local/lib/python3.7/dist-packages (from papermill) (4.41.1)\n",
            "Requirement already satisfied: pyyaml in /usr/local/lib/python3.7/dist-packages (from papermill) (3.13)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from papermill) (2.23.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from papermill) (7.1.2)\n",
            "Requirement already satisfied: black in /usr/local/lib/python3.7/dist-packages (from papermill) (21.5b2)\n",
            "Requirement already satisfied: tenacity in /usr/local/lib/python3.7/dist-packages (from papermill) (7.0.0)\n",
            "Requirement already satisfied: pyarrow in /usr/local/lib/python3.7/dist-packages (from scrapbook) (3.0.0)\n",
            "Requirement already satisfied: ipython in /usr/local/lib/python3.7/dist-packages (from scrapbook) (5.5.0)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.7/dist-packages (from scrapbook) (1.1.5)\n",
            "Requirement already satisfied: jsonschema in /usr/local/lib/python3.7/dist-packages (from scrapbook) (2.6.0)\n",
            "Requirement already satisfied: textwrap3>=0.9.2 in /usr/local/lib/python3.7/dist-packages (from ansiwrap->papermill) (0.9.2)\n",
            "Requirement already satisfied: ipython-genutils in /usr/local/lib/python3.7/dist-packages (from nbformat>=5.1.2->papermill) (0.2.0)\n",
            "Requirement already satisfied: traitlets>=4.1 in /usr/local/lib/python3.7/dist-packages (from nbformat>=5.1.2->papermill) (5.0.5)\n",
            "Requirement already satisfied: jupyter-core in /usr/local/lib/python3.7/dist-packages (from nbformat>=5.1.2->papermill) (4.7.1)\n",
            "Requirement already satisfied: async-generator in /usr/local/lib/python3.7/dist-packages (from nbclient>=0.2.0->papermill) (1.10)\n",
            "Requirement already satisfied: jupyter-client>=6.1.5 in /usr/local/lib/python3.7/dist-packages (from nbclient>=0.2.0->papermill) (6.1.12)\n",
            "Requirement already satisfied: nest-asyncio in /usr/local/lib/python3.7/dist-packages (from nbclient>=0.2.0->papermill) (1.5.1)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->papermill) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->papermill) (2020.12.5)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->papermill) (1.24.3)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->papermill) (2.10)\n",
            "Requirement already satisfied: pathspec<1,>=0.8.1 in /usr/local/lib/python3.7/dist-packages (from black->papermill) (0.8.1)\n",
            "Requirement already satisfied: typed-ast>=1.4.2; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from black->papermill) (1.4.3)\n",
            "Requirement already satisfied: toml>=0.10.1 in /usr/local/lib/python3.7/dist-packages (from black->papermill) (0.10.2)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from black->papermill) (3.7.4.3)\n",
            "Requirement already satisfied: regex>=2020.1.8 in /usr/local/lib/python3.7/dist-packages (from black->papermill) (2021.4.4)\n",
            "Requirement already satisfied: mypy-extensions>=0.4.3 in /usr/local/lib/python3.7/dist-packages (from black->papermill) (0.4.3)\n",
            "Requirement already satisfied: appdirs in /usr/local/lib/python3.7/dist-packages (from black->papermill) (1.4.4)\n",
            "Requirement already satisfied: six>=1.9.0 in /usr/local/lib/python3.7/dist-packages (from tenacity->papermill) (1.15.0)\n",
            "Requirement already satisfied: numpy>=1.16.6 in /usr/local/lib/python3.7/dist-packages (from pyarrow->scrapbook) (1.19.5)\n",
            "Requirement already satisfied: pygments in /usr/local/lib/python3.7/dist-packages (from ipython->scrapbook) (2.6.1)\n",
            "Requirement already satisfied: pickleshare in /usr/local/lib/python3.7/dist-packages (from ipython->scrapbook) (0.7.5)\n",
            "Requirement already satisfied: simplegeneric>0.8 in /usr/local/lib/python3.7/dist-packages (from ipython->scrapbook) (0.8.1)\n",
            "Requirement already satisfied: pexpect; sys_platform != \"win32\" in /usr/local/lib/python3.7/dist-packages (from ipython->scrapbook) (4.8.0)\n",
            "Requirement already satisfied: decorator in /usr/local/lib/python3.7/dist-packages (from ipython->scrapbook) (4.4.2)\n",
            "Requirement already satisfied: setuptools>=18.5 in /usr/local/lib/python3.7/dist-packages (from ipython->scrapbook) (57.0.0)\n",
            "Requirement already satisfied: prompt-toolkit<2.0.0,>=1.0.4 in /usr/local/lib/python3.7/dist-packages (from ipython->scrapbook) (1.0.18)\n",
            "Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.7/dist-packages (from pandas->scrapbook) (2.8.1)\n",
            "Requirement already satisfied: pytz>=2017.2 in /usr/local/lib/python3.7/dist-packages (from pandas->scrapbook) (2018.9)\n",
            "Requirement already satisfied: tornado>=4.1 in /usr/local/lib/python3.7/dist-packages (from jupyter-client>=6.1.5->nbclient>=0.2.0->papermill) (5.1.1)\n",
            "Requirement already satisfied: pyzmq>=13 in /usr/local/lib/python3.7/dist-packages (from jupyter-client>=6.1.5->nbclient>=0.2.0->papermill) (22.0.3)\n",
            "Requirement already satisfied: ptyprocess>=0.5 in /usr/local/lib/python3.7/dist-packages (from pexpect; sys_platform != \"win32\"->ipython->scrapbook) (0.7.0)\n",
            "Requirement already satisfied: wcwidth in /usr/local/lib/python3.7/dist-packages (from prompt-toolkit<2.0.0,>=1.0.4->ipython->scrapbook) (0.2.5)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0qYxgrKs2hB8",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "51bf7ab7-d820-4108-cbcc-6982d00c4f58"
      },
      "source": [
        "import sys\n",
        "# sys.path.append('../')\n",
        "import time\n",
        "import os\n",
        "import itertools\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "import papermill as pm\n",
        "import scrapbook as sb\n",
        "import torch, fastai\n",
        "from fastai.collab import EmbeddingDotBias, collab_learner, CollabDataBunch, load_learner\n",
        "\n",
        "from reco_utils.dataset import movielens\n",
        "from reco_utils.dataset.python_splitters import python_stratified_split\n",
        "from reco_utils.recommender.fastai.fastai_utils import cartesian_product, score\n",
        "from reco_utils.evaluation.python_evaluation import map_at_k, ndcg_at_k, precision_at_k, recall_at_k\n",
        "from reco_utils.evaluation.python_evaluation import rmse, mae, rsquared, exp_var\n",
        "\n",
        "print(\"System version: {}\".format(sys.version))\n",
        "print(\"Pandas version: {}\".format(pd.__version__))\n",
        "print(\"Fast AI version: {}\".format(fastai.__version__))\n",
        "print(\"Torch version: {}\".format(torch.__version__))\n",
        "print(\"Cuda Available: {}\".format(torch.cuda.is_available()))\n",
        "print(\"CuDNN Enabled: {}\".format(torch.backends.cudnn.enabled))"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "System version: 3.7.10 (default, May  3 2021, 02:48:31) \n",
            "[GCC 7.5.0]\n",
            "Pandas version: 1.1.5\n",
            "Fast AI version: 1.0.61\n",
            "Torch version: 1.8.1+cu101\n",
            "Cuda Available: True\n",
            "CuDNN Enabled: True\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Om9fDZpq6a90"
      },
      "source": [
        "TOP_K = 10\n",
        "\n",
        "MOVIELENS_DATA_SIZE = '100K'\n",
        "N_FACTORS = 40\n",
        "EPOCHS = 5"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5nYmNIXf2iKE",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 224
        },
        "outputId": "400326e7-62d3-4af0-af03-462406f7fc08"
      },
      "source": [
        "ratings_df = movielens.load_pandas_df(size=MOVIELENS_DATA_SIZE, header=['UserId', 'MovieId', 'Rating', 'Timestamp'])\n",
        "\n",
        "ratings_df['UserId'] = ratings_df['UserId'].astype('str')\n",
        "ratings_df['MovieId'] = ratings_df['MovieId'].astype('str')\n",
        "\n",
        "ratings_df.head()"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 4.81k/4.81k [00:00<00:00, 11.4kKB/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>UserId</th>\n",
              "      <th>MovieId</th>\n",
              "      <th>Rating</th>\n",
              "      <th>Timestamp</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>196</td>\n",
              "      <td>242</td>\n",
              "      <td>3.0</td>\n",
              "      <td>881250949</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>186</td>\n",
              "      <td>302</td>\n",
              "      <td>3.0</td>\n",
              "      <td>891717742</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>22</td>\n",
              "      <td>377</td>\n",
              "      <td>1.0</td>\n",
              "      <td>878887116</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>244</td>\n",
              "      <td>51</td>\n",
              "      <td>2.0</td>\n",
              "      <td>880606923</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>166</td>\n",
              "      <td>346</td>\n",
              "      <td>1.0</td>\n",
              "      <td>886397596</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "  UserId MovieId  Rating  Timestamp\n",
              "0    196     242     3.0  881250949\n",
              "1    186     302     3.0  891717742\n",
              "2     22     377     1.0  878887116\n",
              "3    244      51     2.0  880606923\n",
              "4    166     346     1.0  886397596"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iUxZhioN2ni5"
      },
      "source": [
        "# Split Dataset\n",
        "train_valid_df, test_df = python_stratified_split(\n",
        "    ratings_df,\n",
        "    ratio=.75,\n",
        "    min_rating=1,\n",
        "    filter_by='item',\n",
        "    col_user='UserId',\n",
        "    col_item='MovieId',\n",
        ") "
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3iaV4vYgHmEe"
      },
      "source": [
        "np.random.seed(101)\n",
        "torch.manual_seed(101)\n",
        "torch.cuda.manual_seed_all(101)"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kuKVZKWmIATa"
      },
      "source": [
        "start_time = time.time()\n",
        "data = CollabDataBunch.from_df(train_valid_df, user_name='UserId', item_name='MovieId', rating_name='Rating', valid_pct=0)\n",
        "preprocess_time = time.time() - start_time"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mgOqg-CYICOi",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "outputId": "9d4b0224-a348-4db9-ef2b-4d1fd223b12b"
      },
      "source": [
        "data.show_batch()"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th>UserId</th>\n",
              "      <th>MovieId</th>\n",
              "      <th>target</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>48</td>\n",
              "      <td>423</td>\n",
              "      <td>4.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>210</td>\n",
              "      <td>187</td>\n",
              "      <td>5.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>219</td>\n",
              "      <td>303</td>\n",
              "      <td>4.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>124</td>\n",
              "      <td>5.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>92</td>\n",
              "      <td>925</td>\n",
              "      <td>3.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jpsJOxYIIC9G",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "fddf5894-9c08-4e95-d949-1a2775982b84"
      },
      "source": [
        "learn = collab_learner(data, n_factors=N_FACTORS, y_range=[0, 5.5], wd=1e-1)\n",
        "learn.model"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "EmbeddingDotBias(\n",
              "  (u_weight): Embedding(944, 40)\n",
              "  (i_weight): Embedding(1683, 40)\n",
              "  (u_bias): Embedding(944, 1)\n",
              "  (i_bias): Embedding(1683, 1)\n",
              ")"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QhZvQiWZIDAD",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 224
        },
        "outputId": "9c462561-36bb-427e-9c66-29e53b5336a8"
      },
      "source": [
        "start_time = time.time()\n",
        "\n",
        "learn.fit_one_cycle(EPOCHS, max_lr=5e-3)\n",
        "\n",
        "train_time = time.time() - start_time + preprocess_time\n",
        "\n",
        "print(f'Took {train_time} seconds for training')"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: left;\">\n",
              "      <th>epoch</th>\n",
              "      <th>train_loss</th>\n",
              "      <th>valid_loss</th>\n",
              "      <th>time</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>0</td>\n",
              "      <td>0.937871</td>\n",
              "      <td>#na#</td>\n",
              "      <td>00:06</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>0.877238</td>\n",
              "      <td>#na#</td>\n",
              "      <td>00:06</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>0.772738</td>\n",
              "      <td>#na#</td>\n",
              "      <td>00:06</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>0.652344</td>\n",
              "      <td>#na#</td>\n",
              "      <td>00:06</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>0.536503</td>\n",
              "      <td>#na#</td>\n",
              "      <td>00:06</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Took 32.39713501930237 seconds for training\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IbgdW0mMIDGA"
      },
      "source": [
        "learn.export('movilens_model.pkl')"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "v6SEhEw6K4dI"
      },
      "source": [
        "learner = load_learner(path='.', file='movilens_model.pkl')"
      ],
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2FibNYPk5m16"
      },
      "source": [
        "total_users, total_items = learner.data.train_ds.x.classes.values()\n",
        "total_items = total_items[1:]\n",
        "total_users = total_users[1:]"
      ],
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DQRq32RO5nAo"
      },
      "source": [
        "test_users = test_df['UserId'].unique()\n",
        "test_users = np.intersect1d(test_users, total_users)"
      ],
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZF-NPh6L5nFg"
      },
      "source": [
        "users_items = cartesian_product(np.array(test_users), np.array(total_items))\n",
        "users_items = pd.DataFrame(users_items, columns=['UserId', 'MovieId'])"
      ],
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "E4hRDx8z5nJY"
      },
      "source": [
        "training_removed = pd.merge(users_items, train_valid_df.astype(str), on=['UserId', 'MovieId'], how='left')\n",
        "training_removed = training_removed[training_removed['Rating'].isna()][['UserId', 'MovieId']]"
      ],
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Ix1PPqA35nNg",
        "outputId": "d1c3a436-1ac3-403c-a91f-02e0278af1f5"
      },
      "source": [
        "start_time = time.time()\n",
        "\n",
        "top_k_scores = score(learner,\n",
        "                     test_df=training_removed,\n",
        "                     user_col='UserId',\n",
        "                     item_col='MovieId',\n",
        "                     prediction_col='Prediction')\n",
        "\n",
        "test_time = time.time() - start_time\n",
        "print(f'Took {test_time} seconds for {len(training_removed)} predictions.')"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Took 1.8492753505706787 seconds for 1511060 predictions.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2vXtc84a5zy5"
      },
      "source": [
        "eval_map = map_at_k(test_df, top_k_scores, col_user='UserId', col_item='MovieId',\n",
        "                   col_rating='Rating', col_prediction='Prediction',\n",
        "                   relevancy_method='top_k', k=TOP_K)"
      ],
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Q3FYMvTP50Bj"
      },
      "source": [
        "eval_ndcg = ndcg_at_k(test_df, top_k_scores, col_user='UserId', col_item='MovieId',\n",
        "                     col_rating='Rating', col_prediction='Prediction',\n",
        "                     relevancy_method='top_k', k=TOP_K)"
      ],
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hExpQKA550FC"
      },
      "source": [
        "eval_precision = precision_at_k(test_df, top_k_scores, col_user='UserId', col_item='MovieId',\n",
        "                     col_rating='Rating', col_prediction='Prediction',\n",
        "                     relevancy_method='top_k', k=TOP_K)"
      ],
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6dOqKMbc50Ju"
      },
      "source": [
        "eval_recall = recall_at_k(test_df, top_k_scores, col_user='UserId', col_item='MovieId',\n",
        "                     col_rating='Rating', col_prediction='Prediction',\n",
        "                     relevancy_method='top_k', k=TOP_K)"
      ],
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zAnY-eAg50M0",
        "outputId": "b0ce1b77-a371-463b-da0e-336c879785a6"
      },
      "source": [
        "print(\"Model:\\t\" + learn.__class__.__name__,\n",
        "      \"Top K:\\t%d\" % TOP_K,\n",
        "      \"MAP:\\t%f\" % eval_map,\n",
        "      \"NDCG:\\t%f\" % eval_ndcg,\n",
        "      \"Precision@K:\\t%f\" % eval_precision,\n",
        "      \"Recall@K:\\t%f\" % eval_recall, sep='\\n')"
      ],
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model:\tCollabLearner\n",
            "Top K:\t10\n",
            "MAP:\t0.027680\n",
            "NDCG:\t0.158812\n",
            "Precision@K:\t0.139661\n",
            "Recall@K:\t0.057563\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7P8T3XA98LET"
      },
      "source": [
        "scores = score(learner,\n",
        "               test_df=test_df.copy(),\n",
        "               user_col='UserId',\n",
        "               item_col='MovieId',\n",
        "               prediction_col='Prediction')"
      ],
      "execution_count": 23,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2OtKbvZj8LOI",
        "outputId": "34a510a3-033a-4b93-e8b5-72cd14c7523a"
      },
      "source": [
        "eval_r2 = rsquared(test_df, scores, col_user='UserId', col_item='MovieId', col_rating='Rating', col_prediction='Prediction')\n",
        "eval_rmse = rmse(test_df, scores, col_user='UserId', col_item='MovieId', col_rating='Rating', col_prediction='Prediction')\n",
        "eval_mae = mae(test_df, scores, col_user='UserId', col_item='MovieId', col_rating='Rating', col_prediction='Prediction')\n",
        "eval_exp_var = exp_var(test_df, scores, col_user='UserId', col_item='MovieId', col_rating='Rating', col_prediction='Prediction')\n",
        "\n",
        "print(\"Model:\\t\" + learn.__class__.__name__,\n",
        "      \"RMSE:\\t%f\" % eval_rmse,\n",
        "      \"MAE:\\t%f\" % eval_mae,\n",
        "      \"Explained variance:\\t%f\" % eval_exp_var,\n",
        "      \"R squared:\\t%f\" % eval_r2, sep='\\n')"
      ],
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model:\tCollabLearner\n",
            "RMSE:\t0.902230\n",
            "MAE:\t0.712558\n",
            "Explained variance:\t0.346533\n",
            "R squared:\t0.345887\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17
        },
        "id": "DcpVx1-E8LR-",
        "outputId": "e6bae276-5ca0-4d17-b94f-ab17eb7dda75"
      },
      "source": [
        "# Record results with papermill for tests\n",
        "sb.glue(\"map\", eval_map)\n",
        "sb.glue(\"ndcg\", eval_ndcg)\n",
        "sb.glue(\"precision\", eval_precision)\n",
        "sb.glue(\"recall\", eval_recall)\n",
        "sb.glue(\"rmse\", eval_rmse)\n",
        "sb.glue(\"mae\", eval_mae)\n",
        "sb.glue(\"exp_var\", eval_exp_var)\n",
        "sb.glue(\"rsquared\", eval_r2)\n",
        "sb.glue(\"train_time\", train_time)\n",
        "sb.glue(\"test_time\", test_time)"
      ],
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/scrapbook.scrap.json+json": {
              "version": 1,
              "data": 0.027680044544179286,
              "name": "map",
              "encoder": "json"
            }
          },
          "metadata": {
            "tags": [],
            "scrapbook": {
              "data": true,
              "name": "map",
              "display": false
            }
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "application/scrapbook.scrap.json+json": {
              "version": 1,
              "data": 0.1588123076990557,
              "name": "ndcg",
              "encoder": "json"
            }
          },
          "metadata": {
            "tags": [],
            "scrapbook": {
              "data": true,
              "name": "ndcg",
              "display": false
            }
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "application/scrapbook.scrap.json+json": {
              "version": 1,
              "data": 0.13966065747613995,
              "name": "precision",
              "encoder": "json"
            }
          },
          "metadata": {
            "tags": [],
            "scrapbook": {
              "data": true,
              "name": "precision",
              "display": false
            }
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "application/scrapbook.scrap.json+json": {
              "version": 1,
              "data": 0.05756329495350074,
              "name": "recall",
              "encoder": "json"
            }
          },
          "metadata": {
            "tags": [],
            "scrapbook": {
              "data": true,
              "name": "recall",
              "display": false
            }
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "application/scrapbook.scrap.json+json": {
              "version": 1,
              "data": 0.9022304905554323,
              "name": "rmse",
              "encoder": "json"
            }
          },
          "metadata": {
            "tags": [],
            "scrapbook": {
              "data": true,
              "name": "rmse",
              "display": false
            }
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "application/scrapbook.scrap.json+json": {
              "version": 1,
              "data": 0.7125582958416156,
              "name": "mae",
              "encoder": "json"
            }
          },
          "metadata": {
            "tags": [],
            "scrapbook": {
              "data": true,
              "name": "mae",
              "display": false
            }
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "application/scrapbook.scrap.json+json": {
              "version": 1,
              "data": 0.34653328414404316,
              "name": "exp_var",
              "encoder": "json"
            }
          },
          "metadata": {
            "tags": [],
            "scrapbook": {
              "data": true,
              "name": "exp_var",
              "display": false
            }
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "application/scrapbook.scrap.json+json": {
              "version": 1,
              "data": 0.34588745793012965,
              "name": "rsquared",
              "encoder": "json"
            }
          },
          "metadata": {
            "tags": [],
            "scrapbook": {
              "data": true,
              "name": "rsquared",
              "display": false
            }
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "application/scrapbook.scrap.json+json": {
              "version": 1,
              "data": 32.39713501930237,
              "name": "train_time",
              "encoder": "json"
            }
          },
          "metadata": {
            "tags": [],
            "scrapbook": {
              "data": true,
              "name": "train_time",
              "display": false
            }
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "application/scrapbook.scrap.json+json": {
              "version": 1,
              "data": 1.8492753505706787,
              "name": "test_time",
              "encoder": "json"
            }
          },
          "metadata": {
            "tags": [],
            "scrapbook": {
              "data": true,
              "name": "test_time",
              "display": false
            }
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2tCTYJ8B8LVg"
      },
      "source": [
        ""
      ],
      "execution_count": 25,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HtOWaJaM8LZY"
      },
      "source": [
        ""
      ],
      "execution_count": 25,
      "outputs": []
    }
  ]
}