diff --git a/notebooks/nn_response_evaluation.ipynb b/notebooks/nn_response_evaluation.ipynb
new file mode 100644
index 0000000..7689c1f
--- /dev/null
+++ b/notebooks/nn_response_evaluation.ipynb
@@ -0,0 +1,1671 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "cellId": "5nmr5qtyi4l6dv4003f5hk",
+ "execution_id": "b4c3a56f-dd79-4dc8-9c15-e451a726c6d8"
+ },
+ "source": [
+ "# RS evaluation with a custom response model"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "cellId": "a23y5u7l746q73y0tqxss",
+ "execution_id": "b0740c39-9cda-411d-b832-490de8695080"
+ },
+ "source": [
+ "## Imports and preparations"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "%load_ext autoreload\n",
+ "%autoreload 2\n",
+ "# !pip install jupyter-black\n",
+ "%load_ext jupyter_black"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import os\n",
+ "import sys\n",
+ "\n",
+ "os.environ[\"JAVA_HOME\"] = \"/usr\"\n",
+ "os.environ[\"PYSPARK_PYTHON\"] = sys.executable\n",
+ "os.environ[\"PYSPARK_DRIVER_PYTHON\"] = sys.executable\n",
+ "os.environ[\"CUDA_LAUNCH_BLOCKING\"] = \"1\"\n",
+ "os.environ[\"OMP_NUM_THREADS\"] = \"32\"\n",
+ "os.environ[\"NUMBA_NUM_THREADS\"] = \"4\""
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "metadata": {
+ "cellId": "jlz0twee16goe1fisulv5"
+ },
+ "outputs": [],
+ "source": [
+ "import random\n",
+ "import time\n",
+ "import scipy\n",
+ "import torch\n",
+ "import numpy as np\n",
+ "import pandas as pd\n",
+ "import warnings\n",
+ "\n",
+ "from pyspark import SparkConf, StorageLevel\n",
+ "from pyspark.ml import Pipeline\n",
+ "from pyspark.sql import SparkSession\n",
+ "import pyspark.sql.functions as sf\n",
+ "from replay.models import UCB\n",
+ "from tqdm.auto import tqdm\n",
+ "\n",
+ "from experiments.response_models.utils import plot_metric, calc_metric\n",
+ "from pyspark.sql.functions import col"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 4,
+ "metadata": {
+ "cellId": "jlz0twee16goe1fisulv5"
+ },
+ "outputs": [],
+ "source": [
+ "from sim4rec.modules import RealDataGenerator\n",
+ "from sim4rec.modules import Simulator\n",
+ "from sim4rec.response import BernoulliResponse, ActionModelTransformer\n",
+ "\n",
+ "\n",
+ "def warn(*args, **kwargs):\n",
+ " pass\n",
+ "\n",
+ "\n",
+ "warnings.warn = warn"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 5,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# path to spark tmp folder and simulator checkpoints\n",
+ "SPARK_LOCAL_DIR = \"./tmp/task_1\"\n",
+ "CHECKPOINT_DIR = \"./tmp/task_1_checkpoints\""
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 6,
+ "metadata": {
+ "cellId": "ku55idy38nqtlrc187ucq"
+ },
+ "outputs": [],
+ "source": [
+ "%%bash -s \"$CHECKPOINT_DIR\" \"$SPARK_LOCAL_DIR\"\n",
+ "# simulator saves the interaction history between users and recommender system\n",
+ "# to rerun the simulation cycle or begin a new one, clear the directory or use another CHECKPOINT_DIR\n",
+ "rm -rf $1 $2"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 7,
+ "metadata": {
+ "cellId": "v1uc08ym8h8d7qxpmsop4"
+ },
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "WARNING: An illegal reflective access operation has occurred\n",
+ "WARNING: Illegal reflective access by org.apache.spark.unsafe.Platform (file:/root/.cache/pypoetry/virtualenvs/sim4rec-lKa0R1gD-py3.9/lib/python3.9/site-packages/pyspark/jars/spark-unsafe_2.12-3.1.3.jar) to constructor java.nio.DirectByteBuffer(long,int)\n",
+ "WARNING: Please consider reporting this to the maintainers of org.apache.spark.unsafe.Platform\n",
+ "WARNING: Use --illegal-access=warn to enable warnings of further illegal reflective access operations\n",
+ "WARNING: All illegal access operations will be denied in a future release\n",
+ "24/11/24 21:38:08 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n",
+ "Using Spark's default log4j profile: org/apache/spark/log4j-defaults.properties\n",
+ "Setting default log level to \"WARN\".\n",
+ "To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).\n",
+ "24/11/24 21:38:09 WARN SparkConf: Note that spark.local.dir will be overridden by the value set by the cluster manager (via SPARK_LOCAL_DIRS in mesos/standalone/kubernetes and LOCAL_DIRS in YARN).\n"
+ ]
+ }
+ ],
+ "source": [
+ "NUM_THREADS = 4\n",
+ "\n",
+ "spark = (\n",
+ " SparkSession.builder.appName(\"simulator\")\n",
+ " .master(f\"local[{NUM_THREADS}]\")\n",
+ " .config(\"spark.sql.shuffle.partitions\", f\"{NUM_THREADS * 3}\")\n",
+ " .config(\"spark.default.parallelism\", f\"{NUM_THREADS * 3}\")\n",
+ " .config(\"spark.driver.memory\", \"4G\")\n",
+ " .config(\"spark.executor.memory\", \"4G\")\n",
+ " .config(\"spark.driver.extraJavaOptions\", \"-XX:+UseG1GC\")\n",
+ " .config(\"spark.executor.extraJavaOptions\", \"-XX:+UseG1GC\")\n",
+ " .config(\"spark.local.dir\", SPARK_LOCAL_DIR)\n",
+ " .getOrCreate()\n",
+ ")\n",
+ "\n",
+ "spark.sparkContext.setLogLevel(\"ERROR\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 8,
+ "metadata": {
+ "cellId": "9vf3sqixr4vgau073549pv"
+ },
+ "outputs": [],
+ "source": [
+ "K = 10 # number of iterations\n",
+ "NUM_ITER = 10\n",
+ "SEED = 1234\n",
+ "np.random.seed(SEED)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "cellId": "ffl9l89trncxzwbk5zq7gn",
+ "execution_id": "cbbf5328-af84-4368-957e-3c000bd80332"
+ },
+ "source": [
+ "## Data"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Load ContentWise in tabular form. Uncomment for the first run.\n",
+ "\n",
+ "This preprocessing is similat to preprocessing in old `ContentWise` class: \n",
+ "join interactions with impressions, binarize clicks, etc.\n",
+ "\n",
+ "ContentWise consists of following files:\n",
+ "1. interactions.csv.gz:\n",
+ " * `utc_ts_milliseconds` -- interaction timestamp \n",
+ " * `user_id` -- user id\n",
+ " * `item_id`, `series_id` -- video id, video series id. As long as only series were recommended, we threat 'series' as primary items\n",
+ " * `recommendation_id` -- key to match with recommendations\n",
+ " * `episode_number`, `series_length`, `item_type`, `interaction_type`, `vision_factor`, `explicit_rating` are ignored\n",
+ "\n",
+ "2. impressions-direct-link.csv.gz:\n",
+ " * `recommendation_id` -- impression id to match with responses, `-1` for unknown recommendation\n",
+ " * `recommendation_list_length`\n",
+ " * `recommended_series_list` -- string representation of python list of series ids\n",
+ " * `row_position` -- a position of recommendation slate on the site"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 9,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# cw_impr = \"data/ContentWiseImpressions/CW10M-CSV/impressions-direct-link.csv\"\n",
+ "# impressions = spark.read.option(\"header\", \"true\").csv(cw_impr)\n",
+ "# impressions = impressions.filter(col(\"recommendation_list_length\") == K)\n",
+ "# impressions = impressions.withColumn(\n",
+ "# \"tmp\", sf.regexp_replace(\"recommended_series_list\", r\"\\[\\s*\", \"\")\n",
+ "# )\n",
+ "# impressions = impressions.withColumn(\"tmp\", sf.regexp_replace(\"tmp\", r\"\\s*\\]\", \"\"))\n",
+ "# impressions = impressions.withColumn(\"tmp\", sf.regexp_replace(\"tmp\", r\"\\s+\", \" \"))\n",
+ "# impressions = impressions.withColumn(\"tmp\", sf.split(\"tmp\", \" \"))\n",
+ "# impressions = impressions.select(\n",
+ "# \"recommendation_id\", sf.posexplode(\"tmp\").alias(\"slate_pos\", \"item_id\")\n",
+ "# ).drop(\"tmp\", \"recommendation_list_length\", \"recommended_series_list\", \"row_position\")\n",
+ "# cw_inter = \"data/ContentWiseImpressions/CW10M-CSV/interactions.csv\"\n",
+ "# interactions = spark.read.option(\"header\", \"true\").csv(cw_inter)\n",
+ "# interactions = interactions.filter(col(\"recommendation_id\") != -1)\n",
+ "# interactions = interactions.drop(\n",
+ "# \"item_id\",\n",
+ "# \"episode_number\",\n",
+ "# \"series_length\",\n",
+ "# \"item_type\",\n",
+ "# \"interaction_type\",\n",
+ "# \"vision_factor\",\n",
+ "# \"explicit_rating\",\n",
+ "# )\n",
+ "# interactions = interactions.withColumnRenamed(\"series_id\", \"item_id\")\n",
+ "\n",
+ "# recommendation_metadata = interactions.groupBy(\"recommendation_id\").agg(\n",
+ "# sf.min(\"utc_ts_milliseconds\").alias(\"recommendation_timestamp\"),\n",
+ "# sf.first(\"user_id\").alias(\"user_id\"),\n",
+ "# )\n",
+ "# # click om item = first click on item\n",
+ "# interactions = interactions.drop(\"user_id\")\n",
+ "# interactions = interactions.groupBy(\"recommendation_id\", \"item_id\").agg(\n",
+ "# sf.min(\"utc_ts_milliseconds\").alias(\"response_timestamp\")\n",
+ "# )\n",
+ "# # add response\n",
+ "# interactions = interactions.withColumn(\"response\", sf.lit(1))\n",
+ "# content_wise_all = impressions.join(\n",
+ "# interactions, how=\"left\", on=[\"recommendation_id\", \"item_id\"]\n",
+ "# ).join(recommendation_metadata, on=\"recommendation_id\")\n",
+ "# content_wise_all = content_wise_all.withColumn(\n",
+ "# \"response\", sf.coalesce(\"response\", sf.lit(0))\n",
+ "# )\n",
+ "\n",
+ "# # split train-val-test by global time:\n",
+ "# total_cnt = content_wise_all.count()\n",
+ "# train_cnt = int(0.6 * total_cnt)\n",
+ "# test_cnt = int(0.2 * total_cnt)\n",
+ "# val_cnt = int(0.2 * total_cnt)\n",
+ "# sorted_df = content_wise_all.orderBy(\"recommendation_timestamp\")\n",
+ "# train = sorted_df.limit(train_cnt)\n",
+ "# val = sorted_df.subtract(train).limit(val_cnt)\n",
+ "# test = sorted_df.subtract(train).subtract(val)\n",
+ "# MAX_TIME = content_wise_all.agg(sf.max(\"recommendation_timestamp\")).collect()[0][0]\n",
+ "# MAX_TIME"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 10,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# def content_wise_to_simlog_format(df: \"DataFrame\"):\n",
+ "# \"\"\"Convert ContentWise dataset to math sim4rec log schema\"\"\"\n",
+ "# df = df.drop(\"response_timestamp\")\n",
+ "# df = df.withColumn(\"response_proba\", sf.col(\"response\").cast(\"float\"))\n",
+ "# df = df.withColumnRenamed(\"item_id\", \"item_idx\")\n",
+ "# df = df.withColumnRenamed(\"user_id\", \"user_idx\")\n",
+ "# # place all historical interactions BEFORE the simulated\n",
+ "# df = df.withColumn(\n",
+ "# \"__iter\",\n",
+ "# (sf.col(\"recommendation_timestamp\").cast(\"bigint\") - int(MAX_TIME)) / 1000,\n",
+ "# ).drop(\"recommendation_timestamp\", \"recommendation_id\")\n",
+ "# return df\n",
+ "\n",
+ "\n",
+ "# train = content_wise_to_simlog_format(train)\n",
+ "# test = content_wise_to_simlog_format(test)\n",
+ "# val = content_wise_to_simlog_format(val)\n",
+ "# val.show(10)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 11,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# from sim4rec.response.nn_response import SIM_LOG_SCHEMA\n",
+ "# print(SIM_LOG_SCHEMA, train.schema)\n",
+ "\n",
+ "# # cast types to match simlog schema\n",
+ "# train = train.withColumn(\"relevance\", 1 / (sf.col(\"slate_pos\") + 1))\n",
+ "# val = train.withColumn(\"relevance\", 1 / (sf.col(\"slate_pos\") + 1))\n",
+ "# test = train.withColumn(\"relevance\", 1 / (sf.col(\"slate_pos\") + 1))\n",
+ "\n",
+ "# for field in SIM_LOG_SCHEMA:\n",
+ "# train = train.withColumn(field.name, train[field.name].cast(field.dataType))\n",
+ "# val = val.withColumn(field.name, val[field.name].cast(field.dataType))\n",
+ "# test = test.withColumn(field.name, test[field.name].cast(field.dataType))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 12,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# # this takes some time\n",
+ "# !rm -rdf parquet && mkdir parquet\n",
+ "# train.write.partitionBy(\"user_idx\").parquet(\"parquet/train\")\n",
+ "# val.write.partitionBy(\"user_idx\").parquet(\"parquet/val\")\n",
+ "# test.write.partitionBy(\"user_idx\").parquet(\"parquet/test\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 13,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ " \r"
+ ]
+ }
+ ],
+ "source": [
+ "# re-read:\n",
+ "train = spark.read.parquet(\"../parquet/train\")\n",
+ "val = spark.read.parquet(\"../parquet/val\")\n",
+ "test = spark.read.parquet(\"../parquet/test\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 14,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "+--------+---------+--------+--------------+--------+-------------------+--------+\n",
+ "|item_idx|slate_pos|response|response_proba| __iter| relevance|user_idx|\n",
+ "+--------+---------+--------+--------------+--------+-------------------+--------+\n",
+ "| 25508| 0| 0| 0.0|-8211853| 1.0| 35783|\n",
+ "| 9245| 8| 0| 0.0|-8211853| 0.1111111111111111| 35783|\n",
+ "| 5368| 6| 1| 1.0|-8211853|0.14285714285714285| 35783|\n",
+ "| 5939| 4| 0| 0.0|-8211853| 0.2| 35783|\n",
+ "| 10685| 5| 0| 0.0|-8211853|0.16666666666666666| 35783|\n",
+ "| 21058| 3| 0| 0.0|-8211853| 0.25| 35783|\n",
+ "| 5018| 9| 0| 0.0|-8211853| 0.1| 35783|\n",
+ "| 22613| 7| 0| 0.0|-8211853| 0.125| 35783|\n",
+ "| 17175| 2| 0| 0.0|-8211853| 0.3333333333333333| 35783|\n",
+ "| 7618| 1| 0| 0.0|-8211853| 0.5| 35783|\n",
+ "+--------+---------+--------+--------------+--------+-------------------+--------+\n",
+ "only showing top 10 rows\n",
+ "\n"
+ ]
+ }
+ ],
+ "source": [
+ "train.show(10)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 15,
+ "metadata": {
+ "cellId": "zh9qtijcn5143mi4887od"
+ },
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ " \r"
+ ]
+ },
+ {
+ "data": {
+ "text/plain": [
+ "(2658, 19205)"
+ ]
+ },
+ "execution_count": 15,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "users = [\n",
+ " [int(row[\"user_idx\"])] for row in train.select(\"user_idx\").distinct().collect()\n",
+ "]\n",
+ "NUM_USERS = len(users)\n",
+ "users = spark.createDataFrame(users, schema=[\"user_idx\"])\n",
+ "\n",
+ "items = [\n",
+ " [int(row[\"item_idx\"])] for row in train.select(\"item_idx\").distinct().collect()\n",
+ "]\n",
+ "NUM_ITEMS = len(items)\n",
+ "items = spark.createDataFrame(items, schema=[\"item_idx\"])\n",
+ "\n",
+ "NUM_ITEMS, NUM_USERS"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 16,
+ "metadata": {
+ "cellId": "3btpzal8hblukqhq46asd"
+ },
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ " \r"
+ ]
+ },
+ {
+ "data": {
+ "text/plain": [
+ "2658"
+ ]
+ },
+ "execution_count": 16,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "item_generator = RealDataGenerator(label=\"items_real\", seed=SEED)\n",
+ "user_generator = RealDataGenerator(label=\"users_real\", seed=SEED)\n",
+ "\n",
+ "item_generator.fit(items)\n",
+ "user_generator.fit(users)\n",
+ "\n",
+ "item_generator.generate(NUM_ITEMS)\n",
+ "user_generator.generate(NUM_USERS)\n",
+ "item_generator.getDataSize()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "cellId": "1crcvslqi2qm4dnavh3obq",
+ "execution_id": "2c640f43-9dd5-445d-9511-d0d100fc830c"
+ },
+ "source": [
+ "# One iteration of simulation cycle step by step"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "cellId": "q2oio9pylodwwu9qptdom",
+ "execution_id": "0f252faf-52c4-4a99-ac83-b425c14d055e"
+ },
+ "source": [
+ "## (1) Choise of users"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 17,
+ "metadata": {
+ "cellId": "erkzmjzvo06nl5bok1qv5n"
+ },
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "1964"
+ ]
+ },
+ "execution_count": 17,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "user_generator.sample(0.1).count()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "cellId": "tnbcus0l6hcbvoht2rnge",
+ "execution_id": "a45bc1e9-f284-4702-bfeb-d525f9349d20"
+ },
+ "source": [
+ "## (2) Choise of items\n",
+ "During the simulation cycle all 5276 items will be available at each iteration."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "cellId": "4kfl2on9vg46431mkek4ot",
+ "execution_id": "85a06070-5f73-407a-ad84-ff781e7d04fb"
+ },
+ "source": [
+ "## (3) Initialization of recommender model"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "cellId": "p7qktnabe0pupwrpgo1vu",
+ "execution_id": "4f4957f4-cea9-47fe-9140-95dafb48e159"
+ },
+ "source": [
+ "### Baseline"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 18,
+ "metadata": {
+ "cellId": "kck71xz164nwlq1tlo81d"
+ },
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ " \r"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "\n",
+ "\n",
+ "
\n",
+ " \n",
+ " \n",
+ " | \n",
+ " user_idx | \n",
+ " item_idx | \n",
+ " relevance | \n",
+ "
\n",
+ " \n",
+ " \n",
+ " \n",
+ " | 0 | \n",
+ " 232 | \n",
+ " 16330 | \n",
+ " 0.000376 | \n",
+ "
\n",
+ " \n",
+ " | 1 | \n",
+ " 232 | \n",
+ " 17915 | \n",
+ " 0.000376 | \n",
+ "
\n",
+ " \n",
+ " | 2 | \n",
+ " 23288 | \n",
+ " 17599 | \n",
+ " 0.000376 | \n",
+ "
\n",
+ " \n",
+ " | 3 | \n",
+ " 23288 | \n",
+ " 14627 | \n",
+ " 0.000376 | \n",
+ "
\n",
+ " \n",
+ "
\n",
+ "
\n",
+ "\n",
+ "
\n",
+ " \n",
+ " \n",
+ " | \n",
+ " user_idx | \n",
+ " item_idx | \n",
+ " relevance | \n",
+ " response_proba | \n",
+ " response | \n",
+ "
\n",
+ " \n",
+ " \n",
+ " \n",
+ " | 0 | \n",
+ " 23288 | \n",
+ " 17599 | \n",
+ " 0.000376 | \n",
+ " 0.192810 | \n",
+ " 0 | \n",
+ "
\n",
+ " \n",
+ " | 1 | \n",
+ " 23288 | \n",
+ " 14627 | \n",
+ " 0.000376 | \n",
+ " 0.198072 | \n",
+ " 0 | \n",
+ "
\n",
+ " \n",
+ " | 2 | \n",
+ " 232 | \n",
+ " 16330 | \n",
+ " 0.000376 | \n",
+ " 0.134092 | \n",
+ " 0 | \n",
+ "
\n",
+ " \n",
+ " | 3 | \n",
+ " 232 | \n",
+ " 17915 | \n",
+ " 0.000376 | \n",
+ " 0.075755 | \n",
+ " 0 | \n",
+ "
\n",
+ " \n",
+ "
\n",
+ "
\n",
+ "\n",
+ "
\n",
+ " \n",
+ " \n",
+ " | \n",
+ " user_idx | \n",
+ " item_idx | \n",
+ " relevance | \n",
+ "
\n",
+ " \n",
+ " \n",
+ " \n",
+ " | 0 | \n",
+ " 23288 | \n",
+ " 17599 | \n",
+ " 0 | \n",
+ "
\n",
+ " \n",
+ " | 1 | \n",
+ " 23288 | \n",
+ " 14627 | \n",
+ " 0 | \n",
+ "
\n",
+ " \n",
+ " | 2 | \n",
+ " 232 | \n",
+ " 16330 | \n",
+ " 0 | \n",
+ "
\n",
+ " \n",
+ " | 3 | \n",
+ " 232 | \n",
+ " 17915 | \n",
+ " 0 | \n",
+ "
\n",
+ " \n",
+ "
\n",
+ "
\n",
+ "\n",
+ "
\n",
+ " \n",
+ " \n",
+ " | \n",
+ " item_idx | \n",
+ " relevance | \n",
+ "
\n",
+ " \n",
+ " \n",
+ " \n",
+ " | 0 | \n",
+ " 16330 | \n",
+ " 0.37233 | \n",
+ "
\n",
+ " \n",
+ " | 1 | \n",
+ " 17915 | \n",
+ " 0.37233 | \n",
+ "
\n",
+ " \n",
+ " | 2 | \n",
+ " 14627 | \n",
+ " 0.37233 | \n",
+ "
\n",
+ " \n",
+ " | 3 | \n",
+ " 17599 | \n",
+ " 0.37233 | \n",
+ "
\n",
+ " \n",
+ "
\n",
+ "