15. 推荐系统当前最流行的Embedding算法.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##  生成Embedding的几种方法"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import findspark\n",
    "findspark.init(\"D:\\software\\spark-2.4.4-bin-hadoop2.7\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.context import SparkContext\n",
    "from pyspark.sql.session import SparkSession\n",
    "sc = SparkContext('local')\n",
    "spark = SparkSession(sc)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1. 内容向量word2vec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Text: [Hi, I, heard, about, Spark] => \n",
      "Vector: [-0.09118835926055908,0.024794609472155574,-0.0023326151072978973]\n",
      "\n",
      "Text: [I, wish, Java, could, use, case, classes] => \n",
      "Vector: [-0.004019131617886679,0.0024854108674584752,0.003071522439963051]\n",
      "\n",
      "Text: [Logistic, regression, models, are, neat] => \n",
      "Vector: [0.05315629169344902,0.02378393579274416,-0.059764140844345094]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from pyspark.ml.feature import Word2Vec\n",
    "\n",
    "# Input data: Each row is a bag of words from a sentence or document.\n",
    "documentDF = spark.createDataFrame([\n",
    "    (\"Hi I heard about Spark\".split(\" \"), ),\n",
    "    (\"I wish Java could use case classes\".split(\" \"), ),\n",
    "    (\"Logistic regression models are neat\".split(\" \"), )\n",
    "], [\"text\"])\n",
    "\n",
    "# Learn a mapping from words to Vectors.\n",
    "word2Vec = Word2Vec(vectorSize=3, minCount=0, inputCol=\"text\", outputCol=\"result\")\n",
    "model = word2Vec.fit(documentDF)\n",
    "\n",
    "result = model.transform(documentDF)\n",
    "for row in result.collect():\n",
    "    text, vector = row\n",
    "    print(\"Text: [%s] => \\nVector: %s\\n\" % (\", \".join(text), str(vector)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "把（文档ID，用户词语列表），变成（用户ID，播放电影ID列表），输入到word2vec，就能得到每个电影的Embedding向量"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. 使用Spark ALS的矩阵分解的行为Embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Root-mean-square error = 1.7636180291295112\n"
     ]
    }
   ],
   "source": [
    "from pyspark.ml.evaluation import RegressionEvaluator\n",
    "from pyspark.ml.recommendation import ALS\n",
    "from pyspark.sql import Row\n",
    "\n",
    "lines = spark.read.text(\"D:/workbench/ant-learn-recsys/datas/als/sample_movielens_ratings.txt\").rdd\n",
    "parts = lines.map(lambda row: row.value.split(\"::\"))\n",
    "ratingsRDD = parts.map(lambda p: Row(userId=int(p[0]), movieId=int(p[1]),\n",
    "                                     rating=float(p[2]), timestamp=int(p[3])))\n",
    "ratings = spark.createDataFrame(ratingsRDD)\n",
    "(training, test) = ratings.randomSplit([0.8, 0.2])\n",
    "\n",
    "# Build the recommendation model using ALS on the training data\n",
    "# Note we set cold start strategy to 'drop' to ensure we don't get NaN evaluation metrics\n",
    "als = ALS(maxIter=5, \n",
    "          regParam=0.01, \n",
    "          userCol=\"userId\", \n",
    "          itemCol=\"movieId\", \n",
    "          ratingCol=\"rating\",\n",
    "          coldStartStrategy=\"drop\")\n",
    "model = als.fit(training)\n",
    "\n",
    "# Evaluate the model by computing the RMSE on the test data\n",
    "predictions = model.transform(test)\n",
    "evaluator = RegressionEvaluator(metricName=\"rmse\", labelCol=\"rating\",\n",
    "                                predictionCol=\"prediction\")\n",
    "rmse = evaluator.evaluate(predictions)\n",
    "print(\"Root-mean-square error = \" + str(rmse))\n",
    "\n",
    "# Generate top 10 movie recommendations for each user\n",
    "userRecs = model.recommendForAllUsers(10)\n",
    "# Generate top 10 user recommendations for each movie\n",
    "movieRecs = model.recommendForAllItems(10)\n",
    "\n",
    "# Generate top 10 movie recommendations for a specified set of users\n",
    "users = ratings.select(als.getUserCol()).distinct().limit(3)\n",
    "userSubsetRecs = model.recommendForUserSubset(users, 10)\n",
    "# Generate top 10 user recommendations for a specified set of movies\n",
    "movies = ratings.select(als.getItemCol()).distinct().limit(3)\n",
    "movieSubSetRecs = model.recommendForItemSubset(movies, 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+------+----------+------+\n",
      "|movieId|rating| timestamp|userId|\n",
      "+-------+------+----------+------+\n",
      "|      0|   1.0|1424380312|     3|\n",
      "|      0|   1.0|1424380312|     5|\n",
      "|      0|   1.0|1424380312|     6|\n",
      "|      0|   1.0|1424380312|     8|\n",
      "|      0|   1.0|1424380312|    11|\n",
      "|      0|   1.0|1424380312|    13|\n",
      "|      0|   1.0|1424380312|    15|\n",
      "|      0|   1.0|1424380312|    19|\n",
      "|      0|   1.0|1424380312|    20|\n",
      "|      0|   1.0|1424380312|    21|\n",
      "+-------+------+----------+------+\n",
      "only showing top 10 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "training.show(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---+------------------------------------------------------------------------------------------------------------------------+\n",
      "|id |features                                                                                                                |\n",
      "+---+------------------------------------------------------------------------------------------------------------------------+\n",
      "|0  |[1.2830094, -0.5151567, 0.18020844, -0.71568125, -2.535846, 0.52966636, -0.9128374, -1.1174475, 0.19547723, 0.698292]   |\n",
      "|10 |[1.0180752, -2.7507377, 1.0930991, -1.134008, -0.5343898, -1.9026369, -1.1077534, -0.8857196, -0.39179775, -0.01721368] |\n",
      "|20 |[1.4512534, -3.517298, 0.7868661, -1.4977869, -0.24221556, -2.037162, 1.0100238, -1.0118681, -0.3201244, -0.18585977]   |\n",
      "|30 |[1.4444151, -3.6120615, 1.6103011, 0.17859526, 0.15473363, -0.7841998, 3.4736896, -0.54864204, -0.19071166, 1.2209741]  |\n",
      "|40 |[1.8021005, -2.4846869, -1.0012007, -0.11796358, -3.8910062, 0.6172575, 0.46259242, 0.20520537, -0.75374764, 0.98922247]|\n",
      "|50 |[0.85364246, -3.47646, 0.36618742, -1.4283884, -1.0077556, -1.321288, -1.2243408, 0.40804875, 0.07967562, 0.14777525]   |\n",
      "|60 |[1.1889794, -2.6609316, 1.0438567, -0.45259616, -0.9141676, 0.13169621, -0.44755557, 1.5527409, -1.880157, -0.53658813] |\n",
      "|70 |[1.7922753, -2.0595644, 0.58770806, -1.4066004, -2.7759798, -1.5907837, 1.471062, -0.028371431, -2.0037463, 0.84014094] |\n",
      "|80 |[-0.40876412, -1.2445711, 2.2364929, -0.8541729, -2.781932, -0.1874267, 0.9496937, 0.4868828, 1.2105056, -0.40545303]   |\n",
      "|90 |[2.0153913, -3.951402, 1.3197321, -0.32983437, 1.661446, -2.995254, 1.9340911, -0.087758064, 1.1283636, -0.11234664]    |\n",
      "+---+------------------------------------------------------------------------------------------------------------------------+\n",
      "only showing top 10 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "model.itemFactors.show(10, truncate=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3. DNN中的Embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(32, 10, 64)\n"
     ]
    }
   ],
   "source": [
    "model = tf.keras.Sequential()\n",
    "# 注意，这一层是Embedding层\n",
    "model.add(tf.keras.layers.Embedding(1000, 64, input_length=10))\n",
    "model.compile('rmsprop', 'mse')\n",
    "\n",
    "input_array = np.random.randint(1000, size=(32, 10))\n",
    "output_array = model.predict(input_array)\n",
    "print(output_array.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[array([[ 0.00781213,  0.03940525, -0.00024771, ...,  0.03611508,\n",
       "          0.02551547, -0.03192703],\n",
       "        [-0.03161997, -0.02198304,  0.03973298, ..., -0.02881846,\n",
       "         -0.03093203, -0.01212269],\n",
       "        [ 0.00935531, -0.01970615,  0.03177864, ...,  0.04194124,\n",
       "         -0.02666444,  0.02423222],\n",
       "        ...,\n",
       "        [-0.04647785,  0.01175867,  0.02346585, ..., -0.00246744,\n",
       "         -0.01744302, -0.00606211],\n",
       "        [-0.01508133, -0.00510512, -0.02035259, ...,  0.04146155,\n",
       "         -0.00624609,  0.03074067],\n",
       "        [ 0.02103826, -0.01366248,  0.01829243, ..., -0.03217832,\n",
       "          0.02095122, -0.03056069]], dtype=float32)]"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 训练完之后，embedding的layer的weights就是embedding向量\n",
    "model.layers[0].get_weights()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}