P9_v0.4_wcosmodb.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "840e601b-b608-4d10-926c-3a355d85b650",
   "metadata": {},
   "source": [
    "### data creation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "9857b01e-2ac6-4e0e-99ee-43ea732acb21",
   "metadata": {},
   "outputs": [],
   "source": [
    "def cut_data_by_qtime(qtime=0.5):\n",
    "    '''le problème principal est que la liste d articles étaient presque complètes avant la première interaction\n",
    "        en cutant par un quantile du click_timestamp, on diminuer le nombre de clics mais pas le nombre d'articles\n",
    "        et si on cut par un quantile items.created_at_ts, on aura plus d interactions\n",
    "        \n",
    "        dans l ideal il faut cutter les items par un quantile items.created_at_ts\n",
    "                                et les interactions par un quantile click_timestamp\n",
    "                                en virant toutes les interactions avec des items cuttés\n",
    "                                mais on peut renconter un grand saut dans le nombre d interactions'''\n",
    "\n",
    "    cut_click=clics.click_timestamp.quantile(qtime)\n",
    "    #cut_items=items.created_at_ts.quantile(q)\n",
    "    #fusion items et items2vec \"embedding\"\n",
    "    items_=pd.concat([items,pd.DataFrame(items2vec)], axis=1)\n",
    "    #data cut\n",
    "    clics__=clics.loc[clics.click_timestamp<cut_click].copy()\n",
    "    items__=items_.loc[items_.created_at_ts<cut_click].copy()\n",
    "    \n",
    "    #merge clics & items, pour retirer les articles non connus\n",
    "    data=clics__.loc[clics__.click_article_id.isin(set(items__.article_id))].copy()\n",
    "    data.rename(columns={'click_article_id':'article_id'}, inplace = True)\n",
    "    #display(data)\n",
    "    #1 ligne par item interagi pour chaque user\n",
    "    data_map=data[['user_id','article_id']].groupby(['user_id', 'article_id']).size().to_frame().reset_index()\n",
    "    data_map.columns=['user_id','article_id','nb']\n",
    "    #display(data_map)\n",
    "    #extraction item2vec\n",
    "    items2vec_=items__.iloc[:,5:].to_numpy()\n",
    "    #creation des dicos inner_id/raw_id et raw_id/inner_id\n",
    "    dico_inner_raw_iid = dict( (inner_iid,iid) for inner_iid,iid in enumerate(items__.article_id ))\n",
    "    dico_raw_inner_iid = dict( (iid,inner_iid) for inner_iid,iid in enumerate(items__.article_id ))\n",
    "    return data_map, items__, items2vec_, dico_inner_raw_iid, dico_raw_inner_iid"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "86e94a89-40aa-4e93-bc90-556f0dd07f2e",
   "metadata": {},
   "outputs": [],
   "source": [
    "#globo user data\n",
    "import pandas as pd\n",
    "#article content embedding : np matrix 364047 articles / 250D\n",
    "#embeddings can be used by Neural Networks to represent their content\n",
    "items2vec = pd.read_pickle(r'data/news-portal-user-interactions-by-globocom/articles_embeddings.pickle')\n",
    "#metadata info about 364047 articles\n",
    "items = pd.read_csv('data/news-portal-user-interactions-by-globocom/articles_metadata.csv', delimiter = ',')\n",
    "#all_click\n",
    "for i in range(365):\n",
    "    temp=pd.read_csv('data/news-portal-user-interactions-by-globocom/clicks/clicks/clicks_hour_'+\"%03d\" % i+'.csv', delimiter = ',')\n",
    "    if i==0:\n",
    "        clics=temp\n",
    "    else:\n",
    "        clics=pd.concat([clics, temp])\n",
    "clics=clics.reset_index(drop=True)\n",
    "\n",
    "data_map,items_df, i2vec, dic_ir, dic_ri=cut_data_by_qtime(qtime=0.01)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cad98633-8b5b-4552-9130-455493b35024",
   "metadata": {},
   "source": [
    "### serialization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "2f89257a-f9b4-40b1-a88b-df90a7aada0a",
   "metadata": {},
   "outputs": [],
   "source": [
    "### dump \n",
    "import os\n",
    "import pickle\n",
    "local_path = \"./2AzBlob_2\"\n",
    "#os.mkdir(local_path)\n",
    "\n",
    "#on binarise/serialise le numpy array du word embedding\n",
    "local_file='data_map.pkl'\n",
    "file_path = os.path.join(local_path, local_file)\n",
    "with open(file_path, 'wb') as f:\n",
    "    pickle.dump(data_map, f)\n",
    "    \n",
    "local_file='items_df.pkl'\n",
    "file_path = os.path.join(local_path, local_file)\n",
    "with open(file_path, 'wb') as f:\n",
    "    pickle.dump(items_df, f)\n",
    "    \n",
    "local_file='i2vec.pkl'\n",
    "file_path = os.path.join(local_path, local_file)\n",
    "with open(file_path, 'wb') as f:\n",
    "    pickle.dump(i2vec, f)\n",
    "\n",
    "local_file='dic_ir.pkl'\n",
    "file_path = os.path.join(local_path, local_file)\n",
    "with open(file_path, 'wb') as f:\n",
    "    pickle.dump(dic_ir, f)\n",
    "\n",
    "local_file='dic_ri.pkl'\n",
    "file_path = os.path.join(local_path, local_file)\n",
    "with open(file_path, 'wb') as f:\n",
    "    pickle.dump(dic_ri, f)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "55f10d18-c41d-47f7-a0b0-6fe6dd331c03",
   "metadata": {},
   "source": [
    "### to blob storage"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "3688b65c-48e9-4144-b08c-4fc66178f339",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Azure Blob Storage v12.12.0 - Python quickstart sample\n",
      "\n",
      "Uploading to Azure Storage as blob:\n",
      "\tdata_map.pkl\n",
      "\n",
      "Uploading to Azure Storage as blob:\n",
      "\titems_df.pkl\n",
      "\n",
      "Uploading to Azure Storage as blob:\n",
      "\ti2vec.pkl\n",
      "\n",
      "Uploading to Azure Storage as blob:\n",
      "\tdic_ir.pkl\n",
      "\n",
      "Uploading to Azure Storage as blob:\n",
      "\tdic_ri.pkl\n"
     ]
    }
   ],
   "source": [
    "import os, uuid\n",
    "from azure.storage.blob import BlobServiceClient, BlobClient, ContainerClient, __version__\n",
    "\n",
    "try:\n",
    "    print(\"Azure Blob Storage v\" + __version__ + \" - Python quickstart sample\")\n",
    "        # recuperation de la chaine de connexion au azure blob storage à partir de la variabale environmentale\n",
    "    connect_str = os.getenv('AZURE_STORAGE_CONNECTION_STRING')\n",
    "        # Creation du service_client blob qui permettra de creer un conteneur\n",
    "    blob_service_client = BlobServiceClient.from_connection_string(connect_str)\n",
    "        # Creation d'un nom unique aleatoire grace a la fonction uuid4 pour le conteneur blob\n",
    "    container_name = str(uuid.uuid4())\n",
    "        # Creation du conteneur\n",
    "    container_client = blob_service_client.create_container(container_name)\n",
    "\n",
    "\n",
    "    \n",
    "    toblob_list=['data_map.pkl','items_df.pkl','i2vec.pkl','dic_ir.pkl','dic_ri.pkl']\n",
    "\n",
    "    # Create a blob client using the local file name as the name for the blob\n",
    "    for i in toblob_list:\n",
    "        local_file = i\n",
    "        upload_file_path = os.path.join(local_path, local_file)\n",
    "        blob_client = blob_service_client.get_blob_client(container=container_name, blob=local_file)\n",
    "\n",
    "        print(\"\\nUploading to Azure Storage as blob:\\n\\t\" + local_file)\n",
    "\n",
    "        # Upload the created file\n",
    "        with open(upload_file_path, \"rb\") as data:\n",
    "            blob_client.upload_blob(data)\n",
    "\n",
    "\n",
    "except Exception as ex:\n",
    "    print('Exception:')\n",
    "    print(ex)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "0e846b05-1416-4f27-844f-e09d389b7518",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Listing blobs...\n",
      "\tdata_map.pkl\n",
      "\tdic_ir.pkl\n",
      "\tdic_ri.pkl\n",
      "\ti2vec.pkl\n",
      "\titems_df.pkl\n"
     ]
    }
   ],
   "source": [
    "print(\"\\nListing blobs...\")\n",
    "\n",
    "# List the blobs in the container\n",
    "blob_list = container_client.list_blobs()\n",
    "for blob in blob_list:\n",
    "    print(\"\\t\" + blob.name)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1da4d4f1-4357-425f-a472-b738053a88a6",
   "metadata": {},
   "source": [
    "### recup des données du azure blob storage\n",
    "    * attention pas les memes fonctions pour les bindings azure functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "3070858c-5ef1-46b6-bf62-f1dee5469ac1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: total: 2.75 s\n",
      "Wall time: 11.6 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "#chargement du blob\n",
    "#blob_client = BlobClient.from_connection_string(connection_string, container_name, blob_name)\n",
    "blob_client = blob_service_client.get_blob_client(container=container_name, blob='data_map.pkl')\n",
    "data_map_=pickle.loads(blob_client.download_blob(0).readall())\n",
    "#blob_client = blob_service_client.get_blob_client(container=container_name, blob='items_df.pkl')\n",
    "#items_df_=pickle.loads(blob_client.download_blob(0).readall())\n",
    "blob_client = blob_service_client.get_blob_client(container=container_name, blob='i2vec.pkl')\n",
    "i2vec_=pickle.loads(blob_client.download_blob(0).readall())\n",
    "blob_client = blob_service_client.get_blob_client(container=container_name, blob='dic_ir.pkl')\n",
    "dic_ir_=pickle.loads(blob_client.download_blob(0).readall())\n",
    "blob_client = blob_service_client.get_blob_client(container=container_name, blob='dic_ri.pkl')\n",
    "dic_ri_=pickle.loads(blob_client.download_blob(0).readall())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "e939d786-b1ae-4af6-9166-c4bf7baa87ed",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'5130f488-4775-417a-b715-1eeafb91e572'"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "container_name"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "98c5f1f6-df4c-4547-979f-c08db16087d2",
   "metadata": {},
   "source": [
    "### content based classe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "d196203d-56a7-4049-b768-a818c93f4694",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from sklearn import preprocessing\n",
    "import scipy\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "\n",
    "class ContentBasedRecommender:\n",
    "    '''le fit calcule les users profiles'''\n",
    "    \n",
    "    MODEL_NAME = 'Content-Based'\n",
    "    \n",
    "\n",
    "    def __init__(self,data_map,i2vec,dic_ri,dic_ir):\n",
    "        '''constructeur'''\n",
    "        #dico inner raw iid\n",
    "        self.dic_ir = dic_ir\n",
    "        #dico raw inner iid\n",
    "        self.dic_ri = dic_ri\n",
    "        self.items_embedding = i2vec\n",
    "        #les train interactions servent à:\n",
    "            #calculer les profils\n",
    "            #ignorer les interactions dans la reco\n",
    "        self.train_user_interact = data_map\n",
    "        \n",
    "    ###############################################################\n",
    "    #profilage des users avec le fit\n",
    "        \n",
    "    def _build_users_profile(self,uid, click_df,emb_matrix,dic_ri):\n",
    "        '''calcul du profil embedding pour un uid\n",
    "        à partir des interactions du train'''\n",
    "        #on recupere le dataframe d'interaction pour l'uid ciblé\n",
    "        click_uid_df = click_df.loc[click_df.user_id==uid]\n",
    "        #on recupere les embeddings des items vus par l'uid\n",
    "        user_item_profiles = np.array([emb_matrix[dic_ri[iid]] for iid in click_uid_df['article_id']])\n",
    "        #on recupere le nombre de click sur les articles par l'uid afin d apporter une ponderation à la moyenne des embedding pour le calcul du profil\n",
    "        user_item_strengths = np.array(click_uid_df['nb']).reshape(-1,1) #-1 veut dire unknow dim\n",
    "        #on pondere la localisation embedding de chaque item par le nombre d'interactions puis on somme afin d'obtenir le profile qui est un barycentre\n",
    "        user_item_strengths_weighted_avg = np.sum(np.multiply(user_item_profiles,user_item_strengths), axis=0) / np.sum(user_item_strengths)\n",
    "        user_profile_norm = preprocessing.normalize(np.reshape(user_item_strengths_weighted_avg,(1,250)))\n",
    "        return user_profile_norm\n",
    "\n",
    "    def _build_users_profiles(self,click_df, emb_matrix,dic_ri): \n",
    "        '''calcul des profils de tous les uid sous forme de dic {uid:profil}\n",
    "        à partir des interactions du train'''\n",
    "        user_profiles = {}\n",
    "        for uid in click_df.user_id.unique():\n",
    "            user_profiles[uid] = self._build_users_profile(uid, click_df, emb_matrix, dic_ri)\n",
    "        return user_profiles\n",
    "    \n",
    "    def fit(self):\n",
    "        '''calcul des profils utilisateurs'''\n",
    "        self.user_profiles=self._build_users_profiles(self.train_user_interact,self.items_embedding,self.dic_ri)\n",
    "        #return self.user_profiles\n",
    "        \n",
    "    ###############################################################\n",
    "    \n",
    "    \n",
    "    def get_model_name(self):\n",
    "        '''renvoie le nom du modèle'''\n",
    "        return self.MODEL_NAME\n",
    "    \n",
    "\n",
    "    def _get_similar_items_to_user_profile(self, uid):#, topn=100000):\n",
    "        '''renvoie une liste d items similaires au profil de l'uid\n",
    "            liste de couples (raw iid, cosine) triés par cosine decroissant'''\n",
    "        #on calcul distances profil uid vs articles : sous forme 2D 1 ligne / nombre iid colonnes dans l'ordre inner du wemb\n",
    "        cosine_similarities = cosine_similarity(self.user_profiles[uid], self.items_embedding)\n",
    "        #on tri par distance: argsort donne les indices par scores croissants /flatten pour passer en 1D\n",
    "        #on recupère un array des inner_iid de taille topn : les plus proches du profil sont à droite\n",
    "        similar_indices = cosine_similarities.argsort().flatten()#[-topn:]\n",
    "                                                        ################################################################################\n",
    "        #on tri : sorted(iterable, key=None, reverse=False)\n",
    "        # l'iterable est une liste de couples (raw_iid, cosine similarites)\n",
    "        #on tri par similarité décroissante\n",
    "        similar_items = [(self.dic_ir[i], cosine_similarities[0,i]) for i in similar_indices[::-1]]\n",
    "        return similar_items\n",
    "    \n",
    "    #methode qui recommande en ignorant les articles deja vus\n",
    "    def recommend_items(self, uid, topn=5):\n",
    "        '''renvoie une liste de reco similaires au profil de l uid\n",
    "            triés par cosine decroissant\n",
    "             sans les interactions du train\n",
    "                sous forme de dataframe'''\n",
    "        #on recupere la liste brute de 1200 reco classés par cosine decroissant\n",
    "        similar_items = self._get_similar_items_to_user_profile(uid)\n",
    "        #on recupere la liste des iid a ignorer car deja vus\n",
    "        #items_to_ignore = self.train_user_interact.loc[self.train_user_interact.user_id==uid,\"article_id\"].tolist()\n",
    "        items_to_ignore = set(self.train_user_interact.loc[self.train_user_interact.user_id==uid].article_id)\n",
    "        #on enleve les deja vus\n",
    "        #similar_items_filtered = list(filter(lambda x: x[0] not in items_to_ignore, similar_items))\n",
    "        recommendations_df = pd.DataFrame(similar_items, columns=['article_id', 'cb_cosine_with_profile'])\n",
    "        reco = recommendations_df.loc[recommendations_df.article_id.isin(items_to_ignore)==False].copy()\n",
    "        if topn>0:\n",
    "            reco=reco.head(topn)\n",
    "        return reco"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ce29b31f-c578-437a-852c-3c0bdd446b6f",
   "metadata": {},
   "source": [
    "### colab filtering classe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "5d43203a-189f-43e9-bbb2-b0862bbe8e97",
   "metadata": {},
   "outputs": [],
   "source": [
    "from surprise import Reader, Dataset,SVDpp\n",
    "class CollabFiltRecommender:\n",
    "    \n",
    "    MODEL_NAME = 'Collaborative-Filtering-SVDpp'\n",
    "    \n",
    "    #constructeur\n",
    "    def __init__(self,data_map):\n",
    "        '''constructeur'''\n",
    "        #les train interactions servent à:\n",
    "            #calculer les profils\n",
    "            #ignorer les interactions dans la reco\n",
    "        self.train_user_interact = data_map\n",
    "        \n",
    "    #methode qui renvoie le nom du modele    \n",
    "    def get_model_name(self):\n",
    "        return self.MODEL_NAME\n",
    "    \n",
    "    #fit du modèle\n",
    "    def fit(self):\n",
    "        'fit du modele SVDpp'\n",
    "        #constructeur du reader\n",
    "        reader = Reader(rating_scale=(0,5))\n",
    "        #lecture du train par surprise\n",
    "        data = Dataset.load_from_df(self.train_user_interact[['user_id','article_id','nb']], reader)\n",
    "        #costruction du trainset surprise \n",
    "        trainset = data.build_full_trainset()\n",
    "        self.algo = SVDpp(n_epochs=20, lr_all=0.007, reg_all=0.1)\n",
    "        self.algo.fit(trainset)\n",
    "\n",
    "    \n",
    "    def recommend_items(self,uid,topn=5):\n",
    "        '''renvoie une liste de prediction pour un uid\n",
    "            le SVD ne peut sortir des prédictions que pour les uid et iid dispo dans le train'''\n",
    "        #prediction fonctionne avec les raws id\n",
    "        #https://towardsdatascience.com/difference-between-apply-and-transform-in-pandas-242e5cf32705\n",
    "        iid_to_ignore=set(self.train_user_interact.loc[self.train_user_interact.user_id==uid].article_id)\n",
    "        items2pred=pd.DataFrame(set(self.train_user_interact.article_id)-iid_to_ignore,columns=['article_id'])\n",
    "        items2pred['pred']=items2pred['article_id'].apply(lambda x:self.algo.predict(uid=uid, iid=x)[3])\n",
    "        #pred['detail']=pred['article_id'].apply(lambda x:cf.predict(uid=uid, iid=x)[4])\n",
    "        if topn==0:\n",
    "            recommendations_df=items2pred.loc[:,['article_id','pred']].sort_values(by='pred', ascending=False)\n",
    "        else:\n",
    "            recommendations_df=items2pred.loc[:,['article_id','pred']].sort_values(by='pred', ascending=False).head(topn)\n",
    "\n",
    "        return recommendations_df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0c5e86ca-b553-450e-9048-ca4cdc8678a3",
   "metadata": {},
   "source": [
    "### pop filtering classe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "02ef4fcc-f0ac-4c5c-b157-93cc8a744dd1",
   "metadata": {},
   "outputs": [],
   "source": [
    "class PopularityFiltRecommender:\n",
    "    \n",
    "    MODEL_NAME = 'Popularity-Filtering'\n",
    "    \n",
    "    #constructeur\n",
    "    def __init__(self,data_map):\n",
    "        '''constructeur'''\n",
    "        self.train_user_interact = data_map\n",
    "        \n",
    "    #methode qui renvoie le nom du modele    \n",
    "    def get_model_name(self):\n",
    "        return self.MODEL_NAME\n",
    "    \n",
    "    #fit du modèle\n",
    "    def fit(self):\n",
    "        self.raw_reco=self.train_user_interact[['nb','article_id']].groupby(['article_id']).sum().sort_values(by=['nb'],ascending=False).reset_index()\n",
    "\n",
    "    def recommend_items(self,uid,topn=5):\n",
    "        '''renvoie une liste de prediction pour un uid'''\n",
    "        if topn==0:\n",
    "            recommendations_df=self.raw_reco\n",
    "        else:\n",
    "            recommendations_df=self.raw_reco.head(topn)\n",
    "        return recommendations_df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a00da321-4016-4f6a-aea6-ff0c8c232d11",
   "metadata": {},
   "source": [
    "### hybrid filtering classe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "24f2feb4-8105-4ad3-8f2a-d7332652ebff",
   "metadata": {},
   "outputs": [],
   "source": [
    "class HybridRecommender:\n",
    "    \n",
    "    MODEL_NAME = 'Hybrid-Filtering'\n",
    "    \n",
    "    #constructeur\n",
    "    def __init__(self,data_map,i2vec,dic_ri,dic_ir):\n",
    "        '''constructeur'''\n",
    "        self.train_user_interact = data_map\n",
    "        self.dic_ir = dic_ir\n",
    "        self.dic_ri = dic_ri\n",
    "        self.items_embedding = i2vec\n",
    "    \n",
    "    def get_model_name(self):\n",
    "        return self.MODEL_NAME\n",
    "    \n",
    "    #fit du modèle\n",
    "    def fit(self):\n",
    "        '''le fit consiste a fitter les sous modèles'''\n",
    "        self.cf_model = CollabFiltRecommender(self.train_user_interact)\n",
    "        self.cf_model.fit()\n",
    "        self.cb_model = ContentBasedRecommender(self.train_user_interact,self.items_embedding,self.dic_ri,self.dic_ir)\n",
    "        self.cb_model.fit()\n",
    "        self.pf_model = PopularityFiltRecommender(self.train_user_interact)\n",
    "        self.pf_model.fit()\n",
    "        \n",
    "    def recommend_items(self,uid,topn=5):\n",
    "        '''renvoie une liste de prediction pour un uid'''\n",
    "        #si uid pas connu du train: seul le popularity recommander fonctionne dans notre cas\n",
    "        #on pourrait également concevoir un social reco basé sur un social profil, encore faudrait il avoir assez d 'infos'\n",
    "        if uid not in set(self.train_user_interact.user_id):\n",
    "            #parfait quand l'uid n'est pas connu du train, n a pas  d'interactions\n",
    "            reco=self.pf_model.recommend_items(uid,0)\n",
    "        else:\n",
    "                # content based : permet de donner des iid qui n'ont pas d'interactions mais qui étaient disponibles\n",
    "            reco_cb=self.cb_model.recommend_items(uid,0)\n",
    "                # pour la normalisation, le but est de garder les meilleurs, donc les outliers positifs\n",
    "                #il vaut mieux dans ce cas diviser par le max que par une standard deviation \n",
    "                #qui ne rendra pas forcement les outliers similaires entre facteurs pour une pondéaration\n",
    "            #reco_cb['norm_cb']=(reco_cb.cb_cosine_with_profile-reco_cb.cb_cosine_with_profile.median())/(reco_cb.cb_cosine_with_profile.max()-reco_cb.cb_cosine_with_profile.median())\n",
    "            reco_cb['rank_cb']=reco_cb.cb_cosine_with_profile.rank(ascending=False)\n",
    "                # donne de bons resultats mais necessite des uid et des iid qui ont des interactions dans le train\n",
    "            reco_cf=self.cf_model.recommend_items(uid,0)\n",
    "            #reco_cf['norm_cf']=(reco_cf.pred-reco_cf.pred.median())/(reco_cf.pred.max()-reco_cf.pred.median())\n",
    "            reco_cf['rank_cf']=reco_cf.pred.rank(ascending=False)\n",
    "                #comment les pondérer? 80/20 normalisé par exemple?\n",
    "                #en gros la pondération est anecdotique: on privilegie CF et quand on a pas de CF, on met la note de CB\n",
    "            reco = reco_cb.merge(reco_cf,how='outer', on='article_id')\n",
    "                #on fill les norm_cf vides (iid inconnu du train) par norm_cv avant pondération\n",
    "                #on pourait également filler les iid peu interagis afin de favoriser le content dans ce cas la\n",
    "            reco.loc[reco['rank_cf'].isnull(),'rank_cf'] = reco['rank_cb']\n",
    "            reco['rank']=0.99999*reco['rank_cf']+0.00001*reco['rank_cb']\n",
    "            \n",
    "            reco=reco.sort_values(by='rank', ascending=True)\n",
    "\n",
    "\n",
    "        if topn==0:\n",
    "            recommendations_df=reco\n",
    "        else:\n",
    "            recommendations_df=reco.head(topn)\n",
    "            \n",
    "        return recommendations_df.reset_index(drop=True)\n",
    "        #return recommendations_df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dc8d775c-7b3a-4f02-a3c4-ee7077a1f07e",
   "metadata": {},
   "source": [
    "### calcul top 5 pour tous les uid"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "06efd3be-d637-4ea9-b353-6a06b1839759",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: total: 6.52 s\n",
      "Wall time: 6.52 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "hr_model = HybridRecommender(data_map_,i2vec_,dic_ri_,dic_ir_)\n",
    "hr_model.fit()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "26dcd48e-1899-4c8d-b113-80cafdda23fd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "nb de users: 10457\n"
     ]
    }
   ],
   "source": [
    "print('nb de users:',len(set(data_map_.user_id)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "42fd42e6-8948-4a0d-9361-9859632ac459",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>article_id</th>\n",
       "      <th>cb_cosine_with_profile</th>\n",
       "      <th>rank_cb</th>\n",
       "      <th>pred</th>\n",
       "      <th>rank_cf</th>\n",
       "      <th>rank</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>274999</td>\n",
       "      <td>0.781554</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>207742</td>\n",
       "      <td>0.600974</td>\n",
       "      <td>2984.0</td>\n",
       "      <td>1.138642</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.02983</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>203789</td>\n",
       "      <td>0.773637</td>\n",
       "      <td>2.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>206967</td>\n",
       "      <td>0.773242</td>\n",
       "      <td>3.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>277263</td>\n",
       "      <td>0.214929</td>\n",
       "      <td>178959.0</td>\n",
       "      <td>1.113861</td>\n",
       "      <td>2.0</td>\n",
       "      <td>3.78957</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   article_id  cb_cosine_with_profile   rank_cb      pred  rank_cf     rank\n",
       "0      274999                0.781554       1.0       NaN      1.0  1.00000\n",
       "1      207742                0.600974    2984.0  1.138642      1.0  1.02983\n",
       "2      203789                0.773637       2.0       NaN      2.0  2.00000\n",
       "3      206967                0.773242       3.0       NaN      3.0  3.00000\n",
       "4      277263                0.214929  178959.0  1.113861      2.0  3.78957"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hr_model.recommend_items(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "e8a414c5-5387-4742-91fb-710067e6dd96",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: total: 2h 26min 47s1019 %%\n",
      "Wall time: 2h 13min 33s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "reco={}\n",
    "#0 n'étant pas un user id valide il servira à stocker le popularity reco\n",
    "reco[0]=hr_model.recommend_items(0).article_id.to_list()\n",
    "for c,i in enumerate(set(data_map_.user_id)):\n",
    "    if c%(int(len(set(data_map_.user_id))/100))==0:\n",
    "        print('users processed:' ,(100*c/len(set(data_map_.user_id))),'%',end='\\r')\n",
    "    reco[i]=hr_model.recommend_items(i).article_id.to_list()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "9343fa90-b221-45d7-bb86-9977233d2308",
   "metadata": {},
   "outputs": [],
   "source": [
    "#il faut obligatoiurement donner un id qui ne soi pas 0, en string ca passe\n",
    "reco_items_to_create=[]\n",
    "for key, value in reco.items():\n",
    "    reco_items_to_create.append({'id':str(key),'userId':key,'reco':value})\n",
    "    \n",
    "local_file='reco_items_to_create.pkl'\n",
    "file_path = os.path.join(local_file)\n",
    "with open(file_path, 'wb') as f:\n",
    "    pickle.dump(reco_items_to_create, f)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "64d1a84c-3583-468a-8d96-c16a41804b5c",
   "metadata": {},
   "source": [
    "### connexion et creation du conteneur cosmodb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "91651442-c967-47f8-9dc0-3a0aa3de9ea2",
   "metadata": {},
   "outputs": [],
   "source": [
    "from azure.cosmos import exceptions, CosmosClient, PartitionKey\n",
    "\n",
    "# Initialize the Cosmos client\n",
    "endpoint = \"https://nab-cosmodb-account.documents.azure.com:443/\"\n",
    "key = 'qMtk0HrT6bHZgHOeKyqAO8Q7LKlwgCdIn1oMIfpxkf9IqvJdT0ZGqfdzVdfRhBA2Sx7DgfMIoyrbqTlp87a4ew=='\n",
    "\n",
    "# <create_cosmos_client>\n",
    "client = CosmosClient(endpoint, key)\n",
    "# Create a database\n",
    "database_name = 'RecoP9Database'\n",
    "database = client.create_database_if_not_exists(id=database_name)\n",
    "# Create a container\n",
    "# en general il faut des partitions keys qui suporteront equitablement les requetes pour optimiser la charge\n",
    "# d'ou le choix d'un bon partition key\n",
    "container_name = 'RecoP9Container'\n",
    "container = database.create_container_if_not_exists(\n",
    "    id=container_name, \n",
    "    partition_key=PartitionKey(path=\"/userId\"),\n",
    "    #offer_throughput=400 -> pas dispo en serverless\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9bcc332a-c320-4ff4-8ef9-5e017419fcdd",
   "metadata": {},
   "source": [
    "### chargement des 10457 items dans cosmo db: item par item, c est pas le plus efficace, il doit exister un autre moyen"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "f66d6c10-d7f6-4a9e-b7d1-773d8f6a4687",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: total: 1min 1s\n",
      "Wall time: 29min 13s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# <create_item>\n",
    "for reco_item in reco_items_to_create:\n",
    "    container.create_item(body=reco_item)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "96cbe0b1-9c1f-438a-9380-abeeb4235ea4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Query returned 2 items. Operation consumed 2.95 request units\n",
      "[157519, 277263, 159495, 156690, 118773]\n"
     ]
    }
   ],
   "source": [
    "#test cosmodb\n",
    "query = \"SELECT c.reco FROM c WHERE c.id in ('7','0')\"\n",
    "\n",
    "items = list(container.query_items(\n",
    "    query=query,\n",
    "    enable_cross_partition_query=True\n",
    "))\n",
    "\n",
    "request_charge = container.client_connection.last_response_headers['x-ms-request-charge']\n",
    "\n",
    "print('Query returned {0} items. Operation consumed {1} request units'.format(len(items), request_charge))\n",
    "print(items[0]['reco'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "de3618ee-3a4e-4d8c-807c-99fa6c450d01",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'reco': [285781, 337465, 97948, 316239, 232088]}\n"
     ]
    }
   ],
   "source": [
    "\n",
    "\n",
    "try:\n",
    "    print(items[2])\n",
    "    pass\n",
    "except IndexError:\n",
    "    print(items[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "feb60c9b-c098-4f17-98a9-8f4b68b79da4",
   "metadata": {},
   "outputs": [],
   "source": [
    "\"SELECT c.reco FROM c WHERE c.userId in (0,StringToNumber({userId}))\","
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "env",
   "language": "python",
   "name": "env"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}