Data_Preprocess.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "framed-snake",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import random\n",
    "\n",
    "def dataset_movielens ():\n",
    "    data_dir=\"Movie-Lens-dataset\"\n",
    "    # my_data = pd.read_csv(r\"Movie-Lens-dataset\\u.data\", '\\t', names=['user_id', 'item_id', 'rating', 'timestamp'],engine='python')\n",
    "\n",
    "    names = ['user_id', 'item_id', 'rating', 'timestamp']\n",
    "    movie_db = pd.read_csv(os.path.join(data_dir, 'u.data'), '\\t', names=names,\n",
    "                           engine='python')\n",
    "    #creating time intervals\n",
    "    movie_db['date'] = pd.to_datetime(movie_db['timestamp'].astype(int), unit='s')\n",
    "    # movie_db['date1'] = pd.to_datetime(movie_db['date'],format=\"%m/%d/%Y  %H:%M:%S\").dt.strftime('%Y-%m-%d  %H:%M:%S')\n",
    "\n",
    "    # print(movie_db)\n",
    "\n",
    "    item_info=movie_db.groupby('item_id', as_index=False).rating.agg({np.mean, np.var})\n",
    "    item_info.reset_index(level=0, inplace=True)\n",
    "\n",
    "    # inject fake ratings\n",
    "    fake_db = pd.DataFrame( columns=['user_id', 'item_id', 'rating', 'date'])\n",
    "    filler_db=pd.DataFrame(columns=['user_id', 'item_id', 'rating', 'date'])\n",
    "    target_db=pd.DataFrame(columns=['user_id', 'item_id', 'rating', 'date'])\n",
    "    selected_db=pd.DataFrame(columns=['user_id', 'item_id', 'rating', 'date'])\n",
    "    fake_profile=[]\n",
    "    filler_size= 0.05 # from 1% to 10% which is 1% of items excluding target items\n",
    "    attack_size= 0.15\n",
    "    target_size=100\n",
    "    selected_item_size=20\n",
    "    attack_date=random.sample(list(movie_db['date'].unique()),1) \n",
    "    #     print('attack day is: ',attack_date)\n",
    "    #'1998-04-19'\n",
    "    #select day randomly, include all days of 8 months\n",
    "    x=0.01 ## percentage of fillers for AOP attack\n",
    "    attack_model='bandwagon' ## can be filled with one of : (random, average, bandwagon,AOP)\n",
    "    avg_rating=movie_db['rating'].mean() \n",
    "    std_rating=movie_db['rating'].std() \n",
    "    \n",
    "    # defining fake profiles\n",
    "    for i in range (round(attack_size*movie_db['user_id'].nunique())):\n",
    "        fake_profile.append(movie_db['user_id'].max()+(i+1))\n",
    "#     print('number of fake profiles generated is: \\n',round(attack_size*movie_db['user_id'].nunique()))\n",
    "\n",
    "    #defining target items\n",
    "    #30 items are choosen randomly out of 1682 items\n",
    "    target_items=random.sample(list(movie_db['item_id'].unique()),target_size)\n",
    "    #     print('number of target items selected is: \\n',target_size)\n",
    "    #defining filler items\n",
    "    fillers = [x for x in list(movie_db['item_id']) if x not in target_items]\n",
    "    #randomly choosed filler items based on filler size\n",
    "    filler_items=random.sample(fillers,round(filler_size*len(set(fillers))))\n",
    "\n",
    "    #insert ratings for filler items\n",
    "    filler_db.drop(filler_db.index, inplace=True)\n",
    "    for i in filler_items:\n",
    "        if attack_model=='random':\n",
    "            filler_db=filler_db.append(pd.DataFrame({'user_id': 0, 'item_id': i ,'rating':round(np.random.normal(avg_rating, std_rating),2),'date':[attack_date[0]]}), ignore_index=True)\n",
    "        if attack_model=='average':\n",
    "            filler_db=filler_db.append(pd.DataFrame({'user_id': 0, 'item_id': i ,'rating':item_info.loc[item_info['item_id']==i,'mean'],'date':[attack_date[0]]}), ignore_index=True)\n",
    "        if attack_model=='bandwagon':\n",
    "            filler_db=filler_db.append(pd.DataFrame({'user_id': 0, 'item_id': i ,'rating':round(np.random.normal(avg_rating, std_rating),2),'date':[attack_date[0]]}), ignore_index=True)\n",
    "        if attack_model=='AOP':\n",
    "            AOP_filler_db=pd.DataFrame(columns=['user_id', 'item_id', 'rating', 'date'])\n",
    "            AOP_fillers=movie_db.groupby('item_id', as_index=False)['rating'].count()\n",
    "            AOP_fillers=AOP_fillers.sort_values('rating',ascending=False)\n",
    "            x=0.01 ## percentage of fillers for AOP attack\n",
    "            AOP_fillers=AOP_fillers.head(int(len(AOP_fillers)*(x)))\n",
    "            AOP_fillers=[x for x in AOP_fillers['item_id']]\n",
    "#             if attack_model=='AOP':\n",
    "            for i in AOP_fillers:\n",
    "                AOP_filler_db=AOP_filler_db.append(pd.DataFrame({'user_id':random.sample(fake_profile,1), 'item_id': i ,'rating':np.random.normal(item_info.loc[item_info['item_id']==i,'mean'],item_info.loc[item_info['item_id']==i,'var']),'date1':[attack_date[0]]}), ignore_index=True)\n",
    "            filler_db=AOP_filler_db\n",
    "\n",
    "#             print('number of filler items selected for AOP attack is: \\n',round(len(set(AOP_fillers))))\n",
    "#         else:\n",
    "#             print('number of filler items selected is: \\n',round(filler_size*len(set(fillers))))\n",
    "\n",
    "\n",
    "    #insert ratings for target items\n",
    "    target_db.drop(target_db.index, inplace=True)\n",
    "    for i in target_items:\n",
    "        #print(random.sample(fake_profile,1),i,5)\n",
    "        target_db=target_db.append(pd.DataFrame({'user_id':0,'item_id':i,'rating':5,'date':[attack_date[0]]}),ignore_index=True)\n",
    "\n",
    "    injected_db=pd.DataFrame()\n",
    "    injected_db=injected_db.append(target_db)\n",
    "    injected_db=injected_db.append(filler_db)\n",
    "\n",
    "    injected_db\n",
    "    movie_db=movie_db.append(injected_db)\n",
    "\n",
    "    movie_db.fillna(0, inplace=True)\n",
    "    item_db=movie_db.groupby(['item_id','date'], as_index=False).rating.agg({np.sum})\n",
    "\n",
    "    item_db.index\n",
    "    item_db=item_db.reset_index(level=['item_id', 'date'])\n",
    "    item_db.columns = ['item_id', 'date','rating']\n",
    "\n",
    "    item_db=item_db.pivot(index='item_id',columns='date',values='rating')\n",
    "    item_db.fillna(0, inplace=True)\n",
    "    item_db=item_db.reset_index(level=['item_id'])\n",
    "    item_db['target']=0\n",
    "    item_db.loc[item_db.item_id.isin(target_items),'target']='1'\n",
    "    y=item_db[['target']]\n",
    "    x=item_db.drop('target', axis=1)\n",
    "    x.set_index('item_id', inplace=True)\n",
    "    x_column_names=x.columns\n",
    "    x.index.names = ['item_id']\n",
    "    y.index.names = ['item_id']\n",
    "    from sklearn import preprocessing\n",
    "\n",
    "#     x1 = x.values #returns a numpy array\n",
    "#     min_max_scaler = preprocessing.MinMaxScaler(feature_range=(0, 5))\n",
    "#     x_scaled = min_max_scaler.fit_transform(x1)\n",
    "#     x_scaled = pd.DataFrame(x_scaled,columns=x_column_names)\n",
    "#     x_scaled.index.names = ['item_id']\n",
    "    return (x,y)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "committed-norfolk",
   "metadata": {},
   "outputs": [],
   "source": [
    "def dataset_movielens_daily (target_size_var,filler_size_var,attack_size_var,attack_model_var):\n",
    "    data_dir=\"Movie-Lens-dataset\"\n",
    "    # my_data = pd.read_csv(r\"Movie-Lens-dataset\\u.data\", '\\t', names=['user_id', 'item_id', 'rating', 'timestamp'],engine='python')\n",
    "\n",
    "    names = ['user_id', 'item_id', 'rating', 'timestamp']\n",
    "    movie_db = pd.read_csv(os.path.join(data_dir, 'u.data'), '\\t', names=names,\n",
    "                           engine='python')\n",
    "    #creating time intervals\n",
    "    movie_db['date'] = pd.to_datetime(movie_db['timestamp'].astype(int), unit='s')\n",
    "    movie_db['date1'] = pd.to_datetime(movie_db['date'],format=\"%m/%d/%Y\").dt.strftime('%Y-%m-%d')\n",
    "\n",
    "    # print(movie_db)\n",
    "\n",
    "    item_info=movie_db.groupby('item_id', as_index=False).rating.agg({np.mean, np.var})\n",
    "    item_info.reset_index(level=0, inplace=True)\n",
    "\n",
    "    # inject fake ratings\n",
    "    fake_db = pd.DataFrame( columns=['user_id', 'item_id', 'rating', 'date1'])\n",
    "    filler_db=pd.DataFrame(columns=['user_id', 'item_id', 'rating', 'date1'])\n",
    "    target_db=pd.DataFrame(columns=['user_id', 'item_id', 'rating', 'date1'])\n",
    "    selected_db=pd.DataFrame(columns=['user_id', 'item_id', 'rating', 'date1'])\n",
    "    fake_profile=[]\n",
    "    filler_size= filler_size_var #0.05 # from 1% to 10% which is 1% of items excluding target items\n",
    "    attack_size= attack_size_var  #0.15\n",
    "    target_size=target_size_var#100\n",
    "    selected_item_size=20\n",
    "    attack_date=random.sample(list(movie_db['date1'].unique()),1) \n",
    "    #     print('attack day is: ',attack_date)\n",
    "    #'1998-04-19'\n",
    "    #select day randomly, include all days of 8 months\n",
    "    x=0.01 ## percentage of fillers for AOP attack\n",
    "    attack_model=attack_model_var  #'bandwagon' ## can be filled with one of : (random, average, bandwagon,AOP)\n",
    "    avg_rating=movie_db['rating'].mean() \n",
    "    std_rating=movie_db['rating'].std() \n",
    "    \n",
    "    # defining fake profiles\n",
    "    for i in range (round(attack_size*movie_db['user_id'].nunique())):\n",
    "        fake_profile.append(movie_db['user_id'].max()+(i+1))\n",
    "        \n",
    "    #defining target items\n",
    "    #30 items are choosen randomly out of 1682 items\n",
    "    target_items=random.sample(list(movie_db['item_id'].unique()),target_size)\n",
    "    #     print('number of target items selected is: \\n',target_size)\n",
    "    #defining filler items\n",
    "    fillers = [x for x in list(movie_db['item_id']) if x not in target_items]\n",
    "    #randomly choosed filler items based on filler size\n",
    "    filler_items=random.sample(fillers,round(filler_size*len(set(fillers))))\n",
    "\n",
    "    #insert ratings for filler items\n",
    "    filler_db.drop(filler_db.index, inplace=True)\n",
    "    for i in filler_items:\n",
    "        if attack_model=='random':\n",
    "            filler_db=filler_db.append(pd.DataFrame({'user_id': 0, 'item_id': i ,'rating':round(np.random.normal(avg_rating, std_rating),2),'date':[attack_date[0]]}), ignore_index=True)\n",
    "        if attack_model=='average':\n",
    "            filler_db=filler_db.append(pd.DataFrame({'user_id': 0, 'item_id': i ,'rating':item_info.loc[item_info['item_id']==i,'mean'],'date':[attack_date[0]]}), ignore_index=True)\n",
    "        if attack_model=='bandwagon':\n",
    "            filler_db=filler_db.append(pd.DataFrame({'user_id': 0, 'item_id': i ,'rating':round(np.random.normal(avg_rating, std_rating),2),'date':[attack_date[0]]}), ignore_index=True)\n",
    "        if attack_model=='AOP':\n",
    "            AOP_filler_db=pd.DataFrame(columns=['user_id', 'item_id', 'rating', 'date1'])\n",
    "            AOP_fillers=movie_db.groupby('item_id', as_index=False)['rating'].count()\n",
    "            AOP_fillers=AOP_fillers.sort_values('rating',ascending=False)\n",
    "            x=0.01 ## percentage of fillers for AOP attack\n",
    "            AOP_fillers=AOP_fillers.head(int(len(AOP_fillers)*(x)))\n",
    "            AOP_fillers=[x for x in AOP_fillers['item_id']]\n",
    "#             if attack_model=='AOP':\n",
    "            for i in AOP_fillers:\n",
    "                AOP_filler_db=AOP_filler_db.append(pd.DataFrame({'user_id':random.sample(fake_profile,1), 'item_id': i ,'rating':np.random.normal(item_info.loc[item_info['item_id']==i,'mean'],item_info.loc[item_info['item_id']==i,'var']),'date1':[attack_date[0]]}), ignore_index=True)\n",
    "            filler_db=AOP_filler_db\n",
    "\n",
    "#             print('number of filler items selected for AOP attack is: \\n',round(len(set(AOP_fillers))))\n",
    "#         else:\n",
    "#             print('number of filler items selected is: \\n',round(filler_size*len(set(fillers))))\n",
    "\n",
    "    #insert ratings for target items\n",
    "    target_db.drop(target_db.index, inplace=True)\n",
    "    for i in target_items:\n",
    "        #print(random.sample(fake_profile,1),i,5)\n",
    "        target_db=target_db.append(pd.DataFrame({'user_id':0,'item_id':i,'rating':5,'date':[attack_date[0]]}),ignore_index=True)\n",
    "\n",
    "    injected_db=pd.DataFrame()\n",
    "    injected_db=injected_db.append(target_db)\n",
    "    injected_db=injected_db.append(filler_db)\n",
    "\n",
    "    injected_db\n",
    "    movie_db=movie_db.append(injected_db)\n",
    "\n",
    "    movie_db.fillna(0, inplace=True)\n",
    "    item_db=movie_db.groupby(['item_id','date1'], as_index=False).rating.agg({np.sum})\n",
    "\n",
    "    item_db.index\n",
    "    item_db=item_db.reset_index(level=['item_id', 'date1'])\n",
    "    item_db.columns = ['item_id', 'date','rating']\n",
    "\n",
    "    item_db=item_db.pivot(index='item_id',columns='date',values='rating')\n",
    "    item_db.fillna(0, inplace=True)\n",
    "    item_db=item_db.reset_index(level=['item_id'])\n",
    "    item_db['target']=0\n",
    "    item_db.loc[item_db.item_id.isin(target_items),'target']='1'\n",
    "    y=item_db[['target']]\n",
    "    x=item_db.drop('target', axis=1)\n",
    "    x.set_index('item_id', inplace=True)\n",
    "    x_column_names=x.columns\n",
    "    x.index.names = ['item_id']\n",
    "    y.index.names = ['item_id']\n",
    "    from sklearn import preprocessing\n",
    "\n",
    "#     x1 = x.values #returns a numpy array\n",
    "#     min_max_scaler = preprocessing.MinMaxScaler(feature_range=(0, 5))\n",
    "#     x_scaled = min_max_scaler.fit_transform(x1)\n",
    "#     x_scaled = pd.DataFrame(x_scaled,columns=x_column_names)\n",
    "#     x_scaled.index.names = ['item_id']\n",
    "    return (x,y)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "saved-friendship",
   "metadata": {},
   "outputs": [],
   "source": [
    "def dataset_Netflix_daily (target_size_var,filler_size_var,attack_size_var,attack_model_var):\n",
    "    import random\n",
    "\n",
    "    \n",
    "    movie_db = pd.read_csv('C:/Users/Mahsa/Shilling_Attack_Detection_SNN/Netflix-dataset/netflix_sample.csv', names = ['user_id','rating','date','item_id'],header = None)\n",
    "#     movie_db = pd.read_csv('C:/Users/Mahsa/Shilling_Attack_Detection_SNN/Netflix-dataset/netflix_sample_complete.csv',skiprows=1, names = ['user_id','rating','date','item_id'],header = None)\n",
    "\n",
    "    selected_item_size=20\n",
    "    x=0.01 ## percentage of fillers for AOP attack\n",
    "    #attack_model='average' ## can be filled with one of : (random, average, bandwagon,AOP)\n",
    "    # avg_rating=movie_db['rating'].mean() \n",
    "    # std_rating=movie_db['rating'].std()  \n",
    "    names = ['user_id', 'item_id', 'rating', 'timestamp']\n",
    "\n",
    "    #creating time intervals\n",
    "    # movie_db['date'] = pd.to_datetime(movie_db['timestamp'].astype(int), unit='s')\n",
    "    movie_db['date1'] = movie_db['date'] #pd.to_datetime(movie_db['date'],format=\"%m/%d/%Y\").dt.strftime('%Y-%m-%d')\n",
    "\n",
    "    # print(movie_db)\n",
    "\n",
    "    item_info=movie_db.groupby('item_id', as_index=False).rating.agg({np.mean, np.var})\n",
    "    item_info.reset_index(level=0, inplace=True)\n",
    "\n",
    "    # inject fake ratings\n",
    "    fake_db = pd.DataFrame( columns=['user_id', 'item_id', 'rating', 'date1'])\n",
    "    filler_db=pd.DataFrame(columns=['user_id', 'item_id', 'rating', 'date1'])\n",
    "    target_db=pd.DataFrame(columns=['user_id', 'item_id', 'rating', 'date1'])\n",
    "    selected_db=pd.DataFrame(columns=['user_id', 'item_id', 'rating', 'date1'])\n",
    "    fake_profile=[]\n",
    "    filler_size= filler_size_var #0.05 # from 1% to 10% which is 1% of items excluding target items\n",
    "    attack_size= attack_size_var  #0.15\n",
    "    target_size=target_size_var#100\n",
    "    selected_item_size=20\n",
    "    attack_date=random.sample(list(movie_db['date1'].unique()),1) \n",
    "    #     print('attack day is: ',attack_date)\n",
    "    #'1998-04-19'\n",
    "    #select day randomly, include all days of 8 months\n",
    "    x=0.01 ## percentage of fillers for AOP attack\n",
    "    attack_model=attack_model_var  #'bandwagon' ## can be filled with one of : (random, average, bandwagon,AOP)\n",
    "    avg_rating=movie_db['rating'].mean() \n",
    "    std_rating=movie_db['rating'].std() \n",
    "\n",
    "    # defining fake profiles\n",
    "    for i in range (round(attack_size*movie_db['user_id'].nunique())):\n",
    "        fake_profile.append(movie_db['user_id'].max()+(i+1))\n",
    "\n",
    "    #defining target items\n",
    "    #30 items are choosen randomly out of 1682 items\n",
    "    target_items=random.sample(list(movie_db['item_id'].unique()),target_size)\n",
    "\n",
    "    #     print('number of target items selected is: \\n',target_size)\n",
    "    #defining filler items\n",
    "    fillers = [x for x in list(movie_db['item_id']) if x not in target_items]\n",
    "    #randomly choosed filler items based on filler size\n",
    "    filler_items=random.sample(fillers,round(filler_size*len(set(fillers))))\n",
    "\n",
    "    #insert ratings for filler items\n",
    "    filler_db.drop(filler_db.index, inplace=True)\n",
    "    for i in filler_items:\n",
    "        if attack_model=='random':\n",
    "            filler_db=filler_db.append(pd.DataFrame({'user_id': 0, 'item_id': i ,'rating':round(np.random.normal(avg_rating, std_rating),2),'date':[attack_date[0]]}), ignore_index=True)\n",
    "        if attack_model=='average':\n",
    "            filler_db=filler_db.append(pd.DataFrame({'user_id': 0, 'item_id': i ,'rating':item_info.loc[item_info['item_id']==i,'mean'],'date':[attack_date[0]]}), ignore_index=True)\n",
    "        if attack_model=='bandwagon':\n",
    "            filler_db=filler_db.append(pd.DataFrame({'user_id': 0, 'item_id': i ,'rating':round(np.random.normal(avg_rating, std_rating),2),'date':[attack_date[0]]}), ignore_index=True)\n",
    "        if attack_model=='AOP':\n",
    "            AOP_filler_db=pd.DataFrame(columns=['user_id', 'item_id', 'rating', 'date1'])\n",
    "            AOP_fillers=movie_db.groupby('item_id', as_index=False)['rating'].count()\n",
    "            AOP_fillers=AOP_fillers.sort_values('rating',ascending=False)\n",
    "            x=0.01 ## percentage of fillers for AOP attack\n",
    "            AOP_fillers=AOP_fillers.head(int(len(AOP_fillers)*(x)))\n",
    "            AOP_fillers=[x for x in AOP_fillers['item_id']]\n",
    "    #             if attack_model=='AOP':\n",
    "            for i in AOP_fillers:\n",
    "                AOP_filler_db=AOP_filler_db.append(pd.DataFrame({'user_id':random.sample(fake_profile,1), 'item_id': i ,'rating':np.random.normal(item_info.loc[item_info['item_id']==i,'mean'],item_info.loc[item_info['item_id']==i,'var']),'date1':[attack_date[0]]}), ignore_index=True)\n",
    "            filler_db=AOP_filler_db\n",
    "\n",
    "    #             print('number of filler items selected for AOP attack is: \\n',round(len(set(AOP_fillers))))\n",
    "    #         else:\n",
    "    #             print('number of filler items selected is: \\n',round(filler_size*len(set(fillers))))\n",
    "\n",
    "    #insert ratings for target items\n",
    "    target_db.drop(target_db.index, inplace=True)\n",
    "    for i in target_items:\n",
    "        #print(random.sample(fake_profile,1),i,5)\n",
    "        target_db=target_db.append(pd.DataFrame({'user_id':0,'item_id':i,'rating':5,'date':[attack_date[0]]}),ignore_index=True)\n",
    "\n",
    "    injected_db=pd.DataFrame()\n",
    "    injected_db=injected_db.append(target_db)\n",
    "    injected_db=injected_db.append(filler_db)\n",
    "\n",
    "    injected_db\n",
    "    movie_db=movie_db.append(injected_db)\n",
    "\n",
    "    movie_db.fillna(0, inplace=True)\n",
    "    item_db=movie_db.groupby(['item_id','date1'], as_index=False).rating.agg({np.sum})\n",
    "\n",
    "    item_db.index\n",
    "    item_db=item_db.reset_index(level=['item_id', 'date1'])\n",
    "    item_db.columns = ['item_id', 'date','rating']\n",
    "\n",
    "    item_db=item_db.pivot(index='item_id',columns='date',values='rating')\n",
    "    item_db.fillna(0, inplace=True)\n",
    "    item_db=item_db.reset_index(level=['item_id'])\n",
    "    item_db['target']=0\n",
    "    item_db.loc[item_db.item_id.isin(target_items),'target']='1'\n",
    "    y=item_db[['target']]\n",
    "    x=item_db.drop('target', axis=1)\n",
    "    x.set_index('item_id', inplace=True)\n",
    "    x_column_names=x.columns\n",
    "    x.index.names = ['item_id']\n",
    "    y.index.names = ['item_id']\n",
    "\n",
    "    return (x,y)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "excellent-macintosh",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}