memory_transofmer_KG_v5-0905.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "intense-router",
   "metadata": {},
   "source": [
    "# notebook for 0905 final dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "sophisticated-consumption",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import roc_curve, auc\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import numpy as np\n",
    "import torch\n",
    "from torch import nn\n",
    "import torch.nn.functional as F\n",
    "import pandas as pd\n",
    "from tqdm import tqdm\n",
    "from torch.utils.data import DataLoader, TensorDataset\n",
    "tqdm.pandas()\n",
    "import torch\n",
    "from torch import nn\n",
    "from torch.nn import Transformer\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "from torch.nn.utils.rnn import pad_sequence\n",
    "from sklearn.metrics import accuracy_score, f1_score, roc_auc_score\n",
    "from sklearn.model_selection import train_test_split\n",
    "import numpy as np\n",
    "\n",
    "from sklearn.metrics import average_precision_score\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "clinical-native",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "coastal-reality",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.0.1\n",
      "11.7\n",
      "Using GPU: NVIDIA A100-SXM4-40GB\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "print(torch.__version__)\n",
    "print(torch.version.cuda)\n",
    "\n",
    "# Check if CUDA (GPU support) is available\n",
    "if torch.cuda.is_available():\n",
    "    device = torch.device(\"cuda\")\n",
    "    print(\"Using GPU:\", torch.cuda.get_device_name(0))\n",
    "else:\n",
    "    device = torch.device(\"cpu\")\n",
    "    print(\"Using CPU\")\n",
    "\n",
    "# Now, you can use `device` to send your tensors to the GPU or CPU.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "valued-script",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GPU 0: NVIDIA A100-SXM4-40GB\n",
      "GPU 1: NVIDIA A100-SXM4-40GB\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "\n",
    "# Print available GPUs\n",
    "for i in range(torch.cuda.device_count()):\n",
    "    print(f\"GPU {i}: {torch.cuda.get_device_name(i)}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "vulnerable-installation",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "device(type='cuda')"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "device"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "hungarian-admission",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total Memory: 1007.45 GB | Used Memory: 1007.07 GB | Available Memory: 988.72 GB\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import psutil\n",
    "def memory_usage():\n",
    "    process = psutil.Process(os.getpid())\n",
    "    total_memory = psutil.virtual_memory().total / (1024**3)  # total RAM in GB\n",
    "    used_memory = total_memory - process.memory_info().rss / (1024**3)  # used RAM in GB\n",
    "    available_memory = psutil.virtual_memory().available / (1024**3)  # available RAM in GB\n",
    "    return f\"Total Memory: {total_memory:.2f} GB | Used Memory: {used_memory:.2f} GB | Available Memory: {available_memory:.2f} GB\"\n",
    "\n",
    "print(memory_usage())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "applied-implement",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# Custom Dataset\n",
    "class MyDataset(Dataset):\n",
    "    def __init__(self, src_data, tgt_data):\n",
    "        self.src_data = [torch.tensor(seq, dtype=torch.float32) for seq in src_data]\n",
    "        self.tgt_data = [torch.tensor(tgt, dtype=torch.float32) for tgt in tgt_data]\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.src_data)\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        return self.src_data[idx], self.tgt_data[idx]\n",
    "\n",
    "# Custom Collate function to handle variable sequence length\n",
    "def collate_fn(batch):\n",
    "    src, tgt = zip(*batch)\n",
    "    src = pad_sequence(src, batch_first=True, padding_value=0)\n",
    "    tgt = pad_sequence(tgt, batch_first=True, padding_value=-1)  # Special padding value for target data\n",
    "    return src, tgt\n",
    "\n",
    "\n",
    "def create_masks(src):\n",
    "    src_key_padding_mask = (src == 0).all(axis=-1)\n",
    "    \n",
    "    # future_mask shape should be (seq_len, seq_len)\n",
    "    future_mask = torch.triu(torch.ones((src.shape[1], src.shape[1])), diagonal=1).bool()\n",
    "\n",
    "    return src_key_padding_mask, future_mask\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "original-giant",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Transformer KG Model\n",
    "# first KG_dims are word embedding vectors, will be put into a linear layer first to reduce the dim to KG_compress_dims\n",
    "# the compressed word embedding will be concated with the other features , and go through transformers \n",
    "class TransformerKGModel(nn.Module):\n",
    "    def __init__(self, KG_dims, KG_compress_dims, input_dim, embedding_size, num_heads, num_layers, dropout):\n",
    "        super(TransformerKGModel, self).__init__()\n",
    "        self.compress  = nn.Linear(KG_dims, KG_compress_dims)\n",
    "        #self.concat_dims = KG_compress_dims\n",
    "        self.embedding = nn.Linear(input_dim, embedding_size)\n",
    "        self.transformer = Transformer(\n",
    "            d_model=embedding_size,\n",
    "            nhead=num_heads,\n",
    "            num_encoder_layers=num_layers,\n",
    "            num_decoder_layers=num_layers,\n",
    "            dropout=dropout,\n",
    "        )\n",
    "        self.fc = nn.Linear(embedding_size, 1)\n",
    "\n",
    "    def forward(self, src, src_key_padding_mask=None, future_mask=None):\n",
    "        \n",
    "        KG_part = src[:,:,:self.compress.in_features] # first KG_dims features are word embeddings\n",
    "        \n",
    "        \n",
    "        other_parts = src[:,:, self.compress.in_features:] # [seq_len, batch_size, feature_size]\n",
    "        # Compress KG_part  to KG_compress_dims\n",
    "        compressed = self.compress(KG_part)  \n",
    "        # Concatenate with second part\n",
    "        src = torch.cat((compressed, other_parts), dim=-1)  \n",
    "        # Reshape for transformer\n",
    "   \n",
    "        #src = src.unsqueeze(1)\n",
    "        \n",
    "\n",
    "        src = self.embedding(src)\n",
    "\n",
    "        src = src.transpose(0, 1) \n",
    "\n",
    "        output = self.transformer.encoder(src, src_key_padding_mask=src_key_padding_mask, mask = future_mask)\n",
    "\n",
    "        output = self.fc(output)\n",
    "\n",
    "        output = torch.sigmoid(output)\n",
    "\n",
    "        return output.transpose(0, 1).squeeze(-1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "former-happiness",
   "metadata": {},
   "outputs": [],
   "source": [
    "import sklearn.metrics as skmetrics\n",
    "def plot_roc(y_test, y_pred_prob, pos_label=True):\n",
    "    \"\"\"\n",
    "    Function to plot the ROC curve and display the AUC of \n",
    "    this predictor.\n",
    "    \"\"\"\n",
    "    fpr, tpr, _ = skmetrics.roc_curve(y_test, y_pred_prob, pos_label=pos_label)\n",
    "    roc_auc = skmetrics.auc(fpr, tpr)\n",
    "    plt.figure()\n",
    "    lw = 2\n",
    "    plt.plot(\n",
    "        fpr[2],\n",
    "        tpr[2],\n",
    "        color=\"darkorange\",\n",
    "        lw=lw,\n",
    "        label=\"ROC curve (area = %0.2f)\" % roc_auc,\n",
    "    )\n",
    "    plt.plot([0, 1], [0, 1], color=\"navy\", lw=lw, linestyle=\"--\")\n",
    "    plt.plot(fpr, tpr)\n",
    "    plt.xlim([0.0, 1.0])\n",
    "    plt.ylim([0.0, 1.05])\n",
    "    plt.xlabel(\"False Positive Rate\")\n",
    "    plt.ylabel(\"True Positive Rate\")\n",
    "    plt.title(\"Receiver operating characteristic example\")\n",
    "    plt.legend(loc=\"lower right\")\n",
    "    plt.show()\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "supreme-poetry",
   "metadata": {},
   "source": [
    "## data processing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "anonymous-chance",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "alpha-pursuit",
   "metadata": {},
   "outputs": [],
   "source": [
    "# read study data\n",
    "dataset = pd.read_csv(\"./data/new_data_sample_0905.csv\", sep=',', index_col=None)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "purple-reunion",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "39931"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(set(dataset.uid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "frequent-painting",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 36452215 entries, 0 to 36452214\n",
      "Data columns (total 18 columns):\n",
      " #   Column                            Dtype  \n",
      "---  ------                            -----  \n",
      " 0   uid                               object \n",
      " 1   register_time                     object \n",
      " 2   wid                               object \n",
      " 3   spelling                          object \n",
      " 4   difficulty                        int64  \n",
      " 5   review_time                       object \n",
      " 6   response                          int64  \n",
      " 7   study_method                      int64  \n",
      " 8   repeat_time_inday                 int64  \n",
      " 9   real_interval_history_byday       object \n",
      " 10  last_real_interval_byday          int64  \n",
      " 11  real_delta_interval_within_inday  int64  \n",
      " 12  review_day_th                     int64  \n",
      " 13  delta_t_between_inday             float64\n",
      " 14  delta_t_within_inday              float64\n",
      " 15  repeat_time_history               int64  \n",
      " 16  delta_interval_within_inday       float64\n",
      " 17  response_history                  object \n",
      "dtypes: float64(3), int64(8), object(7)\n",
      "memory usage: 4.9+ GB\n"
     ]
    }
   ],
   "source": [
    "dataset.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "played-elite",
   "metadata": {},
   "outputs": [],
   "source": [
    " # del dataset['repeat_time_inday']information leakage "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "headed-rainbow",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 36452215 entries, 0 to 36452214\n",
      "Data columns (total 18 columns):\n",
      " #   Column                            Dtype  \n",
      "---  ------                            -----  \n",
      " 0   uid                               object \n",
      " 1   register_time                     object \n",
      " 2   wid                               object \n",
      " 3   spelling                          object \n",
      " 4   difficulty                        int64  \n",
      " 5   review_time                       object \n",
      " 6   response                          int64  \n",
      " 7   study_method                      int64  \n",
      " 8   repeat_time_inday                 int64  \n",
      " 9   real_interval_history_byday       object \n",
      " 10  last_real_interval_byday          int64  \n",
      " 11  real_delta_interval_within_inday  int64  \n",
      " 12  review_day_th                     int64  \n",
      " 13  delta_t_between_inday             float64\n",
      " 14  delta_t_within_inday              float64\n",
      " 15  repeat_time_history               int64  \n",
      " 16  delta_interval_within_inday       float64\n",
      " 17  response_history                  object \n",
      "dtypes: float64(3), int64(8), object(7)\n",
      "memory usage: 4.9+ GB\n"
     ]
    }
   ],
   "source": [
    "dataset.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "advisory-wallet",
   "metadata": {},
   "source": [
    "### only use the words in the book list, there are a lot of phrases in the study logs , but they only take a very small proprotion in the dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "hearing-japan",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "24640"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(set(dataset.wid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "killing-angle",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Ensure the column is of string type\n",
    "dataset['spelling'] = dataset['spelling'].astype(str)\n",
    "\n",
    "# Replace NaNs with a placeholder string\n",
    "dataset['spelling'] = dataset['spelling'].fillna('')\n",
    "\n",
    "#sub_dataset = dataset[~dataset.spelling.str.contains(' ')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "athletic-printer",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 13428 entries, 0 to 13427\n",
      "Data columns (total 6 columns):\n",
      " #   Column     Non-Null Count  Dtype  \n",
      "---  ------     --------------  -----  \n",
      " 0   wid        13428 non-null  object \n",
      " 1   spelling   13428 non-null  object \n",
      " 2   rsp1_rate  13428 non-null  float64\n",
      " 3   rsp2_rate  13428 non-null  float64\n",
      " 4   rsp3_rate  13428 non-null  float64\n",
      " 5   rsp4_rate  13428 non-null  float64\n",
      "dtypes: float64(4), object(2)\n",
      "memory usage: 629.6+ KB\n"
     ]
    }
   ],
   "source": [
    "word_net_diffculty_dataset = pd.read_csv('./data/tmp_word_net_diffculty_dataset.tsv',sep='\\t')\n",
    "#word_net_diffculty_dataset = word_net_diffculty_dataset.drop_duplicates()\n",
    "word_net_diffculty_dataset.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "amateur-sheriff",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 6709 entries, 0 to 11647\n",
      "Data columns (total 2 columns):\n",
      " #   Column            Non-Null Count  Dtype  \n",
      "---  ------            --------------  -----  \n",
      " 0   wid               6709 non-null   object \n",
      " 1   acknowledge_rate  6709 non-null   float64\n",
      "dtypes: float64(1), object(1)\n",
      "memory usage: 157.2+ KB\n"
     ]
    }
   ],
   "source": [
    "word_net_acknowledge_dataset = pd.read_csv('./data/tmp_word_net_acknowledge_dataset.tsv',sep='\\t')\n",
    "word_net_acknowledge_dataset = word_net_acknowledge_dataset.drop_duplicates()\n",
    "word_net_acknowledge_dataset.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "signed-sequence",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>wid</th>\n",
       "      <th>acknowledge_rate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>57067bb0a172044907c61dc0</td>\n",
       "      <td>0.318209</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>57067ba5a172044907c6195b</td>\n",
       "      <td>0.560077</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>57067ba7a172044907c61a8a</td>\n",
       "      <td>0.342722</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>57067b8da172044907c60feb</td>\n",
       "      <td>0.707940</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>57067ba7a172044907c61afa</td>\n",
       "      <td>0.285558</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                        wid  acknowledge_rate\n",
       "0  57067bb0a172044907c61dc0          0.318209\n",
       "1  57067ba5a172044907c6195b          0.560077\n",
       "2  57067ba7a172044907c61a8a          0.342722\n",
       "3  57067b8da172044907c60feb          0.707940\n",
       "4  57067ba7a172044907c61afa          0.285558"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "word_net_acknowledge_dataset.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "documented-crash",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 6709 entries, 0 to 13427\n",
      "Data columns (total 7 columns):\n",
      " #   Column            Non-Null Count  Dtype  \n",
      "---  ------            --------------  -----  \n",
      " 0   wid               6709 non-null   object \n",
      " 1   spelling          6709 non-null   object \n",
      " 2   rsp1_rate         6709 non-null   float64\n",
      " 3   rsp2_rate         6709 non-null   float64\n",
      " 4   rsp3_rate         6709 non-null   float64\n",
      " 5   rsp4_rate         6709 non-null   float64\n",
      " 6   acknowledge_rate  6709 non-null   float64\n",
      "dtypes: float64(5), object(2)\n",
      "memory usage: 419.3+ KB\n"
     ]
    }
   ],
   "source": [
    "word_net_diffculty_ackknowledge_dataset = pd.merge(word_net_diffculty_dataset,word_net_acknowledge_dataset,on=['wid'],how='inner')\n",
    "word_net_diffculty_ackknowledge_dataset = word_net_diffculty_ackknowledge_dataset.drop_duplicates()\n",
    "word_net_diffculty_ackknowledge_dataset.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "oriented-european",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>wid</th>\n",
       "      <th>spelling</th>\n",
       "      <th>rsp1_rate</th>\n",
       "      <th>rsp2_rate</th>\n",
       "      <th>rsp3_rate</th>\n",
       "      <th>rsp4_rate</th>\n",
       "      <th>acknowledge_rate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>57067bb0a172044907c61dc0</td>\n",
       "      <td>preach</td>\n",
       "      <td>0.3624</td>\n",
       "      <td>0.1708</td>\n",
       "      <td>0.4665</td>\n",
       "      <td>0.0003</td>\n",
       "      <td>0.318209</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>57067ba5a172044907c6195b</td>\n",
       "      <td>arise</td>\n",
       "      <td>0.5498</td>\n",
       "      <td>0.1521</td>\n",
       "      <td>0.2977</td>\n",
       "      <td>0.0004</td>\n",
       "      <td>0.560077</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>57067ba7a172044907c61a8a</td>\n",
       "      <td>ingredient</td>\n",
       "      <td>0.3437</td>\n",
       "      <td>0.1730</td>\n",
       "      <td>0.4831</td>\n",
       "      <td>0.0003</td>\n",
       "      <td>0.342722</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>57067b8da172044907c60feb</td>\n",
       "      <td>application</td>\n",
       "      <td>0.6840</td>\n",
       "      <td>0.1079</td>\n",
       "      <td>0.2077</td>\n",
       "      <td>0.0005</td>\n",
       "      <td>0.707940</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>57067ba7a172044907c61afa</td>\n",
       "      <td>ingenious</td>\n",
       "      <td>0.4271</td>\n",
       "      <td>0.1814</td>\n",
       "      <td>0.3912</td>\n",
       "      <td>0.0003</td>\n",
       "      <td>0.285558</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                        wid     spelling  rsp1_rate  rsp2_rate  rsp3_rate  \\\n",
       "0  57067bb0a172044907c61dc0       preach     0.3624     0.1708     0.4665   \n",
       "2  57067ba5a172044907c6195b        arise     0.5498     0.1521     0.2977   \n",
       "4  57067ba7a172044907c61a8a   ingredient     0.3437     0.1730     0.4831   \n",
       "6  57067b8da172044907c60feb  application     0.6840     0.1079     0.2077   \n",
       "8  57067ba7a172044907c61afa    ingenious     0.4271     0.1814     0.3912   \n",
       "\n",
       "   rsp4_rate  acknowledge_rate  \n",
       "0     0.0003          0.318209  \n",
       "2     0.0004          0.560077  \n",
       "4     0.0003          0.342722  \n",
       "6     0.0005          0.707940  \n",
       "8     0.0003          0.285558  "
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "word_net_diffculty_ackknowledge_dataset.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "super-photograph",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 36301382 entries, 0 to 36301381\n",
      "Data columns (total 23 columns):\n",
      " #   Column                            Dtype  \n",
      "---  ------                            -----  \n",
      " 0   uid                               object \n",
      " 1   register_time                     object \n",
      " 2   wid                               object \n",
      " 3   spelling                          object \n",
      " 4   difficulty                        int64  \n",
      " 5   review_time                       object \n",
      " 6   response                          int64  \n",
      " 7   study_method                      int64  \n",
      " 8   repeat_time_inday                 int64  \n",
      " 9   real_interval_history_byday       object \n",
      " 10  last_real_interval_byday          int64  \n",
      " 11  real_delta_interval_within_inday  int64  \n",
      " 12  review_day_th                     int64  \n",
      " 13  delta_t_between_inday             float64\n",
      " 14  delta_t_within_inday              float64\n",
      " 15  repeat_time_history               int64  \n",
      " 16  delta_interval_within_inday       float64\n",
      " 17  response_history                  object \n",
      " 18  rsp1_rate                         float64\n",
      " 19  rsp2_rate                         float64\n",
      " 20  rsp3_rate                         float64\n",
      " 21  rsp4_rate                         float64\n",
      " 22  acknowledge_rate                  float64\n",
      "dtypes: float64(8), int64(8), object(7)\n",
      "memory usage: 6.5+ GB\n"
     ]
    }
   ],
   "source": [
    "dataset = pd.merge(dataset,word_net_diffculty_ackknowledge_dataset, on=['wid','spelling'],how='inner')\n",
    "dataset.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "temporal-product",
   "metadata": {},
   "source": [
    "### add KG embedding and filter thw word sets again"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "objective-ranking",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 6685 entries, 0 to 6684\n",
      "Data columns (total 65 columns):\n",
      " #   Column   Non-Null Count  Dtype  \n",
      "---  ------   --------------  -----  \n",
      " 0   w        6685 non-null   object \n",
      " 1   embed1   6685 non-null   float64\n",
      " 2   embed2   6685 non-null   float64\n",
      " 3   embed3   6685 non-null   float64\n",
      " 4   embed4   6685 non-null   float64\n",
      " 5   embed5   6685 non-null   float64\n",
      " 6   embed6   6685 non-null   float64\n",
      " 7   embed7   6685 non-null   float64\n",
      " 8   embed8   6685 non-null   float64\n",
      " 9   embed9   6685 non-null   float64\n",
      " 10  embed10  6685 non-null   float64\n",
      " 11  embed11  6685 non-null   float64\n",
      " 12  embed12  6685 non-null   float64\n",
      " 13  embed13  6685 non-null   float64\n",
      " 14  embed14  6685 non-null   float64\n",
      " 15  embed15  6685 non-null   float64\n",
      " 16  embed16  6685 non-null   float64\n",
      " 17  embed17  6685 non-null   float64\n",
      " 18  embed18  6685 non-null   float64\n",
      " 19  embed19  6685 non-null   float64\n",
      " 20  embed20  6685 non-null   float64\n",
      " 21  embed21  6685 non-null   float64\n",
      " 22  embed22  6685 non-null   float64\n",
      " 23  embed23  6685 non-null   float64\n",
      " 24  embed24  6685 non-null   float64\n",
      " 25  embed25  6685 non-null   float64\n",
      " 26  embed26  6685 non-null   float64\n",
      " 27  embed27  6685 non-null   float64\n",
      " 28  embed28  6685 non-null   float64\n",
      " 29  embed29  6685 non-null   float64\n",
      " 30  embed30  6685 non-null   float64\n",
      " 31  embed31  6685 non-null   float64\n",
      " 32  embed32  6685 non-null   float64\n",
      " 33  embed33  6685 non-null   float64\n",
      " 34  embed34  6685 non-null   float64\n",
      " 35  embed35  6685 non-null   float64\n",
      " 36  embed36  6685 non-null   float64\n",
      " 37  embed37  6685 non-null   float64\n",
      " 38  embed38  6685 non-null   float64\n",
      " 39  embed39  6685 non-null   float64\n",
      " 40  embed40  6685 non-null   float64\n",
      " 41  embed41  6685 non-null   float64\n",
      " 42  embed42  6685 non-null   float64\n",
      " 43  embed43  6685 non-null   float64\n",
      " 44  embed44  6685 non-null   float64\n",
      " 45  embed45  6685 non-null   float64\n",
      " 46  embed46  6685 non-null   float64\n",
      " 47  embed47  6685 non-null   float64\n",
      " 48  embed48  6685 non-null   float64\n",
      " 49  embed49  6685 non-null   float64\n",
      " 50  embed50  6685 non-null   float64\n",
      " 51  embed51  6685 non-null   float64\n",
      " 52  embed52  6685 non-null   float64\n",
      " 53  embed53  6685 non-null   float64\n",
      " 54  embed54  6685 non-null   float64\n",
      " 55  embed55  6685 non-null   float64\n",
      " 56  embed56  6685 non-null   float64\n",
      " 57  embed57  6685 non-null   float64\n",
      " 58  embed58  6685 non-null   float64\n",
      " 59  embed59  6685 non-null   float64\n",
      " 60  embed60  6685 non-null   float64\n",
      " 61  embed61  6685 non-null   float64\n",
      " 62  embed62  6685 non-null   float64\n",
      " 63  embed63  6685 non-null   float64\n",
      " 64  embed64  6685 non-null   float64\n",
      "dtypes: float64(64), object(1)\n",
      "memory usage: 3.3+ MB\n"
     ]
    }
   ],
   "source": [
    "# add KG embedding and filter thw word sets again\n",
    "KGembedding = pd.read_csv('./data/KGembeddings.csv')\n",
    "KGembedding.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "freelance-japan",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>spelling</th>\n",
       "      <th>embed1</th>\n",
       "      <th>embed2</th>\n",
       "      <th>embed3</th>\n",
       "      <th>embed4</th>\n",
       "      <th>embed5</th>\n",
       "      <th>embed6</th>\n",
       "      <th>embed7</th>\n",
       "      <th>embed8</th>\n",
       "      <th>embed9</th>\n",
       "      <th>...</th>\n",
       "      <th>embed55</th>\n",
       "      <th>embed56</th>\n",
       "      <th>embed57</th>\n",
       "      <th>embed58</th>\n",
       "      <th>embed59</th>\n",
       "      <th>embed60</th>\n",
       "      <th>embed61</th>\n",
       "      <th>embed62</th>\n",
       "      <th>embed63</th>\n",
       "      <th>embed64</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>a</td>\n",
       "      <td>-0.112005</td>\n",
       "      <td>0.175023</td>\n",
       "      <td>0.081303</td>\n",
       "      <td>0.119429</td>\n",
       "      <td>0.234358</td>\n",
       "      <td>-0.285393</td>\n",
       "      <td>0.088960</td>\n",
       "      <td>-0.084222</td>\n",
       "      <td>-0.146858</td>\n",
       "      <td>...</td>\n",
       "      <td>0.121453</td>\n",
       "      <td>0.046255</td>\n",
       "      <td>0.027163</td>\n",
       "      <td>0.253145</td>\n",
       "      <td>0.250952</td>\n",
       "      <td>0.395841</td>\n",
       "      <td>0.263606</td>\n",
       "      <td>-0.024474</td>\n",
       "      <td>-0.125878</td>\n",
       "      <td>-0.034926</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>abandon</td>\n",
       "      <td>-0.400461</td>\n",
       "      <td>0.727612</td>\n",
       "      <td>0.237638</td>\n",
       "      <td>0.407411</td>\n",
       "      <td>1.023871</td>\n",
       "      <td>-1.093308</td>\n",
       "      <td>0.371002</td>\n",
       "      <td>-0.282474</td>\n",
       "      <td>-0.603780</td>\n",
       "      <td>...</td>\n",
       "      <td>0.572360</td>\n",
       "      <td>0.213719</td>\n",
       "      <td>0.139102</td>\n",
       "      <td>0.971377</td>\n",
       "      <td>0.966684</td>\n",
       "      <td>1.702201</td>\n",
       "      <td>1.043585</td>\n",
       "      <td>-0.034144</td>\n",
       "      <td>-0.499250</td>\n",
       "      <td>-0.070479</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>abdomen</td>\n",
       "      <td>-0.485593</td>\n",
       "      <td>1.103594</td>\n",
       "      <td>0.264719</td>\n",
       "      <td>0.511833</td>\n",
       "      <td>1.521983</td>\n",
       "      <td>-1.432766</td>\n",
       "      <td>0.503473</td>\n",
       "      <td>-0.349383</td>\n",
       "      <td>-0.807522</td>\n",
       "      <td>...</td>\n",
       "      <td>0.910728</td>\n",
       "      <td>0.306264</td>\n",
       "      <td>0.270944</td>\n",
       "      <td>1.298496</td>\n",
       "      <td>1.351456</td>\n",
       "      <td>2.541386</td>\n",
       "      <td>1.477359</td>\n",
       "      <td>0.069351</td>\n",
       "      <td>-0.689769</td>\n",
       "      <td>-0.026549</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>abide</td>\n",
       "      <td>-0.479631</td>\n",
       "      <td>1.098984</td>\n",
       "      <td>0.261993</td>\n",
       "      <td>0.506807</td>\n",
       "      <td>1.512208</td>\n",
       "      <td>-1.418678</td>\n",
       "      <td>0.499232</td>\n",
       "      <td>-0.345371</td>\n",
       "      <td>-0.799597</td>\n",
       "      <td>...</td>\n",
       "      <td>0.907186</td>\n",
       "      <td>0.303954</td>\n",
       "      <td>0.270819</td>\n",
       "      <td>1.286539</td>\n",
       "      <td>1.342442</td>\n",
       "      <td>2.527811</td>\n",
       "      <td>1.467622</td>\n",
       "      <td>0.073551</td>\n",
       "      <td>-0.683922</td>\n",
       "      <td>-0.024235</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ability</td>\n",
       "      <td>-0.257335</td>\n",
       "      <td>0.465007</td>\n",
       "      <td>0.156881</td>\n",
       "      <td>0.264888</td>\n",
       "      <td>0.649716</td>\n",
       "      <td>-0.698882</td>\n",
       "      <td>0.233898</td>\n",
       "      <td>-0.185201</td>\n",
       "      <td>-0.381730</td>\n",
       "      <td>...</td>\n",
       "      <td>0.361201</td>\n",
       "      <td>0.133533</td>\n",
       "      <td>0.090380</td>\n",
       "      <td>0.622238</td>\n",
       "      <td>0.620555</td>\n",
       "      <td>1.080868</td>\n",
       "      <td>0.667867</td>\n",
       "      <td>-0.024280</td>\n",
       "      <td>-0.319865</td>\n",
       "      <td>-0.049535</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 65 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "  spelling    embed1    embed2    embed3    embed4    embed5    embed6  \\\n",
       "0        a -0.112005  0.175023  0.081303  0.119429  0.234358 -0.285393   \n",
       "1  abandon -0.400461  0.727612  0.237638  0.407411  1.023871 -1.093308   \n",
       "2  abdomen -0.485593  1.103594  0.264719  0.511833  1.521983 -1.432766   \n",
       "3    abide -0.479631  1.098984  0.261993  0.506807  1.512208 -1.418678   \n",
       "4  ability -0.257335  0.465007  0.156881  0.264888  0.649716 -0.698882   \n",
       "\n",
       "     embed7    embed8    embed9  ...   embed55   embed56   embed57   embed58  \\\n",
       "0  0.088960 -0.084222 -0.146858  ...  0.121453  0.046255  0.027163  0.253145   \n",
       "1  0.371002 -0.282474 -0.603780  ...  0.572360  0.213719  0.139102  0.971377   \n",
       "2  0.503473 -0.349383 -0.807522  ...  0.910728  0.306264  0.270944  1.298496   \n",
       "3  0.499232 -0.345371 -0.799597  ...  0.907186  0.303954  0.270819  1.286539   \n",
       "4  0.233898 -0.185201 -0.381730  ...  0.361201  0.133533  0.090380  0.622238   \n",
       "\n",
       "    embed59   embed60   embed61   embed62   embed63   embed64  \n",
       "0  0.250952  0.395841  0.263606 -0.024474 -0.125878 -0.034926  \n",
       "1  0.966684  1.702201  1.043585 -0.034144 -0.499250 -0.070479  \n",
       "2  1.351456  2.541386  1.477359  0.069351 -0.689769 -0.026549  \n",
       "3  1.342442  2.527811  1.467622  0.073551 -0.683922 -0.024235  \n",
       "4  0.620555  1.080868  0.667867 -0.024280 -0.319865 -0.049535  \n",
       "\n",
       "[5 rows x 65 columns]"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "KGembedding.rename(columns={'w':'spelling'},inplace=True)\n",
    "KGembedding.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "scenic-crossing",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 34690422 entries, 0 to 34690421\n",
      "Data columns (total 87 columns):\n",
      " #   Column                            Dtype  \n",
      "---  ------                            -----  \n",
      " 0   spelling                          object \n",
      " 1   embed1                            float64\n",
      " 2   embed2                            float64\n",
      " 3   embed3                            float64\n",
      " 4   embed4                            float64\n",
      " 5   embed5                            float64\n",
      " 6   embed6                            float64\n",
      " 7   embed7                            float64\n",
      " 8   embed8                            float64\n",
      " 9   embed9                            float64\n",
      " 10  embed10                           float64\n",
      " 11  embed11                           float64\n",
      " 12  embed12                           float64\n",
      " 13  embed13                           float64\n",
      " 14  embed14                           float64\n",
      " 15  embed15                           float64\n",
      " 16  embed16                           float64\n",
      " 17  embed17                           float64\n",
      " 18  embed18                           float64\n",
      " 19  embed19                           float64\n",
      " 20  embed20                           float64\n",
      " 21  embed21                           float64\n",
      " 22  embed22                           float64\n",
      " 23  embed23                           float64\n",
      " 24  embed24                           float64\n",
      " 25  embed25                           float64\n",
      " 26  embed26                           float64\n",
      " 27  embed27                           float64\n",
      " 28  embed28                           float64\n",
      " 29  embed29                           float64\n",
      " 30  embed30                           float64\n",
      " 31  embed31                           float64\n",
      " 32  embed32                           float64\n",
      " 33  embed33                           float64\n",
      " 34  embed34                           float64\n",
      " 35  embed35                           float64\n",
      " 36  embed36                           float64\n",
      " 37  embed37                           float64\n",
      " 38  embed38                           float64\n",
      " 39  embed39                           float64\n",
      " 40  embed40                           float64\n",
      " 41  embed41                           float64\n",
      " 42  embed42                           float64\n",
      " 43  embed43                           float64\n",
      " 44  embed44                           float64\n",
      " 45  embed45                           float64\n",
      " 46  embed46                           float64\n",
      " 47  embed47                           float64\n",
      " 48  embed48                           float64\n",
      " 49  embed49                           float64\n",
      " 50  embed50                           float64\n",
      " 51  embed51                           float64\n",
      " 52  embed52                           float64\n",
      " 53  embed53                           float64\n",
      " 54  embed54                           float64\n",
      " 55  embed55                           float64\n",
      " 56  embed56                           float64\n",
      " 57  embed57                           float64\n",
      " 58  embed58                           float64\n",
      " 59  embed59                           float64\n",
      " 60  embed60                           float64\n",
      " 61  embed61                           float64\n",
      " 62  embed62                           float64\n",
      " 63  embed63                           float64\n",
      " 64  embed64                           float64\n",
      " 65  uid                               object \n",
      " 66  register_time                     object \n",
      " 67  wid                               object \n",
      " 68  difficulty                        int64  \n",
      " 69  review_time                       object \n",
      " 70  response                          int64  \n",
      " 71  study_method                      int64  \n",
      " 72  repeat_time_inday                 int64  \n",
      " 73  real_interval_history_byday       object \n",
      " 74  last_real_interval_byday          int64  \n",
      " 75  real_delta_interval_within_inday  int64  \n",
      " 76  review_day_th                     int64  \n",
      " 77  delta_t_between_inday             float64\n",
      " 78  delta_t_within_inday              float64\n",
      " 79  repeat_time_history               int64  \n",
      " 80  delta_interval_within_inday       float64\n",
      " 81  response_history                  object \n",
      " 82  rsp1_rate                         float64\n",
      " 83  rsp2_rate                         float64\n",
      " 84  rsp3_rate                         float64\n",
      " 85  rsp4_rate                         float64\n",
      " 86  acknowledge_rate                  float64\n",
      "dtypes: float64(72), int64(8), object(7)\n",
      "memory usage: 22.7+ GB\n"
     ]
    }
   ],
   "source": [
    "dataset = pd.merge(KGembedding,dataset, on=['spelling'],how='inner')\n",
    "dataset.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "potential-japan",
   "metadata": {},
   "source": [
    "### check the distribution of word review counts per user\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "horizontal-vulnerability",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "coordinate-comparative",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 percentile: 1.0\n",
      "1 percentile: 3.0\n",
      "5 percentile: 20.0\n",
      "10 percentile: 30.0\n",
      "20 percentile: 73.0\n",
      "30 percentile: 148.0\n",
      "40 percentile: 266.0\n",
      "50 percentile: 437.0\n",
      "60 percentile: 639.0\n",
      "70 percentile: 921.0\n",
      "80 percentile: 1364.0\n",
      "90 percentile: 2265.0\n",
      "95 percentile: 3266.8499999999985\n",
      "99 percentile: 5761.549999999952\n",
      "100 percentile: 27652.0\n"
     ]
    }
   ],
   "source": [
    "data_gp = dataset.groupby(['uid']).count().sort_values(by='register_time')\n",
    "\n",
    "\n",
    "percentiles_list = list(range(0, 101, 10)) + [1,5,95 ,99]  # includes 1st and 99th percentile\n",
    "percentiles_list.sort()  # make sure the percentiles are in ascending order\n",
    "\n",
    "percentiles = {i: data_gp['wid'].quantile(i/100) for i in percentiles_list}\n",
    "\n",
    "# Now percentiles is a dictionary that contains the 0th, 1st, 10th, 20th, ..., 99th, 100th percentile of 'column_name'\n",
    "\n",
    "for percentile in percentiles_list:\n",
    "    print(f'{percentile} percentile: {percentiles[percentile]}')\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "coastal-number",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "billion-filename",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "33700"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# only valid users : filtering out abnormal users\n",
    "valid_uids = list(data_gp[(data_gp.register_time<=3000)&(data_gp.register_time>=30)].index)\n",
    "len(valid_uids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "higher-parent",
   "metadata": {},
   "outputs": [],
   "source": [
    "#subselect the dataset which only has valid users\n",
    "dataset= dataset[dataset.uid.isin(valid_uids)]\n",
    "dataset[\"review_time\"] = pd.to_datetime(dataset[\"review_time\"] )\n",
    "dataset[\"register_time\"] = pd.to_datetime(dataset[\"register_time\"] )\n",
    "dataset = dataset.sort_values(by=['uid', 'review_time'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "contained-tender",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 23656968/23656968 [00:12<00:00, 1924209.04it/s]\n"
     ]
    }
   ],
   "source": [
    "# process Y\n",
    "dataset['Y'] = dataset.response.progress_apply(lambda x : 1 if x ==1 else 0)\n",
    "\n",
    "# lower memory\n",
    "for column in dataset.select_dtypes(include=['int64']):\n",
    "    dataset[column] = dataset[column].astype('int8')\n",
    "for column in dataset.select_dtypes(include=['float64']):\n",
    "    dataset[column] = dataset[column].astype('float32')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "handmade-probe",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6180"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "valid_word = list(set(dataset['spelling'].values))\n",
    "len(valid_word)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "danish-assault",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "coordinate-abuse",
   "metadata": {},
   "source": [
    "### create the last response(0/1) as an input feature for next time step. For the first-time review's last response , use the acknowledge rate for the word instead"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "resistant-programmer",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_last_response(x):\n",
    "    response_history = x[0]\n",
    "    acknowledge_rate = x[1]\n",
    "    try:\n",
    "        last_response = int(response_history[-2])\n",
    "        if last_response ==1 :\n",
    "            None \n",
    "        else:\n",
    "            last_response=0\n",
    "    except : \n",
    "        last_response = int(acknowledge_rate>0.5)\n",
    "    return last_response"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "mental-legend",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 23656968/23656968 [01:28<00:00, 266848.96it/s]\n"
     ]
    }
   ],
   "source": [
    "dataset['last_response'] = dataset[['response_history','acknowledge_rate']].progress_apply(get_last_response, axis =1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "signal-corpus",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>spelling</th>\n",
       "      <th>embed1</th>\n",
       "      <th>embed2</th>\n",
       "      <th>embed3</th>\n",
       "      <th>embed4</th>\n",
       "      <th>embed5</th>\n",
       "      <th>embed6</th>\n",
       "      <th>embed7</th>\n",
       "      <th>embed8</th>\n",
       "      <th>embed9</th>\n",
       "      <th>...</th>\n",
       "      <th>repeat_time_history</th>\n",
       "      <th>delta_interval_within_inday</th>\n",
       "      <th>response_history</th>\n",
       "      <th>rsp1_rate</th>\n",
       "      <th>rsp2_rate</th>\n",
       "      <th>rsp3_rate</th>\n",
       "      <th>rsp4_rate</th>\n",
       "      <th>acknowledge_rate</th>\n",
       "      <th>Y</th>\n",
       "      <th>last_response</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>7021226</th>\n",
       "      <td>console</td>\n",
       "      <td>-0.474705</td>\n",
       "      <td>1.094874</td>\n",
       "      <td>0.258717</td>\n",
       "      <td>0.503144</td>\n",
       "      <td>1.507025</td>\n",
       "      <td>-1.408898</td>\n",
       "      <td>0.496110</td>\n",
       "      <td>-0.342881</td>\n",
       "      <td>-0.794755</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.4154</td>\n",
       "      <td>0.1731</td>\n",
       "      <td>0.4112</td>\n",
       "      <td>0.0003</td>\n",
       "      <td>0.303781</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6836703</th>\n",
       "      <td>consist</td>\n",
       "      <td>-0.470568</td>\n",
       "      <td>0.962991</td>\n",
       "      <td>0.260600</td>\n",
       "      <td>0.480988</td>\n",
       "      <td>1.344495</td>\n",
       "      <td>-1.333904</td>\n",
       "      <td>0.460674</td>\n",
       "      <td>-0.332445</td>\n",
       "      <td>-0.745455</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.5524</td>\n",
       "      <td>0.1576</td>\n",
       "      <td>0.2895</td>\n",
       "      <td>0.0005</td>\n",
       "      <td>0.563981</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6904211</th>\n",
       "      <td>consistent</td>\n",
       "      <td>-0.404658</td>\n",
       "      <td>0.898675</td>\n",
       "      <td>0.222523</td>\n",
       "      <td>0.423722</td>\n",
       "      <td>1.239092</td>\n",
       "      <td>-1.180395</td>\n",
       "      <td>0.411884</td>\n",
       "      <td>-0.290802</td>\n",
       "      <td>-0.661933</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.4417</td>\n",
       "      <td>0.1619</td>\n",
       "      <td>0.3961</td>\n",
       "      <td>0.0003</td>\n",
       "      <td>0.457183</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7120550</th>\n",
       "      <td>constant</td>\n",
       "      <td>-0.438270</td>\n",
       "      <td>0.956392</td>\n",
       "      <td>0.241749</td>\n",
       "      <td>0.456376</td>\n",
       "      <td>1.321626</td>\n",
       "      <td>-1.270601</td>\n",
       "      <td>0.442882</td>\n",
       "      <td>-0.313167</td>\n",
       "      <td>-0.712319</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.4470</td>\n",
       "      <td>0.1674</td>\n",
       "      <td>0.3853</td>\n",
       "      <td>0.0003</td>\n",
       "      <td>0.443431</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7233272</th>\n",
       "      <td>constituent</td>\n",
       "      <td>-0.465048</td>\n",
       "      <td>1.062780</td>\n",
       "      <td>0.253318</td>\n",
       "      <td>0.490704</td>\n",
       "      <td>1.463766</td>\n",
       "      <td>-1.374023</td>\n",
       "      <td>0.482891</td>\n",
       "      <td>-0.335000</td>\n",
       "      <td>-0.774168</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.3894</td>\n",
       "      <td>0.2085</td>\n",
       "      <td>0.4019</td>\n",
       "      <td>0.0003</td>\n",
       "      <td>0.367841</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 89 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            spelling    embed1    embed2    embed3    embed4    embed5  \\\n",
       "7021226      console -0.474705  1.094874  0.258717  0.503144  1.507025   \n",
       "6836703      consist -0.470568  0.962991  0.260600  0.480988  1.344495   \n",
       "6904211   consistent -0.404658  0.898675  0.222523  0.423722  1.239092   \n",
       "7120550     constant -0.438270  0.956392  0.241749  0.456376  1.321626   \n",
       "7233272  constituent -0.465048  1.062780  0.253318  0.490704  1.463766   \n",
       "\n",
       "           embed6    embed7    embed8    embed9  ...  repeat_time_history  \\\n",
       "7021226 -1.408898  0.496110 -0.342881 -0.794755  ...                    1   \n",
       "6836703 -1.333904  0.460674 -0.332445 -0.745455  ...                    1   \n",
       "6904211 -1.180395  0.411884 -0.290802 -0.661933  ...                    1   \n",
       "7120550 -1.270601  0.442882 -0.313167 -0.712319  ...                    1   \n",
       "7233272 -1.374023  0.482891 -0.335000 -0.774168  ...                    1   \n",
       "\n",
       "         delta_interval_within_inday  response_history  rsp1_rate  rsp2_rate  \\\n",
       "7021226                          0.0                 1     0.4154     0.1731   \n",
       "6836703                          0.0                 1     0.5524     0.1576   \n",
       "6904211                          0.0                 1     0.4417     0.1619   \n",
       "7120550                          0.0                 1     0.4470     0.1674   \n",
       "7233272                          0.0                 1     0.3894     0.2085   \n",
       "\n",
       "         rsp3_rate  rsp4_rate  acknowledge_rate  Y  last_response  \n",
       "7021226     0.4112     0.0003          0.303781  1              0  \n",
       "6836703     0.2895     0.0005          0.563981  1              1  \n",
       "6904211     0.3961     0.0003          0.457183  1              0  \n",
       "7120550     0.3853     0.0003          0.443431  1              0  \n",
       "7233272     0.4019     0.0003          0.367841  1              0  \n",
       "\n",
       "[5 rows x 89 columns]"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "excessive-negotiation",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['spelling', 'embed1', 'embed2', 'embed3', 'embed4', 'embed5', 'embed6',\n",
       "       'embed7', 'embed8', 'embed9', 'embed10', 'embed11', 'embed12',\n",
       "       'embed13', 'embed14', 'embed15', 'embed16', 'embed17', 'embed18',\n",
       "       'embed19', 'embed20', 'embed21', 'embed22', 'embed23', 'embed24',\n",
       "       'embed25', 'embed26', 'embed27', 'embed28', 'embed29', 'embed30',\n",
       "       'embed31', 'embed32', 'embed33', 'embed34', 'embed35', 'embed36',\n",
       "       'embed37', 'embed38', 'embed39', 'embed40', 'embed41', 'embed42',\n",
       "       'embed43', 'embed44', 'embed45', 'embed46', 'embed47', 'embed48',\n",
       "       'embed49', 'embed50', 'embed51', 'embed52', 'embed53', 'embed54',\n",
       "       'embed55', 'embed56', 'embed57', 'embed58', 'embed59', 'embed60',\n",
       "       'embed61', 'embed62', 'embed63', 'embed64', 'uid', 'register_time',\n",
       "       'wid', 'difficulty', 'review_time', 'response', 'study_method',\n",
       "       'repeat_time_inday', 'real_interval_history_byday',\n",
       "       'last_real_interval_byday', 'real_delta_interval_within_inday',\n",
       "       'review_day_th', 'delta_t_between_inday', 'delta_t_within_inday',\n",
       "       'repeat_time_history', 'delta_interval_within_inday',\n",
       "       'response_history', 'rsp1_rate', 'rsp2_rate', 'rsp3_rate', 'rsp4_rate',\n",
       "       'acknowledge_rate', 'Y', 'last_response'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset.columns"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "civic-recovery",
   "metadata": {},
   "source": [
    "### create a new feature for estimate the user tenure"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "applicable-limit",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset['user_tenure'] = (dataset[\"review_time\"] -dataset[\"register_time\"]).dt.days"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "minus-budapest",
   "metadata": {},
   "source": [
    "### create the dataframe , which is ready for be transformed to a sequential list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "several-addition",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 23656968 entries, 7021226 to 29463786\n",
      "Data columns (total 83 columns):\n",
      " #   Column                            Dtype         \n",
      "---  ------                            -----         \n",
      " 0   embed1                            float32       \n",
      " 1   embed2                            float32       \n",
      " 2   embed3                            float32       \n",
      " 3   embed4                            float32       \n",
      " 4   embed5                            float32       \n",
      " 5   embed6                            float32       \n",
      " 6   embed7                            float32       \n",
      " 7   embed8                            float32       \n",
      " 8   embed9                            float32       \n",
      " 9   embed10                           float32       \n",
      " 10  embed11                           float32       \n",
      " 11  embed12                           float32       \n",
      " 12  embed13                           float32       \n",
      " 13  embed14                           float32       \n",
      " 14  embed15                           float32       \n",
      " 15  embed16                           float32       \n",
      " 16  embed17                           float32       \n",
      " 17  embed18                           float32       \n",
      " 18  embed19                           float32       \n",
      " 19  embed20                           float32       \n",
      " 20  embed21                           float32       \n",
      " 21  embed22                           float32       \n",
      " 22  embed23                           float32       \n",
      " 23  embed24                           float32       \n",
      " 24  embed25                           float32       \n",
      " 25  embed26                           float32       \n",
      " 26  embed27                           float32       \n",
      " 27  embed28                           float32       \n",
      " 28  embed29                           float32       \n",
      " 29  embed30                           float32       \n",
      " 30  embed31                           float32       \n",
      " 31  embed32                           float32       \n",
      " 32  embed33                           float32       \n",
      " 33  embed34                           float32       \n",
      " 34  embed35                           float32       \n",
      " 35  embed36                           float32       \n",
      " 36  embed37                           float32       \n",
      " 37  embed38                           float32       \n",
      " 38  embed39                           float32       \n",
      " 39  embed40                           float32       \n",
      " 40  embed41                           float32       \n",
      " 41  embed42                           float32       \n",
      " 42  embed43                           float32       \n",
      " 43  embed44                           float32       \n",
      " 44  embed45                           float32       \n",
      " 45  embed46                           float32       \n",
      " 46  embed47                           float32       \n",
      " 47  embed48                           float32       \n",
      " 48  embed49                           float32       \n",
      " 49  embed50                           float32       \n",
      " 50  embed51                           float32       \n",
      " 51  embed52                           float32       \n",
      " 52  embed53                           float32       \n",
      " 53  embed54                           float32       \n",
      " 54  embed55                           float32       \n",
      " 55  embed56                           float32       \n",
      " 56  embed57                           float32       \n",
      " 57  embed58                           float32       \n",
      " 58  embed59                           float32       \n",
      " 59  embed60                           float32       \n",
      " 60  embed61                           float32       \n",
      " 61  embed62                           float32       \n",
      " 62  embed63                           float32       \n",
      " 63  embed64                           float32       \n",
      " 64  Y                                 int8          \n",
      " 65  spelling                          object        \n",
      " 66  review_time                       datetime64[ns]\n",
      " 67  uid                               object        \n",
      " 68  user_tenure                       int64         \n",
      " 69  rsp1_rate                         float32       \n",
      " 70  rsp2_rate                         float32       \n",
      " 71  rsp3_rate                         float32       \n",
      " 72  rsp4_rate                         float32       \n",
      " 73  acknowledge_rate                  float32       \n",
      " 74  difficulty                        int8          \n",
      " 75  study_method                      int8          \n",
      " 76  repeat_time_inday                 int8          \n",
      " 77  real_delta_interval_within_inday  int8          \n",
      " 78  review_day_th                     int8          \n",
      " 79  delta_t_between_inday             float32       \n",
      " 80  delta_t_within_inday              float32       \n",
      " 81  repeat_time_history               int8          \n",
      " 82  delta_interval_within_inday       float32       \n",
      "dtypes: datetime64[ns](1), float32(72), int64(1), int8(7), object(2)\n",
      "memory usage: 7.4+ GB\n"
     ]
    }
   ],
   "source": [
    "df_encoded = dataset[list(KGembedding.columns[1:]) +['Y','spelling','review_time','uid', 'user_tenure','rsp1_rate',\n",
    "       'rsp2_rate', 'rsp3_rate', 'rsp4_rate', 'acknowledge_rate', 'difficulty',  'study_method', 'repeat_time_inday',\n",
    "       'real_delta_interval_within_inday', 'review_day_th',\n",
    "       'delta_t_between_inday', 'delta_t_within_inday', 'repeat_time_history',\n",
    "       'delta_interval_within_inday']]\n",
    "df_encoded.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "hispanic-parent",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "certain-introduction",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_encoded = df_encoded.sort_values(by=['uid', 'review_time'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "gentle-apparel",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>embed1</th>\n",
       "      <th>embed2</th>\n",
       "      <th>embed3</th>\n",
       "      <th>embed4</th>\n",
       "      <th>embed5</th>\n",
       "      <th>embed6</th>\n",
       "      <th>embed7</th>\n",
       "      <th>embed8</th>\n",
       "      <th>embed9</th>\n",
       "      <th>embed10</th>\n",
       "      <th>...</th>\n",
       "      <th>acknowledge_rate</th>\n",
       "      <th>difficulty</th>\n",
       "      <th>study_method</th>\n",
       "      <th>repeat_time_inday</th>\n",
       "      <th>real_delta_interval_within_inday</th>\n",
       "      <th>review_day_th</th>\n",
       "      <th>delta_t_between_inday</th>\n",
       "      <th>delta_t_within_inday</th>\n",
       "      <th>repeat_time_history</th>\n",
       "      <th>delta_interval_within_inday</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>0 rows × 83 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [embed1, embed2, embed3, embed4, embed5, embed6, embed7, embed8, embed9, embed10, embed11, embed12, embed13, embed14, embed15, embed16, embed17, embed18, embed19, embed20, embed21, embed22, embed23, embed24, embed25, embed26, embed27, embed28, embed29, embed30, embed31, embed32, embed33, embed34, embed35, embed36, embed37, embed38, embed39, embed40, embed41, embed42, embed43, embed44, embed45, embed46, embed47, embed48, embed49, embed50, embed51, embed52, embed53, embed54, embed55, embed56, embed57, embed58, embed59, embed60, embed61, embed62, embed63, embed64, Y, spelling, review_time, uid, user_tenure, rsp1_rate, rsp2_rate, rsp3_rate, rsp4_rate, acknowledge_rate, difficulty, study_method, repeat_time_inday, real_delta_interval_within_inday, review_day_th, delta_t_between_inday, delta_t_within_inday, repeat_time_history, delta_interval_within_inday]\n",
       "Index: []\n",
       "\n",
       "[0 rows x 83 columns]"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#check if any null value in any row\n",
    "rows_with_null = df_encoded[df_encoded.isnull().any(axis=1)]\n",
    "rows_with_null"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "thirty-bridge",
   "metadata": {},
   "outputs": [],
   "source": [
    "#df_encoded.to_csv(\"./data/df_encoded_KG_0905.csv\",index=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "checked-boards",
   "metadata": {},
   "outputs": [],
   "source": [
    "del dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ready-contemporary",
   "metadata": {},
   "source": [
    "#### We have over 24 million study records of 3.3W unique users with over 6K words"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "little-controversy",
   "metadata": {},
   "source": [
    "### transform the dataframe to a sequncial list, each element is a list of a user's sequential study records, each record has X features\n",
    "#### Consider the list will be very large, I have to optimize it using deque() and seperate chunks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "allied-hungarian",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "acting-isolation",
   "metadata": {},
   "source": [
    "## data read and transforming"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "missing-crystal",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_encoded = pd.read_csv(\"./data/df_encoded_KG_0905.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "regular-paper",
   "metadata": {},
   "outputs": [],
   "source": [
    "valid_uids= list(set(df_encoded.uid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "behind-investing",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "169\n"
     ]
    }
   ],
   "source": [
    "\n",
    "from collections import deque\n",
    "chunks =200 # chunk size \n",
    "sub_uids=[valid_uids[i:i+chunks] for i in range (0, len(valid_uids),chunks)]\n",
    "print(len(sub_uids))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "focal-thousand",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loop 0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 139.02it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 406.84it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 2.368722677230835\n",
      "loop 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 162.71it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 443.16it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 4.445590019226074\n",
      "loop 2\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 141.13it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 402.05it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 6.7803122997283936\n",
      "loop 3\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 129.90it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 383.87it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 9.250917434692383\n",
      "loop 4\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 132.78it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 427.77it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 11.633249759674072\n",
      "loop 5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 166.88it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 391.61it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 13.752275228500366\n",
      "loop 6\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 125.48it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 406.01it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 16.246171236038208\n",
      "loop 7\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 192.11it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 442.09it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 18.161577463150024\n",
      "loop 8\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 110.38it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 392.59it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 20.9143385887146\n",
      "loop 9\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 178.31it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 420.13it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 22.922293424606323\n",
      "loop 10\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 101.76it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 386.02it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 25.807850122451782\n",
      "loop 11\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 174.78it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 406.37it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 27.86551260948181\n",
      "loop 12\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 183.96it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 427.06it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 29.83044695854187\n",
      "loop 13\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:02<00:00, 86.74it/s] \n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 352.08it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 33.11113667488098\n",
      "loop 14\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 198.48it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 451.07it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 35.00505018234253\n",
      "loop 15\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 193.78it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 445.87it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 36.90705871582031\n",
      "loop 16\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 181.56it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 122.76it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 40.05214834213257\n",
      "loop 17\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 172.97it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 400.83it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 42.114402770996094\n",
      "loop 18\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 179.96it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 430.33it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 44.097957611083984\n",
      "loop 19\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 190.19it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 434.54it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 46.01983404159546\n",
      "loop 20\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 184.12it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 423.49it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 47.98504424095154\n",
      "loop 21\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:02<00:00, 77.57it/s] \n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 410.81it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 51.45806694030762\n",
      "loop 22\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 177.83it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 416.67it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 53.4774169921875\n",
      "loop 23\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 192.04it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 445.04it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 55.38919973373413\n",
      "loop 24\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 176.85it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 418.70it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 57.40373158454895\n",
      "loop 25\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 178.14it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 408.85it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 59.41468954086304\n",
      "loop 26\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 185.69it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 425.02it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 61.36269927024841\n",
      "loop 27\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:02<00:00, 67.72it/s] \n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 397.68it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 65.21607565879822\n",
      "loop 28\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 191.01it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 440.86it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 67.11084842681885\n",
      "loop 29\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 191.40it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 437.58it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 69.00145125389099\n",
      "loop 30\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 205.13it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 462.84it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 70.8049259185791\n",
      "loop 31\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 188.52it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 431.99it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 72.73319220542908\n",
      "loop 32\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 184.43it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 422.66it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 74.68562388420105\n",
      "loop 33\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 191.30it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 444.33it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 76.5759539604187\n",
      "loop 34\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:03<00:00, 60.75it/s] \n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 433.22it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 80.75615906715393\n",
      "loop 35\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 182.09it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 417.72it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 82.74276900291443\n",
      "loop 36\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 170.43it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 396.58it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 84.81818008422852\n",
      "loop 37\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 187.87it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 435.64it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 86.75633263587952\n",
      "loop 38\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 188.78it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 441.50it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 88.69233870506287\n",
      "loop 39\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 193.95it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 445.62it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 90.58519268035889\n",
      "loop 40\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 176.19it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 408.39it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 92.61860013008118\n",
      "loop 41\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 170.80it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 398.14it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 94.69883751869202\n",
      "loop 42\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:03<00:00, 51.94it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 432.47it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 99.41123414039612\n",
      "loop 43\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 165.23it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 391.98it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 101.56163001060486\n",
      "loop 44\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 177.77it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 418.61it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 103.58449125289917\n",
      "loop 45\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 164.94it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 394.85it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 105.72409844398499\n",
      "loop 46\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 181.19it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 427.98it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 107.70204401016235\n",
      "loop 47\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 186.48it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 438.44it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 109.6333544254303\n",
      "loop 48\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 162.38it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 382.61it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 111.80030751228333\n",
      "loop 49\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 172.75it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 408.09it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 113.85761904716492\n",
      "loop 50\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 168.69it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 402.41it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 115.95631003379822\n",
      "loop 51\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 171.52it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 410.96it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 118.03614330291748\n",
      "loop 52\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:04<00:00, 43.93it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 444.16it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 123.45870637893677\n",
      "loop 53\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 176.16it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 418.36it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 125.48518705368042\n",
      "loop 54\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 185.23it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 431.13it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 127.44250512123108\n",
      "loop 55\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 183.96it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 424.47it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 129.400297164917\n",
      "loop 56\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 200.04it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 459.82it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 131.24204874038696\n",
      "loop 57\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 173.17it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 400.85it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 133.29587769508362\n",
      "loop 58\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 192.04it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 439.77it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 135.19502782821655\n",
      "loop 59\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 176.83it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 414.08it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 137.22990584373474\n",
      "loop 60\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 183.23it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 420.17it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 139.21366214752197\n",
      "loop 61\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 196.45it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 454.38it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 141.08641386032104\n",
      "loop 62\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 182.58it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 424.72it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 143.05185055732727\n",
      "loop 63\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 172.76it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 400.36it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 145.10553002357483\n",
      "loop 64\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 213.31it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:04<00:00, 43.19it/s] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 151.0654158592224\n",
      "loop 65\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 185.70it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 426.50it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 153.01034879684448\n",
      "loop 66\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 183.38it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 422.61it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 154.97523999214172\n",
      "loop 67\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 173.27it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 404.57it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 157.02660989761353\n",
      "loop 68\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 195.19it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 451.09it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 158.8956172466278\n",
      "loop 69\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 192.24it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 438.58it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 160.7951214313507\n",
      "loop 70\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 200.45it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 458.83it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 162.6497642993927\n",
      "loop 71\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 173.56it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 402.08it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 164.70322847366333\n",
      "loop 72\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 191.61it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 440.80it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 166.60644602775574\n",
      "loop 73\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 198.65it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 457.12it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 168.46116256713867\n",
      "loop 74\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 185.06it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 432.22it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 170.41905522346497\n",
      "loop 75\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 173.31it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 412.16it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 172.47154569625854\n",
      "loop 76\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 192.56it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 438.70it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 174.37301683425903\n",
      "loop 77\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 178.81it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 414.98it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 176.4172706604004\n",
      "loop 78\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 204.10it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 464.54it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 178.22554302215576\n",
      "loop 79\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 186.18it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 429.68it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 180.16635298728943\n",
      "loop 80\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:06<00:00, 31.72it/s] \n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 456.29it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 187.30933833122253\n",
      "loop 81\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 164.24it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 383.88it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 189.45545744895935\n",
      "loop 82\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 177.52it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 413.23it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 191.47838854789734\n",
      "loop 83\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 192.86it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 446.77it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 193.37535953521729\n",
      "loop 84\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 181.22it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 411.45it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 195.38119649887085\n",
      "loop 85\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 190.52it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 440.42it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 197.285569190979\n",
      "loop 86\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 192.69it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 439.44it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 199.1763870716095\n",
      "loop 87\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 176.54it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 411.41it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 201.18614768981934\n",
      "loop 88\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 178.88it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 408.48it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 203.19553542137146\n",
      "loop 89\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 177.78it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 407.52it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 205.20798516273499\n",
      "loop 90\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 201.61it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 461.36it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 207.0219123363495\n",
      "loop 91\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 171.78it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 395.59it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 209.09276461601257\n",
      "loop 92\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 175.82it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 403.20it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 211.11822772026062\n",
      "loop 93\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 175.77it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 405.11it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 213.14703965187073\n",
      "loop 94\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 206.17it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 465.02it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 214.93692255020142\n",
      "loop 95\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 171.07it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 392.46it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 217.0097062587738\n",
      "loop 96\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 176.16it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 411.88it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 219.03227758407593\n",
      "loop 97\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 183.09it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 425.18it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 220.9927270412445\n",
      "loop 98\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 186.70it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 435.25it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 222.9320616722107\n",
      "loop 99\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:08<00:00, 24.77it/s] \n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 417.34it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 231.90669631958008\n",
      "loop 100\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 171.09it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 407.94it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 233.9818253517151\n",
      "loop 101\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 193.38it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 457.98it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 235.8613476753235\n",
      "loop 102\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 190.61it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 441.79it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 237.75726532936096\n",
      "loop 103\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 164.79it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 402.56it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 239.8999183177948\n",
      "loop 104\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 180.74it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 424.62it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 241.90177822113037\n",
      "loop 105\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 191.27it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 451.49it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 243.79837584495544\n",
      "loop 106\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 176.07it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 423.55it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 245.8205235004425\n",
      "loop 107\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 159.80it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 384.23it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 248.01560187339783\n",
      "loop 108\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 165.71it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 395.53it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 250.14988827705383\n",
      "loop 109\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 186.44it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 439.56it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 252.09237599372864\n",
      "loop 110\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 194.05it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 451.52it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 253.9798982143402\n",
      "loop 111\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 187.21it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 442.97it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 255.91320300102234\n",
      "loop 112\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 188.34it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 439.20it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 257.84719157218933\n",
      "loop 113\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 186.79it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 437.59it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 259.79329323768616\n",
      "loop 114\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 181.45it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 426.75it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 261.7778477668762\n",
      "loop 115\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 187.51it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 439.34it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 263.722158908844\n",
      "loop 116\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 205.24it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 474.93it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 265.53350019454956\n",
      "loop 117\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 181.95it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 438.75it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 267.50134110450745\n",
      "loop 118\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 161.18it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 386.35it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 269.68507623672485\n",
      "loop 119\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 197.01it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 463.85it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 271.56451416015625\n",
      "loop 120\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 182.69it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 434.62it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 273.53939986228943\n",
      "loop 121\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 174.03it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 413.22it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 275.59279799461365\n",
      "loop 122\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 163.31it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 398.04it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 277.75541377067566\n",
      "loop 123\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:11<00:00, 16.99it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 479.02it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 290.3748927116394\n",
      "loop 124\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 169.82it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 403.90it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 292.4936294555664\n",
      "loop 125\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 184.43it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 439.78it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 294.4498414993286\n",
      "loop 126\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 155.74it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 374.96it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 296.7239260673523\n",
      "loop 127\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 193.57it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 451.73it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 298.60267329216003\n",
      "loop 128\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 180.64it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 425.40it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 300.61306285858154\n",
      "loop 129\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 169.39it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 406.22it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 302.69371485710144\n",
      "loop 130\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 182.89it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 430.69it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 304.66243743896484\n",
      "loop 131\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 196.24it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 458.32it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 306.5491065979004\n",
      "loop 132\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 195.57it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 456.31it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 308.4379873275757\n",
      "loop 133\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 170.88it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 407.48it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 310.52758717536926\n",
      "loop 134\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 178.05it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 416.74it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 312.5331025123596\n",
      "loop 135\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 170.48it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 410.03it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 314.64603567123413\n",
      "loop 136\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 165.95it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 397.42it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 316.7684030532837\n",
      "loop 137\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 174.41it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 411.48it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 318.82059717178345\n",
      "loop 138\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 180.54it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 428.55it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 320.82845759391785\n",
      "loop 139\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 196.84it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 458.05it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 322.7003664970398\n",
      "loop 140\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 192.31it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 456.16it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 324.6154406070709\n",
      "loop 141\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 174.27it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 406.67it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 326.6791024208069\n",
      "loop 142\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 169.52it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 410.46it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 328.79107093811035\n",
      "loop 143\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 183.66it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 432.55it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 330.7650496959686\n",
      "loop 144\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 203.47it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 469.78it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 332.59498858451843\n",
      "loop 145\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 190.39it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 446.00it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 334.5112552642822\n",
      "loop 146\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 189.51it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 444.13it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 336.4276297092438\n",
      "loop 147\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 181.39it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 429.19it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 338.4313585758209\n",
      "loop 148\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 169.10it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 402.57it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 340.53135895729065\n",
      "loop 149\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 179.28it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 422.87it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 342.5356502532959\n",
      "loop 150\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 175.58it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 418.95it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 344.57234168052673\n",
      "loop 151\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 167.87it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:12<00:00, 15.81it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 358.83077478408813\n",
      "loop 152\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 187.97it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 433.89it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 360.7759885787964\n",
      "loop 153\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 183.29it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 432.02it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 362.7446713447571\n",
      "loop 154\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 188.17it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 436.89it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 364.68108773231506\n",
      "loop 155\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 184.94it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 428.67it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 366.6377160549164\n",
      "loop 156\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 198.85it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 459.69it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 368.4794600009918\n",
      "loop 157\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 177.87it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 415.98it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 370.49483013153076\n",
      "loop 158\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 185.93it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 433.07it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 372.44580698013306\n",
      "loop 159\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 164.87it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 388.41it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 374.60196232795715\n",
      "loop 160\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 165.67it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 394.46it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 376.73806858062744\n",
      "loop 161\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 189.84it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 443.72it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 378.6634941101074\n",
      "loop 162\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 201.43it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 470.99it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 380.4976050853729\n",
      "loop 163\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 181.03it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 426.60it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 382.50461411476135\n",
      "loop 164\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 173.82it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 409.98it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 384.56031250953674\n",
      "loop 165\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 171.59it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 402.13it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 386.63916301727295\n",
      "loop 166\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 176.35it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 414.24it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 388.6738955974579\n",
      "loop 167\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 200/200 [00:01<00:00, 197.03it/s]\n",
      "Processing data: 100%|██████████| 200/200 [00:00<00:00, 456.09it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 390.55384278297424\n",
      "loop 168\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing data: 100%|██████████| 100/100 [00:00<00:00, 185.95it/s]\n",
      "Processing data: 100%|██████████| 100/100 [00:00<00:00, 435.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total time 391.7118365764618\n",
      "Total Memory: 1007.45 GB | Used Memory: 917.95 GB | Available Memory: 899.42 GB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "import itertools\n",
    "import sys\n",
    "import pickle\n",
    "starttime=time.time()\n",
    "\n",
    "src_data=[]\n",
    "tgt_data=[]\n",
    "\n",
    "for i,sub_uid in enumerate(sub_uids):\n",
    "    print('loop',i)\n",
    "    try:\n",
    "        sub_df_encoded = df_encoded[df_encoded.uid.isin(sub_uid)]\n",
    "        sub_src_data = [group.drop(columns=['uid',\"Y\",'spelling','review_time']).apply(list, axis=1).tolist() for _, group in tqdm(sub_df_encoded.groupby('uid'), desc='Processing data')]   \n",
    "        src_data.append(sub_src_data)\n",
    "\n",
    "        sub_tgt_data = [[elem[0] for elem in group.drop(columns=['uid','spelling','review_time']).apply(list, axis=1).tolist() ]for _, group in tqdm(sub_df_encoded[[\"uid\",'spelling','review_time',\"Y\"]].groupby('uid'), desc='Processing data')]\n",
    "        tgt_data.append(sub_tgt_data)\n",
    "\n",
    "\n",
    "#         with open(f'./data/sub_src_data_list_{i}.pkl', 'wb') as file1:\n",
    "#             pickle.dump(sub_src_data, file1)\n",
    "\n",
    "#         with open(f'./data/sub_tgt_data_list_{i}.pkl', 'wb') as file2:\n",
    "#             pickle.dump(sub_tgt_data, file2)\n",
    "        #size_in_bytes = sys.getsizeof(src_data)\n",
    "        #size_in_gigabytes = size_in_bytes / (1024 * 1024 * 1024)\n",
    "        #print(f\"The list takes up {size_in_gigabytes:.2f} GB.\")\n",
    "        print('total time',time.time() -starttime)\n",
    "        #print(memory_usage())\n",
    "\n",
    "    except: \n",
    "        print('error happens at loop',i )\n",
    "        sub_df_encoded.to_csv(f\"./data/sub_df_encoded_{i}.csv\",index=None)\n",
    "        \n",
    "src_data =  list(itertools.chain(*src_data))\n",
    "tgt_data =list(itertools.chain(*tgt_data))\n",
    "\n",
    "print(memory_usage())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "behavioral-insulin",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "33700 33700\n"
     ]
    }
   ],
   "source": [
    "print(len(src_data), len(tgt_data))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "minute-mexico",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Split data into training and testing sets\n",
    "src_train, src_test, tgt_train, tgt_test = train_test_split(src_data, tgt_data, test_size=0.2, random_state=0)\n",
    "\n",
    "# Create datasets\n",
    "train_dataset = MyDataset(src_train, tgt_train)\n",
    "test_dataset = MyDataset(src_test, tgt_test)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "binary-kingston",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create data loaders\n",
    "train_dataloader = DataLoader(train_dataset, batch_size=16, shuffle=True, collate_fn=collate_fn)\n",
    "test_dataloader = DataLoader(test_dataset, batch_size=16, shuffle=False, collate_fn=collate_fn)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "circular-applicant",
   "metadata": {},
   "source": [
    "### model with KG training "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "funded-expert",
   "metadata": {},
   "outputs": [],
   "source": [
    "# hyperparamter tuning for our model\n",
    "# param_grid = {\n",
    "#     'embedding_size': [32, 64, 128],\n",
    "#     'num_heads': [2, 4, 8],\n",
    "#     'num_layers': [ 2, 3, 4],\n",
    "#     'dropout': [0.1, 0.2, 0.3],\n",
    "#     'lr': [1e-3, 1e-4, 1e-5]\n",
    "# }\n",
    "\n",
    "# from sklearn.metrics import f1_score\n",
    "# import itertools\n",
    "\n",
    "# # Helper function to get all parameter combinations\n",
    "# def get_param_combinations(param_grid):\n",
    "#     keys = param_grid.keys()\n",
    "#     values = (param_grid[key] for key in keys)\n",
    "#     for instance in itertools.product(*values):\n",
    "#         yield dict(zip(keys, instance))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "pregnant-emerald",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "diagnostic-relaxation",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch.nn as nn\n",
    "# Transformer model\n",
    "KG_dims =64\n",
    "KG_compress_dims = 16\n",
    "input_dim = len(src_train[0][0]) -KG_dims + KG_compress_dims\n",
    "embedding_size = 16*2\n",
    "num_heads =4\n",
    "num_layers = 2\n",
    "dropout = 0.1\n",
    "\n",
    "model= TransformerKGModel(KG_dims, KG_compress_dims, input_dim, embedding_size, num_heads, num_layers, dropout)\n",
    "\n",
    "# if torch.cuda.device_count() > 1:\n",
    "#     print(f\"Using {torch.cuda.device_count()} GPUs!\")\n",
    "#     model = nn.DataParallel(model)\n",
    "\n",
    "model.to(device)\n",
    "\n",
    "\n",
    "criterion = nn.BCELoss(reduction='none')\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=0.0001)\n",
    "\n",
    "#device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "# device = torch.device(\"cuda:1\" if torch.cuda.is_available() else \"cpu\")\n",
    "# model = model.to(device)\n",
    "#next(model.parameters()).device\n",
    "\n",
    "\n",
    "# Training Loop\n",
    "num_epochs = 1000\n",
    "patience = 30  # number of epochs to wait for improvement before stopping\n",
    "min_delta = 0.001  # minimum improvement required to reset patience counter\n",
    "\n",
    "#best_loss = None\n",
    "#patience_counter = 0\n",
    "\n",
    "\n",
    "best_f1 = None\n",
    "patience_counter = 0\n",
    "best_AUC=None\n",
    "\n",
    "best_model_AUC = 0\n",
    "best_model = None\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "green-oracle",
   "metadata": {},
   "outputs": [],
   "source": [
    "# def check_memory(src):\n",
    "#     bytes_per_element = src.element_size() # Returns the size of a single element in bytes.\n",
    "#     num_elements = src.numel() # Returns the total number of elements in the tensor.\n",
    "#     total_memory_bytes = bytes_per_element * num_elements\n",
    "        \n",
    "#     total_memory_kB = total_memory_bytes / 1024\n",
    "#     total_memory_MB = total_memory_kB / 1024\n",
    "#     total_memory_GB = total_memory_MB / 1024\n",
    "#     print(total_memory_GB) \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "ahead-jewel",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 0, timecost:201.07 seconds, Loss: 0.5571009640347002, Accuracy: 0.7339584279275215, F1 Score: 0.8293229022363893, ROC AUC: 0.7206060565306357, Average Precision Score: 0.8560682660220055\n",
      "best AUC: 0.7206060565306357\n",
      "Epoch: 1, timecost:396.31 seconds, Loss: 0.5374710150040929, Accuracy: 0.7329229186866422, F1 Score: 0.8203643335820736, ROC AUC: 0.7341519258328222, Average Precision Score: 0.8628554223176953\n",
      "best AUC: 0.7341519258328222\n",
      "Epoch: 2, timecost:588.64 seconds, Loss: 0.5317547624295119, Accuracy: 0.7390309653392809, F1 Score: 0.8232074265047301, ROC AUC: 0.7480338738271014, Average Precision Score: 0.8715178712349129\n",
      "best AUC: 0.7480338738271014\n",
      "Epoch: 3, timecost:780.47 seconds, Loss: 0.5252390283682226, Accuracy: 0.7393770506558612, F1 Score: 0.8247357848393027, ROC AUC: 0.743585941421181, Average Precision Score: 0.8680181645391933\n",
      "best AUC: 0.7480338738271014\n",
      "Epoch: 4, timecost:971.82 seconds, Loss: 0.5236478974981902, Accuracy: 0.7362893273246612, F1 Score: 0.8193754668918843, ROC AUC: 0.7491859836555815, Average Precision Score: 0.8715170819058276\n",
      "best AUC: 0.7491859836555815\n",
      "Epoch: 5, timecost:1162.75 seconds, Loss: 0.5226028168590557, Accuracy: 0.7401812024964626, F1 Score: 0.8225500716030129, ROC AUC: 0.7541837889171783, Average Precision Score: 0.874687208555763\n",
      "best AUC: 0.7541837889171783\n",
      "Epoch: 6, timecost:1353.99 seconds, Loss: 0.518864348734168, Accuracy: 0.7399844958849019, F1 Score: 0.8221793201027127, ROC AUC: 0.7551713095970486, Average Precision Score: 0.8756002548488947\n",
      "best AUC: 0.7541837889171783\n",
      "Epoch: 7, timecost:1546.35 seconds, Loss: 0.5154551904173209, Accuracy: 0.7496721380402909, F1 Score: 0.8315506706062968, ROC AUC: 0.7637091985201304, Average Precision Score: 0.8801415392950217\n",
      "best AUC: 0.7637091985201304\n",
      "Epoch: 8, timecost:1740.18 seconds, Loss: 0.5124010297771023, Accuracy: 0.748110528409738, F1 Score: 0.8295660197093646, ROC AUC: 0.7650203379628044, Average Precision Score: 0.8817303125702697\n",
      "best AUC: 0.7650203379628044\n",
      "Epoch: 9, timecost:1933.22 seconds, Loss: 0.5089464934006643, Accuracy: 0.7459182321674681, F1 Score: 0.8257135465693896, ROC AUC: 0.7678425975347144, Average Precision Score: 0.8833362928891992\n",
      "best AUC: 0.7678425975347144\n",
      "Epoch: 10, timecost:2124.60 seconds, Loss: 0.5064563167201306, Accuracy: 0.7542109687916826, F1 Score: 0.8361480890535523, ROC AUC: 0.7722264829448173, Average Precision Score: 0.8862237591069831\n",
      "best AUC: 0.7722264829448173\n",
      "Epoch: 11, timecost:2314.87 seconds, Loss: 0.503580431626883, Accuracy: 0.752001135869757, F1 Score: 0.8337931377256582, ROC AUC: 0.7709151780834979, Average Precision Score: 0.8862777274582043\n",
      "best AUC: 0.7722264829448173\n",
      "Epoch: 12, timecost:2505.93 seconds, Loss: 0.5014580778445971, Accuracy: 0.7565241202238695, F1 Score: 0.8363945455572835, ROC AUC: 0.7765158254617431, Average Precision Score: 0.8884045163768243\n",
      "best AUC: 0.7765158254617431\n",
      "Epoch: 13, timecost:2698.61 seconds, Loss: 0.5001189581718218, Accuracy: 0.7571622018209835, F1 Score: 0.838599435810845, ROC AUC: 0.7735743095446876, Average Precision Score: 0.8865659825230237\n",
      "best AUC: 0.7765158254617431\n",
      "Epoch: 14, timecost:2890.22 seconds, Loss: 0.49881978063272087, Accuracy: 0.7522540443703349, F1 Score: 0.8313691279491844, ROC AUC: 0.7734682487110441, Average Precision Score: 0.8869256065999305\n",
      "best AUC: 0.7765158254617431\n",
      "Epoch: 15, timecost:3081.44 seconds, Loss: 0.4975025606615253, Accuracy: 0.7564176324341524, F1 Score: 0.8363809717908113, ROC AUC: 0.7758471084494092, Average Precision Score: 0.8883478776478514\n",
      "best AUC: 0.7765158254617431\n",
      "Epoch: 16, timecost:3274.00 seconds, Loss: 0.4956676515698079, Accuracy: 0.7615248206346292, F1 Score: 0.8428668562623659, ROC AUC: 0.7810075592546165, Average Precision Score: 0.8909411932862533\n",
      "best AUC: 0.7810075592546165\n",
      "Epoch: 17, timecost:3465.66 seconds, Loss: 0.4937023871553404, Accuracy: 0.7615554570027026, F1 Score: 0.8426013676213572, ROC AUC: 0.7789969955625198, Average Precision Score: 0.8899389395511706\n",
      "best AUC: 0.7810075592546165\n",
      "Epoch: 18, timecost:3656.08 seconds, Loss: 0.4914446598521326, Accuracy: 0.7651371653584234, F1 Score: 0.8447570370699343, ROC AUC: 0.783621018474535, Average Precision Score: 0.8918497280782828\n",
      "best AUC: 0.783621018474535\n",
      "Epoch: 19, timecost:3847.83 seconds, Loss: 0.49036019400248543, Accuracy: 0.7616758896220254, F1 Score: 0.8455706676914739, ROC AUC: 0.776171634929811, Average Precision Score: 0.8880864804671115\n",
      "best AUC: 0.783621018474535\n",
      "Epoch: 20, timecost:4038.63 seconds, Loss: 0.4903056400466036, Accuracy: 0.7655056469164918, F1 Score: 0.8453277309167685, ROC AUC: 0.7871067118118351, Average Precision Score: 0.8937241039770601\n",
      "best AUC: 0.7871067118118351\n",
      "Epoch: 21, timecost:4233.55 seconds, Loss: 0.4856891687970487, Accuracy: 0.7674959544147746, F1 Score: 0.8481051499565956, ROC AUC: 0.7913858330095186, Average Precision Score: 0.8964502736885633\n",
      "best AUC: 0.7913858330095186\n",
      "Epoch: 22, timecost:4426.57 seconds, Loss: 0.4785826969748081, Accuracy: 0.7687387345320675, F1 Score: 0.8437823135737385, ROC AUC: 0.7992423179239992, Average Precision Score: 0.9010511071175082\n",
      "best AUC: 0.7992423179239992\n",
      "Epoch: 23, timecost:4618.95 seconds, Loss: 0.475633052124114, Accuracy: 0.7701987159347357, F1 Score: 0.8476567900290404, ROC AUC: 0.8012597739771361, Average Precision Score: 0.9026304186504611\n",
      "best AUC: 0.8012597739771361\n",
      "Epoch: 24, timecost:4809.20 seconds, Loss: 0.47399172554737734, Accuracy: 0.7733441201520325, F1 Score: 0.8495523094727737, ROC AUC: 0.8043561289382558, Average Precision Score: 0.903810484094588\n",
      "best AUC: 0.8043561289382558\n",
      "Epoch: 25, timecost:5001.40 seconds, Loss: 0.4710973690366179, Accuracy: 0.7703609830428759, F1 Score: 0.8429078071076511, ROC AUC: 0.8061619132747037, Average Precision Score: 0.9050970998307919\n",
      "best AUC: 0.8061619132747037\n",
      "Epoch: 26, timecost:5193.47 seconds, Loss: 0.46966935053069797, Accuracy: 0.7730244454975843, F1 Score: 0.8470239067465576, ROC AUC: 0.8070432637264294, Average Precision Score: 0.9055678653092953\n",
      "best AUC: 0.8061619132747037\n",
      "Epoch: 27, timecost:5385.96 seconds, Loss: 0.4669167468385102, Accuracy: 0.7742507453617067, F1 Score: 0.8477779602964438, ROC AUC: 0.8089976932454418, Average Precision Score: 0.9065958603837794\n",
      "best AUC: 0.8089976932454418\n",
      "Epoch: 28, timecost:5578.15 seconds, Loss: 0.4653696817354208, Accuracy: 0.7759228572026207, F1 Score: 0.8500479388825554, ROC AUC: 0.8110600814745682, Average Precision Score: 0.9079052132351595\n",
      "best AUC: 0.8110600814745682\n",
      "Epoch: 29, timecost:5768.64 seconds, Loss: 0.4640827079908784, Accuracy: 0.7733153853516326, F1 Score: 0.8445582252512257, ROC AUC: 0.8114592908736542, Average Precision Score: 0.9077743252792616\n",
      "best AUC: 0.8110600814745682\n",
      "Epoch: 30, timecost:5958.98 seconds, Loss: 0.46182178212910097, Accuracy: 0.7773052969012687, F1 Score: 0.8505192258654496, ROC AUC: 0.8135788626879643, Average Precision Score: 0.9090314579638682\n",
      "best AUC: 0.8135788626879643\n",
      "Epoch: 31, timecost:6151.31 seconds, Loss: 0.4612419293789906, Accuracy: 0.7750039774457171, F1 Score: 0.8466702029687658, ROC AUC: 0.8125963969323519, Average Precision Score: 0.9083620624043313\n",
      "best AUC: 0.8135788626879643\n",
      "Epoch: 32, timecost:6343.74 seconds, Loss: 0.4591844149442384, Accuracy: 0.7765879833177581, F1 Score: 0.8495844491150206, ROC AUC: 0.8134309091665044, Average Precision Score: 0.909047910794325\n",
      "best AUC: 0.8135788626879643\n",
      "Epoch: 33, timecost:6534.77 seconds, Loss: 0.45815241947018426, Accuracy: 0.7771658486052107, F1 Score: 0.8499777952457075, ROC AUC: 0.8144179360409005, Average Precision Score: 0.9097018274178164\n",
      "best AUC: 0.8135788626879643\n",
      "Epoch: 34, timecost:6726.95 seconds, Loss: 0.45714917221833407, Accuracy: 0.7777470944574162, F1 Score: 0.8507150191270189, ROC AUC: 0.8141947471141224, Average Precision Score: 0.9095380731465057\n",
      "best AUC: 0.8135788626879643\n",
      "Epoch: 35, timecost:6918.75 seconds, Loss: 0.45681806673635716, Accuracy: 0.7751468063065281, F1 Score: 0.8459353129136753, ROC AUC: 0.8150114699622307, Average Precision Score: 0.9100920050410095\n",
      "best AUC: 0.8150114699622307\n",
      "Epoch: 36, timecost:7111.17 seconds, Loss: 0.45566762735298905, Accuracy: 0.7756293819250076, F1 Score: 0.8458714264772689, ROC AUC: 0.8155191940757686, Average Precision Score: 0.9096503782126104\n",
      "best AUC: 0.8150114699622307\n",
      "Epoch: 37, timecost:7303.74 seconds, Loss: 0.4547519478847436, Accuracy: 0.775520358711726, F1 Score: 0.8484662464685558, ROC AUC: 0.8104730243219189, Average Precision Score: 0.9069952405691275\n",
      "best AUC: 0.8150114699622307\n",
      "Epoch: 38, timecost:7494.67 seconds, Loss: 0.4537975935978423, Accuracy: 0.773786340278774, F1 Score: 0.8455933857485083, ROC AUC: 0.8120235404837775, Average Precision Score: 0.9082489829484955\n",
      "best AUC: 0.8150114699622307\n",
      "Epoch: 39, timecost:7686.40 seconds, Loss: 0.4543076071434983, Accuracy: 0.7746836689353778, F1 Score: 0.845171309169553, ROC AUC: 0.8159787034173113, Average Precision Score: 0.9106007492937506\n",
      "best AUC: 0.8150114699622307\n",
      "Epoch: 40, timecost:7875.49 seconds, Loss: 0.45280043408495735, Accuracy: 0.7764172647977355, F1 Score: 0.8483481213320506, ROC AUC: 0.8160343827282146, Average Precision Score: 0.910516421924515\n",
      "best AUC: 0.8160343827282146\n",
      "Epoch: 41, timecost:8066.55 seconds, Loss: 0.4507973072259999, Accuracy: 0.7755364163943023, F1 Score: 0.8445115552444924, ROC AUC: 0.818551415611689, Average Precision Score: 0.9115300988950156\n",
      "best AUC: 0.818551415611689\n",
      "Epoch: 42, timecost:8257.53 seconds, Loss: 0.4503759411393004, Accuracy: 0.7751366646122693, F1 Score: 0.8448388926227823, ROC AUC: 0.8182555661895777, Average Precision Score: 0.9120366356935158\n",
      "best AUC: 0.818551415611689\n",
      "Epoch: 43, timecost:8449.21 seconds, Loss: 0.44940682014654937, Accuracy: 0.7773435395400361, F1 Score: 0.84788620049011, ROC AUC: 0.8177872441110722, Average Precision Score: 0.9112724510748281\n",
      "best AUC: 0.818551415611689\n",
      "Epoch: 44, timecost:8639.59 seconds, Loss: 0.4483897283982806, Accuracy: 0.7786966105823931, F1 Score: 0.8492049540804419, ROC AUC: 0.8189785948758529, Average Precision Score: 0.9118064342315182\n",
      "best AUC: 0.818551415611689\n",
      "Epoch: 45, timecost:8832.35 seconds, Loss: 0.447128287868613, Accuracy: 0.7785480770185617, F1 Score: 0.8509280956956635, ROC AUC: 0.8171031814253964, Average Precision Score: 0.9110859017793866\n",
      "best AUC: 0.818551415611689\n",
      "Epoch: 46, timecost:9024.45 seconds, Loss: 0.44668177932235537, Accuracy: 0.7702132946202327, F1 Score: 0.8399289401216026, ROC AUC: 0.8138542847963885, Average Precision Score: 0.9096500823182669\n",
      "best AUC: 0.818551415611689\n",
      "Epoch: 47, timecost:9215.26 seconds, Loss: 0.4462957231687155, Accuracy: 0.7775541909812026, F1 Score: 0.8476382688549491, ROC AUC: 0.8191052674279933, Average Precision Score: 0.9121380252330531\n",
      "best AUC: 0.818551415611689\n",
      "Epoch: 48, timecost:9405.43 seconds, Loss: 0.44578088288491136, Accuracy: 0.7803704127056836, F1 Score: 0.8516651642817584, ROC AUC: 0.8197326748759253, Average Precision Score: 0.9122260114858698\n",
      "best AUC: 0.8197326748759253\n",
      "Epoch: 49, timecost:9596.95 seconds, Loss: 0.4443793115789176, Accuracy: 0.7796480282750436, F1 Score: 0.8493270577982789, ROC AUC: 0.820350069435433, Average Precision Score: 0.9123833247734707\n",
      "best AUC: 0.8197326748759253\n",
      "Epoch: 50, timecost:9788.57 seconds, Loss: 0.4447794666453002, Accuracy: 0.77986227156626, F1 Score: 0.8499005385669485, ROC AUC: 0.8207813020438626, Average Precision Score: 0.9128035701153123\n",
      "best AUC: 0.8207813020438626\n",
      "Epoch: 51, timecost:9981.43 seconds, Loss: 0.44341419859173037, Accuracy: 0.7732158699767184, F1 Score: 0.8412827623103066, ROC AUC: 0.819688906739906, Average Precision Score: 0.9126078847603125\n",
      "best AUC: 0.8207813020438626\n",
      "Epoch: 52, timecost:10171.88 seconds, Loss: 0.44348849785080297, Accuracy: 0.7794940012934886, F1 Score: 0.8484137740509855, ROC AUC: 0.8219338287097474, Average Precision Score: 0.9136053490722008\n",
      "best AUC: 0.8219338287097474\n",
      "Epoch: 53, timecost:10362.79 seconds, Loss: 0.4425825853729814, Accuracy: 0.779612743630435, F1 Score: 0.8499138184946561, ROC AUC: 0.8202021438336433, Average Precision Score: 0.9127558230672922\n",
      "best AUC: 0.8219338287097474\n",
      "Epoch: 54, timecost:10554.40 seconds, Loss: 0.4409534384836429, Accuracy: 0.7813425363574457, F1 Score: 0.8518426682670283, ROC AUC: 0.8227890500233241, Average Precision Score: 0.914167917173892\n",
      "best AUC: 0.8219338287097474\n",
      "Epoch: 55, timecost:10747.32 seconds, Loss: 0.44019003429115705, Accuracy: 0.7793258181970307, F1 Score: 0.8511856919912272, ROC AUC: 0.8200998457940709, Average Precision Score: 0.913256884443705\n",
      "best AUC: 0.8219338287097474\n",
      "Epoch: 56, timecost:10937.99 seconds, Loss: 0.43954180606394916, Accuracy: 0.7789778313127769, F1 Score: 0.8462831126022444, ROC AUC: 0.824185823335205, Average Precision Score: 0.9147186705812329\n",
      "best AUC: 0.824185823335205\n",
      "Epoch: 57, timecost:11129.11 seconds, Loss: 0.4386515953774268, Accuracy: 0.7809472215666509, F1 Score: 0.8494749219725818, ROC AUC: 0.8248830936582497, Average Precision Score: 0.9155383700453741\n",
      "best AUC: 0.824185823335205\n",
      "Epoch: 58, timecost:11322.09 seconds, Loss: 0.4385091460423229, Accuracy: 0.779803745538975, F1 Score: 0.8484494635473827, ROC AUC: 0.8249044809797267, Average Precision Score: 0.9158510663538634\n",
      "best AUC: 0.824185823335205\n",
      "Epoch: 59, timecost:11513.33 seconds, Loss: 0.43665711487082415, Accuracy: 0.7786835108939756, F1 Score: 0.8468262954502682, ROC AUC: 0.8241170732596685, Average Precision Score: 0.9150948053803222\n",
      "best AUC: 0.824185823335205\n",
      "Epoch: 60, timecost:11703.75 seconds, Loss: 0.43652835297301545, Accuracy: 0.7804908453250065, F1 Score: 0.8484665324691444, ROC AUC: 0.8262098245166042, Average Precision Score: 0.916332950150405\n",
      "best AUC: 0.8262098245166042\n",
      "Epoch: 61, timecost:11894.04 seconds, Loss: 0.436157488274645, Accuracy: 0.7780327521790381, F1 Score: 0.8445375869369529, ROC AUC: 0.8270193855959911, Average Precision Score: 0.9169176960378207\n",
      "best AUC: 0.8262098245166042\n",
      "Epoch: 62, timecost:12086.69 seconds, Loss: 0.4353076789428643, Accuracy: 0.7799565048087477, F1 Score: 0.847949138650205, ROC AUC: 0.8264026615937375, Average Precision Score: 0.9166595153434983\n",
      "best AUC: 0.8262098245166042\n",
      "Epoch: 63, timecost:12278.68 seconds, Loss: 0.4347472556267011, Accuracy: 0.7812983777303607, F1 Score: 0.8492536307033547, ROC AUC: 0.826617610806278, Average Precision Score: 0.9163488218636583\n",
      "best AUC: 0.8262098245166042\n",
      "Epoch: 64, timecost:12470.32 seconds, Loss: 0.4339175468205698, Accuracy: 0.7798895273695805, F1 Score: 0.8479974623662879, ROC AUC: 0.8259793515678807, Average Precision Score: 0.916122857062247\n",
      "best AUC: 0.8262098245166042\n",
      "Epoch: 65, timecost:12662.42 seconds, Loss: 0.43396422137846224, Accuracy: 0.7829721798536511, F1 Score: 0.8526031529966641, ROC AUC: 0.8267127056474053, Average Precision Score: 0.9165246748122423\n",
      "best AUC: 0.8262098245166042\n",
      "Epoch: 66, timecost:12852.56 seconds, Loss: 0.43296483732118807, Accuracy: 0.7802446979539344, F1 Score: 0.8481614047852556, ROC AUC: 0.8266133814317796, Average Precision Score: 0.9168656657585073\n",
      "best AUC: 0.8262098245166042\n",
      "Epoch: 67, timecost:13042.51 seconds, Loss: 0.4321703451677317, Accuracy: 0.7819406850334158, F1 Score: 0.8499590101194631, ROC AUC: 0.8283729600833483, Average Precision Score: 0.9174107543072244\n",
      "best AUC: 0.8283729600833483\n",
      "Epoch: 68, timecost:13233.38 seconds, Loss: 0.4319317880830001, Accuracy: 0.7836185015773504, F1 Score: 0.8533635659176675, ROC AUC: 0.826773148928521, Average Precision Score: 0.9162647884423232\n",
      "best AUC: 0.8283729600833483\n",
      "Epoch: 69, timecost:13423.13 seconds, Loss: 0.43208660653153935, Accuracy: 0.7813220416836311, F1 Score: 0.8492817068423775, ROC AUC: 0.8269777443476124, Average Precision Score: 0.916546888335461\n",
      "best AUC: 0.8283729600833483\n",
      "Epoch: 70, timecost:13613.86 seconds, Loss: 0.43120174986318593, Accuracy: 0.7827951227747169, F1 Score: 0.8502591362629522, ROC AUC: 0.8288893771263462, Average Precision Score: 0.9174850783319389\n",
      "best AUC: 0.8283729600833483\n",
      "Epoch: 71, timecost:13804.32 seconds, Loss: 0.4304945148593948, Accuracy: 0.7819552637189128, F1 Score: 0.8491238001447955, ROC AUC: 0.8292749870679875, Average Precision Score: 0.9178712398780811\n",
      "best AUC: 0.8283729600833483\n",
      "Epoch: 72, timecost:13997.08 seconds, Loss: 0.4310519211370089, Accuracy: 0.7793397630266365, F1 Score: 0.8452714247765492, ROC AUC: 0.8298201648387941, Average Precision Score: 0.9181154023065894\n",
      "best AUC: 0.8298201648387941\n",
      "Epoch: 73, timecost:14186.14 seconds, Loss: 0.4292730273054332, Accuracy: 0.7846928873128883, F1 Score: 0.8542823293721128, ROC AUC: 0.8286696830386361, Average Precision Score: 0.9173207882684006\n",
      "best AUC: 0.8298201648387941\n",
      "Epoch: 74, timecost:14376.91 seconds, Loss: 0.42906360030174256, Accuracy: 0.7796051373597409, F1 Score: 0.8456554429755297, ROC AUC: 0.8283701228819651, Average Precision Score: 0.9169637246127392\n",
      "best AUC: 0.8298201648387941\n",
      "Epoch: 75, timecost:14567.50 seconds, Loss: 0.42879399154943243, Accuracy: 0.7849546697959427, F1 Score: 0.8529507205269636, ROC AUC: 0.830147322989088, Average Precision Score: 0.9179931678065962\n",
      "best AUC: 0.8298201648387941\n",
      "Epoch: 76, timecost:14758.35 seconds, Loss: 0.4289425669508801, Accuracy: 0.780914683630904, F1 Score: 0.8475347393510223, ROC AUC: 0.8285962553931923, Average Precision Score: 0.9173263630452295\n",
      "best AUC: 0.8298201648387941\n",
      "Epoch: 77, timecost:14948.55 seconds, Loss: 0.4276506131409891, Accuracy: 0.7787933792484455, F1 Score: 0.8444131243439856, ROC AUC: 0.8295527905653254, Average Precision Score: 0.9179336259012238\n",
      "best AUC: 0.8298201648387941\n",
      "Epoch: 78, timecost:15138.71 seconds, Loss: 0.4276326810570782, Accuracy: 0.7839548677702661, F1 Score: 0.8509865368383255, ROC AUC: 0.8304267155635922, Average Precision Score: 0.9182223377785985\n",
      "best AUC: 0.8298201648387941\n",
      "Epoch: 79, timecost:15330.25 seconds, Loss: 0.4282996624531307, Accuracy: 0.7840750891042919, F1 Score: 0.8511776152833614, ROC AUC: 0.8303458991281107, Average Precision Score: 0.9181743158723078\n",
      "best AUC: 0.8298201648387941\n",
      "Epoch: 80, timecost:15520.69 seconds, Loss: 0.42699860737656276, Accuracy: 0.7828847077406693, F1 Score: 0.8502200145379113, ROC AUC: 0.8286554660469826, Average Precision Score: 0.9169923870527084\n",
      "best AUC: 0.8298201648387941\n",
      "Epoch: 81, timecost:15709.55 seconds, Loss: 0.4266703070448131, Accuracy: 0.7802780810308694, F1 Score: 0.846511321732744, ROC AUC: 0.8292505642088597, Average Precision Score: 0.9178242568857383\n",
      "best AUC: 0.8298201648387941\n",
      "Epoch: 82, timecost:15899.41 seconds, Loss: 0.4265675724083309, Accuracy: 0.7847028177218501, F1 Score: 0.8523162990231601, ROC AUC: 0.8296548588571041, Average Precision Score: 0.9174394232559334\n",
      "best AUC: 0.8298201648387941\n",
      "Epoch: 83, timecost:16090.84 seconds, Loss: 0.4260975664376505, Accuracy: 0.7815519200868298, F1 Score: 0.8483867850418801, ROC AUC: 0.8290926590261192, Average Precision Score: 0.9173826516938594\n",
      "best AUC: 0.8298201648387941\n",
      "Epoch: 84, timecost:16283.23 seconds, Loss: 0.4256672678963013, Accuracy: 0.7848805086566756, F1 Score: 0.8529097630556605, ROC AUC: 0.8306979157051129, Average Precision Score: 0.9183983515979603\n",
      "best AUC: 0.8298201648387941\n",
      "Epoch: 85, timecost:16475.40 seconds, Loss: 0.4258684576265182, Accuracy: 0.7822436681493965, F1 Score: 0.8480753795252075, ROC AUC: 0.8309707954577078, Average Precision Score: 0.9182792797814873\n",
      "best AUC: 0.8309707954577078\n",
      "Epoch: 86, timecost:16665.91 seconds, Loss: 0.4247095514564203, Accuracy: 0.7838559862512432, F1 Score: 0.8500196749152746, ROC AUC: 0.8313585561832276, Average Precision Score: 0.9187274412764665\n",
      "best AUC: 0.8309707954577078\n",
      "Epoch: 87, timecost:16857.61 seconds, Loss: 0.42497893478113397, Accuracy: 0.7849542472253487, F1 Score: 0.8525788468624638, ROC AUC: 0.8304121255381472, Average Precision Score: 0.9181840010835333\n",
      "best AUC: 0.8309707954577078\n",
      "Epoch: 88, timecost:17049.38 seconds, Loss: 0.42475544504312807, Accuracy: 0.78667537725518, F1 Score: 0.8539855861578061, ROC AUC: 0.8318812267537424, Average Precision Score: 0.9185061333762616\n",
      "best AUC: 0.8309707954577078\n",
      "Epoch: 89, timecost:17240.78 seconds, Loss: 0.4240994859431547, Accuracy: 0.788061620089175, F1 Score: 0.8566614580765656, ROC AUC: 0.8324559630507729, Average Precision Score: 0.9192043137039588\n",
      "best AUC: 0.8324559630507729\n",
      "Epoch: 90, timecost:17431.98 seconds, Loss: 0.42461899888621596, Accuracy: 0.7877229297579917, F1 Score: 0.8546583807802463, ROC AUC: 0.8326090210999425, Average Precision Score: 0.9189596686739683\n",
      "best AUC: 0.8324559630507729\n",
      "Epoch: 91, timecost:17625.31 seconds, Loss: 0.42401014829955397, Accuracy: 0.7825525672536947, F1 Score: 0.8491636077973869, ROC AUC: 0.8301265160482846, Average Precision Score: 0.9179506051759622\n",
      "best AUC: 0.8324559630507729\n",
      "Epoch: 92, timecost:17816.38 seconds, Loss: 0.4236213928691004, Accuracy: 0.7836455460953737, F1 Score: 0.851087116171862, ROC AUC: 0.8301080395935138, Average Precision Score: 0.9181552183043277\n",
      "best AUC: 0.8324559630507729\n",
      "Epoch: 93, timecost:18007.48 seconds, Loss: 0.42272331373274147, Accuracy: 0.7892929908004269, F1 Score: 0.8572220913809655, ROC AUC: 0.8327723579426999, Average Precision Score: 0.9188218208022715\n",
      "best AUC: 0.8324559630507729\n",
      "Epoch: 94, timecost:18198.61 seconds, Loss: 0.42317133917299155, Accuracy: 0.7829375290649336, F1 Score: 0.8486335971022791, ROC AUC: 0.8317717953602942, Average Precision Score: 0.9184319498198361\n",
      "best AUC: 0.8324559630507729\n",
      "Epoch: 95, timecost:18391.51 seconds, Loss: 0.42230436076750033, Accuracy: 0.7866297396310156, F1 Score: 0.8531746054823918, ROC AUC: 0.8328643307037165, Average Precision Score: 0.9190514324526008\n",
      "best AUC: 0.8324559630507729\n",
      "Epoch: 96, timecost:18583.48 seconds, Loss: 0.4218745355613154, Accuracy: 0.7895843532250694, F1 Score: 0.8569628292715715, ROC AUC: 0.8349160257845106, Average Precision Score: 0.9202454167193502\n",
      "best AUC: 0.8349160257845106\n",
      "Epoch: 97, timecost:18775.67 seconds, Loss: 0.4219088986471782, Accuracy: 0.7878201209946382, F1 Score: 0.8551949121235043, ROC AUC: 0.8330504124851019, Average Precision Score: 0.9193373644439327\n",
      "best AUC: 0.8349160257845106\n",
      "Epoch: 98, timecost:18966.73 seconds, Loss: 0.4215612148849476, Accuracy: 0.7885285605956724, F1 Score: 0.8568484697484567, ROC AUC: 0.8326337489408518, Average Precision Score: 0.9189475769890953\n",
      "best AUC: 0.8349160257845106\n",
      "Epoch: 99, timecost:19157.97 seconds, Loss: 0.420840641422866, Accuracy: 0.7879261862137611, F1 Score: 0.855025976633307, ROC AUC: 0.8326066922024984, Average Precision Score: 0.9188129495618947\n",
      "best AUC: 0.8349160257845106\n",
      "Epoch: 100, timecost:19348.42 seconds, Loss: 0.42073831963256136, Accuracy: 0.7884129875381819, F1 Score: 0.8549201329702237, ROC AUC: 0.8334824011789881, Average Precision Score: 0.918868684420461\n",
      "best AUC: 0.8349160257845106\n",
      "Epoch: 101, timecost:19540.02 seconds, Loss: 0.42058876438027676, Accuracy: 0.7903868147832942, F1 Score: 0.8565115544468306, ROC AUC: 0.8361432397793479, Average Precision Score: 0.9206375073604002\n",
      "best AUC: 0.8361432397793479\n",
      "Epoch: 102, timecost:19730.62 seconds, Loss: 0.4197131047616727, Accuracy: 0.7898444454257473, F1 Score: 0.8563222151489794, ROC AUC: 0.8350125079954042, Average Precision Score: 0.9199786502444363\n",
      "best AUC: 0.8361432397793479\n",
      "Epoch: 103, timecost:19921.10 seconds, Loss: 0.4196961354077393, Accuracy: 0.7896910523000834, F1 Score: 0.8559848915501685, ROC AUC: 0.8355698504062493, Average Precision Score: 0.9202458611839625\n",
      "best AUC: 0.8361432397793479\n",
      "Epoch: 104, timecost:20113.23 seconds, Loss: 0.42044665721474483, Accuracy: 0.7890417725822254, F1 Score: 0.8559144166973564, ROC AUC: 0.8347969182633975, Average Precision Score: 0.9198958407887521\n",
      "best AUC: 0.8361432397793479\n",
      "Epoch: 105, timecost:20304.25 seconds, Loss: 0.4188093772804702, Accuracy: 0.7872357058629769, F1 Score: 0.8523660430756889, ROC AUC: 0.8355964843172796, Average Precision Score: 0.920172171612674\n",
      "best AUC: 0.8361432397793479\n",
      "Epoch: 106, timecost:20495.92 seconds, Loss: 0.41918146378562426, Accuracy: 0.787705604363633, F1 Score: 0.8545560030664403, ROC AUC: 0.8331541578670619, Average Precision Score: 0.9193593334683163\n",
      "best AUC: 0.8361432397793479\n",
      "Epoch: 107, timecost:20686.49 seconds, Loss: 0.418307662434677, Accuracy: 0.7908062160979535, F1 Score: 0.8560791559482014, ROC AUC: 0.8377616302965251, Average Precision Score: 0.9216196107846168\n",
      "best AUC: 0.8377616302965251\n",
      "Epoch: 108, timecost:20878.79 seconds, Loss: 0.41820841648281504, Accuracy: 0.7909813716092143, F1 Score: 0.8575486383003845, ROC AUC: 0.8356200225549912, Average Precision Score: 0.9202150601914645\n",
      "best AUC: 0.8377616302965251\n",
      "Epoch: 109, timecost:21072.95 seconds, Loss: 0.41742398510700873, Accuracy: 0.7909236907231176, F1 Score: 0.8584203983249298, ROC AUC: 0.8346796329195307, Average Precision Score: 0.9198065164112929\n",
      "best AUC: 0.8377616302965251\n",
      "Epoch: 110, timecost:21264.44 seconds, Loss: 0.41690484660315585, Accuracy: 0.7881376827961158, F1 Score: 0.8536219598518778, ROC AUC: 0.8359863855151248, Average Precision Score: 0.9204460370396843\n",
      "best AUC: 0.8377616302965251\n",
      "Epoch: 111, timecost:21455.19 seconds, Loss: 0.4173891667796172, Accuracy: 0.788677516730098, F1 Score: 0.8543826883330393, ROC AUC: 0.8355138844247046, Average Precision Score: 0.9203111793017242\n",
      "best AUC: 0.8377616302965251\n",
      "Epoch: 112, timecost:21645.12 seconds, Loss: 0.41616599790597175, Accuracy: 0.792263662077057, F1 Score: 0.8576407324645757, ROC AUC: 0.8377132737015506, Average Precision Score: 0.9210756353202735\n",
      "best AUC: 0.8377616302965251\n",
      "Epoch: 113, timecost:21835.72 seconds, Loss: 0.4155602187095127, Accuracy: 0.7917054463222308, F1 Score: 0.8589343173043118, ROC AUC: 0.8365491322958928, Average Precision Score: 0.9207378162912412\n",
      "best AUC: 0.8377616302965251\n",
      "Epoch: 114, timecost:22029.27 seconds, Loss: 0.4153684929411206, Accuracy: 0.7926036201200227, F1 Score: 0.8582726074332256, ROC AUC: 0.8371601467747771, Average Precision Score: 0.9206388932772089\n",
      "best AUC: 0.8377616302965251\n",
      "Epoch: 115, timecost:22220.81 seconds, Loss: 0.4154354010385293, Accuracy: 0.7922972564392892, F1 Score: 0.8582982073723725, ROC AUC: 0.8370351509355686, Average Precision Score: 0.9205763842242716\n",
      "best AUC: 0.8377616302965251\n",
      "Epoch: 116, timecost:22412.87 seconds, Loss: 0.414907010390426, Accuracy: 0.7924977661861968, F1 Score: 0.8576225114438246, ROC AUC: 0.8381080071799742, Average Precision Score: 0.9212198903965998\n",
      "best AUC: 0.8377616302965251\n",
      "Epoch: 117, timecost:22603.15 seconds, Loss: 0.41554263407469505, Accuracy: 0.7929507618630884, F1 Score: 0.8599759746766749, ROC AUC: 0.8367203043714915, Average Precision Score: 0.9206719913836905\n",
      "best AUC: 0.8377616302965251\n",
      "Epoch: 118, timecost:22793.83 seconds, Loss: 0.4147114409979444, Accuracy: 0.7892724961266123, F1 Score: 0.8537865009650714, ROC AUC: 0.8372157081996473, Average Precision Score: 0.9206943970252985\n",
      "best AUC: 0.8377616302965251\n",
      "Epoch: 119, timecost:22985.78 seconds, Loss: 0.4145570810541555, Accuracy: 0.7901782761950983, F1 Score: 0.8558537439636842, ROC AUC: 0.8355787939870445, Average Precision Score: 0.9197266921186522\n",
      "best AUC: 0.8377616302965251\n",
      "Epoch: 120, timecost:23178.48 seconds, Loss: 0.4140958558381134, Accuracy: 0.7931588778806902, F1 Score: 0.8591715720968772, ROC AUC: 0.8377193114757319, Average Precision Score: 0.9207933753030751\n",
      "best AUC: 0.8377616302965251\n",
      "Epoch: 121, timecost:23370.23 seconds, Loss: 0.41421051451643426, Accuracy: 0.7914738776366557, F1 Score: 0.8568640267644013, ROC AUC: 0.8374580782315497, Average Precision Score: 0.920849521752175\n",
      "best AUC: 0.8377616302965251\n",
      "Epoch: 122, timecost:23560.01 seconds, Loss: 0.41391598967840837, Accuracy: 0.79268137310934, F1 Score: 0.8595643289107479, ROC AUC: 0.8368883221401044, Average Precision Score: 0.9205584856155983\n",
      "best AUC: 0.8377616302965251\n",
      "Epoch: 123, timecost:23752.79 seconds, Loss: 0.4133864973702134, Accuracy: 0.7911284261759665, F1 Score: 0.8561608058216857, ROC AUC: 0.8377802647212061, Average Precision Score: 0.9209822720182212\n",
      "best AUC: 0.8377616302965251\n",
      "Epoch: 124, timecost:23945.43 seconds, Loss: 0.4137331532972149, Accuracy: 0.7947502787381281, F1 Score: 0.8610646134201679, ROC AUC: 0.839389066598621, Average Precision Score: 0.9219796711912944\n",
      "best AUC: 0.839389066598621\n",
      "Epoch: 125, timecost:24135.29 seconds, Loss: 0.43047825281096497, Accuracy: 0.7197410402885143, F1 Score: 0.8263881053978728, ROC AUC: 0.6715436626691361, Average Precision Score: 0.831942805445585\n",
      "best AUC: 0.839389066598621\n",
      "Epoch: 126, timecost:24326.50 seconds, Loss: 0.5118792063758352, Accuracy: 0.7420119895954669, F1 Score: 0.8371291895641864, ROC AUC: 0.7576314933575955, Average Precision Score: 0.8806988104809053\n",
      "best AUC: 0.839389066598621\n",
      "Epoch: 127, timecost:24517.59 seconds, Loss: 0.4277163695864578, Accuracy: 0.7924694539563911, F1 Score: 0.8580134526185568, ROC AUC: 0.8388453626831784, Average Precision Score: 0.9219380691266292\n",
      "best AUC: 0.839389066598621\n",
      "Epoch: 128, timecost:24709.33 seconds, Loss: 0.4149145273408833, Accuracy: 0.7903454028650709, F1 Score: 0.8558135385159059, ROC AUC: 0.8369841579956158, Average Precision Score: 0.9206625060899393\n",
      "best AUC: 0.839389066598621\n",
      "Epoch: 129, timecost:24899.50 seconds, Loss: 0.41333604776894656, Accuracy: 0.793935562632674, F1 Score: 0.8605022713489638, ROC AUC: 0.8374023387362994, Average Precision Score: 0.9205736989698192\n",
      "best AUC: 0.839389066598621\n",
      "Epoch: 130, timecost:25090.86 seconds, Loss: 0.41361426297210446, Accuracy: 0.7932034590783693, F1 Score: 0.8590288415074584, ROC AUC: 0.8379035524539413, Average Precision Score: 0.9206786272919629\n",
      "best AUC: 0.839389066598621\n",
      "Epoch: 131, timecost:25281.58 seconds, Loss: 0.4132885607572974, Accuracy: 0.793106901697614, F1 Score: 0.8583020768215599, ROC AUC: 0.8391493466667531, Average Precision Score: 0.9216367724680986\n",
      "best AUC: 0.839389066598621\n",
      "Epoch: 132, timecost:25472.92 seconds, Loss: 0.41201624895061867, Accuracy: 0.7939888065275325, F1 Score: 0.8612882758306729, ROC AUC: 0.838223325423978, Average Precision Score: 0.9212314312838971\n",
      "best AUC: 0.839389066598621\n",
      "Epoch: 133, timecost:25663.36 seconds, Loss: 0.4117530330529199, Accuracy: 0.7938901362938066, F1 Score: 0.8604912940864201, ROC AUC: 0.8373272088409748, Average Precision Score: 0.920600953576538\n",
      "best AUC: 0.839389066598621\n",
      "Epoch: 134, timecost:25854.52 seconds, Loss: 0.41254977422934963, Accuracy: 0.7940682497992262, F1 Score: 0.8615529458823784, ROC AUC: 0.837080781018461, Average Precision Score: 0.919904438667071\n",
      "best AUC: 0.839389066598621\n",
      "Epoch: 135, timecost:26046.85 seconds, Loss: 0.41171783463537515, Accuracy: 0.7945284291762177, F1 Score: 0.8610820096742794, ROC AUC: 0.838165195349243, Average Precision Score: 0.9208580198033669\n",
      "best AUC: 0.839389066598621\n",
      "Epoch: 136, timecost:26238.20 seconds, Loss: 0.41148137812798385, Accuracy: 0.795160383499717, F1 Score: 0.8610931894480496, ROC AUC: 0.8394463449184087, Average Precision Score: 0.9214580677807431\n",
      "best AUC: 0.839389066598621\n",
      "Epoch: 137, timecost:26429.47 seconds, Loss: 0.41111116334132697, Accuracy: 0.7941872034214695, F1 Score: 0.8600701273332982, ROC AUC: 0.8390198077431291, Average Precision Score: 0.9219566712813989\n",
      "best AUC: 0.839389066598621\n",
      "Epoch: 138, timecost:26621.37 seconds, Loss: 0.41156069236622367, Accuracy: 0.7937058955147723, F1 Score: 0.8599539259938608, ROC AUC: 0.8377225941646759, Average Precision Score: 0.9207642879009754\n",
      "best AUC: 0.839389066598621\n",
      "Epoch: 139, timecost:26812.52 seconds, Loss: 0.41139328193240066, Accuracy: 0.793458903002512, F1 Score: 0.8583377532772797, ROC AUC: 0.8392813974310135, Average Precision Score: 0.9214199349329872\n",
      "best AUC: 0.839389066598621\n",
      "Epoch: 140, timecost:27004.86 seconds, Loss: 0.411028074528414, Accuracy: 0.7942588291371722, F1 Score: 0.8608020236997935, ROC AUC: 0.8391353591767073, Average Precision Score: 0.9219214284162531\n",
      "best AUC: 0.839389066598621\n",
      "Epoch: 141, timecost:27197.66 seconds, Loss: 0.41439989072986455, Accuracy: 0.7935108791855882, F1 Score: 0.8595064252126523, ROC AUC: 0.8376258788093325, Average Precision Score: 0.9206371968644247\n",
      "best AUC: 0.839389066598621\n",
      "Epoch: 142, timecost:27389.40 seconds, Loss: 0.4111935306904224, Accuracy: 0.7946511859338081, F1 Score: 0.8602335971280027, ROC AUC: 0.8395331382135625, Average Precision Score: 0.9213953728370107\n",
      "best AUC: 0.839389066598621\n",
      "Epoch: 143, timecost:27578.46 seconds, Loss: 0.412167290127242, Accuracy: 0.7926923599447869, F1 Score: 0.8591134017835579, ROC AUC: 0.8358445745777237, Average Precision Score: 0.9193897492148255\n",
      "best AUC: 0.839389066598621\n",
      "Epoch: 144, timecost:27768.72 seconds, Loss: 0.4108779083727375, Accuracy: 0.7955104832369415, F1 Score: 0.861038699239661, ROC AUC: 0.8397496954293399, Average Precision Score: 0.9214593828429949\n",
      "best AUC: 0.839389066598621\n",
      "Epoch: 145, timecost:27959.27 seconds, Loss: 0.4108196996439812, Accuracy: 0.7908744612489031, F1 Score: 0.8572948629334126, ROC AUC: 0.8351608146973422, Average Precision Score: 0.919134650732885\n",
      "best AUC: 0.839389066598621\n",
      "Epoch: 146, timecost:28150.70 seconds, Loss: 0.41152451843111737, Accuracy: 0.7936112397016906, F1 Score: 0.8598369943667908, ROC AUC: 0.8374816466696663, Average Precision Score: 0.920524590462755\n",
      "best AUC: 0.839389066598621\n",
      "Epoch: 147, timecost:28340.53 seconds, Loss: 0.410304448424178, Accuracy: 0.7943741909093656, F1 Score: 0.8594955773216536, ROC AUC: 0.839820218634154, Average Precision Score: 0.9217721583676292\n",
      "best AUC: 0.839389066598621\n",
      "Epoch: 148, timecost:28533.31 seconds, Loss: 0.4101041864924332, Accuracy: 0.7910356719305581, F1 Score: 0.8547582687037012, ROC AUC: 0.8400418903400526, Average Precision Score: 0.9218371961646679\n",
      "best AUC: 0.839389066598621\n",
      "Epoch: 149, timecost:28724.34 seconds, Loss: 0.4104673629167879, Accuracy: 0.7956537346683465, F1 Score: 0.8611137518004155, ROC AUC: 0.8395609401272571, Average Precision Score: 0.9216124908372855\n",
      "best AUC: 0.839389066598621\n",
      "Epoch: 150, timecost:28916.32 seconds, Loss: 0.4099558587863113, Accuracy: 0.7960439786120119, F1 Score: 0.8624285411135595, ROC AUC: 0.8389698949285183, Average Precision Score: 0.9212865631503873\n",
      "best AUC: 0.839389066598621\n",
      "Epoch: 151, timecost:29107.30 seconds, Loss: 0.4098686552189224, Accuracy: 0.7982183156040319, F1 Score: 0.8652265163181164, ROC AUC: 0.8416035534207345, Average Precision Score: 0.9227391015558374\n",
      "best AUC: 0.8416035534207345\n",
      "Epoch: 152, timecost:29300.59 seconds, Loss: 0.4100203861733926, Accuracy: 0.7949820587090004, F1 Score: 0.8600970589206387, ROC AUC: 0.840457242810706, Average Precision Score: 0.9220426587844575\n",
      "best AUC: 0.8416035534207345\n",
      "Epoch: 153, timecost:29491.54 seconds, Loss: 0.4165897520784101, Accuracy: 0.7952607440158194, F1 Score: 0.8626995849042061, ROC AUC: 0.8387849897206993, Average Precision Score: 0.9213399556611412\n",
      "best AUC: 0.8416035534207345\n",
      "Epoch: 154, timecost:29682.55 seconds, Loss: 0.41098490314950575, Accuracy: 0.7940540936843233, F1 Score: 0.8612030392312241, ROC AUC: 0.8369251165434226, Average Precision Score: 0.9201435093837135\n",
      "best AUC: 0.8416035534207345\n",
      "Epoch: 155, timecost:29873.25 seconds, Loss: 0.409580915626857, Accuracy: 0.7955573885728883, F1 Score: 0.8623650732103095, ROC AUC: 0.8386219422998309, Average Precision Score: 0.9209766678232978\n",
      "best AUC: 0.8416035534207345\n",
      "Epoch: 156, timecost:30065.61 seconds, Loss: 0.40946933731480945, Accuracy: 0.7947952825064014, F1 Score: 0.8599321221973508, ROC AUC: 0.8410757675900896, Average Precision Score: 0.9227691162032672\n",
      "best AUC: 0.8416035534207345\n",
      "Epoch: 157, timecost:30255.41 seconds, Loss: 0.4095981970296769, Accuracy: 0.794474551425468, F1 Score: 0.8594095089242549, ROC AUC: 0.8408326763368075, Average Precision Score: 0.9226954083001667\n",
      "best AUC: 0.8416035534207345\n",
      "Epoch: 158, timecost:30446.17 seconds, Loss: 0.4083373666428671, Accuracy: 0.7961984281641611, F1 Score: 0.8621716685604285, ROC AUC: 0.8404447275226041, Average Precision Score: 0.9222750317331577\n",
      "best AUC: 0.8416035534207345\n",
      "Epoch: 159, timecost:30636.71 seconds, Loss: 0.40847740887003764, Accuracy: 0.7948865577547303, F1 Score: 0.8611365844734123, ROC AUC: 0.8387230238488861, Average Precision Score: 0.921304502362788\n",
      "best AUC: 0.8416035534207345\n",
      "Epoch: 160, timecost:30828.72 seconds, Loss: 0.40869807249006956, Accuracy: 0.7968920777944012, F1 Score: 0.8640120956476238, ROC AUC: 0.8401705255999882, Average Precision Score: 0.9219482909065423\n",
      "best AUC: 0.8416035534207345\n",
      "Epoch: 161, timecost:31020.76 seconds, Loss: 0.40858167698192316, Accuracy: 0.7949562819027594, F1 Score: 0.8602478778884947, ROC AUC: 0.8401998983405548, Average Precision Score: 0.9218507620427251\n",
      "best AUC: 0.8416035534207345\n",
      "Epoch: 162, timecost:31213.33 seconds, Loss: 0.40815696004413354, Accuracy: 0.7909965841506025, F1 Score: 0.8556303155321172, ROC AUC: 0.838799661803167, Average Precision Score: 0.9217224613816721\n",
      "best AUC: 0.8416035534207345\n",
      "Epoch: 163, timecost:31404.09 seconds, Loss: 0.4086014174742939, Accuracy: 0.796055176732756, F1 Score: 0.8616094944060734, ROC AUC: 0.8407299308260655, Average Precision Score: 0.9223193139602494\n",
      "best AUC: 0.8416035534207345\n",
      "Epoch: 164, timecost:31596.16 seconds, Loss: 0.40844114613816007, Accuracy: 0.795945308378286, F1 Score: 0.8614879555741682, ROC AUC: 0.8408291614805165, Average Precision Score: 0.9222377200568614\n",
      "best AUC: 0.8416035534207345\n",
      "Epoch: 165, timecost:31786.62 seconds, Loss: 0.4077713620061337, Accuracy: 0.7956019697705674, F1 Score: 0.8630596767204863, ROC AUC: 0.8376986943807406, Average Precision Score: 0.9206624435462729\n",
      "best AUC: 0.8416035534207345\n",
      "Epoch: 166, timecost:31979.91 seconds, Loss: 0.40816960521373974, Accuracy: 0.7948979671607714, F1 Score: 0.8602784743771303, ROC AUC: 0.8402266132688263, Average Precision Score: 0.9221187724456853\n",
      "best AUC: 0.8416035534207345\n",
      "Epoch: 167, timecost:32173.29 seconds, Loss: 0.4077000672456419, Accuracy: 0.7940898008995261, F1 Score: 0.8596857788449992, ROC AUC: 0.8391302928321904, Average Precision Score: 0.9217173740292993\n",
      "best AUC: 0.8416035534207345\n",
      "Epoch: 168, timecost:32366.08 seconds, Loss: 0.4076081416461871, Accuracy: 0.7964925372976652, F1 Score: 0.8618649609912803, ROC AUC: 0.8395359229900294, Average Precision Score: 0.9211310055329772\n",
      "best AUC: 0.8416035534207345\n",
      "Epoch: 169, timecost:32556.06 seconds, Loss: 0.40777929922239714, Accuracy: 0.7976879895084172, F1 Score: 0.8643596872905255, ROC AUC: 0.8417485086219405, Average Precision Score: 0.9228308214002816\n",
      "best AUC: 0.8416035534207345\n",
      "Epoch: 170, timecost:32746.61 seconds, Loss: 0.40867127896416433, Accuracy: 0.7972679543378668, F1 Score: 0.863457035096647, ROC AUC: 0.8405740774486108, Average Precision Score: 0.922080277649409\n",
      "best AUC: 0.8416035534207345\n",
      "Epoch: 171, timecost:32938.55 seconds, Loss: 0.40760474583515427, Accuracy: 0.7982085964803672, F1 Score: 0.8632248688409976, ROC AUC: 0.8435121113259806, Average Precision Score: 0.9238486911868684\n",
      "best AUC: 0.8435121113259806\n",
      "Epoch: 172, timecost:33129.55 seconds, Loss: 0.40815089188449816, Accuracy: 0.7987458949907847, F1 Score: 0.8643975116939197, ROC AUC: 0.8431023030336068, Average Precision Score: 0.9234828087947494\n",
      "best AUC: 0.8435121113259806\n",
      "Epoch: 173, timecost:33320.82 seconds, Loss: 0.406973115679772, Accuracy: 0.7942357990397928, F1 Score: 0.8595592653817351, ROC AUC: 0.8397536014938999, Average Precision Score: 0.9217062677606652\n",
      "best AUC: 0.8435121113259806\n",
      "Epoch: 174, timecost:33513.33 seconds, Loss: 0.4067364042254513, Accuracy: 0.7965601485927237, F1 Score: 0.8623713789276702, ROC AUC: 0.8415273092219997, Average Precision Score: 0.9229460937697467\n",
      "best AUC: 0.8435121113259806\n",
      "Epoch: 175, timecost:33705.60 seconds, Loss: 0.4073589657286862, Accuracy: 0.7945975194683554, F1 Score: 0.8600334281794455, ROC AUC: 0.8395371898529311, Average Precision Score: 0.9218401095946417\n",
      "best AUC: 0.8435121113259806\n",
      "Epoch: 176, timecost:33896.29 seconds, Loss: 0.40637605425512047, Accuracy: 0.7945776586504321, F1 Score: 0.8588795753778243, ROC AUC: 0.841497363836174, Average Precision Score: 0.9228857944048567\n",
      "best AUC: 0.8435121113259806\n",
      "Epoch: 177, timecost:34089.31 seconds, Loss: 0.40933470069831485, Accuracy: 0.7949746637236034, F1 Score: 0.8616923506707488, ROC AUC: 0.8388525581450241, Average Precision Score: 0.9215738575916825\n",
      "best AUC: 0.8435121113259806\n",
      "Epoch: 178, timecost:34280.53 seconds, Loss: 0.40717277063460305, Accuracy: 0.7965675435781208, F1 Score: 0.8639474709411056, ROC AUC: 0.8405048316320227, Average Precision Score: 0.9223025598044876\n",
      "best AUC: 0.8435121113259806\n",
      "Epoch: 179, timecost:34470.05 seconds, Loss: 0.40602005942992714, Accuracy: 0.7757504484002217, F1 Score: 0.8462694421120677, ROC AUC: 0.8201437464206662, Average Precision Score: 0.9132440010542275\n",
      "best AUC: 0.8435121113259806\n",
      "Epoch: 180, timecost:34662.95 seconds, Loss: 0.4056613800433694, Accuracy: 0.7980452729457417, F1 Score: 0.8636002276947023, ROC AUC: 0.8437114665007174, Average Precision Score: 0.9240851657609633\n",
      "best AUC: 0.8435121113259806\n",
      "Epoch: 181, timecost:34855.07 seconds, Loss: 0.40677449376010044, Accuracy: 0.7983624121766252, F1 Score: 0.8642001203267378, ROC AUC: 0.8423573109008006, Average Precision Score: 0.9232604609229351\n",
      "best AUC: 0.8435121113259806\n",
      "Epoch: 182, timecost:35046.85 seconds, Loss: 0.40571739173961324, Accuracy: 0.7956355641327996, F1 Score: 0.859768088542739, ROC AUC: 0.843185433473801, Average Precision Score: 0.9235737481050387\n",
      "best AUC: 0.8435121113259806\n",
      "Epoch: 183, timecost:35238.81 seconds, Loss: 0.40552792705661816, Accuracy: 0.7977505299563463, F1 Score: 0.8632729900900721, ROC AUC: 0.8439038467868776, Average Precision Score: 0.9243329932587502\n",
      "best AUC: 0.8435121113259806\n",
      "Epoch: 184, timecost:35427.82 seconds, Loss: 0.40594782476432245, Accuracy: 0.7979370948736483, F1 Score: 0.8640862866070239, ROC AUC: 0.8424909432933836, Average Precision Score: 0.9232075880593703\n",
      "best AUC: 0.8435121113259806\n",
      "Epoch: 185, timecost:35617.18 seconds, Loss: 0.4056462161441939, Accuracy: 0.7966619881059055, F1 Score: 0.8613432895601677, ROC AUC: 0.8433919708516869, Average Precision Score: 0.9239841649125367\n",
      "best AUC: 0.8435121113259806\n",
      "Epoch: 186, timecost:35807.11 seconds, Loss: 0.40485877734263503, Accuracy: 0.7928973066829328, F1 Score: 0.8574103959936586, ROC AUC: 0.8398099237362829, Average Precision Score: 0.9225663242768775\n",
      "best AUC: 0.8435121113259806\n",
      "Epoch: 187, timecost:35998.75 seconds, Loss: 0.4046621675837995, Accuracy: 0.7974794509202214, F1 Score: 0.8641200700807624, ROC AUC: 0.8406006330220325, Average Precision Score: 0.9222352757536308\n",
      "best AUC: 0.8435121113259806\n",
      "Epoch: 188, timecost:36191.74 seconds, Loss: 0.40493557591112855, Accuracy: 0.7970308922345681, F1 Score: 0.8625721766453384, ROC AUC: 0.8421454011476798, Average Precision Score: 0.9233211639585609\n",
      "best AUC: 0.8435121113259806\n",
      "Epoch: 189, timecost:36383.44 seconds, Loss: 0.4050395674097078, Accuracy: 0.7960422883296355, F1 Score: 0.8623599226447477, ROC AUC: 0.8417593694826835, Average Precision Score: 0.9233403680550634\n",
      "best AUC: 0.8435121113259806\n",
      "Epoch: 190, timecost:36575.48 seconds, Loss: 0.4060284334580339, Accuracy: 0.7980471745134152, F1 Score: 0.8645189766698739, ROC AUC: 0.8422834853127923, Average Precision Score: 0.9231248669402821\n",
      "best AUC: 0.8435121113259806\n",
      "Epoch: 191, timecost:36767.44 seconds, Loss: 0.4042955491769208, Accuracy: 0.7965443021954444, F1 Score: 0.8625337850714846, ROC AUC: 0.8421039584194016, Average Precision Score: 0.9233943018930779\n",
      "best AUC: 0.8435121113259806\n",
      "Epoch: 192, timecost:36958.47 seconds, Loss: 0.4036230713394346, Accuracy: 0.7973345092064399, F1 Score: 0.8638274668737074, ROC AUC: 0.84166120742056, Average Precision Score: 0.9229545436482619\n",
      "best AUC: 0.8435121113259806\n",
      "Epoch: 193, timecost:37150.24 seconds, Loss: 0.403589829210004, Accuracy: 0.7988033645915845, F1 Score: 0.8639646286977546, ROC AUC: 0.8438963444604429, Average Precision Score: 0.9239598259654499\n",
      "best AUC: 0.8435121113259806\n",
      "Epoch: 194, timecost:37342.74 seconds, Loss: 0.4046770538173019, Accuracy: 0.7993381276784373, F1 Score: 0.8649745309538339, ROC AUC: 0.8442206853698216, Average Precision Score: 0.9244585963419515\n",
      "best AUC: 0.8435121113259806\n",
      "Epoch: 195, timecost:37533.53 seconds, Loss: 0.40356994388011513, Accuracy: 0.7965901511049059, F1 Score: 0.8616667648542202, ROC AUC: 0.8420634640911591, Average Precision Score: 0.9232084661417792\n",
      "best AUC: 0.8435121113259806\n",
      "Epoch: 196, timecost:37725.76 seconds, Loss: 0.405085331679098, Accuracy: 0.797657564425641, F1 Score: 0.8638413948752245, ROC AUC: 0.8422304070023694, Average Precision Score: 0.9232547407152\n",
      "best AUC: 0.8435121113259806\n",
      "Epoch: 197, timecost:37914.95 seconds, Loss: 0.4029629350062294, Accuracy: 0.7979153324880512, F1 Score: 0.8628949324734027, ROC AUC: 0.8431722275498371, Average Precision Score: 0.9236773154514586\n",
      "best AUC: 0.8435121113259806\n",
      "Epoch: 198, timecost:38105.29 seconds, Loss: 0.4052476817758571, Accuracy: 0.7970442032082827, F1 Score: 0.863967274325212, ROC AUC: 0.8412878842150554, Average Precision Score: 0.9227893416369717\n",
      "best AUC: 0.8435121113259806\n",
      "Epoch: 199, timecost:38294.57 seconds, Loss: 0.4054274760474083, Accuracy: 0.7991076154193474, F1 Score: 0.865264544225183, ROC AUC: 0.8424072040513159, Average Precision Score: 0.9228462020924559\n",
      "best AUC: 0.8435121113259806\n",
      "Epoch: 200, timecost:38485.87 seconds, Loss: 0.402786720099718, Accuracy: 0.7987279357405349, F1 Score: 0.8641476240580187, ROC AUC: 0.8424884389430525, Average Precision Score: 0.9230848122315516\n",
      "best AUC: 0.8435121113259806\n",
      "Epoch: 201, timecost:38679.49 seconds, Loss: 0.4033431587215701, Accuracy: 0.7995115929073217, F1 Score: 0.8657734725156433, ROC AUC: 0.8437678141738638, Average Precision Score: 0.9238703814214094\n",
      "Early stopping on epoch 201\n"
     ]
    }
   ],
   "source": [
    "\n",
    "start_time = time.time()\n",
    "for epoch in range(num_epochs):\n",
    "    #print(\"epoch:\",epoch)\n",
    "    epoch_loss = 0\n",
    "    model.train()\n",
    "\n",
    "    \n",
    "    # # # # # # ## # #\n",
    "    # traning \n",
    "    # # # # # # ## # #\n",
    "    for i,(src, tgt) in enumerate(train_dataloader):\n",
    "        #print(\"dataloader:\" ,i )\n",
    "        src = src.to(device)\n",
    "        tgt = tgt.to(device)\n",
    "    \n",
    "        #print(\"src\")\n",
    "        #check_memory(src)\n",
    "        # Create masks\n",
    "        #print(\"Create masks:\" ,i )\n",
    "        src_key_padding_mask, future_mask = create_masks(src)\n",
    "        \n",
    "        #print(\"src_key_padding_mask\")\n",
    "        #check_memory(src_key_padding_mask)\n",
    "        #print(\"future_mask\")\n",
    "       # check_memory(future_mask) \n",
    "        \n",
    "        src_key_padding_mask = src_key_padding_mask.to(device)\n",
    "        \n",
    "        future_mask = future_mask.to(device)\n",
    "\n",
    "        #print(\"Create masks done:\" ,i )\n",
    "        # Pass your sequences and mask through the model\n",
    "        output = model(src, src_key_padding_mask, future_mask)\n",
    "        # Calculate loss, perform backpropagation and update weights\n",
    "        tgt_mask = (tgt != -1).to(device)\n",
    "        loss = criterion(output[tgt_mask], tgt[tgt_mask])\n",
    "        loss = loss.mean()\n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        epoch_loss += loss.item()\n",
    "    \n",
    "    epoch_loss /= len(train_dataloader)  # calculate mean epoch loss\n",
    "    \n",
    "    # Early stopping check\n",
    "    #if best_loss is None or epoch_loss < best_loss - min_delta:\n",
    "    #    best_loss = epoch_loss\n",
    "    #    patience_counter = 0\n",
    "    #else:\n",
    "    #    patience_counter += 1\n",
    "    #    if patience_counter >= patience:\n",
    "    #        print(f'Early stopping on epoch {epoch}')\n",
    "    #        break\n",
    "    \n",
    "    # # # # # # ## # #\n",
    "    # Testing Loop\n",
    "    # # # # # # ## # #\n",
    "    model.eval()\n",
    "    with torch.no_grad():\n",
    "        all_probs, all_preds, all_targets = [], [],[]\n",
    "        for src, tgt in test_dataloader:\n",
    "            src = src.to(device)\n",
    "            \n",
    "            tgt = tgt.to(device)\n",
    "            # Create masks\n",
    "            src_key_padding_mask = (src == 0).all(axis=-1).to(device)\n",
    "\n",
    "            # Pass your sequences and mask through the model\n",
    "            output = model(src, src_key_padding_mask)\n",
    "\n",
    "            # Save predictions and targets\n",
    "            tgt_mask = (tgt != -1).to(device)\n",
    "            probs = output[tgt_mask].cpu().numpy()\n",
    "            preds = (output > 0.5).float()\n",
    "            all_probs.extend(probs)\n",
    "            all_preds.extend(preds[tgt_mask].cpu().numpy())\n",
    "            all_targets.extend(tgt[tgt_mask].cpu().numpy())\n",
    "\n",
    "        # Compute metrics\n",
    "        accuracy = accuracy_score(all_targets, all_preds)\n",
    "        f1 = f1_score(all_targets, all_preds)\n",
    "        roc_auc = roc_auc_score(all_targets, all_probs) if len(np.unique(all_targets)) > 1 else np.nan\n",
    "        avg_precision = average_precision_score(all_targets, all_probs)\n",
    "\n",
    "    epoch_time= time.time() - start_time\n",
    "    print(f\"Epoch: {epoch}, timecost:{epoch_time:.2f} seconds, Loss: {epoch_loss}, Accuracy: {accuracy}, F1 Score: {f1}, ROC AUC: {roc_auc}, Average Precision Score: {avg_precision}\")\n",
    "\n",
    "    # # # # # # # ## # #\n",
    "        # Early stopping check , find the best AUC \n",
    "     # # # # # # # # # # # # #\n",
    "    \n",
    "            \n",
    "    if best_AUC is None or roc_auc > best_AUC + min_delta:\n",
    "        best_AUC = roc_auc\n",
    "        patience_counter = 0\n",
    "    else:\n",
    "        patience_counter += 1\n",
    "        if patience_counter >= patience:\n",
    "            print(f'Early stopping on epoch {epoch}')\n",
    "            break\n",
    "\n",
    "\n",
    "\n",
    "    if best_AUC > best_model_AUC:\n",
    "        best_model_AUC = best_AUC\n",
    "        best_model = model\n",
    "        torch.save(best_model.state_dict(), \"./data/memory_transformer_KG_v2_0905.pth\")\n",
    "    print(\"best AUC:\",best_model_AUC)\n",
    "\n",
    "    \n",
    "    \n",
    "#     if best_f1 is None or f1 > best_f1 + min_delta:\n",
    "#         best_f1 = f1\n",
    "#         patience_counter = 0\n",
    "#     else:\n",
    "#         patience_counter += 1\n",
    "#         if patience_counter >= patience:\n",
    "#             print(f'Early stopping on epoch {epoch}')\n",
    "#             break\n",
    "\n",
    "            \n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "verbal-medline",
   "metadata": {},
   "source": [
    "### Save model and data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "found-chinese",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "\n",
    "# Assume you have a model instance called 'model'\n",
    "torch.save(best_model.state_dict(), \"./data/memory_transformer_KG_v2_0905.pth\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "known-fellowship",
   "metadata": {},
   "outputs": [],
   "source": [
    "# with open(f'./data/src_data_KG.pkl', 'wb') as file1:\n",
    "#     pickle.dump(src_data, file1)\n",
    "    \n",
    "# with open(f'./data/tgt_data_KG.pkl', 'wb') as file1:\n",
    "#     pickle.dump(tgt_data, file1)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "sunset-bahrain",
   "metadata": {},
   "source": [
    "## release memory in GPU"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "union-foster",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ycsong/zhai0045/.conda/envs/cz/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:293: UserWarning: torch.distributed.reduce_op is deprecated, please use torch.distributed.ReduceOp instead\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "1401"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch\n",
    "import gc\n",
    "\n",
    "# Function to check if an object is a tensor on the GPU\n",
    "def is_tensor_on_gpu(obj):\n",
    "    return torch.is_tensor(obj) and obj.is_cuda\n",
    "\n",
    "# Inspect all objects and delete tensors on GPU\n",
    "for obj in gc.get_objects():\n",
    "    try:\n",
    "        if is_tensor_on_gpu(obj):\n",
    "            del obj\n",
    "    except Exception as e:\n",
    "        # Handle any exception that occurs during deletion\n",
    "        print(f\"Error while deleting object: {e}\")\n",
    "\n",
    "# Clear up all unused memory\n",
    "torch.cuda.empty_cache()\n",
    "\n",
    "# Optional: run garbage collector\n",
    "gc.collect()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "accredited-retirement",
   "metadata": {},
   "outputs": [],
   "source": [
    "# del src_data\n",
    "# del tgt_data"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "cz",
   "language": "python",
   "name": "cz"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}