Prov2Vec.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.4.1\n",
      "[PhysicalDevice(name='/physical_device:CPU:0', device_type='CPU'), PhysicalDevice(name='/physical_device:GPU:0', device_type='GPU')]\n"
     ]
    }
   ],
   "source": [
    "# Import all the necessary packages\n",
    "import os\n",
    "import numpy as np \n",
    "import tensorflow as tf\n",
    "import json\n",
    "import matplotlib.pyplot as plt \n",
    "import pandas as pd\n",
    "\n",
    "print(tf.__version__)\n",
    "print(tf.config.list_physical_devices())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import regarding database handles\n",
    "import db_classes as orm\n",
    "from sqlalchemy import create_engine\n",
    "from sqlalchemy.orm import sessionmaker"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import graph_construction as gc\n",
    "import random"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# File locations for specific files.\n",
    "graph_file_path = '/mnt/8tb/csenrc/representation_learning_codes/graph_file.json'\n",
    "index_file_path = '/mnt/8tb/csenrc/representation_learning_codes/index_file.json'\n",
    "# use the database connection URL if needed.\n",
    "connection_url = 'postgresql+psycopg2://csephase2:csephase@@localhost/darpa_tc3'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of subjects in the graph: 224146\n"
     ]
    }
   ],
   "source": [
    "# Let's load the graph first. \n",
    "\n",
    "provenance_graph = json.load(open(graph_file_path))\n",
    "print(\"Number of subjects in the graph: {}\".format(len(provenance_graph)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Please note that the graph loading time is obnoxiously large. \n",
    "# Therefore, at this point, please refrain from doing anything that needs loading the graph from file multiple times.\n",
    "\n",
    "max_len = -1\n",
    "min_len = 1000000000000\n",
    "largest_subject_subgraph = -1\n",
    "smallest_subject_subgraph = -1\n",
    "count = 0\n",
    "small_graphs = []\n",
    "\n",
    "for subject in provenance_graph:\n",
    "    if len(provenance_graph[subject]['events']) > max_len:\n",
    "        max_len = len(provenance_graph[subject]['events'])\n",
    "        largest_subject_subgraph = subject\n",
    "    if len(provenance_graph[subject]['events']) < min_len:\n",
    "        min_len = len(provenance_graph[subject]['events'])\n",
    "        smallest_subject_subgraph = subject\n",
    "    if len(provenance_graph[subject]['events']) < 32:\n",
    "        count += 1\n",
    "        small_graphs.append(subject)  \n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Currently the largest subgraph is rooted at subject 74188 has the size of: 3049408\n",
      "Currently the smallest subgraph is rooted at subject 4 has the size of: 1\n",
      "Currently number of small subgraph is 124874\n"
     ]
    }
   ],
   "source": [
    "print(\"Currently the largest subgraph is rooted at subject {} has the size of: {}\".format(largest_subject_subgraph, max_len))\n",
    "print(\"Currently the smallest subgraph is rooted at subject {} has the size of: {}\".format(smallest_subject_subgraph, min_len))\n",
    "print(\"Currently number of small subgraph is {}\".format(count))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Wow that is a lot of events for a node. \n",
    "# Makes me wonder what a histogram for this data will look like. \n",
    "# In other words, what ate the node densities in the graoh. \n",
    "# Let's try it out. \n",
    "\n",
    "lengths = [] # Simple list will do, we are not tracking, we just want to see the distribution. \n",
    "\n",
    "for subject in provenance_graph:\n",
    "    lengths.append(len(provenance_graph[subject]['events']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA1kAAAGsCAYAAAAvwW2wAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAyzUlEQVR4nO3dfWxVdZ7H8U+nD5e2aa+Upr1cLVoTtoJlZtyiWHAEAm11Kews2SFO9SqRrZgitRZ8YJjZKWZpx/K4264orBFjYWs2iOOCU29x3GLT8jCFjhYI7GaQh5FaZ6gtT95ey9k/DCdzKc/zaw9t36/ExHvO957f93xztfn0nHsaZlmWJQAAAACAEd9zugEAAAAAGEgIWQAAAABgECELAAAAAAwiZAEAAACAQYQsAAAAADCIkAUAAAAABhGyAAAAAMCgCKcbuNmdP39eX3zxheLi4hQWFuZ0OwAAAAAcYlmWTp06Ja/Xq+997/LXqwhZV/HFF18oJSXF6TYAAAAA3CSOHTum22677bL7CVlXERcXJ+m7QcbHxzvaSzAYlN/vV3Z2tiIjIx3tZbBh9s5g7s5g7s5h9s5g7s5g7s5h9jeus7NTKSkpdka4HELWVVy4RTA+Pv6mCFkxMTGKj4/nP4g+xuydwdydwdydw+ydwdydwdydw+z/elf7GhEPvgAAAAAAgwhZAAAAAGAQIQsAAAAADCJkAQAAAIBBhCwAAAAAMIiQBQAAAAAGEbIAAAAAwCBCFgAAAAAYRMgCAAAAAIMIWQAAAABgECELAAAAAAwiZAEAAACAQYQsAAAAADCIkAUAAAAABhGyAAAAAMAgQhYAAAAAGBThdAMAetcdL23t0/U+/9W0Pl0PAADgZsOVLAAAAAAwiJAFAAAAAAYRsgAAAADAIEIWAAAAABhEyAIAAAAAgwhZAAAAAGAQIQsAAAAADCJkAQAAAIBBhCwAAAAAMIiQBQAAAAAGEbIAAAAAwCBCFgAAAAAYRMgCAAAAAIMIWQAAAABgECELAAAAAAwiZAEAAACAQYQsAAAAADCIkAUAAAAABhGyAAAAAMAgQhYAAAAAGETIAgAAAACDCFkAAAAAYBAhCwAAAAAMImQBAAAAgEGELAAAAAAwiJAFAAAAAAYRsgAAAADAIEIWAAAAABhEyAIAAAAAgwhZAAAAAGAQIQsAAAAADCJkAQAAAIBBhCwAAAAAMIiQBQAAAAAGEbIAAAAAwKDrDlnbt2/X9OnT5fV6FRYWpvfeey9kv2VZKikpkdfrVXR0tCZNmqR9+/aF1AQCAc2fP1+JiYmKjY3VjBkzdPz48ZCa9vZ2+Xw+ud1uud1u+Xw+ff311yE1R48e1fTp0xUbG6vExEQVFhaqq6srpOazzz7TxIkTFR0drVtvvVUvv/yyLMu63tMGAAAAgGty3SHrzJkz+sEPfqDKyspL7i8vL9fKlStVWVmp3bt3y+PxKCsrS6dOnbJrioqKtHnzZlVXV6u+vl6nT59Wbm6uuru77Zq8vDw1NzerpqZGNTU1am5uls/ns/d3d3dr2rRpOnPmjOrr61VdXa1NmzZpwYIFdk1nZ6eysrLk9Xq1e/duVVRUaPny5Vq5cuX1njYAAAAAXJOI633Dww8/rIcffviS+yzL0urVq7V48WLNnDlTkvTWW28pOTlZGzdu1Ny5c9XR0aE33nhDb7/9tqZOnSpJqqqqUkpKirZt26acnBwdOHBANTU12rFjh8aNGydJWrdunTIzM3Xw4EGlpaXJ7/dr//79OnbsmLxeryRpxYoVmj17tpYuXar4+Hht2LBB33zzjdavXy+Xy6X09HQdOnRIK1euVHFxscLCwm5oaAAAAABwOdcdsq7k8OHDam1tVXZ2tr3N5XJp4sSJamho0Ny5c9XU1KRgMBhS4/V6lZ6eroaGBuXk5KixsVFut9sOWJJ0//33y+12q6GhQWlpaWpsbFR6erodsCQpJydHgUBATU1Nmjx5shobGzVx4kS5XK6QmkWLFunzzz9Xampqj3MIBAIKBAL2687OTklSMBhUMBg0M6gbdGF9p/sYjPrz7F3hfXt7rMkZ9ee592fM3TnM3hnM3RnM3TnM/sZd68yMhqzW1lZJUnJycsj25ORkHTlyxK6JiorS0KFDe9RceH9ra6uSkpJ6HD8pKSmk5uJ1hg4dqqioqJCaO+64o8c6F/ZdKmSVlZVpyZIlPbb7/X7FxMRc+sT7WG1trdMtDFr9cfbl9/Xteh988IHxY/bHuQ8EzN05zN4ZzN0ZzN05zP76nT179prqjIasCy6+Dc+yrKvemndxzaXqTdRceOjF5fpZtGiRiouL7dednZ1KSUlRdna24uPjr3gOvS0YDKq2tlZZWVmKjIx0tJfBpj/PPr3kwz5dr6Ukx9ix+vPc+zPm7hxm7wzm7gzm7hxmf+Mu3OV2NUZDlsfjkfTdVaLhw4fb29va2uwrSB6PR11dXWpvbw+5mtXW1qbx48fbNV9++WWP43/11Vchx9m5c2fI/vb2dgWDwZCaC1e1/nIdqefVtgtcLlfI7YUXREZG3jQfwpupl8GmP84+0N233z3sjfn0x7kPBMzdOczeGczdGczdOcz++l3rvIz+nazU1FR5PJ6QS49dXV2qq6uzA1RGRoYiIyNDak6cOKGWlha7JjMzUx0dHdq1a5dds3PnTnV0dITUtLS06MSJE3aN3++Xy+VSRkaGXbN9+/aQx7r7/X55vd4etxECAAAAgAnXHbJOnz6t5uZmNTc3S/ruYRfNzc06evSowsLCVFRUpNLSUm3evFktLS2aPXu2YmJilJeXJ0lyu92aM2eOFixYoI8++kh79+7VY489pjFjxthPGxw1apQeeugh5efna8eOHdqxY4fy8/OVm5urtLQ0SVJ2drZGjx4tn8+nvXv36qOPPtLChQuVn59v39aXl5cnl8ul2bNnq6WlRZs3b1ZpaSlPFgQAAADQa677dsHf/e53mjx5sv36wveXnnjiCa1fv14vvPCCzp07p4KCArW3t2vcuHHy+/2Ki4uz37Nq1SpFRERo1qxZOnfunKZMmaL169crPDzcrtmwYYMKCwvtpxDOmDEj5G9zhYeHa+vWrSooKNCECRMUHR2tvLw8LV++3K5xu92qra3VvHnzNHbsWA0dOlTFxcUh37kCAAAAAJOuO2RNmjTJfnjEpYSFhamkpEQlJSWXrRkyZIgqKipUUVFx2ZqEhARVVVVdsZcRI0Zoy5YtV6wZM2aMtm/ffsUaAAAAADDF6HeyAAAAAGCwI2QBAAAAgEGELAAAAAAwiJAFAAAAAAYRsgAAAADAIEIWAAAAABhEyAIAAAAAgwhZAAAAAGAQIQsAAAAADCJkAQAAAIBBhCwAAAAAMIiQBQAAAAAGEbIAAAAAwCBCFgAAAAAYRMgCAAAAAIMIWQAAAABgECELAAAAAAwiZAEAAACAQYQsAAAAADCIkAUAAAAABhGyAAAAAMAgQhYAAAAAGETIAgAAAACDCFkAAAAAYBAhCwAAAAAMImQBAAAAgEGELAAAAAAwiJAFAAAAAAYRsgAAAADAIEIWAAAAABhEyAIAAAAAgwhZAAAAAGAQIQsAAAAADCJkAQAAAIBBhCwAAAAAMIiQBQAAAAAGEbIAAAAAwCBCFgAAAAAYRMgCAAAAAIMIWQAAAABgECELAAAAAAwiZAEAAACAQYQsAAAAADCIkAUAAAAABhGyAAAAAMAgQhYAAAAAGETIAgAAAACDCFkAAAAAYBAhCwAAAAAMImQBAAAAgEGELAAAAAAwiJAFAAAAAAYRsgAAAADAIEIWAAAAABhkPGR9++23+vnPf67U1FRFR0frzjvv1Msvv6zz58/bNZZlqaSkRF6vV9HR0Zo0aZL27dsXcpxAIKD58+crMTFRsbGxmjFjho4fPx5S097eLp/PJ7fbLbfbLZ/Pp6+//jqk5ujRo5o+fbpiY2OVmJiowsJCdXV1mT5tAAAAAJDUCyHrlVde0WuvvabKykodOHBA5eXlWrZsmSoqKuya8vJyrVy5UpWVldq9e7c8Ho+ysrJ06tQpu6aoqEibN29WdXW16uvrdfr0aeXm5qq7u9uuycvLU3Nzs2pqalRTU6Pm5mb5fD57f3d3t6ZNm6YzZ86ovr5e1dXV2rRpkxYsWGD6tAEAAABAkhRh+oCNjY36+7//e02bNk2SdMcdd+g///M/9bvf/U7Sd1exVq9ercWLF2vmzJmSpLfeekvJycnauHGj5s6dq46ODr3xxht6++23NXXqVElSVVWVUlJStG3bNuXk5OjAgQOqqanRjh07NG7cOEnSunXrlJmZqYMHDyotLU1+v1/79+/XsWPH5PV6JUkrVqzQ7NmztXTpUsXHx5s+fQAAAACDnPGQ9cADD+i1117ToUOH9Dd/8zf6/e9/r/r6eq1evVqSdPjwYbW2tio7O9t+j8vl0sSJE9XQ0KC5c+eqqalJwWAwpMbr9So9PV0NDQ3KyclRY2Oj3G63HbAk6f7775fb7VZDQ4PS0tLU2Nio9PR0O2BJUk5OjgKBgJqamjR58uQe/QcCAQUCAft1Z2enJCkYDCoYDBqb0424sL7TfQxG/Xn2rnCrT9czOaP+PPf+jLk7h9k7g7k7g7k7h9nfuGudmfGQ9eKLL6qjo0N33XWXwsPD1d3draVLl+qnP/2pJKm1tVWSlJycHPK+5ORkHTlyxK6JiorS0KFDe9RceH9ra6uSkpJ6rJ+UlBRSc/E6Q4cOVVRUlF1zsbKyMi1ZsqTHdr/fr5iYmKuef1+ora11uoVBqz/Ovvy+vl3vgw8+MH7M/jj3gYC5O4fZO4O5O4O5O4fZX7+zZ89eU53xkPXOO++oqqpKGzdu1N13363m5mYVFRXJ6/XqiSeesOvCwsJC3mdZVo9tF7u45lL1N1LzlxYtWqTi4mL7dWdnp1JSUpSdne347YXBYFC1tbXKyspSZGSko70MNv159uklH/bpei0lOcaO1Z/n3p8xd+cwe2cwd2cwd+cw+xt34S63qzEesp5//nm99NJLeuSRRyRJY8aM0ZEjR1RWVqYnnnhCHo9H0ndXmYYPH26/r62tzb7q5PF41NXVpfb29pCrWW1tbRo/frxd8+WXX/ZY/6uvvgo5zs6dO0P2t7e3KxgM9rjCdYHL5ZLL5eqxPTIy8qb5EN5MvQw2/XH2ge4r//LCtN6YT3+c+0DA3J3D7J3B3J3B3J3D7K/ftc7L+NMFz549q+99L/Sw4eHh9iPcU1NT5fF4Qi5PdnV1qa6uzg5QGRkZioyMDKk5ceKEWlpa7JrMzEx1dHRo165dds3OnTvV0dERUtPS0qITJ07YNX6/Xy6XSxkZGYbPHAAAAAB64UrW9OnTtXTpUo0YMUJ333239u7dq5UrV+rJJ5+U9N3te0VFRSotLdXIkSM1cuRIlZaWKiYmRnl5eZIkt9utOXPmaMGCBRo2bJgSEhK0cOFCjRkzxn7a4KhRo/TQQw8pPz9fr7/+uiTpqaeeUm5urtLS0iRJ2dnZGj16tHw+n5YtW6aTJ09q4cKFys/Pd/zWPwAAAAADk/GQVVFRoV/84hcqKChQW1ubvF6v5s6dq3/+53+2a1544QWdO3dOBQUFam9v17hx4+T3+xUXF2fXrFq1ShEREZo1a5bOnTunKVOmaP369QoPD7drNmzYoMLCQvsphDNmzFBlZaW9Pzw8XFu3blVBQYEmTJig6Oho5eXlafny5aZPGwAAAAAk9ULIiouL0+rVq+1Htl9KWFiYSkpKVFJSctmaIUOGqKKiIuSPGF8sISFBVVVVV+xnxIgR2rJly9XaBgAAAAAjjH8nCwAAAAAGM0IWAAAAABhEyAIAAAAAgwhZAAAAAGAQIQsAAAAADCJkAQAAAIBBhCwAAAAAMIiQBQAAAAAGEbIAAAAAwCBCFgAAAAAYRMgCAAAAAIMIWQAAAABgECELAAAAAAwiZAEAAACAQYQsAAAAADCIkAUAAAAABhGyAAAAAMAgQhYAAAAAGETIAgAAAACDCFkAAAAAYBAhCwAAAAAMImQBAAAAgEGELAAAAAAwiJAFAAAAAAYRsgAAAADAIEIWAAAAABhEyAIAAAAAgwhZAAAAAGAQIQsAAAAADCJkAQAAAIBBhCwAAAAAMIiQBQAAAAAGEbIAAAAAwCBCFgAAAAAYRMgCAAAAAIMIWQAAAABgECELAAAAAAwiZAEAAACAQYQsAAAAADCIkAUAAAAABhGyAAAAAMAgQhYAAAAAGETIAgAAAACDCFkAAAAAYBAhCwAAAAAMImQBAAAAgEGELAAAAAAwiJAFAAAAAAYRsgAAAADAIEIWAAAAABhEyAIAAAAAgwhZAAAAAGAQIQsAAAAADCJkAQAAAIBBvRKy/vjHP+qxxx7TsGHDFBMTox/+8Idqamqy91uWpZKSEnm9XkVHR2vSpEnat29fyDECgYDmz5+vxMRExcbGasaMGTp+/HhITXt7u3w+n9xut9xut3w+n77++uuQmqNHj2r69OmKjY1VYmKiCgsL1dXV1RunDQAAAADmQ1Z7e7smTJigyMhI/eY3v9H+/fu1YsUK3XLLLXZNeXm5Vq5cqcrKSu3evVsej0dZWVk6deqUXVNUVKTNmzerurpa9fX1On36tHJzc9Xd3W3X5OXlqbm5WTU1NaqpqVFzc7N8Pp+9v7u7W9OmTdOZM2dUX1+v6upqbdq0SQsWLDB92gAAAAAgSYowfcBXXnlFKSkpevPNN+1td9xxh/3vlmVp9erVWrx4sWbOnClJeuutt5ScnKyNGzdq7ty56ujo0BtvvKG3335bU6dOlSRVVVUpJSVF27ZtU05Ojg4cOKCamhrt2LFD48aNkyStW7dOmZmZOnjwoNLS0uT3+7V//34dO3ZMXq9XkrRixQrNnj1bS5cuVXx8vOnTBwAAADDIGQ9Z77//vnJycvSTn/xEdXV1uvXWW1VQUKD8/HxJ0uHDh9Xa2qrs7Gz7PS6XSxMnTlRDQ4Pmzp2rpqYmBYPBkBqv16v09HQ1NDQoJydHjY2NcrvddsCSpPvvv19ut1sNDQ1KS0tTY2Oj0tPT7YAlSTk5OQoEAmpqatLkyZN79B8IBBQIBOzXnZ2dkqRgMKhgMGhuUDfgwvpO9zEY9efZu8KtPl3P5Iz689z7M+buHGbvDObuDObuHGZ/4651ZsZD1h/+8AetWbNGxcXF+tnPfqZdu3apsLBQLpdLjz/+uFpbWyVJycnJIe9LTk7WkSNHJEmtra2KiorS0KFDe9RceH9ra6uSkpJ6rJ+UlBRSc/E6Q4cOVVRUlF1zsbKyMi1ZsqTHdr/fr5iYmGsZQa+rra11uoVBqz/Ovvy+vl3vgw8+MH7M/jj3gYC5O4fZO4O5O4O5O4fZX7+zZ89eU53xkHX+/HmNHTtWpaWlkqR77rlH+/bt05o1a/T444/bdWFhYSHvsyyrx7aLXVxzqfobqflLixYtUnFxsf26s7NTKSkpys7Odvz2wmAwqNraWmVlZSkyMtLRXgab/jz79JIP+3S9lpIcY8fqz3Pvz5i7c5i9M5i7M5i7c5j9jbtwl9vVGA9Zw4cP1+jRo0O2jRo1Sps2bZIkeTweSd9dZRo+fLhd09bWZl918ng86urqUnt7e8jVrLa2No0fP96u+fLLL3us/9VXX4UcZ+fOnSH729vbFQwGe1zhusDlcsnlcvXYHhkZedN8CG+mXgab/jj7QPeVf3lhWm/Mpz/OfSBg7s5h9s5g7s5g7s5h9tfvWudl/OmCEyZM0MGDB0O2HTp0SLfffrskKTU1VR6PJ+TyZFdXl+rq6uwAlZGRocjIyJCaEydOqKWlxa7JzMxUR0eHdu3aZdfs3LlTHR0dITUtLS06ceKEXeP3++VyuZSRkWH4zAEAAACgF65kPffccxo/frxKS0s1a9Ys7dq1S2vXrtXatWslfXf7XlFRkUpLSzVy5EiNHDlSpaWliomJUV5eniTJ7XZrzpw5WrBggYYNG6aEhAQtXLhQY8aMsZ82OGrUKD300EPKz8/X66+/Lkl66qmnlJubq7S0NElSdna2Ro8eLZ/Pp2XLlunkyZNauHCh8vPzHb/1DwAAAMDAZDxk3Xvvvdq8ebMWLVqkl19+WampqVq9erUeffRRu+aFF17QuXPnVFBQoPb2do0bN05+v19xcXF2zapVqxQREaFZs2bp3LlzmjJlitavX6/w8HC7ZsOGDSosLLSfQjhjxgxVVlba+8PDw7V161YVFBRowoQJio6OVl5enpYvX276tAEAAABAUi+ELEnKzc1Vbm7uZfeHhYWppKREJSUll60ZMmSIKioqVFFRcdmahIQEVVVVXbGXESNGaMuWLVftGQAAAABMMP6dLAAAAAAYzAhZAAAAAGAQIQsAAAAADCJkAQAAAIBBhCwAAAAAMIiQBQAAAAAGEbIAAAAAwCBCFgAAAAAYRMgCAAAAAIMIWQAAAABgECELAAAAAAwiZAEAAACAQYQsAAAAADCIkAUAAAAABhGyAAAAAMAgQhYAAAAAGETIAgAAAACDCFkAAAAAYBAhCwAAAAAMImQBAAAAgEGELAAAAAAwiJAFAAAAAAYRsgAAAADAIEIWAAAAABhEyAIAAAAAgwhZAAAAAGAQIQsAAAAADCJkAQAAAIBBhCwAAAAAMIiQBQAAAAAGEbIAAAAAwCBCFgAAAAAYRMgCAAAAAIMIWQAAAABgECELAAAAAAwiZAEAAACAQYQsAAAAADCIkAUAAAAABhGyAAAAAMAgQhYAAAAAGETIAgAAAACDCFkAAAAAYBAhCwAAAAAMImQBAAAAgEGELAAAAAAwiJAFAAAAAAYRsgAAAADAIEIWAAAAABhEyAIAAAAAgwhZAAAAAGAQIQsAAAAADCJkAQAAAIBBhCwAAAAAMIiQBQAAAAAGEbIAAAAAwKBeD1llZWUKCwtTUVGRvc2yLJWUlMjr9So6OlqTJk3Svn37Qt4XCAQ0f/58JSYmKjY2VjNmzNDx48dDatrb2+Xz+eR2u+V2u+Xz+fT111+H1Bw9elTTp09XbGysEhMTVVhYqK6urt46XQAAAACDXK+GrN27d2vt2rX6/ve/H7K9vLxcK1euVGVlpXbv3i2Px6OsrCydOnXKrikqKtLmzZtVXV2t+vp6nT59Wrm5ueru7rZr8vLy1NzcrJqaGtXU1Ki5uVk+n8/e393drWnTpunMmTOqr69XdXW1Nm3apAULFvTmaQMAAAAYxHotZJ0+fVqPPvqo1q1bp6FDh9rbLcvS6tWrtXjxYs2cOVPp6el66623dPbsWW3cuFGS1NHRoTfeeEMrVqzQ1KlTdc8996iqqkqfffaZtm3bJkk6cOCAampq9B//8R/KzMxUZmam1q1bpy1btujgwYOSJL/fr/3796uqqkr33HOPpk6dqhUrVmjdunXq7OzsrVMHAAAAMIhF9NaB582bp2nTpmnq1Kn6l3/5F3v74cOH1draquzsbHuby+XSxIkT1dDQoLlz56qpqUnBYDCkxuv1Kj09XQ0NDcrJyVFjY6PcbrfGjRtn19x///1yu91qaGhQWlqaGhsblZ6eLq/Xa9fk5OQoEAioqalJkydP7tF3IBBQIBCwX18IY8FgUMFg0MxwbtCF9Z3uYzDqz7N3hVt9up7JGfXnufdnzN05zN4ZzN0ZzN05zP7GXevMeiVkVVdXa8+ePdq9e3ePfa2trZKk5OTkkO3Jyck6cuSIXRMVFRVyBexCzYX3t7a2Kikpqcfxk5KSQmouXmfo0KGKioqyay5WVlamJUuW9Nju9/sVExNzyff0tdraWqdbGLT64+zL7+vb9T744APjx+yPcx8ImLtzmL0zmLszmLtzmP31O3v27DXVGQ9Zx44d07PPPiu/368hQ4Zcti4sLCzktWVZPbZd7OKaS9XfSM1fWrRokYqLi+3XnZ2dSklJUXZ2tuLj46/YX28LBoOqra1VVlaWIiMjHe1lsOnPs08v+bBP12spyTF2rP489/6MuTuH2TuDuTuDuTuH2d+4a/3KkfGQ1dTUpLa2NmVkZNjburu7tX37dlVWVtrfl2ptbdXw4cPtmra2Nvuqk8fjUVdXl9rb20OuZrW1tWn8+PF2zZdfftlj/a+++irkODt37gzZ397ermAw2OMK1wUul0sul6vH9sjIyJvmQ3gz9TLY9MfZB7qv/MsL03pjPv1x7gMBc3cOs3cGc3cGc3cOs79+1zov4w++mDJlij777DM1Nzfb/4wdO1aPPvqompubdeedd8rj8YRcnuzq6lJdXZ0doDIyMhQZGRlSc+LECbW0tNg1mZmZ6ujo0K5du+yanTt3qqOjI6SmpaVFJ06csGv8fr9cLldICAQAAAAAU4xfyYqLi1N6enrIttjYWA0bNszeXlRUpNLSUo0cOVIjR45UaWmpYmJilJeXJ0lyu92aM2eOFixYoGHDhikhIUELFy7UmDFjNHXqVEnSqFGj9NBDDyk/P1+vv/66JOmpp55Sbm6u0tLSJEnZ2dkaPXq0fD6fli1bppMnT2rhwoXKz893/NY/AAAAAANTrz1d8EpeeOEFnTt3TgUFBWpvb9e4cePk9/sVFxdn16xatUoRERGaNWuWzp07pylTpmj9+vUKDw+3azZs2KDCwkL7KYQzZsxQZWWlvT88PFxbt25VQUGBJkyYoOjoaOXl5Wn58uV9d7IAAAAABpU+CVn/8z//E/I6LCxMJSUlKikpuex7hgwZooqKClVUVFy2JiEhQVVVVVdce8SIEdqyZcv1tAsAAAAAN6zX/hgxAAAAAAxGhCwAAAAAMIiQBQAAAAAGEbIAAAAAwCBCFgAAAAAYRMgCAAAAAIMIWQAAAABgECELAAAAAAwiZAEAAACAQYQsAAAAADCIkAUAAAAABhGyAAAAAMAgQhYAAAAAGETIAgAAAACDCFkAAAAAYBAhCwAAAAAMImQBAAAAgEERTjcAAH+NO17a2udrfv6raX2+JgAA6D+4kgUAAAAABhGyAAAAAMAgQhYAAAAAGETIAgAAAACDCFkAAAAAYBAhCwAAAAAMImQBAAAAgEGELAAAAAAwiJAFAAAAAAYRsgAAAADAIEIWAAAAABhEyAIAAAAAgwhZAAAAAGAQIQsAAAAADCJkAQAAAIBBhCwAAAAAMIiQBQAAAAAGEbIAAAAAwCBCFgAAAAAYRMgCAAAAAIMIWQAAAABgECELAAAAAAwiZAEAAACAQYQsAAAAADCIkAUAAAAABhGyAAAAAMAgQhYAAAAAGETIAgAAAACDCFkAAAAAYBAhCwAAAAAMImQBAAAAgEGELAAAAAAwiJAFAAAAAAYRsgAAAADAIEIWAAAAABgU4XQDANDf3PHS1j5d7/NfTevT9QAAwF/H+JWssrIy3XvvvYqLi1NSUpJ+/OMf6+DBgyE1lmWppKREXq9X0dHRmjRpkvbt2xdSEwgENH/+fCUmJio2NlYzZszQ8ePHQ2ra29vl8/nkdrvldrvl8/n09ddfh9QcPXpU06dPV2xsrBITE1VYWKiuri7Tpw0AAAAAknohZNXV1WnevHnasWOHamtr9e233yo7O1tnzpyxa8rLy7Vy5UpVVlZq9+7d8ng8ysrK0qlTp+yaoqIibd68WdXV1aqvr9fp06eVm5ur7u5uuyYvL0/Nzc2qqalRTU2Nmpub5fP57P3d3d2aNm2azpw5o/r6elVXV2vTpk1asGCB6dMGAAAAAEm9cLtgTU1NyOs333xTSUlJampq0oMPPijLsrR69WotXrxYM2fOlCS99dZbSk5O1saNGzV37lx1dHTojTfe0Ntvv62pU6dKkqqqqpSSkqJt27YpJydHBw4cUE1NjXbs2KFx48ZJktatW6fMzEwdPHhQaWlp8vv92r9/v44dOyav1ytJWrFihWbPnq2lS5cqPj7e9OkDAAAAGOR6/TtZHR0dkqSEhARJ0uHDh9Xa2qrs7Gy7xuVyaeLEiWpoaNDcuXPV1NSkYDAYUuP1epWenq6Ghgbl5OSosbFRbrfbDliSdP/998vtdquhoUFpaWlqbGxUenq6HbAkKScnR4FAQE1NTZo8eXKPfgOBgAKBgP26s7NTkhQMBhUMBg1N5cZcWN/pPgaj/jx7V7jVp+uZnNG1zL2vz88Jff2568+f9/6O2TuDuTuDuTuH2d+4a51Zr4Ysy7JUXFysBx54QOnp6ZKk1tZWSVJycnJIbXJyso4cOWLXREVFaejQoT1qLry/tbVVSUlJPdZMSkoKqbl4naFDhyoqKsquuVhZWZmWLFnSY7vf71dMTMxVz7kv1NbWOt3CoNUfZ19+X9+u98EHHxg/5pXm3tfn54TemOm16I+f94GC2TuDuTuDuTuH2V+/s2fPXlNdr4asZ555Rp9++qnq6+t77AsLCwt5bVlWj20Xu7jmUvU3UvOXFi1apOLiYvt1Z2enUlJSlJ2d7fjthcFgULW1tcrKylJkZKSjvQw2/Xn26SUf9ul6LSU5xo51LXPv6/NzgsmZXov+/Hnv75i9M5i7M5i7c5j9jbtwl9vV9FrImj9/vt5//31t375dt912m73d4/FI+u4q0/Dhw+3tbW1t9lUnj8ejrq4utbe3h1zNamtr0/jx4+2aL7/8sse6X331Vchxdu7cGbK/vb1dwWCwxxWuC1wul1wuV4/tkZGRN82H8GbqZbDpj7MPdF/5lxem9cZ8rjT3vj4/J/T1Zy695EOV3yfds/S3fTJfHlHfU3/8f81AwNydwdydw+yv37XOy/jTBS3L0jPPPKN3331Xv/3tb5WamhqyPzU1VR6PJ+TyZFdXl+rq6uwAlZGRocjIyJCaEydOqKWlxa7JzMxUR0eHdu3aZdfs3LlTHR0dITUtLS06ceKEXeP3++VyuZSRkWH61AEAAADA/JWsefPmaePGjfr1r3+tuLg4+7tPbrdb0dHRCgsLU1FRkUpLSzVy5EiNHDlSpaWliomJUV5enl07Z84cLViwQMOGDVNCQoIWLlyoMWPG2E8bHDVqlB566CHl5+fr9ddflyQ99dRTys3NVVpamiQpOztbo0ePls/n07Jly3Ty5EktXLhQ+fn5jt/6BwAAAGBgMh6y1qxZI0maNGlSyPY333xTs2fPliS98MILOnfunAoKCtTe3q5x48bJ7/crLi7Orl+1apUiIiI0a9YsnTt3TlOmTNH69esVHh5u12zYsEGFhYX2UwhnzJihyspKe394eLi2bt2qgoICTZgwQdHR0crLy9Py5ctNnzYAAAAASOqFkGVZV3+cclhYmEpKSlRSUnLZmiFDhqiiokIVFRWXrUlISFBVVdUV1xoxYoS2bNly1Z4AAAAAwATj38kCAAAAgMGMkAUAAAAABhGyAAAAAMAgQhYAAAAAGETIAgAAAACDCFkAAAAAYBAhCwAAAAAMImQBAAAAgEGELAAAAAAwiJAFAAAAAAYRsgAAAADAIEIWAAAAABhEyAIAAAAAgwhZAAAAAGAQIQsAAAAADCJkAQAAAIBBhCwAAAAAMIiQBQAAAAAGEbIAAAAAwCBCFgAAAAAYFOF0AwAGljte2mrsWK5wS+X3SeklHyrQHWbsuAAAAL2JK1kAAAAAYBAhCwAAAAAMImQBAAAAgEGELAAAAAAwiJAFAAAAAAYRsgAAAADAIEIWAAAAABhEyAIAAAAAg/hjxABwkzP5B56vhSu8T5cDAGDA4UoWAAAAABhEyAIAAAAAg7hdEFfU17cpSdLnv5rW52sCAAAApnAlCwAAAAAMImQBAAAAgEGELAAAAAAwiJAFAAAAAAbx4AsMetfycA9XuKXy+6T0kg8V6A77q9bjwR4AAAADG1eyAAAAAMAgQhYAAAAAGETIAgAAAACDCFkAAAAAYBAhCwAAAAAMImQBAAAAgEGELAAAAAAwiJAFAAAAAAYRsgAAAADAIEIWAAAAABhEyAIAAAAAgwhZAAAAAGAQIQsAAAAADCJkAQAAAIBBhCwAAAAAMIiQBQAAAAAGRTjdAABgcLvjpa1Ot9DrPv/VNKdbAAD0oUFxJevVV19VamqqhgwZooyMDH3yySdOtwQAAABggBrwIeudd95RUVGRFi9erL179+pHP/qRHn74YR09etTp1gAAAAAMQAP+dsGVK1dqzpw5+qd/+idJ0urVq/Xhhx9qzZo1Kisr61EfCAQUCATs1x0dHZKkkydPKhgM9k3TlxEMBnX27Fn9+c9/VmRkZJ+sGfHtmT5Z5y/9+c9/7tP1ruUcI85bOnv2vCKC31P3+bC/ar2b8fxuVibnjmvH3M271v/unfj/PJi7U5i7c5j9jTt16pQkybKsK9aFWVer6Me6uroUExOj//qv/9I//MM/2NufffZZNTc3q66ursd7SkpKtGTJkr5sEwAAAEA/cuzYMd12222X3T+gr2T96U9/Und3t5KTk0O2Jycnq7W19ZLvWbRokYqLi+3X58+f18mTJzVs2DCFhTn7G93Ozk6lpKTo2LFjio+Pd7SXwYbZO4O5O4O5O4fZO4O5O4O5O4fZ3zjLsnTq1Cl5vd4r1g3okHXBxeHIsqzLBiaXyyWXyxWy7ZZbbumt1m5IfHw8/0E4hNk7g7k7g7k7h9k7g7k7g7k7h9nfGLfbfdWaAf3gi8TERIWHh/e4atXW1tbj6hYAAAAAmDCgQ1ZUVJQyMjJUW1sbsr22tlbjx493qCsAAAAAA9mAv12wuLhYPp9PY8eOVWZmptauXaujR4/q6aefdrq16+ZyufTLX/6yx+2M6H3M3hnM3RnM3TnM3hnM3RnM3TnMvvcN6KcLXvDqq6+qvLxcJ06cUHp6ulatWqUHH3zQ6bYAAAAADECDImQBAAAAQF8Z0N/JAgAAAIC+RsgCAAAAAIMIWQAAAABgECELAAAAAAwiZPUjr776qlJTUzVkyBBlZGTok08+cbqlAa2srEz33nuv4uLilJSUpB//+Mc6ePCg020NOmVlZQoLC1NRUZHTrQwKf/zjH/XYY49p2LBhiomJ0Q9/+EM1NTU53daA9u233+rnP/+5UlNTFR0drTvvvFMvv/yyzp8/73RrA8727ds1ffp0eb1ehYWF6b333gvZb1mWSkpK5PV6FR0drUmTJmnfvn3ONDuAXGnuwWBQL774osaMGaPY2Fh5vV49/vjj+uKLL5xreAC52mf+L82dO1dhYWFavXp1n/U3kBGy+ol33nlHRUVFWrx4sfbu3asf/ehHevjhh3X06FGnWxuw6urqNG/ePO3YsUO1tbX69ttvlZ2drTNnzjjd2qCxe/durV27Vt///vedbmVQaG9v14QJExQZGanf/OY32r9/v1asWKFbbrnF6dYGtFdeeUWvvfaaKisrdeDAAZWXl2vZsmWqqKhwurUB58yZM/rBD36gysrKS+4vLy/XypUrVVlZqd27d8vj8SgrK0unTp3q404HlivN/ezZs9qzZ49+8YtfaM+ePXr33Xd16NAhzZgxw4FOB56rfeYveO+997Rz5055vd4+6mwQsNAv3HfffdbTTz8dsu2uu+6yXnrpJYc6Gnza2tosSVZdXZ3TrQwKp06dskaOHGnV1tZaEydOtJ599lmnWxrwXnzxReuBBx5wuo1BZ9q0adaTTz4Zsm3mzJnWY4895lBHg4Mka/Pmzfbr8+fPWx6Px/rVr35lb/vmm28st9ttvfbaaw50ODBdPPdL2bVrlyXJOnLkSN80NUhcbvbHjx+3br31VqulpcW6/fbbrVWrVvV5bwMRV7L6ga6uLjU1NSk7Oztke3Z2thoaGhzqavDp6OiQJCUkJDjcyeAwb948TZs2TVOnTnW6lUHj/fff19ixY/WTn/xESUlJuueee7Ru3Tqn2xrwHnjgAX300Uc6dOiQJOn3v/+96uvr9Xd/93cOdza4HD58WK2trSE/a10ulyZOnMjP2j7W0dGhsLAwrqL3gfPnz8vn8+n555/X3Xff7XQ7A0qE0w3g6v70pz+pu7tbycnJIduTk5PV2trqUFeDi2VZKi4u1gMPPKD09HSn2xnwqqurtWfPHu3evdvpVgaVP/zhD1qzZo2Ki4v1s5/9TLt27VJhYaFcLpcef/xxp9sbsF588UV1dHTorrvuUnh4uLq7u7V06VL99Kc/dbq1QeXCz9NL/aw9cuSIEy0NSt98841eeukl5eXlKT4+3ul2BrxXXnlFERERKiwsdLqVAYeQ1Y+EhYWFvLYsq8c29I5nnnlGn376qerr651uZcA7duyYnn32Wfn9fg0ZMsTpdgaV8+fPa+zYsSotLZUk3XPPPdq3b5/WrFlDyOpF77zzjqqqqrRx40bdfffdam5uVlFRkbxer5544gmn2xt0+FnrnGAwqEceeUTnz5/Xq6++6nQ7A15TU5P+9V//VXv27OEz3gu4XbAfSExMVHh4eI+rVm1tbT1+4wbz5s+fr/fff18ff/yxbrvtNqfbGfCamprU1tamjIwMRUREKCIiQnV1dfq3f/s3RUREqLu72+kWB6zhw4dr9OjRIdtGjRrFA3Z62fPPP6+XXnpJjzzyiMaMGSOfz6fnnntOZWVlTrc2qHg8HkniZ61DgsGgZs2apcOHD6u2tparWH3gk08+UVtbm0aMGGH/vD1y5IgWLFigO+64w+n2+j1CVj8QFRWljIwM1dbWhmyvra3V+PHjHepq4LMsS88884zeffdd/fa3v1VqaqrTLQ0KU6ZM0Weffabm5mb7n7Fjx+rRRx9Vc3OzwsPDnW5xwJowYUKPP1Nw6NAh3X777Q51NDicPXtW3/te6I/j8PBwHuHex1JTU+XxeEJ+1nZ1damuro6ftb3sQsD63//9X23btk3Dhg1zuqVBwefz6dNPPw35eev1evX888/rww8/dLq9fo/bBfuJ4uJi+Xw+jR07VpmZmVq7dq2OHj2qp59+2unWBqx58+Zp48aN+vWvf624uDj7t5tut1vR0dEOdzdwxcXF9fjeW2xsrIYNG8b34XrZc889p/Hjx6u0tFSzZs3Srl27tHbtWq1du9bp1ga06dOna+nSpRoxYoTuvvtu7d27VytXrtSTTz7pdGsDzunTp/V///d/9uvDhw+rublZCQkJGjFihIqKilRaWqqRI0dq5MiRKi0tVUxMjPLy8hzsuv+70ty9Xq/+8R//UXv27NGWLVvU3d1t/7xNSEhQVFSUU20PCFf7zF8caCMjI+XxeJSWltbXrQ48zj7cENfj3//9363bb7/dioqKsv72b/+WR4n3MkmX/OfNN990urVBh0e4953//u//ttLT0y2Xy2Xddddd1tq1a51uacDr7Oy0nn32WWvEiBHWkCFDrDvvvNNavHixFQgEnG5twPn4448v+f/1J554wrKs7x7j/stf/tLyeDyWy+WyHnzwQeuzzz5ztukB4EpzP3z48GV/3n788cdOt97vXe0zfzEe4W5OmGVZVh/lOQAAAAAY8PhOFgAAAAAYRMgCAAAAAIMIWQAAAABgECELAAAAAAwiZAEAAACAQYQsAAAAADCIkAUAAAAABhGyAAAAAMAgQhYAAAAAGETIAgAAAACDCFkAAAAAYND/AyUiluRjhqNEAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 1000x500 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Now, let's plot the histogram. \n",
    "n_bins = 25\n",
    "\n",
    "plt.figure(figsize=(10,5))\n",
    "# We can set the number of bins with the *bins* keyword argument.\n",
    "#plt.scatter(np.arange(len(lengths)),np.log(lengths), marker='.')\n",
    "plt.hist(np.log(lengths), bins=n_bins)\n",
    "plt.grid(True)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Okay, that does not look great at all. \n",
    "A lot of nodes have more than 10^(3 to 6) number of events. Which means the timestamp difference varies wildly and plain normalization may not be an option anymore. Therefore, we resort to sinusoidal coding like the original BERT paper. \n",
    "# So, the question becomes how do we generate paths now. \n",
    "Well, here is another interesting situation. We can subsample the stream of events to generate the paths. But again, the question is now how do we encode the subject information in the path. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Max time delta: 270997933817167, max_sub_origin: 1522759935112293459, max_event_stamp = 1523030933046110626, max_sub = 10623\n",
      "Min time delta: -10001641, min_sub_origin: 1523218713175536589, min_event_stamp = 1523218713165534948, min_sub = 121235\n",
      "Negative count: 2819176, pct: 6.827477012164842\n"
     ]
    }
   ],
   "source": [
    "max_time_delta = 0\n",
    "max_sub_origin =  -1\n",
    "max_event_stamp = -1\n",
    "max_sub = -1\n",
    "\n",
    "min_time_delta = 100000000000000\n",
    "min_sub_origin =  -1\n",
    "min_event_stamp = -1\n",
    "min_sub = -1\n",
    "negative_count = 0\n",
    "total_count = 0\n",
    "\n",
    "for subject in provenance_graph:\n",
    "    subject_timestamp = provenance_graph[subject]['sub'][2]\n",
    "    event_list = provenance_graph[subject]['events']\n",
    "    total_count += len(event_list)\n",
    "    if subject_timestamp != 0:\n",
    "        for event in event_list:\n",
    "            if event[0][3] != 0:\n",
    "                time_delta = event[0][3] - subject_timestamp\n",
    "\n",
    "                if time_delta<=0:\n",
    "                    negative_count +=1 \n",
    "            \n",
    "                if time_delta > max_time_delta:    \n",
    "                    max_time_delta = time_delta\n",
    "                    max_sub_origin = subject_timestamp\n",
    "                    max_event_stamp = event[0][3]\n",
    "                    max_sub = subject\n",
    "\n",
    "                if time_delta < min_time_delta:\n",
    "                    min_time_delta = time_delta\n",
    "                    min_sub_origin = subject_timestamp\n",
    "                    min_event_stamp = event[0][3]\n",
    "                    min_sub = subject\n",
    "        \n",
    "print(\"Max time delta: {}, max_sub_origin: {}, max_event_stamp = {}, max_sub = {}\".format(max_time_delta, max_sub_origin, max_event_stamp, max_sub))\n",
    "print(\"Min time delta: {}, min_sub_origin: {}, min_event_stamp = {}, min_sub = {}\".format(min_time_delta, min_sub_origin, min_event_stamp, min_sub))\n",
    "print(\"Negative count: {}, pct: {}\".format(negative_count, (negative_count/total_count)*100))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Looks like 6.82% events are have negative time delta which is not possible for us to use. In this case, the ususal approach is to drop the data points. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 275,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of events kept: 10796008, number of events removed for negative time delta: 2819176, percentage: 20.706117522906776\n"
     ]
    }
   ],
   "source": [
    "cleaned_provenance_graph = dict() # Pruned graph object\n",
    "remove_count = 0 # Number of events that have been removed from the graph\n",
    "keep_count = 0 # Number of events that have been retained from the graph\n",
    "\n",
    "for subject in provenance_graph:\n",
    "    \n",
    "    subject_timestamp = provenance_graph[subject]['sub'][2]\n",
    "    event_list = provenance_graph[subject]['events'] \n",
    "    temp_event_list = []\n",
    "    condition = lambda x: [0,0] if ( x is None) else x\n",
    "    for event in event_list:\n",
    "        if event[0][3] != 0:\n",
    "            time_delta = event[0][3] - subject_timestamp\n",
    "            if time_delta>0:\n",
    "                temp_event_list.append(event)\n",
    "            else:\n",
    "                remove_count +=1\n",
    "        else:\n",
    "            remove_count+=1 \n",
    "\n",
    "    tuple_event_list = [tuple(condition(x[0])+condition(x[1])+condition(x[2])) for x in temp_event_list]\n",
    "    tuple_event_list = set(tuple_event_list)\n",
    "    tuple_event_list = [ [[x[0],x[1],x[2],x[3],x[4]], [x[5],x[6]], [x[7], x[8]]]  for x in tuple_event_list ]\n",
    "    \n",
    "    if len(tuple_event_list)>0:\n",
    "        \n",
    "        cleaned_provenance_graph[subject] = dict()\n",
    "        cleaned_provenance_graph[subject]['sub'] = provenance_graph[subject]['sub'] # Saving the subject identity.\n",
    "        cleaned_provenance_graph[subject]['events'] = tuple_event_list\n",
    "        keep_count += len(tuple_event_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 281,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of events kept: 10796008, number of events removed for negative time delta: 2819176, reduced graph size percentage: 26.108281332242022\n"
     ]
    }
   ],
   "source": [
    "print(\"Number of events kept: {}, number of events removed for negative time delta: {}, reduced graph size percentage: {}\".format(keep_count, remove_count, ((100*keep_count)/41350895)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 279,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9594416139480517\n"
     ]
    }
   ],
   "source": [
    "for subject in cleaned_provenance_graph:\n",
    "    if len(cleaned_provenance_graph[subject]['events']) == 0:\n",
    "        cleaned_provenance_graph.pop(subject,None)\n",
    "\n",
    "print(len(cleaned_provenance_graph)/len(provenance_graph))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 280,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Max time delta: 270997933817167, max_sub_origin: 1522759935112293459, max_event_stamp = 1523030933046110626, max_sub = 10623\n",
      "Min time delta: 1472570, min_sub_origin: 1523556159254718787, min_event_stamp = 1523556159256191357, min_sub = 200097\n",
      "Negative count: 0, pct: 0.0\n"
     ]
    }
   ],
   "source": [
    "# Running the same code block on the pruned provenance graph\n",
    "max_time_delta = 0\n",
    "max_sub_origin =  -1\n",
    "max_event_stamp = -1\n",
    "max_sub = -1\n",
    "\n",
    "min_time_delta = 100000000000000\n",
    "min_sub_origin =  -1\n",
    "min_event_stamp = -1\n",
    "min_sub = -1\n",
    "negative_count = 0\n",
    "total_count = 0\n",
    "\n",
    "for subject in cleaned_provenance_graph:\n",
    "    subject_timestamp = cleaned_provenance_graph[subject]['sub'][2]\n",
    "    event_list = cleaned_provenance_graph[subject]['events']\n",
    "    total_count += len(event_list)\n",
    "    if subject_timestamp != 0:\n",
    "        for event in event_list:\n",
    "            if event[0][3] != 0:\n",
    "                time_delta = event[0][3] - subject_timestamp\n",
    "\n",
    "                if time_delta<=0:\n",
    "                    negative_count +=1 \n",
    "            \n",
    "                if time_delta > max_time_delta:    \n",
    "                    max_time_delta = time_delta\n",
    "                    max_sub_origin = subject_timestamp\n",
    "                    max_event_stamp = event[0][3]\n",
    "                    max_sub = subject\n",
    "\n",
    "                if time_delta < min_time_delta:\n",
    "                    min_time_delta = time_delta\n",
    "                    min_sub_origin = subject_timestamp\n",
    "                    min_event_stamp = event[0][3]\n",
    "                    min_sub = subject\n",
    "        \n",
    "print(\"Max time delta: {}, max_sub_origin: {}, max_event_stamp = {}, max_sub = {}\".format(max_time_delta, max_sub_origin, max_event_stamp, max_sub))\n",
    "print(\"Min time delta: {}, min_sub_origin: {}, min_event_stamp = {}, min_sub = {}\".format(min_time_delta, min_sub_origin, min_event_stamp, min_sub))\n",
    "print(\"Negative count: {}, pct: {}\".format(negative_count, (negative_count/total_count)*100))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 282,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5.796340601824141e-15\n",
      "1.0\n"
     ]
    }
   ],
   "source": [
    "MAX_TIME_DELTA = 270997933817167\n",
    "NUMERATOR = np.pi * 0.5 * (1/MAX_TIME_DELTA)\n",
    "print(NUMERATOR)\n",
    "print(np.sin(NUMERATOR*MAX_TIME_DELTA))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Encoding of a sequence.\n",
    "The following function handles sampling from teh graph and encoding the sequence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 283,
   "metadata": {},
   "outputs": [],
   "source": [
    "MAX_SUB_ID = 224629\n",
    "def get_random_sequence(subject_id, encoding = None, sequence_size = 31 , imaging = False):\n",
    "    \n",
    "    subject_start_time_stamp = cleaned_provenance_graph[subject_id]['sub'][2]\n",
    "    events = cleaned_provenance_graph[subject_id]['events']\n",
    "    sampled_events = None\n",
    "    \n",
    "    if len(events)>sequence_size:\n",
    "        sampled_events = random.sample(events, k=sequence_size)\n",
    "    else:\n",
    "        sampled_events = events\n",
    "\n",
    "    sampled_events = sorted(sampled_events, key=lambda x:x[0][3])\n",
    "\n",
    "    result_array = None\n",
    "    index = -1\n",
    "    \n",
    "    if imaging:\n",
    "        result_array = np.zeros((sequence_size*8)+8)\n",
    "        index = 8\n",
    "    else:\n",
    "        result_array = np.zeros((sequence_size*8)+2)\n",
    "        index = 2\n",
    "    \n",
    "    for event in sampled_events:\n",
    "        temp = np.zeros(8)\n",
    "\n",
    "        temp[0] = event[0][0] # Normalized event type\n",
    "        temp[1] = event[0][1] # Normalized predicate object 1 type\n",
    "        temp[2] = event[0][2] # Normalized predicate object 2 type\n",
    "        temp[3] = event[0][4] # Normalized event name\n",
    "\n",
    "        if event[1] is not None:\n",
    "            temp[4] = event[1][0] # Details of predicate object 1 if available\n",
    "            temp[5] = event[1][1] \n",
    "        if event[2] is not None:\n",
    "            temp[6] = event[2][0] # Details of predicate object 2 if available\n",
    "            temp[7] = event[2][1]\n",
    "        \n",
    "        \n",
    "        time_delta = event[0][3] - subject_start_time_stamp \n",
    "\n",
    "        if encoding == 'sine':\n",
    "            result_array[index:index+8] = temp * np.sin(NUMERATOR * time_delta)\n",
    "        elif encoding == 'cosine':\n",
    "            result_array[index:index+8] = temp * np.cos(NUMERATOR * time_delta)\n",
    "        elif encoding =='linear':\n",
    "            result_array[index:index+8] = temp * (time_delta/MAX_TIME_DELTA)\n",
    "        elif encoding is None:\n",
    "            result_array[index:index+8] = temp\n",
    "        \n",
    "        index+=8\n",
    "\n",
    "    result_array[0] = cleaned_provenance_graph[subject_id]['sub'][0]\n",
    "    result_array[1] = cleaned_provenance_graph[subject_id]['sub'][1]\n",
    "    if imaging:\n",
    "        result_array[2] = int(subject_id)/(MAX_SUB_ID+1)\n",
    "    return result_array"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 284,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.25      ,  0.        ,  0.3305213 ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.64      ,  0.41666667,\n",
       "        0.        ,  0.33333333,  0.5       , -1.        ,  0.        ,\n",
       "        0.        ,  0.74      ,  0.41666667,  0.        ,  0.26315789,\n",
       "        1.        ,  0.        ,  0.        ,  0.        ,  0.64      ,\n",
       "        0.41666667,  0.        ,  0.33333333,  0.5       , -1.        ,\n",
       "        0.        ,  0.        ,  0.64      ,  0.41666667,  0.        ,\n",
       "        0.33333333,  0.5       , -1.        ,  0.        ,  0.        ,\n",
       "        0.74      ,  0.41666667,  0.        ,  0.26315789,  1.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.74      ,  0.41666667,\n",
       "        0.        ,  0.26315789,  1.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.74      ,  0.41666667,  0.        ,  0.26315789,\n",
       "        1.        ,  0.        ,  0.        ,  0.        ,  0.74      ,\n",
       "        0.41666667,  0.        ,  0.26315789,  1.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.74      ,  0.41666667,  0.        ,\n",
       "        0.26315789,  1.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.74      ,  0.41666667,  0.        ,  0.26315789,  1.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.74      ,  0.41666667,\n",
       "        0.        ,  0.26315789,  1.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.64      ,  0.41666667,  0.        ,  0.33333333,\n",
       "        0.5       , -1.        ,  0.        ,  0.        ,  0.74      ,\n",
       "        0.41666667,  0.        ,  0.26315789,  1.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.64      ,  0.41666667,  0.        ,\n",
       "        0.33333333,  0.5       , -1.        ,  0.        ,  0.        ,\n",
       "        0.74      ,  0.41666667,  0.        ,  0.26315789,  1.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.64      ,  0.41666667,\n",
       "        0.        ,  0.33333333,  0.5       , -1.        ,  0.        ,\n",
       "        0.        ,  0.74      ,  0.41666667,  0.        ,  0.26315789,\n",
       "        1.        ,  0.        ,  0.        ,  0.        ,  0.74      ,\n",
       "        0.41666667,  0.        ,  0.26315789,  1.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.74      ,  0.41666667,  0.        ,\n",
       "        0.26315789,  1.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.74      ,  0.41666667,  0.        ,  0.26315789,  1.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.74      ,  0.41666667,\n",
       "        0.        ,  0.26315789,  1.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.74      ,  0.41666667,  0.        ,  0.26315789,\n",
       "        1.        ,  0.        ,  0.        ,  0.        ,  0.64      ,\n",
       "        0.41666667,  0.        ,  0.33333333,  0.5       , -1.        ,\n",
       "        0.        ,  0.        ,  0.74      ,  0.41666667,  0.        ,\n",
       "        0.26315789,  1.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.74      ,  0.41666667,  0.        ,  0.26315789,  1.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.64      ,  0.41666667,\n",
       "        0.        ,  0.33333333,  0.5       , -1.        ,  0.        ,\n",
       "        0.        ,  0.64      ,  0.41666667,  0.        ,  0.33333333,\n",
       "        0.5       , -1.        ,  0.        ,  0.        ,  0.74      ,\n",
       "        0.41666667,  0.        ,  0.26315789,  1.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.74      ,  0.41666667,  0.        ,\n",
       "        0.26315789,  1.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.64      ,  0.41666667,  0.        ,  0.33333333,  0.5       ,\n",
       "       -1.        ,  0.        ,  0.        ,  0.74      ,  0.41666667,\n",
       "        0.        ,  0.26315789,  1.        ,  0.        ,  0.        ,\n",
       "        0.        ])"
      ]
     },
     "execution_count": 284,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_random_sequence('74245', encoding=None, imaging=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 285,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import traceback\n",
    "\n",
    "# t_condition = lambda x: [0,0] if ( x is None) else x\n",
    "# try:\n",
    "#     uniques = [ t_condition(x[0]) +t_condition(x[1]) +t_condition(x[2]) for x in cleaned_provenance_graph['74189']['events'] ]\n",
    "#     uniques = [tuple(x) for x in uniques]\n",
    "#     uniques = set(uniques)\n",
    "#     uniques = [ [[x[0],x[1],x[2],x[3],x[4]], [x[5],x[6]], [x[7], x[8]]]  for x in uniques]\n",
    "# except:\n",
    "#     traceback.print_exc()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 251,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "155"
      ]
     },
     "execution_count": 251,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(cleaned_provenance_graph['74189']['events'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 286,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "215055\n"
     ]
    }
   ],
   "source": [
    "subject_list = list(cleaned_provenance_graph.keys())\n",
    "print(len(subject_list))\n",
    "random.shuffle(subject_list)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Generators"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generator( datasource , encoding_scheme = None, batch_size = 32, sequence_size = 31):\n",
    "    while True:\n",
    "        x = np.zeros((batch_size, (sequence_size*8)+2))\n",
    "        y = np.zeros((batch_size, (sequence_size*8)+2))\n",
    "\n",
    "        subjects = random.sample(datasource, k=batch_size)\n",
    "\n",
    "        for i in range(batch_size):\n",
    "            x[i] = get_random_sequence(subjects[i], encoding=encoding_scheme ,sequence_size= sequence_size)\n",
    "            y[i] = x[i]\n",
    "\n",
    "        yield x,y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 287,
   "metadata": {},
   "outputs": [],
   "source": [
    "def image_generator(datasource , encoding_scheme = None, batch_size = 32, sequence_size = 31):\n",
    "    while True:\n",
    "        x = np.zeros((batch_size, (sequence_size*8)+8))\n",
    "        subjects = random.sample(datasource, k=batch_size)\n",
    "        for i in range(batch_size):\n",
    "            x[i] = get_random_sequence(subjects[i], encoding=encoding_scheme ,sequence_size= sequence_size, imaging=True)\n",
    "\n",
    "        target = np.reshape(x, (batch_size, (sequence_size+1),8))\n",
    "        target[:,0,2] /=65535\n",
    "\n",
    "        yield target, target "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 288,
   "metadata": {},
   "outputs": [],
   "source": [
    "def skip_pixel_image_generator(datasource , encoding_scheme = None, batch_size = 32, sequence_size = 31, skip_pct = 0.2):\n",
    "    while True:\n",
    "        x = np.zeros((batch_size, (sequence_size*8)+8))\n",
    "        subjects = random.sample(datasource, k=batch_size)\n",
    "        for i in range(batch_size):\n",
    "            x[i] = get_random_sequence(subjects[i], encoding=encoding_scheme ,sequence_size= sequence_size, imaging=True)\n",
    "\n",
    "        samples = np.reshape(x, (batch_size, (sequence_size+1),8))\n",
    "        samples[:,0,2] /=65535\n",
    "        target = np.copy(samples)\n",
    "\n",
    "        number_of_skip = int(batch_size*skip_pct)\n",
    "        skip_indices = random.sample(range(1,batch_size,1), k=number_of_skip)\n",
    "        for index in skip_indices:\n",
    "            samples[index] = np.zeros(((sequence_size+1),8))\n",
    "\n",
    "        yield samples, target "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 337,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_autoencoder(input_length):\n",
    "    input_layer = tf.keras.Input(shape=(input_length+1, 8, 1 ))\n",
    "    \n",
    "    feature_extraction = tf.keras.layers.Conv2D(128, (1,8), activation='relu')(input_layer)\n",
    "    flattening = tf.keras.layers.Flatten()(feature_extraction)\n",
    "    \n",
    "    layer_0_down = tf.keras.layers.Dense(512, activation ='relu')(flattening)\n",
    "    dropout0 = tf.keras.layers.Dropout(rate=0.25)(layer_0_down)\n",
    "    \n",
    "    layer_1_down = tf.keras.layers.Dense(256, activation ='relu')(dropout0)\n",
    "    dropout1 = tf.keras.layers.Dropout(rate=0.25)(layer_1_down)\n",
    "    \n",
    "    layer_2_down =  tf.keras.layers.Dense(128, activation ='relu')(dropout1)\n",
    "    \n",
    "    # layer_3_down = tf.keras.layers.Dense(64, activation ='relu')(layer_2_down)\n",
    "    \n",
    "    bottleneck_layer = tf.keras.layers.Dense(128, activation ='relu')(layer_2_down)\n",
    "    \n",
    "    # layer_1_up = tf.keras.layers.Dense(64, activation ='relu')(bottleneck_layer)\n",
    "    \n",
    "    layer_2_up = tf.keras.layers.Dense(128, activation ='relu')(bottleneck_layer) \n",
    "    \n",
    "    layer_3_up = tf.keras.layers.Dense(256, activation ='relu')(layer_2_up)\n",
    "    dropout2 = tf.keras.layers.Dropout(rate=0.25)(layer_3_up)\n",
    "    \n",
    "    layer_4_up = tf.keras.layers.Dense(512, activation ='relu')(dropout2)\n",
    "    dropout3 = tf.keras.layers.Dropout(rate=0.25)(layer_4_up)\n",
    "\n",
    "    layer_5_up = tf.keras.layers.Dense(3072, activation ='relu')(dropout3)\n",
    "    reshape = tf.keras.layers.Reshape((24,1,128))(layer_5_up)\n",
    "    \n",
    "    #output_layer = tf.keras.layers.Dense(input_length, activation ='relu')(reshape)\n",
    "    output_layer = tf.keras.layers.Conv2DTranspose(1, (1,8), strides=(1,1), activation ='relu')(reshape)\n",
    "\n",
    "    autoencoder_model = tf.keras.Model(input_layer, output_layer)\n",
    "\n",
    "    return autoencoder_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 357,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.keras.backend.clear_session()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 358,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_1 (InputLayer)         [(None, 24, 8, 1)]        0         \n",
      "_________________________________________________________________\n",
      "conv2d (Conv2D)              (None, 24, 1, 128)        1152      \n",
      "_________________________________________________________________\n",
      "flatten (Flatten)            (None, 3072)              0         \n",
      "_________________________________________________________________\n",
      "dense (Dense)                (None, 512)               1573376   \n",
      "_________________________________________________________________\n",
      "dropout (Dropout)            (None, 512)               0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 256)               131328    \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 128)               32896     \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 128)               16512     \n",
      "_________________________________________________________________\n",
      "dense_4 (Dense)              (None, 128)               16512     \n",
      "_________________________________________________________________\n",
      "dense_5 (Dense)              (None, 256)               33024     \n",
      "_________________________________________________________________\n",
      "dropout_2 (Dropout)          (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "dense_6 (Dense)              (None, 512)               131584    \n",
      "_________________________________________________________________\n",
      "dropout_3 (Dropout)          (None, 512)               0         \n",
      "_________________________________________________________________\n",
      "dense_7 (Dense)              (None, 3072)              1575936   \n",
      "_________________________________________________________________\n",
      "reshape (Reshape)            (None, 24, 1, 128)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_transpose (Conv2DTran (None, 24, 8, 1)          1025      \n",
      "=================================================================\n",
      "Total params: 3,513,345\n",
      "Trainable params: 3,513,345\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "seq_length = 23\n",
    "autoencoder = get_autoencoder(seq_length)\n",
    "autoencoder.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 359,
   "metadata": {},
   "outputs": [],
   "source": [
    "total_dataset_length = 224146\n",
    "\n",
    "validation_length = 60000\n",
    "training_length = total_dataset_length-validation_length\n",
    "\n",
    "\n",
    "training_data = subject_list[:training_length]\n",
    "validation_data = subject_list[training_length:]\n",
    "\n",
    "train_seq_generator = image_generator(training_data, encoding_scheme= 'sine', batch_size=128, sequence_size=23)\n",
    "valid_seq_generator = image_generator(validation_data, encoding_scheme='sine', batch_size=128, sequence_size=23)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 360,
   "metadata": {},
   "outputs": [],
   "source": [
    "autoencoder.compile(loss=tf.keras.losses.MSE , optimizer=tf.keras.optimizers.Adam(learning_rate=5e-04))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 361,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/monowar/anaconda3/envs/envphase2/lib/python3.9/site-packages/tensorflow/python/keras/engine/training.py:1844: UserWarning: `Model.fit_generator` is deprecated and will be removed in a future version. Please use `Model.fit`, which supports generators.\n",
      "  warnings.warn('`Model.fit_generator` is deprecated and '\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "1000/1000 [==============================] - 26s 26ms/step - loss: 1449.7998 - val_loss: 205.4621\n",
      "Epoch 2/10\n",
      "1000/1000 [==============================] - 25s 25ms/step - loss: 156.9663 - val_loss: 135.9453\n",
      "Epoch 3/10\n",
      "1000/1000 [==============================] - 26s 26ms/step - loss: 162.7905 - val_loss: 148.7082\n",
      "Epoch 4/10\n",
      "1000/1000 [==============================] - 26s 26ms/step - loss: 554.9325 - val_loss: 59.2964\n",
      "Epoch 5/10\n",
      "1000/1000 [==============================] - 26s 26ms/step - loss: 77.1823 - val_loss: 554.8632\n",
      "Epoch 6/10\n",
      "1000/1000 [==============================] - 27s 27ms/step - loss: 78.1240 - val_loss: 824.1027\n",
      "Epoch 7/10\n",
      "1000/1000 [==============================] - 26s 26ms/step - loss: 228.8014 - val_loss: 101.4425\n",
      "Epoch 8/10\n",
      "1000/1000 [==============================] - 27s 27ms/step - loss: 312.2745 - val_loss: 44.1707\n",
      "Epoch 9/10\n",
      "1000/1000 [==============================] - 26s 26ms/step - loss: 130.4138 - val_loss: 1626.3037\n",
      "Epoch 10/10\n",
      "1000/1000 [==============================] - 26s 26ms/step - loss: 401.5026 - val_loss: 27.2191\n"
     ]
    }
   ],
   "source": [
    "history = autoencoder.fit_generator(train_seq_generator, steps_per_epoch=1000, epochs = 10, validation_data = valid_seq_generator, validation_steps=250)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 345,
   "metadata": {},
   "outputs": [],
   "source": [
    "training_loss = history.history['loss'][:]\n",
    "val_los = history.history['val_loss'][:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 346,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAqQAAADsCAYAAACi0rSBAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAA9hAAAPYQGoP6dpAAA8V0lEQVR4nO3deXwU9f3H8dfuZnMfkIQkBAIGCGeC3CheIBAEwYO2tOIBii39eVQK1gp4oBWotIot/KRVEVBKsf4EsYqY4AEiYgKIJOG+wpWDI+Q+Nrvz+yMQiMgRSDJJ9v18PPIgOzs785l8N8s735n5fi2GYRiIiIiIiJjEanYBIiIiIuLeFEhFRERExFQKpCIiIiJiKgVSERERETGVAqmIiIiImEqBVERERERMpUAqIiIiIqZSIBURERERU3mYXcCVcLlcHD16lICAACwWi9nliIiIiMiPGIZBfn4+kZGRWK0X7wNtkIH06NGjREVFmV2GiIiIiFzCoUOHaNmy5UXXaZCBNCAgAKg4wMDAwDrZp8PhICEhgfj4eOx2e53sU8ylNnc/anP3ozZ3T2r3upGXl0dUVFRlbruYBhlIz5ymDwwMrNNA6uvrS2BgoN68bkJt7n7U5u5Hbe6e1O5163Iur9RNTSIiIiJiKgVSERERETGVAqmIiIiImEqBVERERERMpUAqIiLiZgzDICEtk80Hc8wuRQRooHfZi4iIyJX7++d7mL16F542KyufuIl2Yf5mlyRuTj2kIiIibuTtdfuZvXoXAGVOF1OXp2AYhslVibtTIBUREXET/9l4iBc/3gbAmOtb42O38d3+k7y/8bDJlYm7UyAVERFxAytTMnj6g60APHxjNNPu6MLEwe0BmL5yO8cLSs0sT9ycAqmIiEgjt2bXMZ5Y+j0uA37ZK4qpt3fCYrHw4A3X0CUykNxiBy+d7jkVMYMCqYiISCOWfOAk49/diMNpcHvX5swYGVc5laOHzcrMkXFYLfDhlqN8vfuYydWKu1IgFRERaaRSj+Ty0IJkShwu+ndoxuxR3bBZq84r3rVlE8b0uwaAqctTKS5zmlCpuDsFUhERkUZoT3YBD7ydRH5pOX2ig5l3b088PX76v/1J8R1oHuTNwZNF/P2L3XVcqYgCqYiISKNz6GQR9731HScLy4hrEcT8Mb3w8bRdcH1/Lw9evDMWgDfX7mNHZl5dlSoCKJCKiIg0Ktn5Jdw//zsy80poF+bPoof6EOBtv+TrBncO57YuEZS7DCYvS8Hl0tikUncUSEVERBqJU0Vl3P9WEgdOFBEV7MPicX0J9vO87NdPu6ML/l4efH/wFP/6Lr0WKxWpSoFURESkESgoLWfsgmR2ZuUTFuDFv8ZdR0SQd7W2ERHkzVO3dQBg1qqdZOWV1EapIudRIBUREWngShxOfvPORrYcOkUTXzuLH+5LqxDfK9rWvX1b0y2qCfml5Uz7KK2GKxX5aQqkIiIiDZjD6eKxJd+zfu8J/DxtLHqwD+3DA654ezarhZkj4/CwWvg0NZPEbVk1WK3IT1MgFRERaaBcLoM/vP8Dq7dn4eVh5a0xvbk2qslVb7dT80AevqkNAM+tSKWgtPyqtylyMQqkIiIiDZBhGDz3USofbjmKh9XCvPt6cH3bkBrb/hMDY4gK9iEjt4RXEnbW2HZFfooCqYiISAM067OdLN5wEIsFXv1lN27tGF6j2/fxtDH9rjgAFq0/wNbDp2p0+yLnUiAVERFpYF7/ag/zvtoLwPS74rjj2sha2c/N7ZtxZ7dIXAY8/UEK5U5XrexHRIFURESkAXl3QzqzVlWcQp88tCOj+7aq1f09O7wzQT52tmXkseCbA7W6L3FfCqQiIiINxIffH+G5FakAPDagHeNvaVvr+wz192LKsI4AvJq4i0Mni2p9n+J+FEhFREQagMRtWUx6/wcMA8Zc35pJ8e3rbN+jekXRJzqYYoeT51akYhiaVlRqlgKpiIhIPbd+z3EeXbIZp8tgZPcWPD+iCxaLpc72b7FYmHF3HJ42K1/uPMYnKRl1tm9xDwqkIiIi9dj3B3N4+J2NlJW7iO8czqyfd8Vqrbsweka7MH/+p3/FJQIv/HcbucWOOq9BGi8FUhERkXpqR2YeYxckU1Tm5MZ2ocwZ3R0Pm3n/dT8yoC1tmvlxLL+Ul1ftMK0OaXwUSEVEROqhA8cLue+tJHKLHfRo1YR/3t8TLw+bqTV5ediYcXfF2KRLvjvIxgMnTa1HGo+rCqQzZ87EYrEwYcKEymWGYTBt2jQiIyPx8fGhf//+pKWlVXldaWkpjz/+OKGhofj5+XHHHXdw+PDhqylFRESk0cjILebet77jeEEpHSMCWDC2D35eHmaXBcB1bUIY1aslAJOXpVBWrrFJ5epdcSBNTk7mjTfeoGvXrlWWz5o1i1dffZW5c+eSnJxMREQEgwcPJj8/v3KdCRMmsHz5cpYuXcq6desoKChg+PDhOJ3OKz8SERGRRuBEQSn3vfUdR04VEx3qx7vj+hLkaze7rCqmDOtEiJ8nu7MLeGPtXrPLkUbgigJpQUEB9957L2+++SZNmzatXG4YBq+99hpTp05l5MiRxMbGsmjRIoqKiliyZAkAubm5zJ8/n1deeYVBgwbRvXt3Fi9eTEpKCqtXr66ZoxIREWmA8kocPPB2EnuPFRIZ5M3ih/vSLMDL7LLO08TXk+dGdAbg71/sYf/xQpMrkobuivr/H330UW6//XYGDRrESy+9VLl8//79ZGZmEh8fX7nMy8uLW265hfXr1zN+/Hg2bdqEw+Gosk5kZCSxsbGsX7+eIUOGnLe/0tJSSktLKx/n5eUB4HA4cDjq5i6/M/upq/2J+dTm7kdt7n7qU5sXlzl56J1NpB3NI9jPzoIxPQnz86gXtf2UoZ2b8X67ENbtOcGUZVtZNLZnnQ5FdTXqU7s3ZtX5+VY7kC5dupTNmzeTnJx83nOZmZkAhIeHV1keHh5Oenp65Tqenp5VelbPrHPm9T82c+ZMXnjhhfOWJyQk4OvrW91DuCqJiYl1uj8xn9rc/ajN3Y/ZbV7ugjd3WNmRa8XHZjCubTE7ktdQ3+9jHxAA31ltfLvvJC+8s4o+zRrWgPlmt3tjV1R0+bN6VSuQHjp0iCeeeIKEhAS8vb0vuN6P/0IyDOOSfzVdbJ3JkyczceLEysd5eXlERUURHx9PYGBgNY7gyjkcDhITExk8eDB2e/26lkdqh9rc/ajN3U99aPNyp4vfv5/CjtwsfOxWFo7tRY9WTUyp5UoUh+7nr4m7WXnUm9/9/AaC/TzNLumS6kO7u4MzZ7QvR7UC6aZNm8jOzqZnz56Vy5xOJ2vXrmXu3Lns3LkTqOgFbd68eeU62dnZlb2mERERlJWVkZOTU6WXNDs7m379+v3kfr28vPDyOv8aGrvdXudvJDP2KeZSm7sftbn7MavNXS6DKR9uZVVaFp42K2880Iu+bZvVeR1XY3z/dnycksmOzHxmJezhlVHXml3SZdPveu2qzs+2Wjc1DRw4kJSUFLZs2VL51atXL+699162bNlCmzZtiIiIqNIFXlZWxpo1ayrDZs+ePbHb7VXWycjIIDU19YKBVEREpLExDIOXPtnO+5sOY7XA3+/pxk0xDSuMAthtVmaOjMNigQ82H2b9nuNmlyQNULV6SAMCAoiNja2yzM/Pj5CQkMrlEyZMYMaMGcTExBATE8OMGTPw9fVl9OjRAAQFBTFu3DgmTZpESEgIwcHBPPnkk8TFxTFo0KAaOiwREZH67bXVu3n7m/0AzPr5tdwW2/wSr6i/urdqyv3Xteadb9OZsjyFVRNuxttu7iD+0rDU+Ci7Tz31FMXFxTzyyCPk5OTQt29fEhISCAgIqFxn9uzZeHh4MGrUKIqLixk4cCALFy7EZtObV0REGr+3vt7H3z7fDcC0EZ35ec+WJld09f4wpAOfpWVy4EQRc7/Yw5NDOphdkjQgVx1Iv/rqqyqPLRYL06ZNY9q0aRd8jbe3N3PmzGHOnDlXu3sREZEG5T/Jh3jpk+0ATBrcnrE3RJtcUc0I8Lbzwh1d+O3izfxjzV7u6BZJ+/CAS79QBM1lLyIiUmc+2ZrB08u2AvDrm6J57NZ2JldUs4Z0iWBQp3DKXQaTl6XgcjWsYaDEPAqkIiIideCrndlMeO97XAb8qncUU4Z1ajADyV8ui8XCi3d2wc/Txqb0HP6dfNDskqSBUCAVERGpZUn7T/LbxZtwOA2Gd23O9LvjGl0YPSOyiQ+T4iuuH/3zpzvIzisxuSJpCBRIRUREalHqkVzGLUymxOFiQIdmvDqqGzZr4wyjZ4zpdw1dWwaRX1LOCx9vM7scaQAUSEVERGrJnux8Hng7ifzScvpEBzPvvp54ejT+/3ptVgsz7o7DZrXwydYMvtyRbXZJUs81/t8KERERExw6WcR9byVxsrCMri2DmD+ml1uNzRnbIoiHbrgGgGc+TKWorNzcgqReUyAVERGpYdl5Jdw3/zsy80qICfNn4YN9CPB2vykqfz+4PS2a+HDkVDGzE3eZXY7UYwqkIiIiNehUURn3z08i/UQRUcE+LH64L8F+nmaXZQpfTw9euqtiJse3vzlA6pFckyuS+kqBVEREpIYUlJYzZkEyO7PyCQvw4l/jriM80Nvsskw1oGMYt3dtjtNlMGV5Ck6NTSo/QYFURESkBpQ4nPx60UZ+OHSKpr52Fj/cl1YhvmaXVS88P6IzAd4ebD2cy6L1B8wuR+ohBVIREZGr5HC6eGzJZr7ddwJ/Lw8WPdRH02aeIyzAm6eHdgTglYSdHD1VbHJFUt8okIqIiFwFl8vgyfd/YPX2bLw8rLw1phddWzYxu6x6557erejVuimFZU6eW5GGYejUvZylQCoiInKFDMPg2RWprNhyFA+rhXn39eC6NiFml1UvWa0WZoyMw26zsHp7Fp+lZZpdktQjCqQiIiJX6OVVO/nXdwexWGD2L7txa8dws0uq19qHBzD+5rYAPP9RGnklDpMrkvpCgVREROQKvP7VHv6xZi8AM+6OY8S1kSZX1DA8dms7rgnxJSuvlL9+ttPscqSeUCAVERGppne/PcCsVRVhasqwjtzTp5XJFTUc3nYb0++OA+DdDelsPphjckVSHyiQioiIVMPy7w/z7Io0AB6/tR2/OX0KWi7fDe1CGdmjBYYBU5al4HC6zC5JTKZAKiIicpkS0jJ58v2tAIztdw0TB7c3uaKG65nbO9PU186OzHze+nq/2eWIyRRIRURELsM3e47z2JLvcboMRvZowXPDO2OxWMwuq8EK9vNk6u2dAXht9S7STxSaXJGYSYFURETkEjYfzOHX72ykzOliSJdwZv2sK1arwujV+lmPFvRrG0JpuYtnPkzV2KRuTIFURETkIrZn5DH27SSKypzcFBPK3+/pjodN/33WBIvFwvS74/D0sPL17uOs2HLU7JLEJPqNEhERuYD9xwu5f34SeSXl9GjVhH/e3xMvD5vZZTUq0aF+/O7WdgD86eNtnCoqM7kiMYMCqYiIyE84eqqY+976juMFpXRqHsiCB/vg6+lhdlmN0m9ubkv7cH9OFJYxY+V2s8sREyiQioiI/MjxglLum/8dR04V0ybUj3ce6kOQj93sshotTw8rM06PTfqfjYfZsO+EyRVJXVMgFREROUdusYMH5iex71ghkUHevPtwX5oFeJldVqPX65pgRvetmGBgyvIUSsudJlckdUmBVERE5LSisnLGLUxmW0Yeof6eLH64Ly2a+Jhdltv4420daRbgxb5jhbz+5V6zy5E6pEAqIiIClJY7Gf/uJjam5xDg7cE7D/WlTTN/s8tyK0E+dp4fUTE26byv9rInu8DkiqSuKJCKiIjbK3e6mLB0C1/vPo6P3cbCB3vTOTLQ7LLc0u1xzRnQoRllThdTlqfgcmlsUnegQCoiIm7NZcDUFdv4NDUTT5uVNx7oSc/WwWaX5bYsFgsv3hmLj91G0v6TvL/pkNklSR1QIBUREbdlGAbLD1hZ9v1RbFYLf7+nOzfFNDO7LLcXFezLxMHtAZixcgfHC0pNrkhqmwKpiIi4pcLScqZ9vJ21mRX/Fc76WVdui40wuSo548EbrqFLZCC5xQ7+9PE2s8uRWqZAKiIibueLHVnEz17LkqTDADx7e0d+1rOlyVXJuTxsVmaOjMNqgRVbjrJm1zGzS5JaVK1AOm/ePLp27UpgYCCBgYFcf/31fPrpp5XPG4bBtGnTiIyMxMfHh/79+5OWllZlG6WlpTz++OOEhobi5+fHHXfcweHDh2vmaERERC4iO7+ER5ds5qGFGzlyqpgWTbwZ39HJA9e1Mrs0+QldWzZhTL9rAHjmwxSKyzQ2aWNVrUDasmVL/vznP7Nx40Y2btzIrbfeyp133lkZOmfNmsWrr77K3LlzSU5OJiIigsGDB5Ofn1+5jQkTJrB8+XKWLl3KunXrKCgoYPjw4TidepOJiEjtcLkM/p10kEGvrOGTrRlYLfCbm9uw8vF+dG6qu7jrs0nxHWge5M2hk8X87fPdZpcjtaRagXTEiBEMGzaM9u3b0759e6ZPn46/vz8bNmzAMAxee+01pk6dysiRI4mNjWXRokUUFRWxZMkSAHJzc5k/fz6vvPIKgwYNonv37ixevJiUlBRWr15dKwcoIiLubU92Pr96YwOTl6WQV1JOXIsgPnrsRqYM66S56RsAfy8PXrwzFoA3v97H9ow8kyuS2nDF15A6nU6WLl1KYWEh119/Pfv37yczM5P4+PjKdby8vLjllltYv349AJs2bcLhcFRZJzIyktjY2Mp1REREakJpuZPZibsY+revSTpwEl9PG8/c3onlj/QjtkWQ2eVJNQzuHM5tXSJwugwmL0vBqbFJG51q/2mYkpLC9ddfT0lJCf7+/ixfvpzOnTtXBsrw8PAq64eHh5Oeng5AZmYmnp6eNG3a9Lx1MjMzL7jP0tJSSkvPDvmQl1fx15HD4cDhcFT3EK7Imf3U1f7EfGpz96M2bzySDpzk2RXb2He8CID+7UOZNqITLZr4YLicOFwVl4mpzRuOqcPa8/WeY2w5dIp31u/jvr5Xft2v2r1uVOfnW+1A2qFDB7Zs2cKpU6f44IMPGDNmDGvWrKl83mKxVFnfMIzzlv3YpdaZOXMmL7zwwnnLExIS8PX1reYRXJ3ExMQ63Z+YT23uftTmDVdROXyUbuXb7IoTgAF2g59d46JbcCY/rM/khwu8Tm3eMAyNtPB/+238+dPtWI+m0sTr6randq9dRUVFl71utQOpp6cn7dq1A6BXr14kJyfzt7/9jT/+8Y9ARS9o8+bNK9fPzs6u7DWNiIigrKyMnJycKr2k2dnZ9OvX74L7nDx5MhMnTqx8nJeXR1RUFPHx8QQG1s3Ubg6Hg8TERAYPHozdbq+TfYq51ObuR23ecBmGwcrULF5ZuYPjBWUA/LJXS/4QH0OQz4XbUm3esAxxGex+M4kfDufyTUkk/3t3tyvajtq9bpw5o305rvpqbsMwKC0tJTo6moiICBITE+nevTsAZWVlrFmzhpdffhmAnj17YrfbSUxMZNSoUQBkZGSQmprKrFmzLrgPLy8vvLzO/zPIbrfX+RvJjH2KudTm7kdt3rAczini2Q9T+XJnxTiVbZv5MXNkV/pEX/70n2rzhsEO/PlnXRkxZx0J27L5ctcJ4rtc+WQGavfaVZ2fbbUC6ZQpUxg6dChRUVHk5+ezdOlSvvrqK1atWoXFYmHChAnMmDGDmJgYYmJimDFjBr6+vowePRqAoKAgxo0bx6RJkwgJCSE4OJgnn3ySuLg4Bg0aVL2jFBERt1budLFw/QFeSdhFscOJp83KIwPa8j/92+LlYTO7PKklnZoH8vBNbfjHmr08/1Ea/dqF4u+l0RIaumq1YFZWFvfffz8ZGRkEBQXRtWtXVq1axeDBgwF46qmnKC4u5pFHHiEnJ4e+ffuSkJBAQEBA5TZmz56Nh4cHo0aNori4mIEDB7Jw4UJsNn14iIjI5Uk5nMvk5VtJPVJxSrBPdDAz7o6jXZi/yZVJXXhiYAyfpBzl0Mli/vrZTqbd0cXskuQqVSuQzp8//6LPWywWpk2bxrRp0y64jre3N3PmzGHOnDnV2bWIiAiFpeXMTtzF29/sx2VAoLcHU2/vxC96RmG1XvwGWmk8fDxtTL8rjgfeTmLRtwe4u3sLro1qYnZZDUJpubNenkHQXPYiItIgfLkjm/jZa3lrXUUYHXFtJKsn3cIve7dSGHVDN7dvxp3dIjEMmLwshXKny+yS6q2ThWX8J/kQDy1MpseLieQW1b/hrnTRhYiI1GvH8kt58eNt/PeHowC0aOLDS3fHMqBDmMmVidmeHd6Zr3YeY1tGHm9/s5/f3NzW7JLqjSOniklIy2RVaibJB05y7lwC6/ceZ2hc8wu/2AQKpCIiUi+5XAb/2XiIGSu3k1dSjtUC426M5veD22vKTwEg1N+LKcM68scPUpiduJuhsc2JCq7b8cnrkz3Z+XyWlsWq1ExSjuRWea5LZCC3dYlgSGwEMfXwWmv9RouISL2zJ7uAKctTSNp/EoDYFoH8eWRXTfkp5xnVK4oPNh8haf9Jnl2RyoKxvS85IU9jYRgGWw/n8llaJqvSMtl3rLDyOYsFercOZkhsBPGdw+t9UFcgFRGReqO03Mm8r/by+pd7KXO68LHbmBTfnrH9rsHDptse5HwWi4UZd8cx7G9f89XOY3y8NYMR10aaXVatKXe6SDpwkoS0LD5LyyQjt6TyOU+blX7tQritSwSDOocT6n+VU1nVIQVSERGpF5L2n2Tysq3sPd3LM6BDM168M7be9+yI+dqF+fM//dvyt89388J/t3FzTDOCfBvPgPclDifrdh/ns7RMVm/PIuecm5J8PW0M6BDGkNgIBnRoRoB3wzxuBVIRETFVbpGDP6/azr+TDgEV1wVOu6Mzt8c1d5tTr3L1HhnQlv9uPcq+Y4X8edUOZo6MM7ukq5JX4uDLHdkkpGXx5c5sisqclc819bUzqFM4t8VGcEO7ULzt9W8Yp+pSIBUREVMYhsEnKRlM+2gbxwtKAbinTxRP39apUfVuSd3w8rAx4+44fvXGBv6ddJCRPVrQ+5rLnz62PjheUEritopT8d/sOY7DefbW+OZB3gzpEsGQLhH0vqZpo7uERYFURETq3OGcIp5bkcYXO7KBK5t/XuTHrmsTwi97RfHexkNMWZbCJ7+7CU+P+h3cDp0s4rO0TBLSskhOP4lxzvBMbZv5MaRLBLfFRhDXIqhRnzFQIBURkTqj+eeltk0e1pHPd2SxO7uAf67Zy+MDY8wuqQrDMNiVVcBnaZl8lpZJ2tG8Ks93bRl0uic0nHZhARfYSuOjQCoiInUi9Uguk5elVI6P2OeaYGaM1PzzUrOa+Hry7PDOPLF0C3O+3MPtXZvTppm57zGXy2DL4VOVPaH7j58dnslqgT7RwQzpEkF8lwhaNPExsVLzKJCKiEitKio7M//8AZwug0BvD6YM68SoXpp/XmrHHddG8n+bDvP17uNMXZ7Kkl/3rfPT3Q6ni+/2nawIodsyycorrXzO02blpphQhnSJYGCnMEIa0PBMtUWBVEREas2XO7N5ZnkqR04VAzC8a3OeG9GZsABvkyuTxsxisTD9rjjiX1vDt/tO8MHmI/y8Z8ta329xmZO1u4/xWVomn2/PJrf47PBM/l4eDOgYxpAu4fTvEIa/lyLYufTTEBGRGncsv5Q/fbyNj86df/6uWAZ01PzzUjdahfjyxMD2vLxqB9M/2catHcMI9vOs8f3kFjv4YkcWn6VmsWbXMYodZ4dnCvHzZHDncIZ0iaBfuxBdJ30RCqQiIlJjDOPM/PM7yC12YLXAQzdUzD/vpx4hqWMP3xTNii1H2JGZz0ufbOPVUd1qZLvZeSUknB6e6du9Jyh3nb01vkUTn8qbknpdE4xNl6VcFn06iIhIjdh7rIApy1L47pz552fe3ZW4lpp/Xsxht1mZOTKOkfPWs2zzEX7WoyU3tAu9om2lnyg8fWd8FpsP5lQZnikmzJ/bYivGCO0SGdioh2eqLQqkIiJyVUrLnfzjq33875d7NP+81DvdWzXl/uta88636UxdnsKqCTdzOSfODcNgR2Y+q1IrhmfakZlf5flro5pw2+meULPv4m8MFEhFROSKJR84yeRlKezJLgCgf4dm/Enzz0s984chHfgsLZMDJ4qY88VuJtza9ifXc7kMvj+UczqEZnHwZFHlczarhb6VwzOF0zzIPYdnqi0KpCIiUm25xQ5eXrWDJd8dBCDU35PnR3RheFfNPy/1T4C3nRfu6MJvF2/mn2v2MazL2ZvryspdbNh3glVpmSRuy+JY/tnhmbw8rNwU04zbYiMY2DGMprVwU5RUUCAVEZHLZhgGK1MymfbftMr/uDX/vDQEQ7pEMKhTOKu3Z/HsR9vp5m3h8/dT+HLXMfJLyivXC/Dy4NZOYdzWJYKb2zfTzXh1RD9lERG5LEdOFfPch6l8fnr++TbN/Jh5dxx924SYXJnIpVksFl68swvf7j3O5oOn2IwNyAAg1N/r9PBM4fRrG4qnh659rmsKpCIiclFOl3F6/vmdFJU5sdssPNK/HY8M0Pzz0rBENvFh6u2dmbI8hRAvgzt7XsPQrpH0aNVUwzOZTIFUREQuKPVILlOWp7D18Lnzz8fSLizA5MpErszovq24rXMz1n6ewO1DO2C361KT+kCBVEREzlNUVs7fVu/mrXX7cboMAk7PP/9LzT8vjUCAtwe6965+USAVEZEq1uw6xtTlKRzO0fzzIlI3FEhFRASA4wUV88+v2HJ2/vk/3dWFWzuGm1yZiDR2CqQiIm7OMAze33iY6Su3V84//+AN0UzU/PMiUkf0SSMi4ub+mrCT//1yLwBdIgP580jNPy8idUuBVETEjb29bn9lGP3DkA6Mv7mN5p8XkTqnQCoi4qZWbDnCix9vAyrC6KMD2plckYi4K/0ZLCLihtbsOsak//wAwNh+1/BI/7YmVyQi7kyBVETEzWw5dIr/WbyJcpfBHddG8tzwzlg0KKOImEiBVETEjezJLuDBBUkUlTm5uX0z/vqLazXQvYiYrlqBdObMmfTu3ZuAgADCwsK466672LlzZ5V1DMNg2rRpREZG4uPjQ//+/UlLS6uyTmlpKY8//jihoaH4+flxxx13cPjw4as/GhERuaCM3GLGvJ1ETpGDa6OaMO/eHnh6qF9CRMxXrU+iNWvW8Oijj7JhwwYSExMpLy8nPj6ewsLCynVmzZrFq6++yty5c0lOTiYiIoLBgweTn59fuc6ECRNYvnw5S5cuZd26dRQUFDB8+HCcTmfNHZmIiFQ6VVTGmLeTOHKqmDbN/FgwtrfGGBWReqNan0arVq2q8njBggWEhYWxadMmbr75ZgzD4LXXXmPq1KmMHDkSgEWLFhEeHs6SJUsYP348ubm5zJ8/n3fffZdBgwYBsHjxYqKioli9ejVDhgypoUMTERGA4jIn4xZtZFdWARGB3rzzUB+C/TzNLktEpNJV/Xmcm5sLQHBwMAD79+8nMzOT+Pj4ynW8vLy45ZZbWL9+PePHj2fTpk04HI4q60RGRhIbG8v69et/MpCWlpZSWlpa+TgvLw8Ah8OBw+G4mkO4bGf2U1f7E/Opzd1PY2xzh9PFI0u2sCk9h0BvD+Y/0J1wf3ujOsar0RjbXC5N7V43qvPzveJAahgGEydO5MYbbyQ2NhaAzMxMAMLDq857HB4eTnp6euU6np6eNG3a9Lx1zrz+x2bOnMkLL7xw3vKEhAR8fX2v9BCuSGJiYp3uT8ynNnc/jaXNDQOW7LWSdMyK3WLwYNsS9mz6mj1mF1YPNZY2l+pRu9euoqKiy173igPpY489xtatW1m3bt15z/14+BDDMC45pMjF1pk8eTITJ06sfJyXl0dUVBTx8fEEBgZeQfXV53A4SExMZPDgwdjt9jrZp5hLbe5+Glubv/zZLpKOHcBmtTB3dHdu7dDM7JLqncbW5nJ51O5148wZ7ctxRYH08ccf56OPPmLt2rW0bNmycnlERARQ0QvavHnzyuXZ2dmVvaYRERGUlZWRk5NTpZc0Ozubfv36/eT+vLy88PLyOm+53W6v8zeSGfsUc6nN3U9jaPM31+7jrXUHAPjzyDiGxEaaW1A91xjaXKpP7V67qvOzrdZd9oZh8Nhjj7Fs2TK++OILoqOjqzwfHR1NRERElS7wsrIy1qxZUxk2e/bsid1ur7JORkYGqampFwykIiJy+T7YdJjpK7cD8PTQjvyiV5TJFYmIXFy1ekgfffRRlixZwooVKwgICKi85jMoKAgfHx8sFgsTJkxgxowZxMTEEBMTw4wZM/D19WX06NGV644bN45JkyYREhJCcHAwTz75JHFxcZV33YuIyJX5ckc2T32wFYBf3xTN+JvbmFyRiMilVSuQzps3D4D+/ftXWb5gwQLGjh0LwFNPPUVxcTGPPPIIOTk59O3bl4SEBAICAirXnz17Nh4eHowaNYri4mIGDhzIwoULsdlsV3c0IiJubFN6Dv/zr004XQYju7dg8tBOmhJURBqEagVSwzAuuY7FYmHatGlMmzbtgut4e3szZ84c5syZU53di4jIBezOyuehhcmUOFwM6NCMl3/eVVOCikiDoTnjREQauCOninng7SRyix10b9WE/723B3abPt5FpOHQJ5aISAN2srCMB+Z/R0ZuCe3C/Hl7TG98PTUlqIg0LAqkIiINVFFZOQ8tTGbvsUKaB1VMCdpUU4KKSAOkQCoi0gA5nC7+Z/Fmthw6RRNfO++O60NkEx+zyxIRuSIKpCIiDYzLZfCH939gza5j+NhtvD22N+3CAi79QhGRekqBVESkATEMg+krt/PhlqN4WC28fl8PerRqeukXiojUYwqkIiINyD/W7GP+uv0A/OUXXRnQIczkikRErp4CqYhIA/GfjYd4edUOAJ65vRN3d29pckUiIjVDgVREpAFYvS2LyctSAPjtLW15+CZNCSoijYcCqYhIPZd84CSPLtmM02Xwi54t+eNtHcwuSUSkRimQiojUYzsy83hoYTKl5S4GdQpj5sg4zU8vIo2OAqmISD116GQRD8xPIr+knF6tmzLnnh54aEpQEWmE9MkmIlIPnSgoZczbSWTnl9IhPID5Y3rj42kzuywRkVqhQCoX5HIZlDicOF2G2aWIuJWC0nIeXJjMvuOFtGjiw6KH+hDkaze7LBGRWuNhdgHuwjAMyl0G5U6DMqcLx+mvKo/LDRwuF45yFw6ngcPpouz0Ome+d/z4cblBuevs9w6nq+Lx6e/Pfp19XOY0KD9neVl5xWscTgNH+el9uozKIBrg5cHdPVpwb9/WdIjQbDAitams3MVv393E1sO5BPt58s64PkQEeZtdlohIrVIgvQwrUzL4LDWDg4etfHxqC06D84JeRag7HfrKK0JfRXg8HfqcLrMP44rll5bzzrfpvPNtOr2vacp917XmttgIvDx0+lCkJrlcBpPe/4F1e47j62ljwdjetG3mb3ZZIiK1ToH0MmzPyGPFDxmAFU5k19h27TYLdpsVD6sFTw8rdlvFl4fNgqftzGMLHjbr6ceWynUqv/ewYree873t9OPT33uefv2Z13jarKcfn96HR8X+7TZrZQ3n1uNhs7D1UC6LN6STuD2L5AM5JB/IIdjPk1/0asm9fVrTKsS3xn4mIu7KMAxe/Hgb//3hKHabhX/c15Nro5qYXZaISJ1QIL0MN7dvhp+nld07tnNt11i87XbsHj8RDn/isefpUHfme7uHBQ9rxToNZeiWG2NCuTEmlMzcEpYmH2Rp0iEy80r455p9vLF2HzfHNOPevq24tWOY7gAWuUL/++UeFq4/AMBff3EtN7dvZm5BIiJ1SIH0MvS+JphuLQJYmbuNYb2jsNvd8+aCiCBvJgxqz2MD2vH5jmwWb0jn693HWbPrGGt2HaN5kDf39GnFr3pHERaoa95ELte/kw7y14RdADw/ojN3dmthckUiInVLgVSqzcNmZUiXCIZ0iSD9RCFLvjvIfzYeIiO3hFcTd/H3z3czuHM4913Xmn5tQxpMT7CIGValZjJ1ecWUoI8NaMeDN0SbXJGISN1TIJWr0jrEj8nDOvH7we1ZlZrJ4g3pbEzP4dPUTD5NzaRNqB+j+7bi5z1b0sTX0+xyReqVDftO8Lul3+My4J4+UUyKb292SSIiplAglRrhbbdxV/cW3NW9Bdsz8vjXd+ks33yEfccLeemT7fzls50M7xrJvde1ontUE/WaittLO5rLrxdtpKzcRXzncP50Z6x+L0TEbSmQSo3r1DyQl+6K4+mhnVix5QiLNxxke0YeH2w+zAebD9O5eSD3XdeaO7tF4uelt6C4n4MnihjzdjL5peX0iQ7m7/d01w2BIuLW9Akotcbfy4N7+7Zm5e9uZNkj/RjZowWeHla2ZeQxZXkKfWd8zrMfprIzM9/sUkXqzLH8Uu5/+zuOF5TSMSKANx/ohbddY/qKiHtT95TUOovFQo9WTenRqinP3t6ZDzYf5l/fHWT/8ULe3ZDOuxvS6dW6YsD9oXEacF8ar/wSB2MXJJF+ooioYB/eeagPQT7uOWqHiMi5FEilTjX18+Thm9rw0A3RrN97gn99l07Ctiw2puewMT2HFz+uGHB/dJ9WtA7xM7tckRpTWu5k/LubSDuaR4ifJ+881FfDo4mInKZAKqawWi2VA+5n5ZWwNOkQ/046WDng/j/X7OPm9s24TwPuSyPgdBn8/r0trN97Aj9PGwsf7EN0qP7gEhE5Q4FUTBce6M0Tg2J4dEBbPt+Rzb++O8jaXccqv5oHefOr3q34VZ8owtWjJA2MYRg8/1EqK1My8bRZeeOBXsS1DDK7LBGRekWBVOqNiw24P3v1Lv7+xW4Gdzo74L7VqiFypP772+e7WbzhIBYLzP5lN25oF2p2SSIi9Y4CqdRLFxpwf1VaJqvSMokO9WN0n4oB95v6acB9qZ/e3ZDOa6t3A/DinbHc3rW5yRWJiNRPCqRSr5074P6OzDz+teEgy78/wv7jhUxfuZ2/JOxkeNfm3Hddaw24L/XKJ1szeG5FKgBPDIzh/utam1yRiEj9pUAqDUbHiED+dFcsfxzakY+2HGXxhnS2ZeSxbPMRlm0+Qqfmgdx3XSvu6tZCA+6LqdbvOc7v39uCYcC9fVsxYVCM2SWJiNRr1b51ee3atYwYMYLIyEgsFgsffvhhlecNw2DatGlERkbi4+ND//79SUtLq7JOaWkpjz/+OKGhofj5+XHHHXdw+PDhqzoQcR/+Xh6M7tuKT3404P72jDymLk+l74zPeebDFHZk5pldqrih1CO5/PqdjZQ5XQyLi+BFTQkqInJJ1Q6khYWFXHvttcydO/cnn581axavvvoqc+fOJTk5mYiICAYPHkx+/tnZeCZMmMDy5ctZunQp69ato6CggOHDh+N0Oq/8SMTtnBlw/9VR3UiaMpBnbu9EdKgfBaXlLN5wkNte+5qfz1vP8u8PU+LQe0tq34HjhYxdkERhmZPr24Qw+5fdsOnmOxGRS6r2ec2hQ4cydOjQn3zOMAxee+01pk6dysiRIwFYtGgR4eHhLFmyhPHjx5Obm8v8+fN59913GTRoEACLFy8mKiqK1atXM2TIkKs4HHFXTXzPDrj/7b4TLN7wowH3/7uNUb2iGN1XA+5L7cjOKzk9JWgZXSIDeeOBnpp1TETkMtXohXb79+8nMzOT+Pj4ymVeXl7ccsstrF+/nvHjx7Np0yYcDkeVdSIjI4mNjWX9+vU/GUhLS0spLS2tfJyXV3Eq1uFw4HA4avIQLujMfupqf3Ll+rQOok/rrmTllfD+piO8t/EwmXml/HPtPv65dh83tgthdO8oBnQIveiA+2pz93OlbZ5f4uCBtzdy6GQxrYJ9eOv+7njb9N5pCPR77p7U7nWjOj/fGg2kmZmZAISHh1dZHh4eTnp6euU6np6eNG3a9Lx1zrz+x2bOnMkLL7xw3vKEhAR8fX1rovTLlpiYWKf7k6vTBniqM2zLsfBNloUdpyys23OCdXtOEORpcH2Yi+vDDJp4XXgbanP3U502d7jgH9tt7MmzEGA3GNM6n6S1n9didVIb9HvuntTutauoqOiy162VW5F/fAG/YRiXvKj/YutMnjyZiRMnVj7Oy8sjKiqK+Ph4AgMDr77gy+BwOEhMTGTw4MHY7fY62afUnBGn/z14soilyYf5v81HyClysOqwjcSjFgZ2bMY9vaPo1ya4csB9tbn7qW6blztd/O69rezJy8bfy4PF43rRuXndfCZJzdDvuXtSu9eNM2e0L0eNBtKIiAigohe0efOzA0BnZ2dX9ppGRERQVlZGTk5OlV7S7Oxs+vXr95Pb9fLywsvr/C4su91e528kM/YpNadteBBThwfx5G0dKwfcTz6QQ8K2bBK2ZXNNiC/39m3Nz3u2xN+zop3V5u7nctrcMAye/SiFxO3ZeHpYefOBXlzbKqSOKpSapt9z96R2r13V+dnWaCCNjo4mIiKCxMREunfvDkBZWRlr1qzh5ZdfBqBnz57Y7XYSExMZNWoUABkZGaSmpjJr1qyaLEfkgrw8bNzZrQV3dqsYcH/JdwdZtvkIB04UVQ64P6xLOB55Fgo3HcbHy46nzYanh7Xiy1bxr9ePHld53mbV9KaN2CsJu1iafAirBf7+q25c31ZhVETkSlU7kBYUFLBnz57Kx/v372fLli0EBwfTqlUrJkyYwIwZM4iJiSEmJoYZM2bg6+vL6NGjAQgKCmLcuHFMmjSJkJAQgoODefLJJ4mLi6u8616kLnWMCOTFO2P5420dWXHOgPsf/pAB2Pi//duueNt2m+Wnw6pHRbj1sv14edXHFw28VQLx5YVlD6tFY2LWgIXf7GfulxWfg9PvjuO2WE0JKiJyNaodSDdu3MiAAQMqH5+5tnPMmDEsXLiQp556iuLiYh555BFycnLo27cvCQkJBAQEVL5m9uzZeHh4MGrUKIqLixk4cCALFy7EZtMQKWIev9MD7t/TJ4oth07xfxsPkbonneBmYZS7oLTcRdmZL+eFvz+Xw2ngcDopLKsf46BaLFQNrBcIun6eHlwT6kdMmD/twvyJCQsgyFentQA++uEoL3xc8UfKpMHtuadPK5MrEhFp+KodSPv3749hGBd83mKxMG3aNKZNm3bBdby9vZkzZw5z5syp7u5Fap3FYqF7q6bENvdn5cr9DBvW47KvgzEMozKg/lSALf3JEOus/L70x8/96HHpRZ6rug9n5WOXcW59FcG6tNxF/oUP4yc1C/A6J6D60y4sgJhwf0L8PN2m1/Xr3ceY9J+KKUHHXN+ax25tZ3ZJIiKNgib8FqlBFosFLw8bXh42Ai69ep0od/4o1F6kh/fMc3nFDvYdK2R3dj57sgvIyC3hWH4px/JLWb/3RJXtN/G1VwbUM2E1JtyfiEDvRhVUfzh0ivHvbsLhNBjetTnPj+jSqI5PRMRMCqQijZyHzYqHzYqv55VvI7/Ewd5jhezOqgioe7IL2J1dwKGcIk4VOUg+kEPygZwqr/H38qDtmYB6zqn/lk19GtzNXnuPFfDgwmSKypzc2C6UV0Zd2+COQUSkPlMgFZFLCvC20y2qCd2imlRZXlzmZN/x0wE160xQzefAiSIKSsv54dApfjh0qsprvO1W2oRW9KKe27PaOsQX+0VmzjJLVl4JD8xP4mRhGV1bBvGP+zUlqIhITVMgFZEr5uNpo0tkEF0ig6osLyt3kX6ikN2ng+qZU//7jhVS4nCxLSOPbRlVB0y22yxEh/rR7sz1qadP/UeH+pkWAHOLHDwwP4kjp4qJDvVjwdje+HvpY1NEpKbpk1VEapynh5WY8ABiwgMg7uzycqeLQznFlT2pe7IK2HOsome1qMzJrqwCdmUVAGenEbZaoHXImaB65hKAANqG+eHrWXsfYcVlTsYtSmZnVj5hAV6881AfQvwvMsesiIhcMQVSEakzHjYr0aF+RIf6MbhzeOVyl8vgaG7x2etTT/eq7s4uIL+knP3HC9l/vJDEbVlVtteiic85p/7Pnv4P8rm6IarKnS4eW7qFjek5BHh7sOihPkQF+17VNkVE5MIUSEXEdFarhZZNfWnZ1Jf+HcIqlxuGwbH8UnZnn70+dXdWAXuPFXC8oIwjp4o5cqqYr3Yeq7K9sACv00E1oErP6uX0cBoGPPPRNj7fkY2Xh5X5Y3rTSfPTi4jUKgVSEam3LBYLYYHehAV6c0O70CrPnSwsO+eO/7N3/2fklpCdX0p2finf7Kk6RFWwnyftmvnTLvzsqf92Yf6EB3pVDuH034NWPj96FJvVwtzRPegTHVxnxysi4q4USEWkQQr286RPdPB5gTGvxMHe08NSnfl3d3Y+h3OKOVlYRlLhSZIOnKzymgAvD9qF+9PUx84XRyvu9J95d1yVywpERKT2KJCKSKMS6G2ne6umdG/VtMry4jIne0/fQFV5+j+7gPQTReSXlvP9wVOV6z45OIZRvaPquHIREfelQCoibsHH00ZsiyBiW5w/RNWBE4XszipgZ2Yux9J38ZubrjGnSBERN6VAKiJuzdPDSvvwANqHBxDfKZSVK3dqSlARkTpW/6ZFERERERG3okAqIiIiIqZSIBURERERUymQioiIiIipFEhFRERExFQN8i57wzAAyMvLq7N9OhwOioqKyMvLw26/unmypWFQm7sftbn7UZu7J7V73TiT087ktotpkIE0Pz8fgKgoDVwtIiIiUp/l5+cTFBR00XUsxuXE1nrG5XJx9OhRAgIC6my8wLy8PKKiojh06BCBgYF1sk8xl9rc/ajN3Y/a3D2p3euGYRjk5+cTGRmJ1Xrxq0QbZA+p1WqlZcuWpuw7MDBQb143ozZ3P2pz96M2d09q99p3qZ7RM3RTk4iIiIiYSoFUREREREylQHqZvLy8eP755/Hy8jK7FKkjanP3ozZ3P2pz96R2r38a5E1NIiIiItJ4qIdUREREREylQCoiIiIiplIgFRERERFTKZCKiIiIiKkUSC/D66+/TnR0NN7e3vTs2ZOvv/7a7JKklsycOZPevXsTEBBAWFgYd911Fzt37jS7LKlDM2fOxGKxMGHCBLNLkVp25MgR7rvvPkJCQvD19aVbt25s2rTJ7LKklpSXl/PMM88QHR2Nj48Pbdq04cUXX8TlcpldmqBAeknvvfceEyZMYOrUqXz//ffcdNNNDB06lIMHD5pdmtSCNWvW8Oijj7JhwwYSExMpLy8nPj6ewsJCs0uTOpCcnMwbb7xB165dzS5FallOTg433HADdrudTz/9lG3btvHKK6/QpEkTs0uTWvLyyy/zj3/8g7lz57J9+3ZmzZrFX/7yF+bMmWN2aYKGfbqkvn370qNHD+bNm1e5rFOnTtx1113MnDnTxMqkLhw7doywsDDWrFnDzTffbHY5UosKCgro0aMHr7/+Oi+99BLdunXjtddeM7ssqSVPP/0033zzjc54uZHhw4cTHh7O/PnzK5f97Gc/w9fXl3fffdfEygTUQ3pRZWVlbNq0ifj4+CrL4+PjWb9+vUlVSV3Kzc0FIDg42ORKpLY9+uij3H777QwaNMjsUqQOfPTRR/Tq1Ytf/OIXhIWF0b17d958802zy5JadOONN/L555+za9cuAH744QfWrVvHsGHDTK5MADzMLqA+O378OE6nk/Dw8CrLw8PDyczMNKkqqSuGYTBx4kRuvPFGYmNjzS5HatHSpUvZvHkzycnJZpcidWTfvn3MmzePiRMnMmXKFJKSkvjd736Hl5cXDzzwgNnlSS344x//SG5uLh07dsRms+F0Opk+fTr33HOP2aUJCqSXxWKxVHlsGMZ5y6Txeeyxx9i6dSvr1q0zuxSpRYcOHeKJJ54gISEBb29vs8uROuJyuejVqxczZswAoHv37qSlpTFv3jwF0kbqvffeY/HixSxZsoQuXbqwZcsWJkyYQGRkJGPGjDG7PLenQHoRoaGh2Gy283pDs7Ozz+s1lcbl8ccf56OPPmLt2rW0bNnS7HKkFm3atIns7Gx69uxZuczpdLJ27Vrmzp1LaWkpNpvNxAqlNjRv3pzOnTtXWdapUyc++OADkyqS2vaHP/yBp59+ml/96lcAxMXFkZ6ezsyZMxVI6wFdQ3oRnp6e9OzZk8TExCrLExMT6devn0lVSW0yDIPHHnuMZcuW8cUXXxAdHW12SVLLBg4cSEpKClu2bKn86tWrF/feey9btmxRGG2kbrjhhvOGdNu1axetW7c2qSKpbUVFRVitVWOPzWbTsE/1hHpIL2HixIncf//99OrVi+uvv5433niDgwcP8tvf/tbs0qQWPProoyxZsoQVK1YQEBBQ2TseFBSEj4+PydVJbQgICDjvGmE/Pz9CQkJ07XAj9vvf/55+/foxY8YMRo0aRVJSEm+88QZvvPGG2aVJLRkxYgTTp0+nVatWdOnShe+//55XX32Vhx56yOzSBA37dFlef/11Zs2aRUZGBrGxscyePVtDADVSF7o2eMGCBYwdO7ZuixHT9O/fX8M+uYGPP/6YyZMns3v3bqKjo5k4cSK//vWvzS5Lakl+fj7PPvssy5cvJzs7m8jISO655x6ee+45PD09zS7P7SmQioiIiIipdA2piIiIiJhKgVRERERETKVAKiIiIiKmUiAVEREREVMpkIqIiIiIqRRIRURERMRUCqQiIiIiYioFUhERERExlQKpiIiIiJhKgVRERERETKVAKiIiIiKmUiAVEREREVP9P8pUMp9/CJ0IAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 800x250 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(8,2.5))\n",
    "plt.grid()\n",
    "plt.plot((training_loss))\n",
    "# plt.plot(np.log(val_los))\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 347,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_conv_autoencoder(sequence_size):\n",
    "    # Encoder\n",
    "    input_layer = tf.keras.layers.Input(shape=((sequence_size+1),8,1)) \n",
    "    feature_extractor = tf.keras.layers.Conv2D(256, (1,8), activation='relu')(input_layer) # out: (seq_size, 1 , 256) \n",
    "    pooling1 = tf.keras.layers.MaxPooling2D((2,1), padding= 'same')(feature_extractor) # out : (seq_size/2, 1, 256)\n",
    "    conv_0_down = tf.keras.layers.Conv2D(128, (4,1), activation='relu')(pooling1) # out: (seq_size/2 - 3, 1 , 128)\n",
    "    pooling2 = tf.keras.layers.MaxPooling2D((2,1), padding= 'same')(conv_0_down) # out: ((seq_size/2 - 3)/2, 1 , 128)\n",
    "    #Decoder\n",
    "    conv_0_up = tf.keras.layers.Conv2DTranspose(128, (4,1), strides=(2,1), activation='relu', output_padding=(1,0))(pooling2)\n",
    "    conv_1_up = tf.keras.layers.Conv2DTranspose(256, (2,8), strides=(2,1), activation='relu', output_padding=(0,0))(conv_0_up)\n",
    "    output_layer = tf.keras.layers.Conv2D(1, (1,1), activation = 'relu')(conv_1_up)\n",
    "\n",
    "    conv_autoencoder = tf.keras.Model(input_layer, output_layer)\n",
    "\n",
    "    return conv_autoencoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 348,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_conv_autoencoder_no_pooling(sequence_size):\n",
    "    input_layer = tf.keras.layers.Input(shape=((sequence_size+1),8,1)) \n",
    "    feature_extractor = tf.keras.layers.Conv2D(256, (1,8), activation='relu')(input_layer) # out: (seq_size, 1 , 256)\n",
    "    \n",
    "    down_sample1 = tf.keras.layers.Conv2D(64, (11,1), activation= 'relu')(feature_extractor)\n",
    "    down_sample2 = tf.keras.layers.Conv2D(32, (11,1), activation= 'relu')(down_sample1)\n",
    "    \n",
    "    bottleneck = tf.keras.layers.Conv2D(16, (21,1), activation='relu')(down_sample2)\n",
    "\n",
    "    up_sample1 = tf.keras.layers.Conv2DTranspose(32, (21,1), strides=(1,1), activation='relu', output_padding=(0,0))(bottleneck)\n",
    "    up_sample2 = tf.keras.layers.Conv2DTranspose(64, (11,1), strides=(1,1), activation='relu', output_padding=(0,0))(up_sample1)\n",
    "    up_sample3 = tf.keras.layers.Conv2DTranspose(256, (11,8), strides=(1,1), activation='relu', output_padding=(0,0))(up_sample2)\n",
    "\n",
    "    reconstruction = tf.keras.layers.Conv2D(1, (1,1), activation='relu')(up_sample3)\n",
    "\n",
    "    conv_no_pool_autoenecoder = tf.keras.Model(input_layer, reconstruction)\n",
    "\n",
    "    return conv_no_pool_autoenecoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 349,
   "metadata": {},
   "outputs": [],
   "source": [
    "total_dataset_length = 224146\n",
    "\n",
    "validation_length = 60000\n",
    "training_length = total_dataset_length-validation_length\n",
    "\n",
    "\n",
    "training_data = subject_list[:training_length]\n",
    "validation_data = subject_list[training_length:]\n",
    "\n",
    "train_seq_generator_conv = image_generator(training_data, encoding_scheme= None, batch_size=128, sequence_size=69)\n",
    "valid_seq_generator_conv = image_generator(validation_data, encoding_scheme=None, batch_size=128, sequence_size=69)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 350,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.keras.backend.clear_session()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 351,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_1 (InputLayer)         [(None, 70, 8, 1)]        0         \n",
      "_________________________________________________________________\n",
      "conv2d (Conv2D)              (None, 70, 1, 256)        2304      \n",
      "_________________________________________________________________\n",
      "max_pooling2d (MaxPooling2D) (None, 35, 1, 256)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_1 (Conv2D)            (None, 32, 1, 128)        131200    \n",
      "_________________________________________________________________\n",
      "max_pooling2d_1 (MaxPooling2 (None, 16, 1, 128)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_transpose (Conv2DTran (None, 35, 1, 128)        65664     \n",
      "_________________________________________________________________\n",
      "conv2d_transpose_1 (Conv2DTr (None, 70, 8, 256)        524544    \n",
      "_________________________________________________________________\n",
      "conv2d_2 (Conv2D)            (None, 70, 8, 1)          257       \n",
      "=================================================================\n",
      "Total params: 723,969\n",
      "Trainable params: 723,969\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "conv_based_autoencoder = get_conv_autoencoder(69)\n",
    "#conv_based_autoencoder = get_conv_autoencoder_no_pooling(63)\n",
    "conv_based_autoencoder.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 352,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Default GPU Device:/device:GPU:0\n"
     ]
    }
   ],
   "source": [
    "if tf.test.gpu_device_name(): \n",
    "    print('Default GPU Device:{}'.format(tf.test.gpu_device_name()))\n",
    "else:\n",
    "   print(\"Please install GPU version of TF\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "def SSIMLoss(y_true, y_pred):\n",
    "    return 1 - tf.reduce_mean(tf.image.ssim(y_true, y_pred, 2.0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 353,
   "metadata": {},
   "outputs": [],
   "source": [
    "conv_based_autoencoder.compile(loss=tf.keras.losses.MSE, optimizer=tf.keras.optimizers.Adam(learning_rate=5e-04))\n",
    "#conv_based_autoencoder.compile(loss=tf.keras.losses.MSE, optimizer=tf.keras.optimizers.Adam(learning_rate=5e-04))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 354,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/monowar/anaconda3/envs/envphase2/lib/python3.9/site-packages/tensorflow/python/keras/engine/training.py:1844: UserWarning: `Model.fit_generator` is deprecated and will be removed in a future version. Please use `Model.fit`, which supports generators.\n",
      "  warnings.warn('`Model.fit_generator` is deprecated and '\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/50\n",
      "500/500 [==============================] - 20s 39ms/step - loss: 412.5180 - val_loss: 0.0640\n",
      "Epoch 2/50\n",
      "500/500 [==============================] - 19s 39ms/step - loss: 0.0842 - val_loss: 0.0433\n",
      "Epoch 3/50\n",
      "500/500 [==============================] - 19s 39ms/step - loss: 0.0464 - val_loss: 0.0429\n",
      "Epoch 4/50\n",
      "500/500 [==============================] - 20s 39ms/step - loss: 0.0425 - val_loss: 0.0457\n",
      "Epoch 5/50\n",
      "500/500 [==============================] - 19s 39ms/step - loss: 0.0461 - val_loss: 0.0837\n",
      "Epoch 6/50\n",
      "500/500 [==============================] - 19s 38ms/step - loss: 0.3967 - val_loss: 0.0400\n",
      "Epoch 7/50\n",
      "500/500 [==============================] - 19s 38ms/step - loss: 4.1701 - val_loss: 0.0441\n",
      "Epoch 8/50\n",
      "500/500 [==============================] - 19s 39ms/step - loss: 0.1069 - val_loss: 0.0416\n",
      "Epoch 9/50\n",
      "500/500 [==============================] - 19s 39ms/step - loss: 0.1685 - val_loss: 0.0707\n",
      "Epoch 10/50\n",
      "500/500 [==============================] - 19s 39ms/step - loss: 0.0641 - val_loss: 0.0277\n",
      "Epoch 11/50\n",
      "500/500 [==============================] - 20s 39ms/step - loss: 0.0395 - val_loss: 0.6451\n",
      "Epoch 12/50\n",
      "500/500 [==============================] - 20s 39ms/step - loss: 0.0976 - val_loss: 0.0271\n",
      "Epoch 13/50\n",
      "500/500 [==============================] - 19s 39ms/step - loss: 0.0285 - val_loss: 0.0273\n",
      "Epoch 14/50\n",
      "500/500 [==============================] - 19s 39ms/step - loss: 0.0481 - val_loss: 46.9040\n",
      "Epoch 15/50\n",
      "500/500 [==============================] - 20s 39ms/step - loss: 1.6836 - val_loss: 0.0294\n",
      "Epoch 16/50\n",
      "500/500 [==============================] - 19s 39ms/step - loss: 0.0351 - val_loss: 0.0446\n",
      "Epoch 17/50\n",
      "500/500 [==============================] - 19s 39ms/step - loss: 0.0883 - val_loss: 0.0268\n",
      "Epoch 18/50\n",
      "500/500 [==============================] - 19s 39ms/step - loss: 0.0273 - val_loss: 0.0270\n",
      "Epoch 19/50\n",
      "500/500 [==============================] - 19s 39ms/step - loss: 0.0271 - val_loss: 0.0266\n",
      "Epoch 20/50\n",
      "500/500 [==============================] - 19s 39ms/step - loss: 0.0267 - val_loss: 0.0266\n",
      "Epoch 21/50\n",
      "500/500 [==============================] - 19s 39ms/step - loss: 8.8653 - val_loss: 23.2773\n",
      "Epoch 22/50\n",
      "500/500 [==============================] - 19s 39ms/step - loss: 2.6964 - val_loss: 0.0607\n",
      "Epoch 23/50\n",
      "500/500 [==============================] - 19s 39ms/step - loss: 0.1810 - val_loss: 0.0715\n",
      "Epoch 24/50\n",
      "500/500 [==============================] - 19s 39ms/step - loss: 0.1564 - val_loss: 0.0917\n",
      "Epoch 25/50\n",
      "500/500 [==============================] - 20s 39ms/step - loss: 4.7542 - val_loss: 0.0302\n",
      "Epoch 26/50\n",
      "500/500 [==============================] - 19s 39ms/step - loss: 46.2956 - val_loss: 0.0698\n",
      "Epoch 27/50\n",
      "500/500 [==============================] - 19s 39ms/step - loss: 0.0386 - val_loss: 0.0330\n",
      "Epoch 28/50\n",
      "500/500 [==============================] - 19s 39ms/step - loss: 0.0346 - val_loss: 0.0301\n",
      "Epoch 29/50\n",
      "500/500 [==============================] - 19s 39ms/step - loss: 0.0489 - val_loss: 0.0311\n",
      "Epoch 30/50\n",
      "500/500 [==============================] - 19s 39ms/step - loss: 0.0368 - val_loss: 0.0310\n",
      "Epoch 31/50\n",
      "500/500 [==============================] - 19s 39ms/step - loss: 0.0400 - val_loss: 0.0279\n",
      "Epoch 32/50\n",
      "500/500 [==============================] - 19s 39ms/step - loss: 0.0318 - val_loss: 0.0272\n",
      "Epoch 33/50\n",
      "500/500 [==============================] - 19s 39ms/step - loss: 0.0275 - val_loss: 0.0269\n",
      "Epoch 34/50\n",
      "500/500 [==============================] - 19s 39ms/step - loss: 0.0270 - val_loss: 0.0268\n",
      "Epoch 35/50\n",
      "500/500 [==============================] - 19s 39ms/step - loss: 0.0273 - val_loss: 0.0271\n",
      "Epoch 36/50\n",
      "500/500 [==============================] - 19s 39ms/step - loss: 0.0268 - val_loss: 0.0267\n",
      "Epoch 37/50\n",
      "500/500 [==============================] - 19s 38ms/step - loss: 0.0268 - val_loss: 0.0268\n",
      "Epoch 38/50\n",
      "500/500 [==============================] - 20s 39ms/step - loss: 0.0268 - val_loss: 0.0267\n",
      "Epoch 39/50\n",
      "500/500 [==============================] - 19s 39ms/step - loss: 6.0063 - val_loss: 0.0955\n",
      "Epoch 40/50\n",
      "500/500 [==============================] - 19s 39ms/step - loss: 0.1059 - val_loss: 0.1471\n",
      "Epoch 41/50\n",
      "500/500 [==============================] - 19s 39ms/step - loss: 0.4532 - val_loss: 0.0610\n",
      "Epoch 42/50\n",
      "500/500 [==============================] - 20s 39ms/step - loss: 0.3351 - val_loss: 0.1351\n",
      "Epoch 43/50\n",
      "500/500 [==============================] - 19s 39ms/step - loss: 0.2274 - val_loss: 0.0759\n",
      "Epoch 44/50\n",
      "500/500 [==============================] - 19s 39ms/step - loss: 0.0824 - val_loss: 0.0610\n",
      "Epoch 45/50\n",
      "500/500 [==============================] - 19s 39ms/step - loss: 0.0757 - val_loss: 0.0599\n",
      "Epoch 46/50\n",
      "500/500 [==============================] - 19s 39ms/step - loss: 0.0726 - val_loss: 0.0600\n",
      "Epoch 47/50\n",
      "500/500 [==============================] - 19s 39ms/step - loss: 0.4557 - val_loss: 0.0616\n",
      "Epoch 48/50\n",
      "500/500 [==============================] - 19s 39ms/step - loss: 0.0615 - val_loss: 0.0620\n",
      "Epoch 49/50\n",
      "500/500 [==============================] - 19s 39ms/step - loss: 0.0615 - val_loss: 0.0621\n",
      "Epoch 50/50\n",
      "500/500 [==============================] - 19s 39ms/step - loss: 0.0629 - val_loss: 0.0614\n"
     ]
    }
   ],
   "source": [
    "history = conv_based_autoencoder.fit_generator(train_seq_generator_conv, steps_per_epoch=500, epochs=50, validation_data = valid_seq_generator_conv, validation_steps=250)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 356,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgkAAADFCAYAAADe12EeAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAnhElEQVR4nO3de3Ab5b038O+urrYjOfEllp04iXOBALlAbHASKEkJcZsWaEo7paRvJ+e0U6BchkxgGFLmHcwZJmGYOSmFFAqFKaF9OeHMW+Dw9kCxC8SBEwK5FhNCMNhxnMSOY8e2ZMvWbZ/3j9WuLFuyJVuyZOv7mZEl7a7kxz+tpZ+eqySEECAiIiIaQk51AYiIiCg9MUkgIiKiiJgkEBERUURMEoiIiCgiJglEREQUEZMEIiIiiohJAhEREUVkTHUBxkJRFJw7dw42mw2SJKW6OERERJOGEAIulwslJSWQ5ZHrCiZlknDu3DmUlpamuhhERESTVktLC2bPnj3iMZMySbDZbADUP9ButyfkOX0+H2pqalBVVQWTyZSQ5yQVY5scjGtyMK7Jw9gmR7xxdTqdKC0t1T9LRzIpkwSticFutyc0ScjOzobdbufJm2CMbXIwrsnBuCYPY5scY41rLM317LhIREREETFJICIiooiYJBAREVFETBKCXv64GU99bsCeg2dSXRQiIqK0wCQhqLV7AE0uCac6+1JdFCIiorTAJCHIZlUHevR6/CkuCRERUXpgkhBkz1KHjTj7mSQQEREBTBJ09mBNgnOASQIRERHAJEFns6hJgmvAl+KSEBERpQcmCUG2LC1JYE0CERERwCRBZ7cG+yQwSSAiIgLAJEEX6pPggxAixaUhIiJKPSYJQbZgTYIvIODxKykuDRERUeoxSQjKMRsgQa1BcLLzIhEREZMEjSxLsBrU25wrgYiIiElCmOAAB9YkEBERgUlCmCy9JoFJAhEREZOEQUI1CWxuICIiYpIwSJZB7bjIWReJiIiYJITRaxLYcZGIiIhJwmB6nwTWJBARETFJGCxUk8AkgYiIiEnCIFa9TwKbG4iIiOJOEvbt24ebb74ZJSUlkCQJb775Zth+IQSqq6tRUlKCrKwsrF27FsePHw87xuPx4L777kNBQQFycnJwyy234MyZM+P6QxKB8yQQERGFxJ0k9PX1Yfny5di1a1fE/U8++SR27tyJXbt24eDBg3A4HFi/fj1cLpd+zJYtW/DGG29gz549+Oijj9Db24ubbroJgUBg7H9JAnCeBCIiohBjvA/YsGEDNmzYEHGfEAJPPfUUHnnkEdx6660AgN27d6OoqAivvvoq7rzzTvT09OCll17Cn//8Z9x4440AgL/85S8oLS3FP/7xD3znO98Zx58zPtmcJ4GIiEgXd5IwkqamJrS1taGqqkrfZrFYsGbNGuzfvx933nknDh8+DJ/PF3ZMSUkJlixZgv3790dMEjweDzwej37f6XQCAHw+H3y+xHzr9/l8ep8EZ3/inpegx5IxTSzGNTkY1+RhbJMj3rjGE/+EJgltbW0AgKKiorDtRUVFaG5u1o8xm82YMWPGsGO0xw+1Y8cOPPbYY8O219TUIDs7OxFFBxDqk9DdN4C33347Yc9Lqtra2lQXYUpiXJODcU0exjY5Yo2r2+2O+TkTmiRoJEkKuy+EGLZtqJGO2bZtG7Zu3arfdzqdKC0tRVVVFex2+/gLDDWzevNtNcBeRcKNVd+F2cjBH4ng8/lQW1uL9evXw2Qypbo4UwbjmhyMa/IwtskRb1y12vhYJDRJcDgcANTaguLiYn17e3u7XrvgcDjg9XrR1dUVVpvQ3t6O1atXR3xei8UCi8UybLvJZEroiWYdFI2BAJCTxZM4kRL9epGKcU0OxjV5GNvkiDWu8cQ+oV+Vy8rK4HA4wqo8vF4v6urq9ASgvLwcJpMp7JjW1lZ8/vnnUZOEiWKQgByLOsSBnReJiCjTxV2T0Nvbi6+//lq/39TUhGPHjiEvLw9z5szBli1bsH37dixatAiLFi3C9u3bkZ2djU2bNgEAcnNz8ctf/hIPPPAA8vPzkZeXhwcffBBLly7VRzukks1iRJ8nwEWeiIgo48WdJBw6dAjf/va39ftaX4HNmzfj5ZdfxkMPPYT+/n7cfffd6OrqQmVlJWpqamCz2fTH/Pa3v4XRaMRPfvIT9Pf3Y926dXj55ZdhMBgS8CeNj91qQpvTw0WeiIgo48WdJKxduxZCiKj7JUlCdXU1qqurox5jtVrxzDPP4Jlnnon31yedPTjEgbMuEhFRpmP3/SFswd6LnHWRiIgyHZOEIexWtdcnaxKIiCjTMUkYQqtJ4EqQRESU6ZgkDMHmBiIiIhWThCFCzQ2sSSAioszGJGEIO2sSiIiIADBJGIZ9EoiIiFRMEoawcXQDERERACYJw7C5gYiISMUkYYjQjItsbiAioszGJGEIrbmh1+NHQIk+/TQREdFUxyRhCJsltJxFL2sTiIgogzFJGMJslGE1qWFh50UiIspkTBIi0CZU6mHnRSIiymBMEiKwZ3EYJBEREZOECDihEhEREZOEiPT1G9jcQEREGYxJQgSh5gbWJBARUeZikhABZ10kIiJikhAROy4SERExSYiIHReJiIiYJETEjotERERMEiJicwMRERGThIhCHRfZ3EBERJmLSUIE2kqQLg9rEoiIKHMxSYggN4s1CUREREwSItA6LroGfFAUkeLSEBERpQaThAi0jouKAPq8rE0gIqLMxCQhAotRhtmghoZTMxMRUaZikhCBJEmDJlRi50UiIspMTBKi0OdKYOdFIiLKUEwSouAiT0RElOmYJETBWReJiCjTMUmIgos8ERFRpmOSEAUXeSIiokzHJCEKNjcQEVGmY5IQBRd5IiKiTMckIQrWJBARUaZjkhAFOy4SEVGmY5IQhd5xkTUJRESUoZgkRBGacZFJAhERZSYmCVGEahLY3EBERJmJSUIU9qzQAk9CiBSXhoiIaOIxSYjCFqxJ8AUEBnxKiktDREQ08ZgkRJFjNkCW1NvsvEhERJmISUIUkiSx8yIREWU0Jgkj4DBIIiLKZEwSRqBNqMQRDkRElIkSniRUV1dDkqSwi8Ph0PcLIVBdXY2SkhJkZWVh7dq1OH78eKKLkRBcCZKIiDJZUmoSrrjiCrS2tuqX+vp6fd+TTz6JnTt3YteuXTh48CAcDgfWr18Pl8uVjKKMizYMkjUJRESUiZKSJBiNRjgcDv1SWFgIQK1FeOqpp/DII4/g1ltvxZIlS7B792643W68+uqrySjKuLAmgYiIMpkxGU/a0NCAkpISWCwWVFZWYvv27Zg/fz6amprQ1taGqqoq/ViLxYI1a9Zg//79uPPOOyM+n8fjgcfj0e87nU4AgM/ng8+XmA9w7XkGP980iwEA0OP2JOz3ZKJIsaXxY1yTg3FNHsY2OeKNazzxl0SCpxN855134Ha7cckll+D8+fN4/PHH8eWXX+L48eM4efIkrr32Wpw9exYlJSX6Y+644w40Nzfj3Xffjfic1dXVeOyxx4Ztf/XVV5GdnZ3I4of/LS0S/n7GgNVFCm6bzwmViIho8nO73di0aRN6enpgt9tHPDbhNQkbNmzQby9duhSrVq3CggULsHv3bqxcuRKAOgfBYEKIYdsG27ZtG7Zu3arfdzqdKC0tRVVV1ah/YKx8Ph9qa2uxfv16mExqM0P7x834+5mTmFFYgu99b1lCfk8mihRbGj/GNTkY1+RhbJMj3rhqtfGxSEpzw2A5OTlYunQpGhoasHHjRgBAW1sbiouL9WPa29tRVFQU9TksFgssFsuw7SaTKeEn2uDnnJFjBQC4vAGe0AmQjNdL1/E18H//FbhuC7DkR8n5HWkqqXHNYIxr8jC2yRFrXOOJfdLnSfB4PDhx4gSKi4tRVlYGh8OB2tpafb/X60VdXR1Wr16d7KLEza7Nk8COi+nvizeBts+AQ39KdUmIiKaMhNckPPjgg7j55psxZ84ctLe34/HHH4fT6cTmzZshSRK2bNmC7du3Y9GiRVi0aBG2b9+O7OxsbNq0KdFFGTdtkScXZ1xMfx0N4ddERDRuCU8Szpw5g9tvvx0dHR0oLCzEypUrceDAAcydOxcA8NBDD6G/vx933303urq6UFlZiZqaGthstkQXZdw4T8Ik0vGVet3bBgz0ANbc1JaHiGgKSHiSsGfPnhH3S5KE6upqVFdXJ/pXJxznSZgkhAivQehoAGZXpK48RERTBNduGIG2CqTHr8DjD6S4NBSVqxXwDpqxU6tVICKicWGSMAKbxQhtZKaLTQ7pa2hSwCSBiCghmCSMQJYlTDNzhEPaG9pZkZ0XiYgSgknCKLQmB3ZeTGNazYFjafh9IiIaFyYJo7BxroT0pyUFi29Sry82AgG+XkRE48UkYRShmgR+6KQtrXmhbA1gygEUP9B1KqVFIiKaCpgkjEKbdZEdF9OUxwU4z6q3Cy8FChaqt9nkQEQ0bkwSRsG5EtJc59fqdXYBkJ0HFFyi3meSQEQ0bkwSRsHmhjSnNTVoyUHBpeHbiYhozJgkjCK0yBObG9KSVmNQqCUJi8K3ExHRmDFJGIVWk8BFntKUlgzoNQmDmhuESE2ZiIimCCYJo9CHQLLjYnoa2tyQNx+QZHWRp74LqSsXEdEUwCRhFOy4mMaUQKjjotbMYLIC09UVR9nkQEQ0PkwSRsGOi2msuxkIeAGjFcgtDW3nCIe0c99/HMXK7e/hYp831UUhojgwSRhFqCaBzQ1pR2tqyF8IyIbQdr3zIkc4pAPngA9/++wc2pwD2PcVm4CIJhMmCaOw6ZMpsSYh7eidFheFb2dNQlo5dOqi3of0UPPF1BaGiOLCJGEUWnNDnzcAf0BJcWkozNCRDRomCWnlk8ZQYnDoVFcKS0JE8WKSMAqtJgHg1MxpZ+jIBo12v7sF8Lontkw0zCdNoSTh5HkXetgJmGjSYJIwCpNBRrZZbe9m58U0E625IScfyMoDIEKjHygl+jx+1J/tAQDkZpkgBHD0NGsTiCYLJgkx0DovsiYhjfR1Au5O9Xb+wuH72eSQFg43dyGgCMyanoV1l83UtxHR5MAkIQb6hEqsJk0f2od/bilgzhm+nyMc0sKnwaaGyrI8VMzNA8B+CUSTCZOEGHCuhDQUralBw5qEtPBJk1rbUzk/DxXzZgAAjrZ0wcdOwESTApOEGHCRpzQUbWSDRk8SpnBNghIARPp+2A74Avhni9ofobIsHwsLpyE3y4QBn4IvzjlTXDoiigWThBiwJiEN6SMbotUkBLd3NgBK+n6QjlnAD+NL63DDiW2AfyDVpYno6OlueAMKZtosmJufDVmWUD5XrU04xH4JRJMCk4QY6LMusuNi+hitJmH6XMBgVj9Ae1omrlwT5fTHkNo/h83TCqmpLtWliSjU1JAPSZIAQE8SDnNSJaJJgUlCDNhxMc34BtR1G4DoSYLBGBr1MBWbHL78m35TPvl2CgsSndZp8ZqyPH1bhVaTcKoLgkt5E6U9JgkxYHNDmrnYqLbFW3KBaUXRj9NHOEyxzotCAF/+t35Xavg7EEivWi6vX8GR4HwIKwclCctLp8NkkNDu8qDlYn+qikdEMWKSEAMu8pRmBo9sCFZjRzRVRzi0HgN6WtAPC3pEDiR3J9ByINWlCvPZmW4M+BTk5ZixcOY0fbvVZMCSWbkAuI4D0WTAJCEG9iwu8pRWok3HPNRUHeEQrEXYG1iGWqVc3XbibyM8YOJpUzFfMy9P74+gqWDnRaJJg0lCDGzsuJheRpsjQTNVmxuCCcG7gavxbqBC3fbl34A0auPXkoTK+XnD9pUHJ1U6zEmViNIek4QY2NlxMb2MNrJBkx9MEvragf4p8oHU+Q1w4QR8woD3lSvxobIUbmFRR3C0fZbq0gEA/AEFh09pMy3mD9uvjXA4ed6FHjf/p4jSGZOEGLDjYhoRIvbmBss0wD5Lvd0xRRZ6OvH/AAAHlMswd1YJFs0wYZ+yLLgvPZocjp9zos8bgN1qxKUO27D9hTYLygrUqbSPcLGnSeHd42349V8O44LLk+qi0ARjkhADreNir8cPRUmfKt2M5DwH+PoA2QjklY1+/FRrcgj2R3hXuRobryzBqiKhNzkowQQi1bT5Ea4py4NBjtyxNDSpEjsvprsBXwC/eb0e73zehl3vT7H+PTQqJgkx0OZJEALo9bJfQkppH/YzygCDafTjp9IIB1cbcOZTAMAHogI3LXVg8XSB+uyV8AkD5Asn1OaIFPukMXpTg2bwfAmU3l4/chadfV4AwGuHWtDZy9qETMIkIQZWkwFmoxoq9ktIsVibGjRTaYRDsBbhmLIAly9ejLwcMwwSsL78EhxQLgsek9omh4Ai8Omp4ZMoDaUt9nSspRte/xScNnuKUBSBFz9sBACYDTIGfAp2f9yc4lLRRGKSECPOlZAmYh3ZoJlCzQ1i0KiGH62YpW//8YpZeFe5GgDg+fytlJRN82WbE64BP6ZZjLiixB71uAWF0zAj2wSPX8Hxcz0TWEKKx3tftqOxow92qxGP/3AJAOCVj0/BzRrVjMEkIUbaXAnsvJhisY5s0GjHXWwE/N7klGki9HdDNO0DAOw3r8S3F8/Ud83Jy0ZX6Y0AAEvrIbVZIkW0pobyuTNgNER/e5EkadA6DmxySFd/3KfWIvxs5Vz8aMVszMvPRrfbhz2fTsH1UCgiJgkx0moSXJwrIbXibW6wFQPmaYAIAF1NyStXsjXUQhZ+NCizsGz51bAYDWG7q1ZehaOKulaF8mXq1nKItF5DNNp8CeyXkJ6Onu7Cp6cuwmSQ8C+r58EgS/jV9fMBAC9+2AhfgM1EmYBJQoy4yFMaGHACrnPq7YKFsT1GkqZEk4P/C7UZ4V2lArcOamrQfOcKB+rkSgBA95HXJ7RsGiFC/RFWRphEaSitX8KhZi72lI5e/FBNqn9w5SwU2a0AgB+tmI2CaRac6xnAW8fOpbJ4NEGYJMSIcyWkgc5gLULOTCBrRuyPm+wjHHz9QEMtAOC4/Vu4snT6sEOsJgMMl98EALC3fgwMTHw7/9ftvbjY54XVJGPprOmjHr90Vi7MBhkdvR40d7qTX0CK2elON975vBUA8Ktvzde3W00G/Ou18wAAz+/7hkPCMwCThBix42IaiLepQaPXJEzSEQ6Ne2EM9OOcyMOSirXD1kLQrLvuOjQos2CEH676/454TDIdCDY1rJgzQx8NNBKryYCls7XFntjkkE5e+qgRigDWXFI4bEKs/7VyLqZZjPjqfC8+ONmeohLSRGGSECMu8pQG4h3ZoCm4NPzxk0zfP/8LAFCrVGDjitlRj7u8xI6jOdcCAC4c/OuElG2wTxrVSZRGmh9hqAq98yInVUoXXX1e/OehMwCAO66fP2x/bpYJmyrnAAD+UJf6eTkouZgkxEivSWCSkDrxjmzQDJ4rYbK1fQf8kBveAQC0FK3DrOlZIx4+7cofAgCK2z+C8E5cFb4QQu+0GGlRp2jKOalS2vk/nzSj3xfA5cV2rF4QOeH75XVlMBtkHDzVxQRvimOSEKPQIk9sbkgZrbmgMM4kIa8MkAyAxwn0nk98uZJInP4YWb5udIscXFb53VGPv+76G3FO5CMLA2j8dOKaHE51utHu8sBskCP2mYhGSxIa2nvR7Z7EQ1SniAFfAC/vVydLuuP6+VGbtorsVvzwKrUD7XN7GyesfDTxmCTEiB0XUyzgD005HG9NgtECzJin3p5kTQ5as8FeUY7vLIve1KCxZ5nxTf4aAMDFQ28ktWyDaU0NV5ZOh9VkGOXokPxpFswvVBd74nwJqfdfx86io9eDklwrvr+seMRj71gzH5IE/OPEeTScd01QCWmiMUmIEZsbUqy7GVB8gDELsI/+YTnMZBzhIARMwaaGzjlVmGYxxvSwgoofAwAWdO1Db/9A0oo32FiaGjT6Og5MElJKUQT+GBz2+IvrymAaYTIsQJ01s+ryIgDA8/tYmzBVMUmIUajjIpsbUkLvj7AQkMdw2k7CEQ7es8cww9eGfmHGZddujPlxiyur0AMb8iQXPq2bmCaHT+KYRGmoiuCkSofZLyGl9n7Vjq/be2GzGHHb1aUxPeauNQsAqDUQrT39ySwepQiThBjZ9CGQrElIibF2WtRMwpqE0x/9JwDgE/kqVF4ae+2JZDChzbEWQGhkRDK1XHTjbHc/jHJoquV4lGuLPZ3phscfSHTxKEYvBGsDNlXO0d/vRnPVnBmoLMuDLyDw0oeTeEZTiiqlScKzzz6LsrIyWK1WlJeX48MPP0xlcUYUam7wp+XscB29Hnxwsh3PvNeAX71yCKt2vIfL/vffsemPB/D0ew34tOni5H4DjiFJ6PP4EYg2ucskXA3S+o3a1NBb9l0Y5MgdyKIpqvwRAOAq9//gqzZnwss24AvgcPNFvPRRE37zRj0AYOnsXGSbY2sSGWx+QQ7ycszw+hV8fjbxZaXRfXamGwcaL8IoS/iX4GRJsbprrVqb8B+fnkaPm1+ippr4/6MT5LXXXsOWLVvw7LPP4tprr8Xzzz+PDRs24IsvvsCcOXNSVayotOaGgCLg9gaQE2P78HgJIeDxK+jz+OH2BtDr8cPt9aPb7cOJVifqz/ag/kwPzvVEbnve/00n9n+jdiqzGGWUz52BlfPzsXJ+PpaX5g5bA2CchQU8LqDvAuDuVK/7LkB2XcD89kZIn7sBuwPIKQByCoHsfMAQ2zcW7cPdP2MhTrW78M2FPjR19KHxQi8aL/ShsaMPF/u8MMgS8nPMmGm3oHCaBTNtVsy0WzDbYsFtANDTghPNbcjKsSHbbEC2xYgskyHuD+Fk62o5gdm+JviFjMvW/Djux09f8l143rJiNjrw0+dfRX/+EjhyrSjOzUJxrhXF09Vrh92KQpsFUTqxA1Bf1lOdffhnSzeOtfTgny3dOHneNSwhu+HSmVGeYWTaYk+1X5zH4eaLY6qNoPHR+iLcsrwExbkjD7Mdau0lhVjssOHLNheefr8BNy8vgVGWYDRI6rUswyBLMBlkBAJ+tLnV/icuj4Jutw9dbi+63D709HvR1eeDL6BgerYZeTkmzMgxIy/bjLwc9aLdzzIbYDbIkCf4/1ZRBAb8AQz4FAz4AvAHBKwmGVazAVkmw6j9OCYjSaToa3FlZSVWrFiB5557Tt922WWXYePGjdixY0fYsR6PBx6PR7/vdDpRWlqKjo4O2O3Rl6ONh/ifp9H/ycuw2eyhYT+SBIHQ7fqzLggIWIwGSJKEiKdnDOesFAy5BAUSRPC+UG9DAEJAQCAgJAQUQIEMAWhHQ4EU8RdZjDKyzAZkmwzIMqsffH0eNbHo8/jhH/KmLkmASZaCvxvBOQSCZQneFgAkSMFYyHpMJEmCkGRIkgSz8MEuejBd6YYJ8fXZcEnT0CPlwiVNgwI5+GcNj+1CfwPM8OH7vidwPDC2JPKQ5S4USE58rsyDByY11lB/pSwJyJIEWZIgSeo2/bUPXgspdFu/jlDesMPGKCfQgznKWRwzLccVD70X8Rifz4fa2lqsX78eJtPwZKvnlU0oaKnBWZGPbjFtcKn1v10lwrYPjot2Wzvv1HNQhgIJsizDajYhy2xCtsUEm8UAQNHPI0kooXkphKL/BjXAwf8tST2n2l1enO3xwCDLMBvlmMIngj9E6F7YNBihxEfSb8f6svj9fhiMpvDj0yuPTCi3xw8JChbPzIHVJEFSAsFgKuprpwTUaxF6fUP3FQx4/egd8EJAQgAyApChCPW2Eryv3ZaDZ1DoWoEBivo/GHx309/rhKQ/PnQOyoPO3uD/a/AFVq+113v8L5gQgCIEFKhf2Eb7tJSCZZAlhN5LEnTe+L7771h45bci7xvlvWAop9OJgoIC9PT0jPoZmpKaBK/Xi8OHD+Phhx8O215VVYX9+/cPO37Hjh147LHHhm2vqalBdnZ2Qsq05MwBLOg/DQzpezP49V2uJYkTufhZPImpAmAgeAnKBlA40nNFO+mjndhiyHUEvcKKi8KGTuSiU9jQJWywSl7kw4k8yYV8qQd5cMEgCdhEL2yiN/qTDdIjsvF1wAGLLFCYBcy0CszMEpgZvF1gBbwK4PQCTp8UvAZ6vOrtkwMLUSCOYIl8Kvov0fKkNNGQXYHmt0de1bG2tjbi9mLjYhSgBrOkTsySOpNRPMAbvMT2EoYZfIo5ADi08zNd+ganSzkmghb7jrE9PAtA1tD3jER8OMb7HDG8P41Z5O9mI5clgeV47cin+OrcyENNo70XDOV2xz7RWkqShI6ODgQCARQVFYVtLyoqQltb27Djt23bhq1bt+r3tZqEqqqqhNUk+NvmYf/eZVixYgWMBsOgb9XQb/d5AzjVMfzdUP+yFPwx2nkhAMiyDCn4LUrNPA2ADMjBb+dGg4QsowyLUYLVKMNqANQv/Qr0LH4c/4UCwHmnB25fAAZJhiRLkCW1+k6WZUjBKkJJkqEoAfj9CgKKgkAgAH9AQUAJhO7DCJ+1AP6sPPgseRDGrOC3cQlGCcjz+3HkyBGsWLEC541GtAEQgQBMvh4YBzph9nTC6HECEGq2PiiWWjyFAJSZV+C9ssWYabNEneRlRAOr4D+9X3/BBABfQG3O8QQEvD4FAwFFrUbXalL0FzVYLhG6DSGgBJ9LCR2m/w3jZcyejltWboBsiNwkNOq3B7EB/nM3AQPd6n09ZqHYCQD9vuC5NDimg4+VJFiNkvr9TQh12e1B3yLVb5oKQl+bQjUE4bcx6Fvo4G+j6u2zF93o7vfqr7d6eCiS2kuhfXOUg9dS8Jvj0Jpn/bwZ9Nppr8/Q4wZTFAVNTU2YN28eDHKU5jiRmG8K6ZKPypKERUU2ZFnM6uslG4Kv25CLbMDg11QM3hesf4MSgKSdI0pAPV+CNRF+nxdHjhzFioqrYTQN/11i0PPoNVFDzje/3w+vXyCgqNX9fkUgoCjB9yUBn6IgEFDCvvVr//Fi8PuzEOGnvH4t6XdMBlm/mI0STAYDzAYJZqMMOfhgAQF/QMAbUOD1K6FrvwKPX9HfI8brhstXYXq+I+K+sdQkxCplfRIADHujF0JEfPO3WCywWCzDtptMppgCEhPH5bhgPwXDJVUwRnnOXADLE/Pb0sIYZhsYE5/Ph95vBK6/tGjI6zV8yeOkMhUAV9wSvglqbctkNuL/wbzK0R+f4PKM1dzgJdV8Ph/Ovf02rqr6XuLeXwgAIHw+XPjaB8PCb0d9nx2NEYA1scUaNxPU2pRUi/UzMZ7zOiW9LAoKCmAwGIbVGrS3tw+rXSAiIqLUSEmSYDabUV5ePqz9pLa2FqtXr05FkYiIiGiIlDU3bN26FT//+c9RUVGBVatW4YUXXsDp06dx1113papIRERENEjKkoTbbrsNnZ2d+Ld/+ze0trZiyZIlePvttzF37uitklrHo3g6X4zG5/PB7XbD6XSyHTLBGNvkYFyTg3FNHsY2OeKNq/bZGcsMCCmbJ2E8zpw5g9LS2OYWJyIiouFaWlowe/bIXdgnZZKgKArOnTsHm802tqFwEWjDKltaWhI2rJJUjG1yMK7JwbgmD2ObHPHGVQgBl8uFkpISyKMsmJfSIZBjJcvyqNnPWNntdp68ScLYJgfjmhyMa/IwtskRT1xzc3NjOm7qTTRNRERECcEkgYiIiCJikhBksVjw6KOPRpzZkcaHsU0OxjU5GNfkYWyTI5lxnZQdF4mIiCj5WJNAREREETFJICIiooiYJBAREVFETBKIiIgoIiYJREREFBGThKBnn30WZWVlsFqtKC8vx4cffpjqIk0q+/btw80334ySkhJIkoQ333wzbL8QAtXV1SgpKUFWVhbWrl2L48ePp6awk8iOHTtw9dVXw2azYebMmdi4cSNOnjwZdgxjG7/nnnsOy5Yt02eoW7VqFd555x19P2OaGDt27IAkSdiyZYu+jbEdm+rqakiSFHZxOBz6/mTFlUkCgNdeew1btmzBI488gqNHj+Jb3/oWNmzYgNOnT6e6aJNGX18fli9fjl27dkXc/+STT2Lnzp3YtWsXDh48CIfDgfXr18Plck1wSSeXuro63HPPPThw4ABqa2vh9/tRVVWFvr4+/RjGNn6zZ8/GE088gUOHDuHQoUO44YYb8IMf/EB/U2VMx+/gwYN44YUXsGzZsrDtjO3YXXHFFWhtbdUv9fX1+r6kxVWQuOaaa8Rdd90Vtm3x4sXi4YcfTlGJJjcA4o033tDvK4oiHA6HeOKJJ/RtAwMDIjc3V/zhD39IQQknr/b2dgFA1NXVCSEY20SaMWOGePHFFxnTBHC5XGLRokWitrZWrFmzRtx///1CCJ6v4/Hoo4+K5cuXR9yXzLhmfE2C1+vF4cOHUVVVFba9qqoK+/fvT1Gpppampia0tbWFxdhisWDNmjWMcZx6enoAAHl5eQAY20QIBALYs2cP+vr6sGrVKsY0Ae655x58//vfx4033hi2nbEdn4aGBpSUlKCsrAw//elP0djYCCC5cZ2Uq0AmUkdHBwKBAIqKisK2FxUVoa2tLUWlmlq0OEaKcXNzcyqKNCkJIbB161Zcd911WLJkCQDGdjzq6+uxatUqDAwMYNq0aXjjjTdw+eWX62+qjOnY7NmzB0eOHMHBgweH7eP5OnaVlZV45ZVXcMkll+D8+fN4/PHHsXr1ahw/fjypcc34JEEjSVLYfSHEsG00Pozx+Nx777347LPP8NFHHw3bx9jG79JLL8WxY8fQ3d2Nv/71r9i8eTPq6ur0/Yxp/FpaWnD//fejpqYGVqs16nGMbfw2bNig3166dClWrVqFBQsWYPfu3Vi5ciWA5MQ145sbCgoKYDAYhtUatLe3D8vKaGy0HriM8djdd999eOutt/DBBx9g9uzZ+nbGduzMZjMWLlyIiooK7NixA8uXL8fvfvc7xnQcDh8+jPb2dpSXl8NoNMJoNKKurg5PP/00jEajHj/GdvxycnKwdOlSNDQ0JPWczfgkwWw2o7y8HLW1tWHba2trsXr16hSVamopKyuDw+EIi7HX60VdXR1jPAohBO699168/vrreP/991FWVha2n7FNHCEEPB4PYzoO69atQ319PY4dO6ZfKioq8LOf/QzHjh3D/PnzGdsE8Xg8OHHiBIqLi5N7zo6r2+MUsWfPHmEymcRLL70kvvjiC7FlyxaRk5MjTp06leqiTRoul0scPXpUHD16VAAQO3fuFEePHhXNzc1CCCGeeOIJkZubK15//XVRX18vbr/9dlFcXCycTmeKS57efv3rX4vc3Fyxd+9e0draql/cbrd+DGMbv23btol9+/aJpqYm8dlnn4nf/OY3QpZlUVNTI4RgTBNp8OgGIRjbsXrggQfE3r17RWNjozhw4IC46aabhM1m0z+nkhVXJglBv//978XcuXOF2WwWK1as0IeYUWw++OADAWDYZfPmzUIIdYjOo48+KhwOh7BYLOL6668X9fX1qS30JBAppgDEn/70J/0YxjZ+v/jFL/T/98LCQrFu3To9QRCCMU2koUkCYzs2t912myguLhYmk0mUlJSIW2+9VRw/flzfn6y4SkIIMb66CCIiIpqKMr5PAhEREUXGJIGIiIgiYpJAREREETFJICIiooiYJBAREVFETBKIiIgoIiYJREREFBGTBCIiIoqISQIRERFFxCSBiIiIImKSQERERBH9fw5EOI15rRzcAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 600x200 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "training_loss = history.history['loss'][:]\n",
    "val_los = history.history['val_loss'][:]\n",
    "\n",
    "plt.figure(figsize=(6,2))\n",
    "plt.grid()\n",
    "\n",
    "plt.plot((training_loss))\n",
    "plt.plot((val_los))\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Before we proceed any further, let us process the labeled data. We have two files for that. Let us extract them and create a label holder."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "file_event_labels = '/mnt/8tb/csenrc/representation_learning_codes/File_events_labels.csv'\n",
    "network_event_labels = '/mnt/8tb/csenrc/representation_learning_codes/IP_events_labels.csv'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "malicious_file_events = pd.read_csv(file_event_labels)\n",
    "malicious_ip_events = pd.read_csv(network_event_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>event_uuid</th>\n",
       "      <th>event_name</th>\n",
       "      <th>event_type</th>\n",
       "      <th>time_stamp_nanos</th>\n",
       "      <th>subject_uuid</th>\n",
       "      <th>predicate_object</th>\n",
       "      <th>prediacte_object_path</th>\n",
       "      <th>predicate_object_2</th>\n",
       "      <th>predicate_object_path_2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>7DBB1800-D1DD-5300-9D44-1A13E9981557</td>\n",
       "      <td>aue_open_rwtc</td>\n",
       "      <td>EVENT_OPEN</td>\n",
       "      <td>1523625074974644925</td>\n",
       "      <td>11C65665-3DC3-11E8-A5CA-3FA3753A265A</td>\n",
       "      <td>0C773AFD-8F2A-3555-AA8F-AFF835357207</td>\n",
       "      <td>/tmp/eWq10bVcx</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>484BEB8C-FA55-54C0-BC5B-0BA0CCA84818</td>\n",
       "      <td>aue_chmod</td>\n",
       "      <td>EVENT_MODIFY_FILE_ATTRIBUTES</td>\n",
       "      <td>1523625074974644925</td>\n",
       "      <td>11C65665-3DC3-11E8-A5CA-3FA3753A265A</td>\n",
       "      <td>0C773AFD-8F2A-3555-AA8F-AFF835357207</td>\n",
       "      <td>/tmp/eWq10bVcx</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>401ECBFF-F199-5CB7-BB3F-33421209E979</td>\n",
       "      <td>aue_open_rwtc</td>\n",
       "      <td>EVENT_OPEN</td>\n",
       "      <td>1523625384084637086</td>\n",
       "      <td>4FB0BFEA-3F1C-11E8-A5CB-3FA3753A265A</td>\n",
       "      <td>0C773AFD-8F2A-3555-AA8F-AFF835357207</td>\n",
       "      <td>/tmp/eWq10bVcx</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>C069C841-DBAF-5D41-BA83-17B97B6B8108</td>\n",
       "      <td>aue_openat_rwtc</td>\n",
       "      <td>EVENT_OPEN</td>\n",
       "      <td>1523625386674637936</td>\n",
       "      <td>2CA83801-3E80-11E8-A5CB-3FA3753A265A</td>\n",
       "      <td>0C773AFD-8F2A-3555-AA8F-AFF835357207</td>\n",
       "      <td>/tmp/eWq10bVcx</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>B6316F5A-21F9-53DB-986F-E993F7FF778C</td>\n",
       "      <td>aue_openat_rwtc</td>\n",
       "      <td>EVENT_OPEN</td>\n",
       "      <td>1523625429714638405</td>\n",
       "      <td>4FB0BFEA-3F1C-11E8-A5CB-3FA3753A265A</td>\n",
       "      <td>0C773AFD-8F2A-3555-AA8F-AFF835357207</td>\n",
       "      <td>/tmp/eWq10bVcx</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                             event_uuid       event_name  \\\n",
       "0  7DBB1800-D1DD-5300-9D44-1A13E9981557    aue_open_rwtc   \n",
       "1  484BEB8C-FA55-54C0-BC5B-0BA0CCA84818        aue_chmod   \n",
       "2  401ECBFF-F199-5CB7-BB3F-33421209E979    aue_open_rwtc   \n",
       "3  C069C841-DBAF-5D41-BA83-17B97B6B8108  aue_openat_rwtc   \n",
       "4  B6316F5A-21F9-53DB-986F-E993F7FF778C  aue_openat_rwtc   \n",
       "\n",
       "                     event_type     time_stamp_nanos  \\\n",
       "0                    EVENT_OPEN  1523625074974644925   \n",
       "1  EVENT_MODIFY_FILE_ATTRIBUTES  1523625074974644925   \n",
       "2                    EVENT_OPEN  1523625384084637086   \n",
       "3                    EVENT_OPEN  1523625386674637936   \n",
       "4                    EVENT_OPEN  1523625429714638405   \n",
       "\n",
       "                           subject_uuid                      predicate_object  \\\n",
       "0  11C65665-3DC3-11E8-A5CA-3FA3753A265A  0C773AFD-8F2A-3555-AA8F-AFF835357207   \n",
       "1  11C65665-3DC3-11E8-A5CA-3FA3753A265A  0C773AFD-8F2A-3555-AA8F-AFF835357207   \n",
       "2  4FB0BFEA-3F1C-11E8-A5CB-3FA3753A265A  0C773AFD-8F2A-3555-AA8F-AFF835357207   \n",
       "3  2CA83801-3E80-11E8-A5CB-3FA3753A265A  0C773AFD-8F2A-3555-AA8F-AFF835357207   \n",
       "4  4FB0BFEA-3F1C-11E8-A5CB-3FA3753A265A  0C773AFD-8F2A-3555-AA8F-AFF835357207   \n",
       "\n",
       "  prediacte_object_path predicate_object_2 predicate_object_path_2  \n",
       "0        /tmp/eWq10bVcx                NaN                     NaN  \n",
       "1        /tmp/eWq10bVcx                NaN                     NaN  \n",
       "2        /tmp/eWq10bVcx                NaN                     NaN  \n",
       "3        /tmp/eWq10bVcx                NaN                     NaN  \n",
       "4        /tmp/eWq10bVcx                NaN                     NaN  "
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "malicious_file_events.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "93\n",
      "27\n"
     ]
    }
   ],
   "source": [
    "print(len(malicious_file_events))\n",
    "print(len(malicious_file_events['subject_uuid'].unique()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [],
   "source": [
    "malicious_file_subjects = malicious_file_events['subject_uuid'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Setting up a db connection. I was trying to avoid this. But, well. It is what it is.\n",
    "\n",
    "psql_connection_url = 'postgresql+psycopg2://csephase2:csephase@@localhost/darpa_tc3'\n",
    "psql_engine = create_engine(psql_connection_url)\n",
    "Session = sessionmaker(bind=psql_engine)\n",
    "session = Session()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "11C65665-3DC3-11E8-A5CA-3FA3753A265A  has database primary key :  201998  and currently has :  546  events in the graph.\n",
      "4FB0BFEA-3F1C-11E8-A5CB-3FA3753A265A  has database primary key :  216121  and currently has :  20799  events in the graph.\n",
      "2CA83801-3E80-11E8-A5CB-3FA3753A265A  has database primary key :  200770  and currently has :  662  events in the graph.\n",
      "E0C1A46B-3F1B-11E8-A5CB-3FA3753A265A  has database primary key :  216033  and currently has :  276  events in the graph.\n",
      "042ADADF-3F1C-11E8-A5CB-3FA3753A265A  has database primary key :  216056  and currently has :  275  events in the graph.\n",
      "8603F11E-3F1C-11E8-A5CB-3FA3753A265A  has database primary key :  216144  and currently has :  290  events in the graph.\n",
      "E24D97B4-3F1C-11E8-A5CB-3FA3753A265A  has database primary key :  216180  and currently has :  279  events in the graph.\n",
      "0A5AA28E-3F1D-11E8-A5CB-3FA3753A265A  has database primary key :  216199  and currently has :  289  events in the graph.\n",
      "11AA4698-3F1D-11E8-A5CB-3FA3753A265A  has database primary key :  216217  and currently has :  284  events in the graph.\n",
      "2720E25F-39BE-11E8-B8CE-15D78AC88FB6  has database primary key :  121208  and currently has :  7087  events in the graph.\n",
      "11C64B2C-3DC3-11E8-A5CA-3FA3753A265A  has database primary key :  182633  and currently has :  2465  events in the graph.\n",
      "219DE547-3E7E-11E8-A5CB-3FA3753A265A  has database primary key :  200446  and currently has :  63  events in the graph.\n",
      "48289024-3E7E-11E8-A5CB-3FA3753A265A  has database primary key :  200487  and currently has :  663  events in the graph.\n",
      "BF8EF43F-3E7D-11E8-A5CB-3FA3753A265A  has database primary key :  200406  and currently has :  2  events in the graph.\n",
      "5CD609A7-3E7F-11E8-A5CB-3FA3753A265A  has database primary key :  200600  and currently has :  48  events in the graph.\n",
      "17C4C0C3-3E7D-11E8-A5CB-3FA3753A265A  has database primary key :  200306  and currently has :  53  events in the graph.\n",
      "7B30E76D-3E7D-11E8-A5CB-3FA3753A265A  has database primary key :  200377  and currently has :  54  events in the graph.\n",
      "DCBB8D5C-3E7F-11E8-A5CB-3FA3753A265A  has database primary key :  200665  and currently has :  55  events in the graph.\n",
      "0001A528-3E80-11E8-A5CB-3FA3753A265A  has database primary key :  200702  and currently has :  54  events in the graph.\n",
      "327621AE-3E80-11E8-A5CB-3FA3753A265A  has database primary key :  200771  and currently has :  54  events in the graph.\n",
      "47E61FFC-3E80-11E8-A5CB-3FA3753A265A  has database primary key :  200787  and currently has :  43477  events in the graph.\n",
      "C4F4FF22-3E7B-11E8-A5CB-3FA3753A265A  has database primary key :  200105  and currently has :  44  events in the graph.\n",
      "42E4A848-3E7C-11E8-A5CB-3FA3753A265A  has database primary key :  200186  and currently has :  54  events in the graph.\n",
      "B6E5EA8F-3E7C-11E8-A5CB-3FA3753A265A  has database primary key :  200230  and currently has :  108  events in the graph.\n",
      "CF565BC0-3E7E-11E8-A5CB-3FA3753A265A  has database primary key :  200548  and currently has :  120  events in the graph.\n",
      "15267234-3E7F-11E8-A5CB-3FA3753A265A  has database primary key :  200569  and currently has :  51  events in the graph.\n",
      "1D3ACF79-3E7F-11E8-A5CB-3FA3753A265A  has database primary key :  200577  and currently has :  35  events in the graph.\n"
     ]
    }
   ],
   "source": [
    "for subject in malicious_file_subjects:\n",
    "    sub = gc.get_node(session, subject, gc.mapping_dict[\"SUBJECT\"])\n",
    "    print(sub.uuid,\" has database primary key : \", sub.id, \" and currently has : \", len(cleaned_provenance_graph[str(sub.id)]['events']), \" events in the graph.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.78',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.147',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.114',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.92',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.103',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.128',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.186',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.66',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.47',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.169',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.155',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.67',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.95',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.174',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.62',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.195',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.5',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.132',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.107',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.202',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.53',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.127',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.63',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.9',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.120',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.182',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.123',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.21',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.40',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.134',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.51',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.13',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.145',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.46',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.199',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.88',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.33',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.74',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.151',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.168',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.22',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.96',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.137',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.1',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.38',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.98',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.28',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.108',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.91',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.70',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.60',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.19',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.167',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.143',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.192',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.29',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.94',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.150',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.190',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.11',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.172',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.106',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.139',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.68',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.35',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.163',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.80',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.73',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.20',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.105',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.61',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.136',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.121',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.2',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.18',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.90',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.178',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.58',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.34',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official-1.json.2',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.69',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.30',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.93',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.112',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.54',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.135',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.59',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.50',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.101',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.75',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.85',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.191',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.32',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.43',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.187',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.131',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.72',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.77',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.82',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official-1.json.1',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.176',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.57',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.157',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.23',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.181',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official-1.json.4',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.180',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.100',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.197',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.26',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.119',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.12',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.25',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.110',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.171',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.111',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.104',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.116',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.41',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.39',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.142',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.27',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.196',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.65',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.154',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.159',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.194',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.141',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.15',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.201',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.189',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.97',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.117',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.161',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.203',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.118',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.8',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.162',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.52',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.24',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.173',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.6',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.81',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.4',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.122',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.76',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.84',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.153',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.198',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.164',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.185',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official-1.json.3',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.170',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.109',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.42',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.44',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.115',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.125',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.156',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.146',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.184',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.166',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.140',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.148',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.64',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.7',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.86',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.129',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.56',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.102',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.55',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.16',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.49',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.37',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.113',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.71',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.200',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.133',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.126',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.144',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.149',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.14',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.183',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.31',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.79',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.138',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.175',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.17',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.158',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.165',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.10',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.48',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.177',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.87',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.152',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.3',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.124',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.45',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.179',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official-1.json',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.89',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.83',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.193',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official-1.json.5',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.36',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.130',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official-1.json.6',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.160',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.188',\n",
       " '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/ta1-trace-e3-official.json.99']"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dir = '/mnt/8tb/csenrc/tc3/data/trace/extracted_json/'\n",
    "files = os.listdir('/mnt/8tb/csenrc/tc3/data/trace/extracted_json')\n",
    "files = [dir+x for x in files]\n",
    "files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "a01f599b34edce01dd96621430605e3428cfca8a7f3f7e0aec5d26beeaa95b07"
  },
  "kernelspec": {
   "display_name": "Python 3.9.5 ('envphase2')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}