News_Preprocess.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import zipfile\n",
    "with zipfile.ZipFile('./data/ReutersNews106521.zip', 'r') as zip_ref:\n",
    "    zip_ref.extractall('./data/2006-2013')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import pickle\n",
    "from striprtf.striprtf import rtf_to_text\n",
    "import string\n",
    "from os import listdir\n",
    "from datetime import datetime\n",
    "from tqdm import tqdm\n",
    "import cathay.re.SVO_final as SVO\n",
    "import re\n",
    "from cathay.config import ApplicationConfig\n",
    "import boto3\n",
    "from multiprocessing import Pool\n",
    "import torch.multiprocessing as mp\n",
    "import nltk\n",
    "from transformers import *\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.utils.data import DataLoader, Dataset\n",
    "from torch import optim\n",
    "import torch.nn.functional as F"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2006-2013"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [],
   "source": [
    "def Preprocess(dataset):\n",
    "    datas = []\n",
    "    for folder in tqdm(dataset):\n",
    "        files = listdir(path + folder)\n",
    "        for file in files:\n",
    "            if file[0] != '.':\n",
    "                with open(path + folder + '/' + file, 'r') as f:\n",
    "                    new = f.readlines()\n",
    "\n",
    "                if new != []:\n",
    "                    data = {}\n",
    "                    new = [x.replace('\\n', '') for x in new if x[:2] != '\\n' and len(x) > 3]\n",
    "                    data['title'] = new[0].replace('-- ', '')\n",
    "                    data['source'] = 'Reuters News'\n",
    "                    date = new[2].replace('-- ', '')\n",
    "                    data['date'] = datetime.strptime(' '.join(date.split(' ')[:4]), '%a %b %d, %Y').strftime('%Y%m%d')\n",
    "                    data['text'] = ''.join(new[4:])\n",
    "                    datas.append(data)\n",
    "                \n",
    "    return datas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 365/365 [14:37<00:00,  2.40s/it]\n",
      "100%|██████████| 365/365 [15:54<00:00,  2.61s/it]\n",
      "100%|██████████| 365/365 [19:24<00:00,  3.19s/it]\n",
      "100%|██████████| 365/365 [20:41<00:00,  3.40s/it]\n",
      "100%|██████████| 365/365 [20:46<00:00,  3.42s/it]\n"
     ]
    }
   ],
   "source": [
    "path = './financial-news-dataset-master/ReutersNews106521/'\n",
    "folders = [x for x in listdir(path) if len(x) == 8]\n",
    "\n",
    "n_workers = 5\n",
    "results = [None] * n_workers\n",
    "with Pool(processes=n_workers) as pool:\n",
    "    for i in range(n_workers):\n",
    "        batch_start = (len(folders) // n_workers) * i\n",
    "        if i == n_workers - 1:\n",
    "            batch_end = len(folders)\n",
    "        else:\n",
    "            batch_end = (len(folders) // n_workers) * (i + 1)\n",
    "\n",
    "        batch = folders[batch_start: batch_end]\n",
    "        results[i] = pool.apply_async(Preprocess, [batch])\n",
    "\n",
    "    pool.close()\n",
    "    pool.join()\n",
    "\n",
    "processed = []\n",
    "for result in results:\n",
    "    processed += result.get()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "69944"
      ]
     },
     "execution_count": 125,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(processed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 152/152 [14:04<00:00,  5.56s/it]\n",
      "100%|██████████| 152/152 [14:05<00:00,  5.57s/it]\n",
      "100%|██████████| 155/155 [14:34<00:00,  5.64s/it]\n",
      "100%|██████████| 152/152 [15:28<00:00,  6.11s/it]\n",
      "100%|██████████| 152/152 [15:52<00:00,  6.26s/it]\n"
     ]
    }
   ],
   "source": [
    "path = './data/2006-2013/'\n",
    "folders = [x for x in listdir(path) if len(x) == 8]\n",
    "\n",
    "n_workers = 5\n",
    "results = [None] * n_workers\n",
    "with Pool(processes=n_workers) as pool:\n",
    "    for i in range(n_workers):\n",
    "        batch_start = (len(folders) // n_workers) * i\n",
    "        if i == n_workers - 1:\n",
    "            batch_end = len(folders)\n",
    "        else:\n",
    "            batch_end = (len(folders) // n_workers) * (i + 1)\n",
    "\n",
    "        batch = folders[batch_start: batch_end]\n",
    "        results[i] = pool.apply_async(Preprocess, [batch])\n",
    "\n",
    "    pool.close()\n",
    "    pool.join()\n",
    "\n",
    "for result in results:\n",
    "    processed += result.get()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "106494"
      ]
     },
     "execution_count": 127,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(processed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [],
   "source": [
    "news = pd.DataFrame({'id':range(len(processed)), 'title':[x['title'] for x in processed], 'date':[x['date'] for x in processed], 'source':[x['source'] for x in processed], 'document_body':[x['text'] for x in processed]})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('2006-2013.pkl', 'wb') as f:\n",
    "    pickle.dump(news, f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# RTF to DataFrame (2014-2019)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "years = ['2014', '2015', '2016', '2017', '2018', '2019']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 152/152 [02:38<00:00,  1.04s/it]\n",
      "100%|██████████| 174/174 [03:26<00:00,  1.19s/it]\n",
      "100%|██████████| 168/168 [02:20<00:00,  1.19it/s]\n",
      "100%|██████████| 114/114 [01:42<00:00,  1.12it/s]\n",
      "100%|██████████| 114/114 [01:41<00:00,  1.13it/s]\n",
      "100%|██████████| 97/97 [01:18<00:00,  1.23it/s]\n"
     ]
    }
   ],
   "source": [
    "all_news = []\n",
    "for year in years:\n",
    "    mypath = './data/' + year + '/'\n",
    "    files = listdir(mypath)\n",
    "    for file in tqdm(files):\n",
    "        with open(mypath + file, 'r') as f:\n",
    "            data = f.readlines()\n",
    "            \n",
    "        for i in data:\n",
    "            if i[:4] == '\\\\par' and rtf_to_text(i) != '\\n':\n",
    "                new = {}\n",
    "                text = rtf_to_text(i)\n",
    "                text = text.split('\\n')\n",
    "                text = [x for x in text if x != '']\n",
    "                if len(text) < 5:\n",
    "                    continue\n",
    "                    \n",
    "                count = 0\n",
    "                # 總字數\n",
    "                while '字' not in text[count]:\n",
    "                    count += 1\n",
    "                # 作者\n",
    "                if 'By' in text[count-1] and count-2 >= 0:\n",
    "                    new['title'] = text[count-2]\n",
    "                else:\n",
    "                    new['title'] = text[count-1]\n",
    "                new['date'] = ''.join([text[count+1].split(' ')[x].zfill(2) for x in [0, 2, 4]])\n",
    "                new['source'] = text[count+2]\n",
    "                count = count + 2\n",
    "                \n",
    "                while '(c)' not in text[count] and 'Copyright' not in text[count]:\n",
    "                    count += 1\n",
    "                if text[count+1][-1] in string.ascii_letters:\n",
    "                    s = text[count+1] + '.'\n",
    "                else:\n",
    "                    s = text[count+1]\n",
    "                for j in range(count+2, len(text)-2):\n",
    "                    if text[j][-1] in string.ascii_letters:\n",
    "                        s += ' ' + text[j] + '.'\n",
    "                    else:\n",
    "                        s += ' ' + text[j]\n",
    "                new['text'] = s\n",
    "                all_news.append(new)\n",
    "news = pd.DataFrame({'id':range(len(all_news)), 'title':[x['title'] for x in all_news], 'date':[x['date'] for x in all_news], 'source':[x['source'] for x in all_news], 'document_body':[x['text'] for x in all_news]})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "with open('2014-2019.pkl', 'wb') as f:\n",
    "    pickle.dump(news, f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Event Extraction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# with open('2014-2019.pkl', 'rb') as f:\n",
    "#     news = pickle.load(f)\n",
    "with open('2006-2013.pkl', 'rb') as f:\n",
    "    news = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>title</th>\n",
       "      <th>date</th>\n",
       "      <th>source</th>\n",
       "      <th>document_body</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>Exxon Mobil offers plan to end Alaska dispute</td>\n",
       "      <td>20061020</td>\n",
       "      <td>Reuters News</td>\n",
       "      <td>ANCHORAGE, Alaska  (Reuters) - Exxon Mobil ( ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>Hey buddy, can you spare $600 for a Google share?</td>\n",
       "      <td>20061020</td>\n",
       "      <td>Reuters News</td>\n",
       "      <td>SAN FRANCISCO/NEW YORK  (Reuters) - Wall Stre...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>AOL CEO says sales may shrink for two years -p...</td>\n",
       "      <td>20061021</td>\n",
       "      <td>Reuters News</td>\n",
       "      <td>FRANKFURT  (Reuters) - Internet service provi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>Fed to keep hawkish tone, hold rates steady</td>\n",
       "      <td>20061022</td>\n",
       "      <td>Reuters News</td>\n",
       "      <td>WASHINGTON  (Reuters) - The central bank is e...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>Pluspetrol says losing $2.4 mln/day in Peru pr...</td>\n",
       "      <td>20061021</td>\n",
       "      <td>Reuters News</td>\n",
       "      <td>LIMA, Peru  (Reuters) - Argentine oil company...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id                                              title      date  \\\n",
       "0   0      Exxon Mobil offers plan to end Alaska dispute  20061020   \n",
       "1   1  Hey buddy, can you spare $600 for a Google share?  20061020   \n",
       "2   2  AOL CEO says sales may shrink for two years -p...  20061021   \n",
       "3   3        Fed to keep hawkish tone, hold rates steady  20061022   \n",
       "4   4  Pluspetrol says losing $2.4 mln/day in Peru pr...  20061021   \n",
       "\n",
       "         source                                      document_body  \n",
       "0  Reuters News   ANCHORAGE, Alaska  (Reuters) - Exxon Mobil ( ...  \n",
       "1  Reuters News   SAN FRANCISCO/NEW YORK  (Reuters) - Wall Stre...  \n",
       "2  Reuters News   FRANKFURT  (Reuters) - Internet service provi...  \n",
       "3  Reuters News   WASHINGTON  (Reuters) - The central bank is e...  \n",
       "4  Reuters News   LIMA, Peru  (Reuters) - Argentine oil company...  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "news.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "news_dict = news.to_dict('records')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Title"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "aws_nlu_config =  ApplicationConfig.get_aws_nlu_config()\n",
    "comprehend = boto3.client(aws_access_key_id=aws_nlu_config['access_key'], aws_secret_access_key=aws_nlu_config['secret_key'], service_name='comprehend', region_name=aws_nlu_config['region'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def title_preprocess(sent, comprehend):\n",
    "    if sent.find('-') == 8 and sent[:6] == 'UPDATE':\n",
    "        sent = sent[9:]\n",
    "    if sent.find('-') == 8 and sent[:6] == 'WRAPUP':\n",
    "        sent = sent[9:]\n",
    "    if sent.find('-') == 3 and sent[:3] == 'RPT':\n",
    "        sent = sent[5:]\n",
    "    while sent.find('-') != -1 and sent[:sent.find('-')].isupper():\n",
    "        sent = sent[sent.find('-')+1:]\n",
    "    sent = sent.replace(' - ',' ')\n",
    "    sent = sent.replace(\"''\",' ')\n",
    "    sent = re.sub(\"[+\\!\\/\\\\_$%^*()+.:\\\"“”]+|[+——！，。？、~@#￥%……&*（）：`]+\", '', sent)\n",
    "    sent = sent.replace('\\\\',' ')\n",
    "    sent = sent.replace('  ',' ')\n",
    "    if sent[0] in ['-', ' ']:\n",
    "        sent = sent[1:]\n",
    "        \n",
    "    # 只留句首以及專有名詞大寫\n",
    "    idx = sent.find(' ')\n",
    "    # 找專有名詞\n",
    "    entity = comprehend.detect_entities(Text=sent[idx:], LanguageCode='en')['Entities']\n",
    "    b = sent[idx:].lower()\n",
    "    b = b.lower()\n",
    "    s = ''\n",
    "    end = 0\n",
    "    for i in entity:\n",
    "        s += b[end:i['BeginOffset']]\n",
    "        s += i['Text']\n",
    "        end = i['EndOffset']\n",
    "    s += b[end:]\n",
    "    sent = sent[:idx] + s\n",
    "    return sent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 使用Title，過濾掉 'The Year Ahead' 和 'Quick Takes'\n",
    "def Event_extrations(dataset):\n",
    "    processed = []\n",
    "    for sample in tqdm(dataset):\n",
    "        sent = re.split(':|;|---', sample['title'])[-1]\n",
    "        if sent != '' and 'The Year Ahead' not in sent and 'Quick Takes' not in sent:\n",
    "            tmp = {}\n",
    "            try:\n",
    "                sent = title_preprocess(sent, comprehend)\n",
    "                svo = SVO.SVO(sent)\n",
    "                svo_result = svo.find_svo()\n",
    "                tmp['id'] = sample['id']\n",
    "                tmp['date'] = sample['date']\n",
    "                tmp['title'] = sample['title']\n",
    "                tmp['title_SVO'] = svo_result\n",
    "                processed.append(tmp)\n",
    "            except:\n",
    "                print(sent)\n",
    "    return processed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  3%|▎         | 442/13311 [04:45<2:03:09,  1.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Some 4 trillion wiped off world stocks in 2 weeks\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 18%|█▊        | 2434/13311 [26:11<2:24:25,  1.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SocGen boss survives and says bank can too\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 26%|██▋       | 3527/13311 [38:07<1:33:21,  1.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fannie, Freddie appraisal deal slips but lives on\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 38%|███▊      | 5037/13311 [54:06<1:32:32,  1.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Some optimistic about retail sales in 2011\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 38%|███▊      | 5093/13311 [54:15<1:17:00,  1.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Walgreen tops view, store comments weigh on shares\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 48%|████▊     | 6343/13311 [1:07:31<1:22:38,  1.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Paul Soros, shipping titan and older brother to George Soros, dies at 87\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 54%|█████▍    | 7232/13311 [1:16:32<47:33,  2.13it/s]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Adelphia founder and son to be resentenced\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 64%|██████▍   | 8561/13311 [1:26:25<36:08,  2.19it/s]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fed's Lacker says must let ailing big firms fail\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 67%|██████▋   | 8944/13311 [1:29:47<29:02,  2.51it/s]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Obama on attack in foreign policy debate, but Romney steady\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 97%|█████████▋| 12971/13311 [2:00:06<02:30,  2.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "US Airways fined 12 million over disabilities infractions\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 13311/13311 [2:00:54<00:00,  1.83it/s]\n",
      "100%|██████████| 13311/13311 [2:01:34<00:00,  1.82it/s]\n",
      "100%|██████████| 13311/13311 [2:01:36<00:00,  1.82it/s]\n",
      "100%|██████████| 13311/13311 [2:01:36<00:00,  1.82it/s]\n",
      "100%|██████████| 13311/13311 [2:02:02<00:00,  1.82it/s]\n",
      "100%|██████████| 13311/13311 [2:02:45<00:00,  1.81it/s]\n",
      "100%|██████████| 13311/13311 [2:04:21<00:00,  1.78it/s]\n",
      "100%|██████████| 13317/13317 [2:05:58<00:00,  1.76it/s]\n"
     ]
    }
   ],
   "source": [
    "n_workers = 8\n",
    "results = [None] * n_workers\n",
    "with Pool(processes=n_workers) as pool:\n",
    "    for i in range(n_workers):\n",
    "        batch_start = (len(news_dict) // n_workers) * i\n",
    "        if i == n_workers - 1:\n",
    "            batch_end = len(news_dict)\n",
    "        else:\n",
    "            batch_end = (len(news_dict) // n_workers) * (i + 1)\n",
    "\n",
    "        batch = news_dict[batch_start: batch_end]\n",
    "        results[i] = pool.apply_async(Event_extrations, [batch])\n",
    "\n",
    "    pool.close()\n",
    "    pool.join()\n",
    "\n",
    "processed = []\n",
    "for result in results:\n",
    "    processed += result.get()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# with open('news_preprocessed.pkl', 'wb') as f:\n",
    "#     pickle.dump(processed, f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Integrate to SVO"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('news_preprocessed.pkl', 'rb') as f:\n",
    "    processed = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 標題的整合\n",
    "class to_SVO():\n",
    "    def __init__(self):\n",
    "        self._be = ['is', 'are', 'am', 'was', 'were']\n",
    "        \n",
    "    def to_SVO(self, data):\n",
    "        if data == 'Sentence can not find SVO.':\n",
    "            return []\n",
    "        \n",
    "        self._data_key = data.keys()\n",
    "        self._results = []\n",
    "        # key: main / which..., value: [{}, {}]->dictionary(keys['subject', 'predicate', 'object']\n",
    "        for key, value in data.items(): \n",
    "            for svos in value:                \n",
    "                # 只有主詞\n",
    "                if svos['subject'] != [] and svos['predicate'] == [] and svos['object'] == []:\n",
    "                    self._Subject_only(svos)\n",
    "                    \n",
    "                # 沒有受詞                \n",
    "                if svos['subject'] != [] and svos['predicate'] != [] and svos['object'] == []:\n",
    "                    self._No_Object(svos)\n",
    "                \n",
    "                # 主動受詞都有\n",
    "                if svos['subject'] != [] and svos['predicate'] != [] and svos['object'] != []:\n",
    "                    self._Complete(svos)\n",
    "        \n",
    "        return self._results\n",
    "    \n",
    "    def _Attr_flatten(self, attrs):\n",
    "        attr_flatten = []\n",
    "        for attr in [x for x in attrs if isinstance(x, dict) == False and x != None]:\n",
    "            attr_flatten.append(attr)\n",
    "        for attr in [x for x in attrs if isinstance(x, dict) == True]:\n",
    "            for i in ['predicate', 'object']:\n",
    "                for j in attr[i]:\n",
    "                    attr_flatten.append(j[0])\n",
    "                    attr_flatten += self._Attr_flatten(j[1])\n",
    "        return attr_flatten\n",
    "    \n",
    "    def _Subject_only(self, svos):\n",
    "        # svo: ('', [])\n",
    "        for svo in svos['subject']:\n",
    "            # 主詞非dic的Attr\n",
    "            S_attr = []\n",
    "            for attr in [x for x in svo[1] if isinstance(x, dict) == False and x != None]:\n",
    "                S_attr.append(attr)\n",
    "                \n",
    "            # 主詞Attr含有動詞，可形成事件\n",
    "            if True in [isinstance(x, dict) for x in svo[1]]:\n",
    "                for attr in [x for x in svo[1] if isinstance(x, dict) == True]:\n",
    "                    if attr['object'] != []:\n",
    "                        self._results.append([(svo[0], ' '.join(S_attr)), \n",
    "                                        (attr['predicate'][0][0], ' '.join(self._Attr_flatten(attr['predicate'][0][1]))), \n",
    "                                        (attr['object'][0][0], ' '.join(self._Attr_flatten(attr['object'][0][1])))])\n",
    "                    \n",
    "                    # dictionary沒有object\n",
    "                    else:\n",
    "                        self._results.append([(svo[0], ' '.join(S_attr)), \n",
    "                                        (attr['predicate'][0][0], ' '.join(self._Attr_flatten(attr['predicate'][0][1])))])\n",
    "                        \n",
    "            # 主詞Attr沒有動詞，無法形成事件\n",
    "            else:\n",
    "                self._results.append([(svo[0], ' '.join(S_attr))])\n",
    "    \n",
    "    def _No_Object(self, svos):\n",
    "        # 連接詞\n",
    "        for subject in svos['subject']:\n",
    "            S = subject[0]\n",
    "            S_attr = self._Attr_flatten(subject[1])\n",
    "            for predicate in svos['predicate']:\n",
    "                P = predicate[0]\n",
    "                P_attr = []\n",
    "                for attr in [x for x in predicate[1] if isinstance(x, dict) == False and x != None]:\n",
    "                    P_attr.append(attr)\n",
    "                \n",
    "                # 動詞Attr可以當受詞\n",
    "                if True in [isinstance(x, dict) for x in predicate[1]]:\n",
    "                    for attr in [x for x in predicate[1] if isinstance(x, dict) == True]:\n",
    "                        if 'predicate' in attr.keys() and 'object' in attr.keys():\n",
    "                            self._results.append([(S, ' '.join(S_attr)), \n",
    "                                             (' '.join([P] + [attr['predicate'][0][0]]), ' '.join(P_attr + self._Attr_flatten(attr['predicate'][0][1]))), \n",
    "                                             (attr['object'][0][0], ' '.join(self._Attr_flatten(attr['object'][0][1])))])\n",
    "                # 動詞Attr不能當受詞\n",
    "                else:        \n",
    "                    self._results.append([(S, ' '.join(S_attr)), (P, ' '.join(P_attr))])\n",
    "    \n",
    "    def _Complete(self, svos):\n",
    "        for subject in svos['subject']:\n",
    "            S = subject[0]\n",
    "            S_attr = self._Attr_flatten(subject[1])\n",
    "            for predicate in svos['predicate']:\n",
    "                P = predicate[0]\n",
    "                P_attr = self._Attr_flatten(predicate[1])\n",
    "                for obj in svos['object']:\n",
    "                    # be動詞 + 受詞是形容詞 + 受詞Attr有dictionary\n",
    "                    if P in self._be and [x for x in nltk.pos_tag([y for y in obj[0].split(' ') if y != '']) if 'NN' in x[1]] == [] and \\\n",
    "                    True in [isinstance(x, dict) for x in obj[1]]:\n",
    "                        tmp_P = [P] + [obj[0]]\n",
    "                        for attr in [x for x in obj[1] if isinstance(x, dict) == False and x != None]:\n",
    "                            tmp_P.append(attr)\n",
    "                        for attr in [x for x in obj[1] if isinstance(x, dict) == True]:\n",
    "                            if 'predicate' in attr.keys() and 'object' in attr.keys():\n",
    "                                self._results.append([(S, ' '.join(S_attr)), \n",
    "                                                     (' '.join(tmp_P + [attr['predicate'][0][0]]), ' '.join(P_attr)), \n",
    "                                                     (attr['object'][0][0], ' '.join(self._Attr_flatten(attr['object'][0][1])))])\n",
    "                            # attr只有predicate\n",
    "                            elif 'predicate' in attr.keys():\n",
    "                                pos = nltk.pos_tag(attr['predicate'][0][0].split(' '))\n",
    "                                self._results.append([(S, ' '.join(S_attr)), \n",
    "                                                     (' '.join(tmp_P + [x[0] for x in pos if 'VB' not in x[1]]), ' '.join(P_attr)), \n",
    "                                                     (' '.join([x[0] for x in pos if 'VB' in x[1]]), '')])\n",
    "                            \n",
    "                            # 受詞在動詞的Attr中\n",
    "                            else:\n",
    "                                for attr in [x for x in predicate[1] if isinstance(x, dict) == True]:\n",
    "                                    self._results.append([(S, ' '.join(S_attr)), \n",
    "                                                         (' '.join(tmp_P + [attr['predicate'][0][0]]), ''), \n",
    "                                                         (attr['object'][0][0], ' '.join(self._Attr_flatten(attr['object'][0][1])))])\n",
    "                    # 正常狀態\n",
    "                    else:\n",
    "                        self._results.append([(S, ' '.join(S_attr)), \n",
    "                                             (P, ' '.join(P_attr)), \n",
    "                                             (obj[0], ' '.join(self._Attr_flatten(obj[1])))])\n",
    "                        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "def Integrate(dataset):\n",
    "    processed = []\n",
    "    for sample in tqdm(dataset):\n",
    "        tmp = {}\n",
    "        try:\n",
    "            tmp['id'] = sample['id']\n",
    "            tmp['date'] = sample['date']\n",
    "            tmp['title'] = sample['title']\n",
    "            tmp['title_SVO'] = sample['title_SVO']\n",
    "            integrate = to_SVO()\n",
    "            tmp['integrate_SVO'] = integrate.to_SVO(sample['title_SVO'])\n",
    "            processed.append(tmp)\n",
    "        except:\n",
    "            print(sample['id'])\n",
    "    return processed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 26620/26620 [00:00<00:00, 29074.47it/s]\n",
      "100%|██████████| 26620/26620 [00:00<00:00, 29514.94it/s]\n",
      "100%|██████████| 26620/26620 [00:00<00:00, 28197.29it/s]\n",
      "100%|██████████| 26623/26623 [00:00<00:00, 27964.61it/s]\n"
     ]
    }
   ],
   "source": [
    "n_workers = 4\n",
    "results = [None] * n_workers\n",
    "with Pool(processes=n_workers) as pool:\n",
    "    for i in range(n_workers):\n",
    "        batch_start = (len(processed) // n_workers) * i\n",
    "        if i == n_workers - 1:\n",
    "            batch_end = len(processed)\n",
    "        else:\n",
    "            batch_end = (len(processed) // n_workers) * (i + 1)\n",
    "\n",
    "        batch = processed[batch_start: batch_end]\n",
    "        results[i] = pool.apply_async(Integrate, [batch])\n",
    "\n",
    "    pool.close()\n",
    "    pool.join()\n",
    "\n",
    "end = []\n",
    "for result in results:\n",
    "    end += result.get()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "# with open('news_preprocessed_integrate.pkl', 'wb') as f:\n",
    "#     pickle.dump(data, f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Event Embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('news_preprocessed_integrate_1.pkl', 'rb') as f:\n",
    "    data = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "106483"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2582"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(set([x['date'] for x in data]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Word Embedding(Bert)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')\n",
    "bert = BertModel.from_pretrained('bert-base-uncased')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Convert to Token"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train = [item for sublist in [x['integrate_SVO'] for x in data] for item in sublist]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2582/2582 [01:17<00:00, 33.15it/s]\n"
     ]
    }
   ],
   "source": [
    "datas = []\n",
    "for i in tqdm(list(set([x['date'] for x in data]))):\n",
    "    tmp = {}\n",
    "    tmp['date'] = i\n",
    "    tmp['SVO'] = [item for sublist in [x['integrate_SVO'] for x in data if x['date'] == i] for item in sublist]\n",
    "    datas.append(tmp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "def Tokenize(dataset):\n",
    "    datas = []\n",
    "    for data in tqdm(dataset):\n",
    "        tmp = {}\n",
    "        tmp['date'] = data['date']\n",
    "        svos = []\n",
    "        for i in data['SVO']:\n",
    "            S = tokenizer.encode(i[0][0], add_special_tokens = False)\n",
    "            S_attr = tokenizer.encode(i[0][1], add_special_tokens = False)\n",
    "\n",
    "            if len(i) == 1:\n",
    "                svos.append([(S, S_attr)])\n",
    "                continue\n",
    "\n",
    "            if len(i) >= 2:\n",
    "                P = tokenizer.encode(i[1][0], add_special_tokens = False)\n",
    "                P_attr = tokenizer.encode(i[1][1], add_special_tokens = False)\n",
    "\n",
    "                if len(i) == 2:\n",
    "                    svos.append([(S, S_attr), (P, P_attr)])\n",
    "                    continue\n",
    "\n",
    "            if len(i) == 3:\n",
    "                O = tokenizer.encode(i[2][0], add_special_tokens = False)\n",
    "                O_attr = tokenizer.encode(i[2][1], add_special_tokens = False)\n",
    "\n",
    "                svos.append([(S, S_attr), (P, P_attr), (O, O_attr)])\n",
    "        tmp['SVO'] = svos\n",
    "        datas.append(tmp)\n",
    "    return datas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 645/645 [00:14<00:00, 43.84it/s]\n",
      "100%|██████████| 647/647 [00:14<00:00, 44.33it/s]\n",
      "100%|██████████| 645/645 [00:15<00:00, 42.76it/s]\n",
      "100%|██████████| 645/645 [00:15<00:00, 42.71it/s]\n"
     ]
    }
   ],
   "source": [
    "n_workers = 4\n",
    "results = [None] * n_workers\n",
    "with Pool(processes=n_workers) as pool:\n",
    "    for i in range(n_workers):\n",
    "        batch_start = (len(datas) // n_workers) * i\n",
    "        if i == n_workers - 1:\n",
    "            batch_end = len(datas)\n",
    "        else:\n",
    "            batch_end = (len(datas) // n_workers) * (i + 1)\n",
    "\n",
    "        batch = datas[batch_start: batch_end]\n",
    "        results[i] = pool.apply_async(Tokenize, [batch])\n",
    "\n",
    "    pool.close()\n",
    "    pool.join()\n",
    "\n",
    "train_token = []\n",
    "for result in results:\n",
    "    train_token += result.get()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2582"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(train_token)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "# with open('news_token.pkl', 'wb') as f:\n",
    "#     pickle.dump(train_token, f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('news_token.pkl', 'rb') as f:\n",
    "    train_token = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4695"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(train_token)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# def Word_Embedding(token, token_attr):\n",
    "#     with torch.no_grad():\n",
    "#         if token.shape[0] == 0:\n",
    "#             return torch.zeros(768)\n",
    "        \n",
    "#         a = bert(token.unsqueeze(-1))[0]\n",
    "#         a = a.view(a.shape[0], a.shape[2])\n",
    "#         a = a.mean(0)\n",
    "        \n",
    "#         # attr空的\n",
    "#         if token_attr.shape[0] == 0:\n",
    "#             return a\n",
    "#         else:\n",
    "#             attr = bert(token_attr.unsqueeze(-1))[0]\n",
    "#             attr = attr.view(attr.shape[0], attr.shape[2])\n",
    "#             attr = attr.mean(0)\n",
    "#             return (a + attr) / 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 4695/4695 [3:07:56<00:00,  2.40s/it]  \n"
     ]
    }
   ],
   "source": [
    "bert.cuda()\n",
    "train_vector = []\n",
    "for data in tqdm(train_token):\n",
    "    tmp = {}\n",
    "    tmp['date'] = data['date']\n",
    "    vectors = []\n",
    "    for t in data['SVO']:\n",
    "        with torch.no_grad():\n",
    "            token = torch.tensor(t[0][0]).cuda()\n",
    "            token_attr = torch.tensor(t[0][1]).cuda()\n",
    "            if token.shape[0] == 0:\n",
    "                S = torch.zeros(768).cuda()\n",
    "            else:\n",
    "                a = bert(token.unsqueeze(0))[0]\n",
    "                a = a.mean(1).flatten()\n",
    "\n",
    "                # attr空的\n",
    "                if token_attr.shape[0] == 0:\n",
    "                    S = a\n",
    "                else:\n",
    "                    attr = bert(token_attr.unsqueeze(0))[0]\n",
    "                    attr = attr.mean(1).flatten()\n",
    "                    S = (a + attr) / 2\n",
    "\n",
    "            if len(t) == 1:\n",
    "                vectors.append(torch.stack((S.cpu(), torch.zeros(768), torch.zeros(768))))\n",
    "\n",
    "            if len(t) >= 2:\n",
    "                token = torch.tensor(t[1][0]).cuda()\n",
    "                token_attr = torch.tensor(t[1][1]).cuda()\n",
    "                if token.shape[0] == 0:\n",
    "                    P = torch.zeros(768).cuda()\n",
    "                else:\n",
    "                    a = bert(token.unsqueeze(0))[0]\n",
    "                    a = a.mean(1).flatten()\n",
    "\n",
    "                    # attr空的\n",
    "                    if token_attr.shape[0] == 0:\n",
    "                        P = a\n",
    "                    else:\n",
    "                        attr = bert(token_attr.unsqueeze(0))[0]\n",
    "                        attr = attr.mean(1).flatten()\n",
    "                        P = (a + attr) / 2\n",
    "\n",
    "                if len(t) == 2:\n",
    "                    vectors.append(torch.stack((S.cpu(), P.cpu(), torch.zeros(768))))\n",
    "\n",
    "            if len(t) == 3:\n",
    "                token = torch.tensor(t[2][0]).cuda()\n",
    "                token_attr = torch.tensor(t[2][1]).cuda()\n",
    "                if token.shape[0] == 0:\n",
    "                    O = torch.zeros(768).cuda()\n",
    "                else:\n",
    "                    a = bert(token.unsqueeze(0))[0]\n",
    "                    a = a.mean(1).flatten()\n",
    "\n",
    "                    # attr空的\n",
    "                    if token_attr.shape[0] == 0:\n",
    "                        O = a\n",
    "                    else:\n",
    "                        attr = bert(token_attr.unsqueeze(0))[0]\n",
    "                        attr = attr.mean(1).flatten()\n",
    "                        O = (a + attr) / 2\n",
    "                vectors.append(torch.stack((S.cpu(), P.cpu(), O.cpu())))\n",
    "    tmp['SVO'] = vectors\n",
    "    train_vector.append(tmp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4695"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(train_vector)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('news_embedding_1.pkl', 'wb') as f:\n",
    "    pickle.dump(train_vector, f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train AutoEncoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('news_embedding_1.pkl', 'rb') as f:\n",
    "    train_vector = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_vector = [x for x in train_vector if x['date'] >= '20140101']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = [item for sublist in [x['SVO'] for x in train_vector] for item in sublist]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "105859"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dataloader = DataLoader(train, batch_size=256, shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# class AutoEncoder(nn.Module):\n",
    "#     def __init__(self):\n",
    "#         super(AutoEncoder, self).__init__()\n",
    "        \n",
    "#         self.encoder = nn.Sequential(\n",
    "#             nn.Conv2d(1, 256, (2,1), stride=(1,1), padding=(1,0)), # 256, 4, 768\n",
    "#             nn.Conv2d(256, 128, (3,1), stride=(1,1), padding=(1,0)), # 128, 4, 768\n",
    "#             nn.Conv2d(128, 32, (3,1), stride=(1,1), padding=(0,0)), # 32, 2, 768\n",
    "#             nn.Conv2d(32, 1, (2,1), stride=(1,1), padding=(0,0)) # 1, 1, 768\n",
    "#         )\n",
    "        \n",
    "#         self.decoder = nn.Sequential(\n",
    "#             nn.ConvTranspose2d(1, 32, (2,1), stride=(1,1), padding=(0,0)), # 32, 2, 768\n",
    "#             nn.ConvTranspose2d(32, 128, (3,1), stride=(1,1), padding=(0,0)), # 128, 4, 768\n",
    "#             nn.ConvTranspose2d(128, 256, (3,1), stride=(1,1), padding=(1,0)), # 256, 4, 768\n",
    "#             nn.ConvTranspose2d(256, 1, (2,1), stride=(1,1), padding=(1,0)) # 1, 3, 768\n",
    "#         )\n",
    "    \n",
    "#     def forward(self, x):\n",
    "#         encoded = self.encoder(x)\n",
    "#         decoded = self.decoder(encoded)\n",
    "        \n",
    "#         return encoded, decoded\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Best\n",
    "class AutoEncoder(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(AutoEncoder, self).__init__()\n",
    "        \n",
    "        self.encoder = nn.Sequential(\n",
    "            nn.Conv2d(1, 256, (2,1), stride=(1,1), padding=(1,0)), # 256, 4, 768\n",
    "            nn.Conv2d(256, 128, (2,1), stride=(1,1), padding=(0,0)), # 128, 3, 768\n",
    "            nn.Conv2d(128, 32, (2,1), stride=(1,1), padding=(0,0)), # 32, 2, 768\n",
    "            nn.Conv2d(32, 1, (2,1), stride=(1,1), padding=(0,0)) # 1, 1, 768\n",
    "        )\n",
    "        \n",
    "        self.decoder = nn.Sequential(\n",
    "            nn.ConvTranspose2d(1, 32, (2,1), stride=(1,1), padding=(0,0)), # 32, 2, 768\n",
    "            nn.ConvTranspose2d(32, 128, (2,1), stride=(1,1), padding=(0,0)), # 128, 3, 768\n",
    "            nn.ConvTranspose2d(128, 256, (2,1), stride=(1,1), padding=(0,0)), # 256, 4, 768\n",
    "            nn.ConvTranspose2d(256, 1, (2,1), stride=(1,1), padding=(1,0)) # 1, 3, 768\n",
    "        )\n",
    "    \n",
    "    def forward(self, x):\n",
    "        encoded = self.encoder(x)\n",
    "        decoded = self.decoder(encoded)\n",
    "        \n",
    "        return encoded, decoded\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# class AutoEncoder(nn.Module):\n",
    "#     def __init__(self):\n",
    "#         super(AutoEncoder, self).__init__()\n",
    "        \n",
    "#         self.encoder = nn.Sequential(\n",
    "#             nn.Conv2d(1, 256, (3,1), stride=(1,1), padding=(1,0)), # 256, 3, 768\n",
    "#             nn.Conv2d(256, 128, (3,1), stride=(1,1), padding=(1,0)),\n",
    "#             nn.Conv2d(128, 32, (2,1), stride=(1,1), padding=(0,0)), # 128, 2, 768\n",
    "#             nn.Conv2d(32, 1, (2,1), stride=(1,1), padding=(0,0)) # 32, 1, 768\n",
    "#         )\n",
    "        \n",
    "#         self.decoder = nn.Sequential(\n",
    "#             nn.ConvTranspose2d(1, 32, (2,1), stride=(1,1), padding=(0,0)), # 32, 2, 768\n",
    "#             nn.ConvTranspose2d(32, 128, (2,1), stride=(1,1), padding=(0,0)), # 128, 3, 768\n",
    "#             nn.ConvTranspose2d(128, 256, (3,1), stride=(1,1), padding=(1,0)), # 256, 4, 768\n",
    "#             nn.ConvTranspose2d(256, 1, (3,1), stride=(1,1), padding=(1,0)) # 1, 3, 768\n",
    "#         )\n",
    "    \n",
    "#     def forward(self, x):\n",
    "#         encoded = self.encoder(x)\n",
    "#         decoded = self.decoder(encoded)\n",
    "        \n",
    "#         return encoded, decoded\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "use_gpu = torch.cuda.is_available()\n",
    "autoencoder = AutoEncoder()\n",
    "if use_gpu:\n",
    "    autoencoder.cuda()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "414it [03:46,  1.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 000: Loss : 0.08622\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "criteria = nn.MSELoss()\n",
    "optimizer = optim.Adam(autoencoder.parameters(), lr=0.001)\n",
    "EPOCH = 1\n",
    "\n",
    "for epoch in range(EPOCH):\n",
    "    cumulate_loss = 0\n",
    "    for idx, x in tqdm(enumerate(train_dataloader)):\n",
    "        x = x.unsqueeze(1)\n",
    "        if use_gpu:\n",
    "            x = x.cuda()\n",
    "        latent, reconstruct = autoencoder(x)\n",
    "        loss = criteria(reconstruct, x)\n",
    "\n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "        cumulate_loss += loss.item() * x.shape[0]\n",
    "#         if (idx % 100) == 0:\n",
    "#             print(loss)\n",
    "            \n",
    "    print(f'Epoch { \"%03d\" % epoch }: Loss : { \"%.5f\" % (cumulate_loss / len(train))}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model saved to autoencoder(1channel_2014)_1.pth\n"
     ]
    }
   ],
   "source": [
    "checkpoint_path = 'autoencoder(1channel_2014)_{}.pth'.format(epoch+1) \n",
    "torch.save(autoencoder.state_dict(), checkpoint_path)\n",
    "print('model saved to %s' % checkpoint_path)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### AutoEncoder Validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Deep Learning"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Day-Vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('news_embedding_1.pkl', 'rb') as f:\n",
    "    data = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Day_vector():\n",
    "    def __init__(self, input_path, model_path, channel):\n",
    "        self.channel = channel\n",
    "        with open(input_path, 'rb') as f:\n",
    "            self.data = pickle.load(f)\n",
    "            \n",
    "        self.autoencoder = AutoEncoder(self.channel)\n",
    "        self.autoencoder.load_state_dict(torch.load(model_path))\n",
    "        self.use_gpu = torch.cuda.is_available()\n",
    "        if self.use_gpu:\n",
    "            self.autoencoder.cuda()\n",
    "    \n",
    "    def DayVector(self, date):\n",
    "        self._vector = [x['SVO'] for x in self.data if x['date'] == date][0]\n",
    "        \n",
    "        if len(self._vector) > 0:\n",
    "            # Event Embedding\n",
    "            with torch.no_grad():\n",
    "                dataloader = DataLoader(self._vector, batch_size=len(self._vector), shuffle=False)\n",
    "                for x in dataloader:\n",
    "                    x = x.unsqueeze(1)\n",
    "                    if self.use_gpu:\n",
    "                        x = x.cuda()\n",
    "                    latent, reconstruct = self.autoencoder(x)\n",
    "\n",
    "                latent = latent.cpu().detach().numpy()\n",
    "                latent = latent.reshape(len(latent), 768*self.channel)\n",
    "\n",
    "            return latent.mean(0)\n",
    "        else:\n",
    "            return []\n",
    "\n",
    "class AutoEncoder(nn.Module):\n",
    "    def __init__(self, channel):\n",
    "        super(AutoEncoder, self).__init__()\n",
    "        \n",
    "#         self.encoder = nn.Sequential(\n",
    "#             nn.Conv2d(1, 256, (3,1), stride=(1,1), padding=(1,0)), # 256, 3, 768\n",
    "#             nn.Conv2d(256, 128, (3,1), stride=(1,1), padding=(1,0)),\n",
    "#             nn.Conv2d(128, 32, (2,1), stride=(1,1), padding=(0,0)), # 128, 2, 768\n",
    "#             nn.Conv2d(32, 1, (2,1), stride=(1,1), padding=(0,0)) # 32, 1, 768\n",
    "#         )\n",
    "        \n",
    "#         self.decoder = nn.Sequential(\n",
    "#             nn.ConvTranspose2d(1, 32, (2,1), stride=(1,1), padding=(0,0)), # 32, 2, 768\n",
    "#             nn.ConvTranspose2d(32, 128, (2,1), stride=(1,1), padding=(0,0)), # 128, 3, 768\n",
    "#             nn.ConvTranspose2d(128, 256, (3,1), stride=(1,1), padding=(1,0)), # 256, 4, 768\n",
    "#             nn.ConvTranspose2d(256, 1, (3,1), stride=(1,1), padding=(1,0)) # 1, 3, 768\n",
    "#         )\n",
    "        \n",
    "        self.encoder = nn.Sequential(\n",
    "            nn.Conv2d(1, 256, (2,1), stride=(1,1), padding=(1,0)), # 256, 4, 768\n",
    "            nn.Conv2d(256, 128, (2,1), stride=(1,1), padding=(0,0)), # 128, 3, 768\n",
    "            nn.Conv2d(128, 32, (2,1), stride=(1,1), padding=(0,0)), # 32, 2, 768\n",
    "            nn.Conv2d(32, 1, (2,1), stride=(1,1), padding=(0,0)) # 1, 1, 768\n",
    "        )\n",
    "        \n",
    "        self.decoder = nn.Sequential(\n",
    "            nn.ConvTranspose2d(1, 32, (2,1), stride=(1,1), padding=(0,0)), # 32, 2, 768\n",
    "            nn.ConvTranspose2d(32, 128, (2,1), stride=(1,1), padding=(0,0)), # 128, 3, 768\n",
    "            nn.ConvTranspose2d(128, 256, (2,1), stride=(1,1), padding=(0,0)), # 256, 4, 768\n",
    "            nn.ConvTranspose2d(256, 1, (2,1), stride=(1,1), padding=(1,0)) # 1, 3, 768\n",
    "        )\n",
    "        \n",
    "#         self.encoder = nn.Sequential(\n",
    "#             nn.Conv2d(1, 256, (2,1), stride=(1,1), padding=(1,0)), # 256, 4, 768\n",
    "#             nn.Conv2d(256, 128, (3,1), stride=(1,1), padding=(1,0)), # 128, 4, 768\n",
    "#             nn.Conv2d(128, 32, (3,1), stride=(1,1), padding=(0,0)), # 32, 2, 768\n",
    "#             nn.Conv2d(32, channel, (2,1), stride=(1,1), padding=(0,0)) # 1, 1, 768\n",
    "#         )\n",
    "        \n",
    "#         self.decoder = nn.Sequential(\n",
    "#             nn.ConvTranspose2d(channel, 32, (2,1), stride=(1,1), padding=(0,0)), # 32, 2, 768\n",
    "#             nn.ConvTranspose2d(32, 128, (3,1), stride=(1,1), padding=(0,0)), # 128, 4, 768\n",
    "#             nn.ConvTranspose2d(128, 256, (3,1), stride=(1,1), padding=(1,0)), # 256, 4, 768\n",
    "#             nn.ConvTranspose2d(256, 1, (2,1), stride=(1,1), padding=(1,0)) # 1, 3, 768\n",
    "#         )\n",
    "    \n",
    "    def forward(self, x):\n",
    "        encoded = self.encoder(x)\n",
    "        decoded = self.decoder(encoded)\n",
    "        \n",
    "        return encoded, decoded"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "day_vector = Day_vector('news_embedding_1.pkl', 'autoencoder(1channel_2014)_1.pth', 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "date = list(set([x['date'] for x in data if x['date'] >= '20140101']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2113/2113 [01:23<00:00, 25.30it/s]\n"
     ]
    }
   ],
   "source": [
    "datas = {}\n",
    "for i in tqdm(date):\n",
    "    day = day_vector.DayVector(i)\n",
    "    if day != []:\n",
    "        datas[i] = day\n",
    "datas = dict([(k,datas[k]) for k in sorted(datas.keys())])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2109"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(datas)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('Event_embedding(1channel_2014).pkl', 'wb') as f:\n",
    "    pickle.dump(datas, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}