bert_yfinance.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "81c42831-0ee4-49de-971a-32781c1642aa",
   "metadata": {},
   "outputs": [],
   "source": [
    "from numba import cuda\n",
    "\n",
    "device = cuda.get_current_device(); device.reset()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "3b129205-90cc-4c12-aad5-7eaa43349e52",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from torch import nn\n",
    "from torch.utils.data import TensorDataset, DataLoader\n",
    "import torch.optim as optim\n",
    "import torch.nn.functional as F\n",
    "from transformers import BertForSequenceClassification, AdamW\n",
    "import os\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.model_selection import train_test_split\n",
    "import re\n",
    "import urllib.request\n",
    "from tqdm import tqdm\n",
    "from transformers import BertTokenizer, TFBertForSequenceClassification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "9b74e1dc-f034-4310-9b7c-2d901c0263d8",
   "metadata": {},
   "outputs": [],
   "source": [
    "max_seq_len = 128\n",
    "tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "76fdee28-a3a7-4a98-8167-fc7ecd9bc7b8",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "path = \"C:/Users/chan/Documents/GitHub/ML/test_AAPL/\" #googlenews file\n",
    "news_df = pd.DataFrame(columns=[\"title\", \"content\"])\n",
    "\n",
    "for txts in os.listdir(path):\n",
    "    full_path = os.path.join(path, txts)  # 파일 전체 경로 생성\n",
    "    if full_path.endswith('header.txt'):\n",
    "        continue\n",
    "    if os.path.isfile(full_path):  # 파일인지 확인\n",
    "        with open(full_path, \"r\", encoding=\"utf-8\") as txt_file:\n",
    "            title = txt_file.readline().strip()\n",
    "            content = txt_file.read().replace('\\n', ' ')\n",
    "            # DataFrame 생성 후 concat 함수를 사용하여 추가\n",
    "            new_row = pd.DataFrame({\"title\": [title], \"content\": [content]})\n",
    "            news_df = pd.concat([news_df, new_row], ignore_index=True)\n",
    "\n",
    "news_df\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "b36d4194-1a44-4a25-b406-ee03d973acf9",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 데이터프레임의 'title'과 'content'를 이용하여 입력 데이터 생성\n",
    "inputs = [content for content in news_df['content']]\n",
    "\n",
    "# 입력 데이터를 BERT 모델의 입력 형식에 맞게 변환\n",
    "max_length = 128\n",
    "input_ids = []\n",
    "attention_masks = []\n",
    "\n",
    "for content in inputs:\n",
    "    encoded_dict = tokenizer.encode_plus(\n",
    "                        content,                    # content\n",
    "                        add_special_tokens = True,  # Add '[CLS]' and '[SEP]'\n",
    "                        max_length = max_length,           # Pad & truncate all sentences\n",
    "                        pad_to_max_length = True,\n",
    "                        return_attention_mask = True,   # Construct attn. masks\n",
    "                        return_tensors = 'pt',     # Return pytorch tensors\n",
    "                   )\n",
    "    \n",
    "    input_ids.append(encoded_dict['input_ids'])\n",
    "    attention_masks.append(encoded_dict['attention_mask'])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "426eb0aa-1cc5-4f82-aee3-6398c9ae52cb",
   "metadata": {},
   "source": [
    "<h1>이미 학습된 경제뉴스기로 학습된 bertmodel 불러와 현재 사용할 ticker관련된 뉴스기사를 model에 학습</h1> "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "c364a7c7-d386-4d6d-b98c-4dc0b95ae0f1",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "\n",
    "predict_model = torch.load(\"bert_model_loss0.34.pth\", map_location=torch.device('cpu')) \n",
    "predict_model.to(torch.device('cpu'))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "a4356817-3d78-4298-984d-f540e6351ac9",
   "metadata": {},
   "outputs": [],
   "source": [
    "predicted_labels = []\n",
    "\n",
    "for inputs in tqdm(zip(input_ids, attention_masks)):\n",
    "    input_ids = inputs[0].to(torch.device('cpu'))\n",
    "    attention_mask = inputs[1].to(torch.device('cpu'))\n",
    "    output = predict_model(input_ids, attention_mask=attention_mask)\n",
    "\n",
    "    ps = F.softmax(output.logits, dim=1)\n",
    "    top_p, top_class = ps.topk(1, dim=1)\n",
    "    predicted_labels.append(top_class.item())\n",
    "\n",
    "predict_df = pd.DataFrame({'predicted_label': predicted_labels})\n",
    "news_df[\"predict\"] = predict_df\n",
    "news_df.to_csv('predicted_news.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "0f379838-2a78-4914-a15c-39fcf60a027c",
   "metadata": {},
   "outputs": [],
   "source": [
    "news_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "d5462aeb-fa7b-4449-82bd-b5f2df76b255",
   "metadata": {},
   "outputs": [],
   "source": [
    "news_df['date'] = pd.to_datetime(news_df['title']).dt.date  # 날짜 추출\n",
    "news_df['time'] = pd.to_datetime(news_df['title']).dt.time  # 시간 추출\n",
    "\n",
    "# 날짜별로 그룹화하고, 'predict' 열에 대한 평균을 계산\n",
    "grouped_df = news_df.groupby('date').agg({\n",
    "    'title': lambda x: x.tolist(),\n",
    "    'content': lambda x: x.tolist(),\n",
    "    'predict': lambda x: round(x.mean(), 3)  \n",
    "})\n",
    "\n",
    "grouped_df.reset_index(inplace=True)  # 인덱스 리셋\n",
    "\n",
    "# 결과 출력\n",
    "grouped_df[['date', 'title', 'content', 'predict']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "id": "a29499d5-fbc9-4e8f-b6c5-0436cabac8ca",
   "metadata": {},
   "outputs": [],
   "source": [
    "result_df = grouped_df[['date', 'predict']]\n",
    "# result_df['date']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "a52954fd-0e9d-4b9e-8777-99d387357bf9",
   "metadata": {},
   "outputs": [],
   "source": [
    "news_df['date'] = pd.to_datetime(news_df['title']).dt.date  # 날짜 추출\n",
    "news_df['time'] = pd.to_datetime(news_df['title']).dt.time  # 시간 추출\n",
    "\n",
    "# 'date' 열을 datetime 형식으로 다시 변환\n",
    "news_df['date'] = pd.to_datetime(news_df['date'])\n",
    "\n",
    "# 날짜별로 그룹화하고, 'predict' 열에 대한 평균을 계산\n",
    "grouped_df = news_df.groupby('date').agg({\n",
    "    'title': lambda x: x.tolist(),\n",
    "    'content': lambda x: x.tolist(),\n",
    "    'predict': lambda x: round(x.mean(), 3)  \n",
    "})\n",
    "\n",
    "grouped_df.reset_index(inplace=True)  # 인덱스 리셋\n",
    "\n",
    "# 모든 날짜를 포함하는 날짜 범위 생성\n",
    "all_dates = pd.date_range(start=grouped_df['date'].min(), end=grouped_df['date'].max(), freq='D')\n",
    "# 새로운 DataFrame 생성 후 기존 데이터와 병합\n",
    "complete_df = pd.DataFrame(all_dates, columns=['date'])\n",
    "complete_df['date'] = pd.to_datetime(complete_df['date'])  # 날짜를 datetime으로 변환\n",
    "complete_df = complete_df.merge(grouped_df, on='date', how='left')\n",
    "# 누락된 'predict' 값을 이전 값으로 채우기\n",
    "complete_df['predict'] = complete_df['predict'].fillna(method='ffill')\n",
    "\n",
    "# 'date'와 'predict' 열만 선택하여 결과 출력\n",
    "result_df_1 = complete_df[['date', 'predict']]\n",
    "result_df_1.tail(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "id": "f9a4d420-7448-4552-905e-6ae2fac69609",
   "metadata": {},
   "outputs": [],
   "source": [
    "ticker = 'AAPL'\n",
    "csv_filename = f\"news_data_{ticker}.csv\"\n",
    "loaded_df = pd.read_csv(csv_filename)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "68c8d969-d8c3-4bd2-b6dd-369467c14dd2",
   "metadata": {},
   "outputs": [],
   "source": [
    "result_df_1['date'] = result_df_1['date'].dt.date\n",
    "result_df = pd.concat([loaded_df, result_df_1], ignore_index=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "d64c7ea6-7634-4c67-8b50-df0b747bd960",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "import pandas_datareader.data as web # 주식 데이터를 얻어오기 위해 사용\n",
    "import datetime # 시간 처리\n",
    "import matplotlib.pyplot as plt\n",
    "import yfinance as yf\n",
    "import FinanceDataReader as fdr\n",
    "from prophet import Prophet\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "c773fd21-00aa-4059-be64-4fbaf543e705",
   "metadata": {},
   "outputs": [],
   "source": [
    "start_date = datetime.datetime( 2021, 4, 5 )\n",
    "end_date = datetime.datetime( 2024, 5, 18)\n",
    "\n",
    "# # # 날짜 기준으로 데이터프레임 정렬\n",
    "# missing_dates = pd.date_range(start=start_date, end=end_date).difference(result_df['date'])\n",
    "\n",
    "# # 새로운 날짜와 예측값 추가\n",
    "# for date in missing_dates:\n",
    "#     result_df = result_df.append({'date': date, 'predict': 1.0}, ignore_index=True)\n",
    "result_df['date'] = pd.to_datetime(result_df['date'])\n",
    "result_df = result_df.sort_values('date').reset_index(drop=True)\n",
    "result_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "2d2727b0-dac6-4582-8c4c-4a6cf3fde235",
   "metadata": {},
   "outputs": [],
   "source": [
    "result_df = result_df.iloc[:-1]\n",
    "# result_df\n",
    "result_df.to_csv(csv_filename, index=False)\n",
    "result_df"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.18"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}