Causality-based classification an prediction.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "e57545ef",
   "metadata": {},
   "source": [
    "# 初始化"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "f2a291e7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date</th>\n",
       "      <th>code</th>\n",
       "      <th>E_RET</th>\n",
       "      <th>FAG</th>\n",
       "      <th>TAG</th>\n",
       "      <th>NPG</th>\n",
       "      <th>TPG</th>\n",
       "      <th>OPG</th>\n",
       "      <th>RG</th>\n",
       "      <th>SG</th>\n",
       "      <th>...</th>\n",
       "      <th>ATURNOVER</th>\n",
       "      <th>AVAR</th>\n",
       "      <th>ABM</th>\n",
       "      <th>RFR</th>\n",
       "      <th>TBR_1Y</th>\n",
       "      <th>TBR_5Y</th>\n",
       "      <th>TBR_10Y</th>\n",
       "      <th>TS</th>\n",
       "      <th>DRS</th>\n",
       "      <th>CPI</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2004-01-31</td>\n",
       "      <td>000001</td>\n",
       "      <td>0.088454</td>\n",
       "      <td>-0.085398</td>\n",
       "      <td>0.529672</td>\n",
       "      <td>-0.088587</td>\n",
       "      <td>-0.353633</td>\n",
       "      <td>-0.144732</td>\n",
       "      <td>-0.066980</td>\n",
       "      <td>0.649646</td>\n",
       "      <td>...</td>\n",
       "      <td>0.62298</td>\n",
       "      <td>0.023678</td>\n",
       "      <td>-1.187669</td>\n",
       "      <td>-0.708848</td>\n",
       "      <td>-0.950604</td>\n",
       "      <td>-0.793329</td>\n",
       "      <td>-0.036616</td>\n",
       "      <td>1.312954</td>\n",
       "      <td>-1.639785</td>\n",
       "      <td>0.547714</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2004-01-31</td>\n",
       "      <td>000002</td>\n",
       "      <td>0.198254</td>\n",
       "      <td>-0.462371</td>\n",
       "      <td>1.278572</td>\n",
       "      <td>0.611947</td>\n",
       "      <td>0.966450</td>\n",
       "      <td>0.584612</td>\n",
       "      <td>2.187683</td>\n",
       "      <td>1.123943</td>\n",
       "      <td>...</td>\n",
       "      <td>0.62298</td>\n",
       "      <td>0.023678</td>\n",
       "      <td>-1.187669</td>\n",
       "      <td>-0.708848</td>\n",
       "      <td>-0.950604</td>\n",
       "      <td>-0.793329</td>\n",
       "      <td>-0.036616</td>\n",
       "      <td>1.312954</td>\n",
       "      <td>-1.639785</td>\n",
       "      <td>0.547714</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2004-01-31</td>\n",
       "      <td>000006</td>\n",
       "      <td>0.089354</td>\n",
       "      <td>-0.133499</td>\n",
       "      <td>-1.131546</td>\n",
       "      <td>-0.488548</td>\n",
       "      <td>-0.694846</td>\n",
       "      <td>-0.133311</td>\n",
       "      <td>0.255228</td>\n",
       "      <td>0.319366</td>\n",
       "      <td>...</td>\n",
       "      <td>0.62298</td>\n",
       "      <td>0.023678</td>\n",
       "      <td>-1.187669</td>\n",
       "      <td>-0.708848</td>\n",
       "      <td>-0.950604</td>\n",
       "      <td>-0.793329</td>\n",
       "      <td>-0.036616</td>\n",
       "      <td>1.312954</td>\n",
       "      <td>-1.639785</td>\n",
       "      <td>0.547714</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2004-01-31</td>\n",
       "      <td>000009</td>\n",
       "      <td>0.177554</td>\n",
       "      <td>-0.121645</td>\n",
       "      <td>-0.213518</td>\n",
       "      <td>0.760117</td>\n",
       "      <td>-0.193851</td>\n",
       "      <td>-0.054944</td>\n",
       "      <td>0.035656</td>\n",
       "      <td>-0.267355</td>\n",
       "      <td>...</td>\n",
       "      <td>0.62298</td>\n",
       "      <td>0.023678</td>\n",
       "      <td>-1.187669</td>\n",
       "      <td>-0.708848</td>\n",
       "      <td>-0.950604</td>\n",
       "      <td>-0.793329</td>\n",
       "      <td>-0.036616</td>\n",
       "      <td>1.312954</td>\n",
       "      <td>-1.639785</td>\n",
       "      <td>0.547714</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2004-01-31</td>\n",
       "      <td>000012</td>\n",
       "      <td>0.244254</td>\n",
       "      <td>0.262967</td>\n",
       "      <td>0.632787</td>\n",
       "      <td>0.019302</td>\n",
       "      <td>0.012431</td>\n",
       "      <td>0.052023</td>\n",
       "      <td>-0.119654</td>\n",
       "      <td>-0.016191</td>\n",
       "      <td>...</td>\n",
       "      <td>0.62298</td>\n",
       "      <td>0.023678</td>\n",
       "      <td>-1.187669</td>\n",
       "      <td>-0.708848</td>\n",
       "      <td>-0.950604</td>\n",
       "      <td>-0.793329</td>\n",
       "      <td>-0.036616</td>\n",
       "      <td>1.312954</td>\n",
       "      <td>-1.639785</td>\n",
       "      <td>0.547714</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 75 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        date    code     E_RET       FAG       TAG       NPG       TPG  \\\n",
       "0 2004-01-31  000001  0.088454 -0.085398  0.529672 -0.088587 -0.353633   \n",
       "1 2004-01-31  000002  0.198254 -0.462371  1.278572  0.611947  0.966450   \n",
       "2 2004-01-31  000006  0.089354 -0.133499 -1.131546 -0.488548 -0.694846   \n",
       "3 2004-01-31  000009  0.177554 -0.121645 -0.213518  0.760117 -0.193851   \n",
       "4 2004-01-31  000012  0.244254  0.262967  0.632787  0.019302  0.012431   \n",
       "\n",
       "        OPG        RG        SG  ...  ATURNOVER      AVAR       ABM       RFR  \\\n",
       "0 -0.144732 -0.066980  0.649646  ...    0.62298  0.023678 -1.187669 -0.708848   \n",
       "1  0.584612  2.187683  1.123943  ...    0.62298  0.023678 -1.187669 -0.708848   \n",
       "2 -0.133311  0.255228  0.319366  ...    0.62298  0.023678 -1.187669 -0.708848   \n",
       "3 -0.054944  0.035656 -0.267355  ...    0.62298  0.023678 -1.187669 -0.708848   \n",
       "4  0.052023 -0.119654 -0.016191  ...    0.62298  0.023678 -1.187669 -0.708848   \n",
       "\n",
       "     TBR_1Y    TBR_5Y   TBR_10Y        TS       DRS       CPI  \n",
       "0 -0.950604 -0.793329 -0.036616  1.312954 -1.639785  0.547714  \n",
       "1 -0.950604 -0.793329 -0.036616  1.312954 -1.639785  0.547714  \n",
       "2 -0.950604 -0.793329 -0.036616  1.312954 -1.639785  0.547714  \n",
       "3 -0.950604 -0.793329 -0.036616  1.312954 -1.639785  0.547714  \n",
       "4 -0.950604 -0.793329 -0.036616  1.312954 -1.639785  0.547714  \n",
       "\n",
       "[5 rows x 75 columns]"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 初始化\n",
    "import sys\n",
    "import time\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from datetime import datetime, timedelta\n",
    "\n",
    "# 数据读取\n",
    "df_raw = pd.read_pickle('data_stock.pkl')\n",
    "# df_raw = df_raw.iloc[:,:63]\n",
    "df_raw.head()\n",
    "# APE pricetoearning ADY股息率 TS 期限利差 DRS 违约利差"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "af6d1d12",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score\n",
    "def eval_class(y_test, y_pred):\n",
    "    acc = accuracy_score(y_test, y_pred) \n",
    "    prec = precision_score(y_test, y_pred) \n",
    "    recall = recall_score(y_test, y_pred)\n",
    "    f1 = f1_score(y_test, y_pred)\n",
    "    df_eval = pd.DataFrame({'Accuracy':acc, 'Precision':prec, 'Recall':recall, 'F1':f1}, index=[0])\n",
    "    return df_eval\n",
    "\n",
    "from sklearn.metrics import r2_score, mean_squared_error, mean_absolute_error, mean_absolute_percentage_error # R2, MSE, MAE, MAPE\n",
    "def eval_result(y_real, y_pred): \n",
    "    y_real, y_pred = np.array(y_real).ravel(), np.array(y_pred).ravel()\n",
    "    scale = np.max(y_real) - np.min(y_real) # scale is important for RMSE and MAE\n",
    "    r2 = r2_score(y_real, y_pred)\n",
    "    rmse = mean_squared_error(y_real, y_pred, squared=False) # RMSE and MAE are various on different scales\n",
    "    mae = mean_absolute_error(y_real, y_pred)\n",
    "    mape = mean_absolute_percentage_error(y_real, y_pred) # Note that dataset cannot have any 0 value.\n",
    "    df_eval = pd.DataFrame({'Scale':scale, 'R2':r2, 'RMSE':rmse, 'MAE':mae, 'MAPE':mape}, index=[0])\n",
    "    return df_eval"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6d9c6680",
   "metadata": {},
   "source": [
    "# 特征选择"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "81851a2f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((339251, 72), (37695, 72))"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 划分测试集\n",
    "df_all = df_raw.iloc[:,2:] # 删除编号\n",
    "df_all.loc[df_all['E_RET']>0,'E_RET'] = 1\n",
    "df_all.loc[df_all['E_RET']<=0,'E_RET'] = 0\n",
    "from sklearn.model_selection import train_test_split\n",
    "X_train, X_test, y_train, y_test = train_test_split(df_all.iloc[:,1:], df_all.E_RET, test_size=0.1, random_state=0, shuffle=False) # 顺序分割\n",
    "X_train.shape, X_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "23393f9d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# # 归一化\n",
    "# from sklearn.preprocessing import MinMaxScaler # 为保证特征选择时，数据的一致性，进行归一化（针对方差选择等）\n",
    "# scalarX = MinMaxScaler(feature_range=(0,1)) \n",
    "# X_train[X_train.columns] = scalarX.fit_transform(X_train)\n",
    "# X_test[X_test.columns] = scalarX.transform(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "2ff6fd2d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 挂载feature_selection包\n",
    "sys.path.append('.\\\\pyFS\\\\')\n",
    "from feature_selection import filter_method as ft"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0f1b2994",
   "metadata": {},
   "source": [
    "## 过滤式特征选择"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "5e4ee7d7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 1、方差特征选择 (标准化后方差为1, 无效)\n",
    "# 作用：删除对大多数/所有观测值显示相同值的特征(常数/准常数特征)\n",
    "from sklearn.feature_selection import VarianceThreshold \n",
    "var = VarianceThreshold(threshold=X_train.var().mean()) # 设置方差阈值 threshold = 所有特征方差的平均值\n",
    "df = var.fit_transform(X_train)\n",
    "selected_feature = var.get_feature_names_out() # 查看被留下特征的字符名称"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "c8c80a49",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 2、卡方特征选择 （只是适用于离散特征）\n",
    "# 作用：卡方检验主要是用来进行 分类变量（离散变量）的关联性、相关性分析\n",
    "from sklearn.feature_selection import chi2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "7038510b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['POA', 'ROE', 'ROIC', 'LTROC', 'ATURNOVER', 'RFR', 'TBR_1Y',\n",
       "       'TBR_5Y', 'TBR_10Y', 'CPI'], dtype=object)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 3、F检验特征选择 \n",
    "# 作用：基于f检验的方法估计两个随机变量之间的线性依赖程度。只适用于非负性特征。\n",
    "from sklearn.feature_selection import SelectKBest, f_classif # f_regression\n",
    "selector = SelectKBest(f_classif, k=10)\n",
    "selector.fit(X_train, y_train)\n",
    "selected_feature = selector.get_feature_names_out() # selector.scores_\n",
    "selected_feature"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "0fee6c15",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['TBR_5Y', 'CPI', 'TBR_1Y', 'ATURNOVER', 'TBR_10Y', 'LTROC', 'RFR',\n",
       "       'ROIC', 'POA', 'ROE'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 4、Pearson系数 特征选择 \n",
    "# 作用：Pearson系数绝对值最大的10个特征\n",
    "df_train = df_all[:-len(df_all)//10]\n",
    "df_corr = df_train.corr()[[\"E_RET\"]].sort_values(by=[\"E_RET\"])\n",
    "df_corr_abs = df_corr.abs().sort_values(by=[\"E_RET\"])\n",
    "selected_feature = df_corr.T[df_corr_abs[-11:].index].T[::-1][1:].index\n",
    "selected_feature"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "28ec3aaa",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['APE', 'ADY', 'ATURNOVER', 'AVAR', 'RFR', 'TBR_1Y', 'TBR_5Y',\n",
       "       'TBR_10Y', 'TS', 'DRS'], dtype=object)"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 5、Mutual information\n",
    "# 作用：互信息度量一个特征的存在/缺失对正确预测Y的贡献。\n",
    "from sklearn.feature_selection import SelectKBest, mutual_info_regression\n",
    "selector = SelectKBest(mutual_info_regression, k=10)\n",
    "selector.fit(X_train, y_train)\n",
    "selected_feature = selector.get_feature_names_out()\n",
    "selected_feature"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b5f7aaa1",
   "metadata": {},
   "source": [
    "### 因果特征选择"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0a69af02",
   "metadata": {},
   "outputs": [],
   "source": [
    "sys.path.append('.\\\\pyCausalFS\\\\')\n",
    "from pyCausalFS import CBD # 使用 pyCausalFS\n",
    "from pyCausalFS.CBD.example.example_MB import * #  注意需要在CBD/example文件夹中的example_MB函数 添加return MB 并修改file.open的位置\n",
    "\n",
    "def show_features(MB):\n",
    "    df_col = pd.DataFrame(df_raw.iloc[:,2:].columns) # 存贮列名\n",
    "    selected_feature = df_col.T[MB].values\n",
    "    return selected_feature\n",
    "\n",
    "df_train, df_test, y_train, y_test = train_test_split(df_all, df_all.E_RET, test_size=0.1, random_state=0, shuffle=False) # 顺序分割\n",
    "df_train.columns = [x for x in range(len(df_train.columns))] # 去除列名\n",
    "df_train.shape # df_train包含target列月超额收益率E_RET"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "26f6cf52",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['OC', 'TBR_5Y', 'CPI', 'ATCDPS', 'DM', 'ERR', 'PE', 'TQ', 'CINT', 'EVM',\n",
       "       'TURNOVER', 'ST_REVERSAL'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 1、IPCMB\n",
    "MB = example(method='IPCMB', data=df_train, target=[0], alpha=0.01, is_discrete=False) # 选择第一列作为target\n",
    "show_features(MB)\n",
    "# 全集选择 ['OC', 'TBR_5Y', 'CPI', 'ATCDPS', 'DM', 'ERR', 'PE', 'TQ', 'CINT', 'EVM', 'TURNOVER', 'ST_REVERSAL']\n",
    "# 训练集选择 收益率转01 ['OC', 'TBR_5Y', 'CPI', 'DM', 'ERR', 'PE', 'FATI', 'CINT', 'EVM', 'TURNOVER', 'BETA_S', 'MOM_3M', 'MOM_6M']\n",
    "# 训练集选择 ['OC', 'TBR_5Y', 'CPI', 'DM', 'ERR', 'PE', 'FATI', 'CINT', 'EVM', 'TURNOVER']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b70ad5d7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 2、PCMB\n",
    "MB = example(method='PCMB', data=df_train, target=[0], alpha=0.01, is_discrete=False) # 选择第一列作为target\n",
    "show_features(MB)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "405a6c83",
   "metadata": {},
   "source": [
    "## 封装式特征选择"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "569a79fe",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "ba9768cc",
   "metadata": {},
   "source": [
    "## 嵌入式特征选择"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "a057be10",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Recall</th>\n",
       "      <th>F1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.505001</td>\n",
       "      <td>0.506229</td>\n",
       "      <td>0.624011</td>\n",
       "      <td>0.558983</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Accuracy  Precision    Recall        F1\n",
       "0  0.505001   0.506229  0.624011  0.558983"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "lr = LogisticRegression(penalty='l2', C=0.5, solver='sag')\n",
    "lr.fit(X_train, y_train)\n",
    "y_pred = lr.predict(X_test)\n",
    "eval_class(y_test, y_pred) \n",
    "# all 0.505054\t0.506271\t0.624116\t0.559051"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "c8cbb1d9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['TBR_5Y', 'TBR_10Y', 'ADY', 'APE', 'RFR', 'TBR_1Y', 'LT_REVERSAL',\n",
       "       'CPI', 'ATURNOVER', 'MOM_6M'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lr_coef = pd.DataFrame(lr.coef_)\n",
    "lr_coef.columns = df_all.columns[1:]\n",
    "lr_coef = lr_coef.T.sort_values(by=[0])\n",
    "lr_coef = lr_coef.abs().sort_values(by=[0])[-10:][::-1]\n",
    "selected_feature = lr_coef.index\n",
    "selected_feature"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "4416953c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Recall</th>\n",
       "      <th>F1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.502799</td>\n",
       "      <td>0.505923</td>\n",
       "      <td>0.468813</td>\n",
       "      <td>0.486661</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Accuracy  Precision    Recall        F1\n",
       "0  0.502799   0.505923  0.468813  0.486661"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn import tree\n",
    "dt = tree.DecisionTreeClassifier()\n",
    "dt.fit(X_train, y_train) \n",
    "y_pred = dt.predict(X_test)\n",
    "eval_class(y_test, y_pred) \n",
    "# all 0.94261917716486\n",
    "# mi 0.730389"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "10f7115b",
   "metadata": {},
   "source": [
    "## 集成式特征选择"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d92bb51d",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "208f5e12",
   "metadata": {},
   "source": [
    "# 对比各模型分类表现"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "45f39269",
   "metadata": {},
   "source": [
    "## 分类"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "025e959b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 记录cfs运行结果\n",
    "# IPCMB MMMB IAMB inter_IAMB IAMBnPC GSMB BAMB MBOR STMB LRH/LCMB\n",
    "def cfs_feature(cfs):\n",
    "    if cfs=='IPCMB': return ['OC', 'TBR_5Y', 'CPI', 'DM', 'ERR', 'PE', 'FATI', 'CINT', 'EVM', 'TURNOVER', 'BETA_S', 'MOM_3M', 'MOM_6M'] # IPCMB 100m \n",
    "    if cfs=='MMMB': return ['OC', 'TBR_5Y', 'ALR', 'CPI', 'DM', 'ERR', 'PE', 'MVTTA', 'FATI', 'CINT', 'EVM', 'TURNOVER', 'BETA_S', 'EBIT'] # MMMB 75m \n",
    "    if cfs=='IAMB': return ['ABM', 'OC', 'TBR_5Y', 'ALR', 'CPI', 'DM', 'ERR', 'FATI', 'MVTTA', 'TQ', 'CINT', 'EVM', 'BETA_S', 'ST_REVERSAL'] # IAMB 1m\n",
    "    if cfs=='inter_IAMB': return ['TBR_5Y'] # inter_IAMB 8s\n",
    "    # if cfs=='IAMBnPC': return ['DM', 'ERR', 'MVTTA', 'EVM', 'FATI', 'CINT', 'BETA_S', 'TBR_5Y', 'CPI'] # IAMBnPC 2m \n",
    "    if cfs=='GSMB': return ['ABM', 'OC', 'TBR_5Y', 'ALR', 'CPI', 'DM', 'ERR', 'MVTTA', 'FATI', 'TQ', 'CINT', 'EVM', 'BETA_S', 'ST_REVERSAL'] # GSMB 1m\n",
    "    if cfs=='BAMB': return ['TBR_5Y', 'CPI', 'DM', 'ERR', 'FATI', 'MVTTA', 'TQ', 'CINT', 'EVM', 'BETA_S'] # BAMB 6m \n",
    "    if cfs=='MBOR': return ['OC', 'TBR_5Y', 'ALR', 'CPI', 'DM', 'ERR', 'PE', 'PS', 'PB', 'FATI', 'MVTTA', 'TQ', 'CINT', 'EVM', 'TURNOVER', 'BETA_S', 'EBIT', 'ST_REVERSAL'] # MBOR 128m\n",
    "    if cfs=='STMB': return ['OC', 'TBR_5Y', 'CPI', 'DM', 'ERR', 'FATI', 'MVTTA', 'TQ', 'CINT', 'EVM', 'BETA_S', 'ST_REVERSAL'] # STMB 3m\n",
    "    if cfs=='LCMB': return ['ABM', 'OC', 'TBR_5Y', 'ALR', 'CPI', 'DM', 'ERR', 'FATI', 'MVTTA', 'TQ', 'CINT', 'EVM', 'BETA_S', 'ST_REVERSAL'] # LRH 12m    \n",
    "    if cfs=='TIE': return [] # TIE 17m   \n",
    "    if cfs=='KIAMB': return ['ABM', 'OC', 'TBR_5Y', 'ALR', 'CPI', 'DM', 'ERR', 'FATI', 'MVTTA', 'TQ', 'CINT', 'EVM', 'BETA_S', 'ST_REVERSAL'] # KIAMB 1m  \n",
    "    if cfs=='FBEDk': return ['TBR_5Y', 'CPI', 'DM', 'ERR', 'FATI', 'MVTTA', 'TQ', 'CINT', 'EVM', 'BETA_S'] # FBEDk 9s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "2583d590",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>OC</th>\n",
       "      <th>TBR_5Y</th>\n",
       "      <th>ALR</th>\n",
       "      <th>CPI</th>\n",
       "      <th>DM</th>\n",
       "      <th>ERR</th>\n",
       "      <th>PE</th>\n",
       "      <th>PS</th>\n",
       "      <th>PB</th>\n",
       "      <th>FATI</th>\n",
       "      <th>MVTTA</th>\n",
       "      <th>TQ</th>\n",
       "      <th>CINT</th>\n",
       "      <th>EVM</th>\n",
       "      <th>TURNOVER</th>\n",
       "      <th>BETA_S</th>\n",
       "      <th>EBIT</th>\n",
       "      <th>ST_REVERSAL</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-0.106573</td>\n",
       "      <td>-0.793329</td>\n",
       "      <td>2.690755</td>\n",
       "      <td>0.547714</td>\n",
       "      <td>4.291879</td>\n",
       "      <td>0.311601</td>\n",
       "      <td>-0.138475</td>\n",
       "      <td>0.067215</td>\n",
       "      <td>0.373770</td>\n",
       "      <td>-0.171218</td>\n",
       "      <td>-1.005916</td>\n",
       "      <td>-0.684939</td>\n",
       "      <td>3.305135</td>\n",
       "      <td>0.227988</td>\n",
       "      <td>-0.762898</td>\n",
       "      <td>0.126015</td>\n",
       "      <td>0.864458</td>\n",
       "      <td>-0.574940</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.302482</td>\n",
       "      <td>-0.793329</td>\n",
       "      <td>0.574912</td>\n",
       "      <td>0.547714</td>\n",
       "      <td>0.719831</td>\n",
       "      <td>-0.207973</td>\n",
       "      <td>-0.300231</td>\n",
       "      <td>-0.448722</td>\n",
       "      <td>-0.500699</td>\n",
       "      <td>-0.553989</td>\n",
       "      <td>-0.613949</td>\n",
       "      <td>-0.478331</td>\n",
       "      <td>-0.308613</td>\n",
       "      <td>-0.285094</td>\n",
       "      <td>-0.473442</td>\n",
       "      <td>-0.502147</td>\n",
       "      <td>-0.025659</td>\n",
       "      <td>-0.073353</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-0.058994</td>\n",
       "      <td>-0.793329</td>\n",
       "      <td>1.452280</td>\n",
       "      <td>0.547714</td>\n",
       "      <td>-0.340210</td>\n",
       "      <td>0.311601</td>\n",
       "      <td>-0.290670</td>\n",
       "      <td>-0.550378</td>\n",
       "      <td>-0.766247</td>\n",
       "      <td>-0.527272</td>\n",
       "      <td>-0.854049</td>\n",
       "      <td>-0.691054</td>\n",
       "      <td>-0.288026</td>\n",
       "      <td>0.071349</td>\n",
       "      <td>-0.133753</td>\n",
       "      <td>-0.104513</td>\n",
       "      <td>-0.144890</td>\n",
       "      <td>-0.601830</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-0.019183</td>\n",
       "      <td>-0.793329</td>\n",
       "      <td>1.279490</td>\n",
       "      <td>0.547714</td>\n",
       "      <td>0.835539</td>\n",
       "      <td>0.311601</td>\n",
       "      <td>-0.210883</td>\n",
       "      <td>-0.189763</td>\n",
       "      <td>0.034569</td>\n",
       "      <td>-0.389905</td>\n",
       "      <td>-0.530074</td>\n",
       "      <td>-0.394632</td>\n",
       "      <td>-0.208324</td>\n",
       "      <td>-0.223464</td>\n",
       "      <td>-0.533321</td>\n",
       "      <td>1.131503</td>\n",
       "      <td>-0.102268</td>\n",
       "      <td>0.350307</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-0.133537</td>\n",
       "      <td>-0.793329</td>\n",
       "      <td>-0.289820</td>\n",
       "      <td>0.547714</td>\n",
       "      <td>-0.489689</td>\n",
       "      <td>-2.051820</td>\n",
       "      <td>-0.269936</td>\n",
       "      <td>-0.139970</td>\n",
       "      <td>-0.368706</td>\n",
       "      <td>0.247612</td>\n",
       "      <td>-0.320430</td>\n",
       "      <td>-0.527791</td>\n",
       "      <td>-0.249045</td>\n",
       "      <td>-0.296841</td>\n",
       "      <td>-0.251305</td>\n",
       "      <td>-0.212079</td>\n",
       "      <td>-0.089145</td>\n",
       "      <td>-0.365854</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         OC    TBR_5Y       ALR       CPI        DM       ERR        PE  \\\n",
       "0 -0.106573 -0.793329  2.690755  0.547714  4.291879  0.311601 -0.138475   \n",
       "1  0.302482 -0.793329  0.574912  0.547714  0.719831 -0.207973 -0.300231   \n",
       "2 -0.058994 -0.793329  1.452280  0.547714 -0.340210  0.311601 -0.290670   \n",
       "3 -0.019183 -0.793329  1.279490  0.547714  0.835539  0.311601 -0.210883   \n",
       "4 -0.133537 -0.793329 -0.289820  0.547714 -0.489689 -2.051820 -0.269936   \n",
       "\n",
       "         PS        PB      FATI     MVTTA        TQ      CINT       EVM  \\\n",
       "0  0.067215  0.373770 -0.171218 -1.005916 -0.684939  3.305135  0.227988   \n",
       "1 -0.448722 -0.500699 -0.553989 -0.613949 -0.478331 -0.308613 -0.285094   \n",
       "2 -0.550378 -0.766247 -0.527272 -0.854049 -0.691054 -0.288026  0.071349   \n",
       "3 -0.189763  0.034569 -0.389905 -0.530074 -0.394632 -0.208324 -0.223464   \n",
       "4 -0.139970 -0.368706  0.247612 -0.320430 -0.527791 -0.249045 -0.296841   \n",
       "\n",
       "   TURNOVER    BETA_S      EBIT  ST_REVERSAL  \n",
       "0 -0.762898  0.126015  0.864458    -0.574940  \n",
       "1 -0.473442 -0.502147 -0.025659    -0.073353  \n",
       "2 -0.133753 -0.104513 -0.144890    -0.601830  \n",
       "3 -0.533321  1.131503 -0.102268     0.350307  \n",
       "4 -0.251305 -0.212079 -0.089145    -0.365854  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "selected_feature = cfs_feature(cfs='MBOR') # IPCMB MMMB IAMB inter_IAMB IAMBnPC GSMB BAMB MBOR STMB LCMB KIAMB FBEDk\n",
    "# selected_feature = ['ST_REVERSAL']\n",
    "X_train_selected, X_test_selected = X_train[selected_feature], X_test[selected_feature]\n",
    "X_train_selected.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1c8835a7",
   "metadata": {},
   "source": [
    "### 逻辑回归"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9c2952c7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 召回率（Recall）是衡量分类模型对正类（关注的类）样本识别能力的指标。它是在所有实际为正类的样本中，被模型正确识别为正类的样本比例。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "71044dbe",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Recall</th>\n",
       "      <th>F1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.510359</td>\n",
       "      <td>0.507055</td>\n",
       "      <td>0.934881</td>\n",
       "      <td>0.6575</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Accuracy  Precision    Recall      F1\n",
       "0  0.510359   0.507055  0.934881  0.6575"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "lr = LogisticRegression(penalty='l2', C=0.5, solver='sag')\n",
    "lr.fit(X_train_selected, y_train)\n",
    "y_pred = lr.predict(X_test_selected)\n",
    "df_test = pd.DataFrame(y_test)\n",
    "df_test['pred'] = y_pred\n",
    "eval_class(y_test, y_pred) \n",
    "# all 0.505001\t0.506229\t0.624011\t0.558983\n",
    "# cfs-y01 0.507017\t0.505218\t0.937573\t0.656614 \n",
    "# cfs-yc 0.507335\t0.505323\t0.949235\t0.659541\n",
    "# MMMB 0.497586\t0.492696\t0.861426\t0.626858\n",
    "# IPCMB(0.3test) 0.495083\t0.491401\t0.87574\t0.629546\n",
    "# f1 0.396976\t0.375682\t0.301478\t0.334514\n",
    "# pearson 0.396843\t0.375477\t0.301214\t0.33427\n",
    "# mi 0.5128\t0.511796\t0.66971\t0.5802\n",
    "# lr 0.482982\t0.489077\t0.636781\t0.55324\n",
    "\n",
    "# IPCMB 0.507335\t0.505323\t0.949235\t0.659541\n",
    "# MMMB 0.508211\t0.505858\t0.938681\t0.657427\n",
    "# IAMB 0.505319\t0.504271\t0.944011\t0.657382\n",
    "# inter_IAMB 0.502719\t0.502719\t1.0\t0.669079\n",
    "# IAMBnPC 0.501048\t0.501997\t0.941636\t0.654874\n",
    "# GSMB 0.505292\t0.504257\t0.943958\t0.657357\n",
    "# BAMB 0.500809\t0.501879\t0.937467\t0.653762\n",
    "# MBOR 0.510359\t0.507055\t0.934881\t0.6575\n",
    "# STMB 0.505611\t0.504437\t0.942005\t0.657036\n",
    "# LRH 0.505292\t0.504257\t0.943958\t0.657357\n",
    "\n",
    "# ST_REVERSAL 0\t0.547685\t0.574323\t0.387388\t0.462688\n",
    "# ST_REVERSAL 0\t0.547685\t0.574323\t0.387388\t0.462688"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ecb162d7",
   "metadata": {},
   "source": [
    "### KNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "5741e323",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Recall</th>\n",
       "      <th>F1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.507163</td>\n",
       "      <td>0.497473</td>\n",
       "      <td>0.589856</td>\n",
       "      <td>0.53974</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Accuracy  Precision    Recall       F1\n",
       "0  0.507163   0.497473  0.589856  0.53974"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "knn = KNeighborsClassifier()   \n",
    "knn.fit(X_train_selected, y_train) \n",
    "y_pred = knn.predict(X_test_selected)\n",
    "eval_class(y_test, y_pred) \n",
    "# cfs 0.521528\t0.518766\t0.666649\t0.583483\n",
    "# cfs-y01 0.509564\t0.509283\t0.670185\t0.578759\n",
    "# cfs-yc 0.530707\t0.526731\t0.655092\t0.583941\n",
    "# f1  0.489534\t0.494814\t0.735145\t0.5915\n",
    "# pearson 0.489534\t0.494814\t0.735145\t0.5915\n",
    "# mi 0.48407\t0.490963\t0.713879\t0.581799\n",
    "# lr 0.506433\t0.505805\t0.79314\t0.617692"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4a33abc9",
   "metadata": {},
   "source": [
    "### 朴素贝叶斯"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "062e268c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Recall</th>\n",
       "      <th>F1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.514684</td>\n",
       "      <td>0.510178</td>\n",
       "      <td>0.867599</td>\n",
       "      <td>0.642528</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Accuracy  Precision    Recall        F1\n",
       "0  0.514684   0.510178  0.867599  0.642528"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.naive_bayes import GaussianNB\n",
    "clf = GaussianNB()\n",
    "clf.fit(X_train_selected, y_train) \n",
    "y_pred = clf.predict(X_test_selected)\n",
    "eval_class(y_test, y_pred) \n",
    "# cfs 0.515029\t0.510163\t0.886121\t0.647527\n",
    "# cfs-y01 0.510041\t0.507276\t0.884855\t0.644861\n",
    "# cfs-yc 0.510121\t0.507153\t0.905488\t0.650159\n",
    "# f1 0.485608\t0.493917\t0.942639\t0.648197\n",
    "# pearson 0.485608\t0.493917\t0.942639\t0.648197\n",
    "# mi 0.51601\t0.512812\t0.745594\t0.607673\n",
    "# lr 0.475819\t0.488803\t0.931821\t0.641235"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b18cc002",
   "metadata": {},
   "source": [
    "### 决策树"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "71b729ff",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Recall</th>\n",
       "      <th>F1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.551001</td>\n",
       "      <td>0.539723</td>\n",
       "      <td>0.725963</td>\n",
       "      <td>0.619141</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Accuracy  Precision    Recall        F1\n",
       "0  0.551001   0.539723  0.725963  0.619141"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "selected_feature = cfs_feature(cfs='FBEDk') # IPCMB MMMB IAMB inter_IAMB IAMBnPC GSMB BAMB MBOR STMB LCMB KIAMB FBEDk\n",
    "X_train_selected, X_test_selected = X_train[selected_feature], X_test[selected_feature]\n",
    "\n",
    "from sklearn import tree\n",
    "clf = tree.DecisionTreeClassifier()\n",
    "clf.fit(X_train_selected, y_train) # X_train\n",
    "y_pred = clf.predict(X_test_selected) # X_test \n",
    "eval_class(y_test, y_pred) \n",
    "# cfs 0.55835\t0.544824\t0.738259\t0.626961\n",
    "# cfs-y01 0\t0.572914\t0.55842\t0.71905\t0.628636\n",
    "# cfs-yc 0.564452\t0.54902\t0.748232\t0.63333\n",
    "# f1 0.50126\t0.503813\t0.522955\t0.513206\n",
    "# pr 0.496432\t0.499149\t0.49504\t    0.497086\n",
    "# mi 0.532829\t0.534737\t0.544274\t0.539463\n",
    "# lr 0.505929\t0.507494\t0.582533\t0.54243\n",
    "\n",
    "# all 0.503701\t0.506773\t0.477731\t0.491824\n",
    "# MBOR 0.567051\t0.570155\t0.552127\t0.767704\t0.64231\n",
    "# LRH 0.485184\t0.489615\t0.56723\t0.525572\n",
    "# STMB 0.557872\t0.54424\t0.741372\t0.627692\n",
    "# KIAMB 0.479904\t0.485012\t0.559261\t0.519497\n",
    "# FBEDk 0.551001\t0.539723\t0.725963\t0.619141"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a8c1ce6a",
   "metadata": {},
   "source": [
    "### SVM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "031724c9",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.svm import SVC\n",
    "import optuna\n",
    "\n",
    "# def objective(trial):\n",
    "#     # kernel = trial.suggest_categorical('kernel', ['linear','rbf','poly','sigmoid'])\n",
    "#     gamma = trial.suggest_loguniform('gamma',1e-5,1e5)\n",
    "#     C = trial.suggest_loguniform('C',1e-5,1e5)\n",
    "#     epsilon = trial.suggest_loguniform('epsilon',1e-5,1e5)\n",
    "#     model = SVC(kernel='rbf', gamma=gamma, C=C, epsilon=epsilon).fit(x_train, y_train)\n",
    "#     score = cross_val_score(model, x_train, y_train, cv=5, scoring=accuracy_score)\n",
    "#     return score.mean() \n",
    "# study = optuna.create_study(study_name='SVR C gamma epsilon', direction='maximize') # TPESampler is used\n",
    "# # optuna.logging.set_verbosity(optuna.logging.WARNING) # not to print\n",
    "# study.optimize(objective, n_trials=100, n_jobs=-1, gc_after_trial=True)  # number of iterations\n",
    "# # best_kernel = study.best_params['kernel']\n",
    "# best_gamma = study.best_params['gamma']\n",
    "# best_C = study.best_params['C']\n",
    "# best_epsilon = study.best_params['epsilon']\n",
    "\n",
    "# Predict\n",
    "# model = SVC(kernel='rbf', gamma=best_gamma, C=best_C, epsilon=best_epsilon)\n",
    "# model = SVC(kernel='rbf', gamma=0.01, C=100)\n",
    "model = SVC(kernel='linear')\n",
    "model.fit(X_train_selected, y_train)\n",
    "y_pred = model.predict(X_test_selected)\n",
    "eval_class(y_test, y_pred) \n",
    "# all "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "777e0dfb",
   "metadata": {},
   "source": [
    "### GBDT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "16d3b962",
   "metadata": {},
   "outputs": [],
   "source": [
    "# GBDT调参\n",
    "# 设置迭代次数的范围\n",
    "param_test1 = {'n_estimators': range(20, 251, 10)}\n",
    "estimator = GradientBoostingClassifier(learning_rate=0.2, min_samples_split=50, min_samples_leaf=5, max_depth=8,\n",
    "                                       max_features='sqrt', subsample=0.8, random_state=10)\n",
    "gsearch1 = GridSearchCV(estimator, param_grid=param_test1, scoring='roc_auc', n_jobs=4, iid=False, cv=5)\n",
    "gsearch1.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7ae65852",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn import tree \n",
    "from sklearn.ensemble import AdaBoostClassifier\n",
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "\n",
    "import optuna\n",
    "def objective(trial):\n",
    "    param_grid = {\n",
    "        'metric': 'rmse', \n",
    "        'random_state': 48,\n",
    "        'n_estimators': 20000,\n",
    "        'max_depth': trial.suggest_categorical('max_depth', [3, 4, 5]),\n",
    "        'num_leaves': trial.suggest_categorical('num_leaves', [5, 6, 7, 12, 13, 14, 15, 28, 29, 30, 31]),\n",
    "        'learning_rate': trial.suggest_float('learning_rate', 0.01, 0.3),\n",
    "        'reg_alpha': trial.suggest_loguniform('reg_alpha', 1e-3, 10.0),\n",
    "        'reg_lambda': trial.suggest_loguniform('reg_lambda', 1e-3, 10.0),\n",
    "        'colsample_bytree': trial.suggest_categorical('colsample_bytree', [0.6,0.7,0.8,0.9,1.0]),\n",
    "        'subsample': trial.suggest_categorical('subsample', [0.6,0.7,0.8,1.0]),\n",
    "    }\n",
    "    model = GradientBoostingClassifier(**param_grid)\n",
    "    model.fit(X_train_selected, y_train, eval_set=[(X_test_selected, y_test)], early_stopping_rounds=100, verbose=False)\n",
    "    score = cross_val_score(model, X_train_selected, y_train, scoring='roc_auc', n_jobs=4, iid=False, cv=5)\n",
    "                            \n",
    "    return score.mean() \n",
    "study = optuna.create_study(study_name='GBDT Classifier hyperparameters', direction='maximize') # TPESampler is used\n",
    "optuna.logging.set_verbosity(optuna.logging.WARNING) # not to print\n",
    "study.optimize(objective, n_trials=100, n_jobs=-1, gc_after_trial=True)  # number of iterations\n",
    "best_params = study.best_params\n",
    "\n",
    "# Predict\n",
    "gbdt = GradientBoostingClassifier(random_state=110)\n",
    "gbdt.fit(X_train_selected, y_train)\n",
    "y_pred = gbdt.predict(X_test_selected)\n",
    "eval_class(y_pred, y_test)\n",
    "\n",
    "model = lgb.LGBMRegressor(**best_params)\n",
    "model.fit(X_train_selected, y_train, eval_set=[(X_test_selected, y_test)], early_stopping_rounds=100, verbose=False)\n",
    "y_pred = model.predict(X_test_selected)\n",
    "eval_class(y_pred, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f04b7674",
   "metadata": {},
   "source": [
    "#### GRU"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5d878d30",
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.layers import Dense, Activation, Dropout, LSTM, GRU\n",
    "from tensorflow.keras.callbacks import ReduceLROnPlateau, EarlyStopping\n",
    "\n",
    "def GRU_model(trainset_shape):# Build GRU model\n",
    "    model = Sequential()\n",
    "    model.add(GRU(128, input_shape=((1,72)), activation='softmax', return_sequences=True)) # input_shape=((timestep, features))\n",
    "    model.add(Dropout(0.2))\n",
    "    model.add(GRU(64,activation='softmax',return_sequences=True))\n",
    "    model.add(Dropout(0.2))\n",
    "    model.add(GRU(32,activation='softmax',return_sequences=False))\n",
    "    model.add(Dropout(0.2))\n",
    "    model.add(Dense(1,activation='softmax'))\n",
    "    model.compile(loss='categorical_crossentropy', optimizer='adam', metrics = ['accuracy'])\n",
    "    return model\n",
    "\n",
    "Epochs = 100\n",
    "predict_duration = 100\n",
    "\n",
    "train_X =  X_train.values.reshape((X_train.shape[0], 1, X_train.shape[1])) # Convert to tensor \n",
    "test_X = X_test.values.reshape((X_test.shape[0], 1, X_test.shape[1]))# Convert to tensor \n",
    "\n",
    "model = GRU_model(train_X.shape) # Build the model # Use model.summary() to show the model structure\n",
    "patience = Epochs//10 \n",
    "EarlyStop = EarlyStopping(monitor='val_loss', patience=5*patience, verbose=0, mode='auto') # Early stop at small learning rate\n",
    "Reduce = ReduceLROnPlateau(monitor='val_loss', patience=patience, verbose=0, mode='auto') # Adaptive learning rate\n",
    "history = model.fit(train_X, y_train, epochs=Epochs, batch_size=16, validation_split=0.1, verbose=1, shuffle=True, callbacks=[EarlyStop,Reduce]) # Train the model\n",
    "\n",
    "y_pred = model.predict(X_test) # Predict\n",
    "eval_class(y_test, y_pred) # Evaluate model"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "881abf6a",
   "metadata": {},
   "source": [
    "## 预测"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "33f4bfc6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['TBR_5Y', 'TBR_10Y', 'ADY', 'APE', 'RFR', 'TBR_1Y', 'LT_REVERSAL',\n",
       "       'CPI', 'ATURNOVER', 'MOM_6M'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "selected_feature"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "d4c5d56d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((339251, 72), (37695, 72))"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 划分测试集\n",
    "df_all = df_raw.iloc[:,2:] # 删除编号\n",
    "from sklearn.model_selection import train_test_split\n",
    "X_train, X_test, y_train, y_test = train_test_split(df_all.iloc[:,1:], df_all.E_RET, test_size=0.1, random_state=0, shuffle=False)\n",
    "\n",
    "# 归一化\n",
    "from sklearn.preprocessing import MinMaxScaler # 为保证特征选择时，数据的一致性，进行归一化（针对方差选择等）\n",
    "scalarX = MinMaxScaler(feature_range=(0,1)) \n",
    "X_train[X_train.columns] = scalarX.fit_transform(X_train)\n",
    "X_test[X_test.columns] = scalarX.transform(X_test)\n",
    "scalarY = MinMaxScaler(feature_range=(0,1)) \n",
    "y_train, y_test = y_train.to_frame(), y_test.to_frame()\n",
    "y_train[y_train.columns] = scalarY.fit_transform(y_train)\n",
    "y_test[y_test.columns] = scalarY.transform(y_test)\n",
    "X_train.shape, X_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "6326c9be",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>TBR_5Y</th>\n",
       "      <th>RG</th>\n",
       "      <th>APE</th>\n",
       "      <th>LT_REVERSAL</th>\n",
       "      <th>ADY</th>\n",
       "      <th>TURNOVER</th>\n",
       "      <th>RFR</th>\n",
       "      <th>TBR_10Y</th>\n",
       "      <th>CINT</th>\n",
       "      <th>ROA</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>151011</th>\n",
       "      <td>0.534817</td>\n",
       "      <td>0.023690</td>\n",
       "      <td>0.130552</td>\n",
       "      <td>0.284517</td>\n",
       "      <td>0.577787</td>\n",
       "      <td>0.041633</td>\n",
       "      <td>0.757655</td>\n",
       "      <td>0.293234</td>\n",
       "      <td>0.001122</td>\n",
       "      <td>0.483300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>183939</th>\n",
       "      <td>0.293179</td>\n",
       "      <td>0.031255</td>\n",
       "      <td>0.158738</td>\n",
       "      <td>0.404748</td>\n",
       "      <td>0.569578</td>\n",
       "      <td>0.111403</td>\n",
       "      <td>0.342511</td>\n",
       "      <td>0.072934</td>\n",
       "      <td>0.010355</td>\n",
       "      <td>0.499544</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52013</th>\n",
       "      <td>0.250478</td>\n",
       "      <td>0.028793</td>\n",
       "      <td>0.486768</td>\n",
       "      <td>0.393565</td>\n",
       "      <td>0.361647</td>\n",
       "      <td>0.052976</td>\n",
       "      <td>0.035129</td>\n",
       "      <td>0.250473</td>\n",
       "      <td>0.002883</td>\n",
       "      <td>0.486869</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>230655</th>\n",
       "      <td>0.617935</td>\n",
       "      <td>0.024450</td>\n",
       "      <td>0.115392</td>\n",
       "      <td>0.273371</td>\n",
       "      <td>0.622685</td>\n",
       "      <td>0.046000</td>\n",
       "      <td>0.600522</td>\n",
       "      <td>0.383359</td>\n",
       "      <td>0.009035</td>\n",
       "      <td>0.469495</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>274364</th>\n",
       "      <td>0.458366</td>\n",
       "      <td>0.025456</td>\n",
       "      <td>0.090385</td>\n",
       "      <td>0.369823</td>\n",
       "      <td>0.725650</td>\n",
       "      <td>0.035918</td>\n",
       "      <td>0.324235</td>\n",
       "      <td>0.215093</td>\n",
       "      <td>0.001214</td>\n",
       "      <td>0.613144</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          TBR_5Y        RG       APE  LT_REVERSAL       ADY  TURNOVER  \\\n",
       "151011  0.534817  0.023690  0.130552     0.284517  0.577787  0.041633   \n",
       "183939  0.293179  0.031255  0.158738     0.404748  0.569578  0.111403   \n",
       "52013   0.250478  0.028793  0.486768     0.393565  0.361647  0.052976   \n",
       "230655  0.617935  0.024450  0.115392     0.273371  0.622685  0.046000   \n",
       "274364  0.458366  0.025456  0.090385     0.369823  0.725650  0.035918   \n",
       "\n",
       "             RFR   TBR_10Y      CINT       ROA  \n",
       "151011  0.757655  0.293234  0.001122  0.483300  \n",
       "183939  0.342511  0.072934  0.010355  0.499544  \n",
       "52013   0.035129  0.250473  0.002883  0.486869  \n",
       "230655  0.600522  0.383359  0.009035  0.469495  \n",
       "274364  0.324235  0.215093  0.001214  0.613144  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train_selected = X_train[selected_feature]\n",
    "X_test_selected = X_test[selected_feature]\n",
    "X_train_selected.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "bd4611cc",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\Software\\Anaconda\\envs\\GPU\\lib\\site-packages\\tqdm\\auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n",
      "\u001b[32m[I 2024-06-21 22:11:15,282]\u001b[0m A new study created in memory with name: LASSO alpha\u001b[0m\n",
      "c:\\Software\\Anaconda\\envs\\GPU\\lib\\site-packages\\sklearn\\metrics\\_regression.py:483: FutureWarning: 'squared' is deprecated in version 1.4 and will be removed in 1.6. To calculate the root mean squared error, use the function'root_mean_squared_error'.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Scale</th>\n",
       "      <th>R2</th>\n",
       "      <th>RMSE</th>\n",
       "      <th>MAE</th>\n",
       "      <th>MAPE</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.667071</td>\n",
       "      <td>-0.001459</td>\n",
       "      <td>0.030922</td>\n",
       "      <td>0.022084</td>\n",
       "      <td>0.152329</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      Scale        R2      RMSE       MAE      MAPE\n",
       "0  0.667071 -0.001459  0.030922  0.022084  0.152329"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.linear_model import Lasso, LassoCV\n",
    "from sklearn.model_selection import cross_val_score, GridSearchCV\n",
    "import optuna\n",
    "\n",
    "def objective(trial):\n",
    "    alpha = trial.suggest_float('alpha', 0, 1) \n",
    "    model = Lasso(alpha=alpha).fit(X_train, y_train)\n",
    "    score = cross_val_score(model, X_train, y_train, cv=5, scoring='r2')\n",
    "    return score.mean() \n",
    "study = optuna.create_study(study_name='LASSO alpha', direction='maximize') # TPESampler is used\n",
    "optuna.logging.set_verbosity(optuna.logging.WARNING) # not to print\n",
    "study.optimize(objective, n_trials=100, n_jobs=-1, gc_after_trial=True)  # number of iterations\n",
    "best_alpha = study.best_params['alpha']\n",
    "\n",
    "model = Lasso(alpha=best_alpha)\n",
    "model.fit(X_train, y_train)\n",
    "y_pred = model.predict(X_test)\n",
    "eval_result(y_test, y_pred) \n",
    "# all 3.797127\t0.035358\t0.142767\t0.10041\t2.114145\n",
    "# var 3.797127\t-1.441297e-07\t0.145359\t0.102538\t1.84601"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "3506b82d",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\Software\\Anaconda\\envs\\GPU\\lib\\site-packages\\sklearn\\metrics\\_regression.py:483: FutureWarning: 'squared' is deprecated in version 1.4 and will be removed in 1.6. To calculate the root mean squared error, use the function'root_mean_squared_error'.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Scale</th>\n",
       "      <th>R2</th>\n",
       "      <th>RMSE</th>\n",
       "      <th>MAE</th>\n",
       "      <th>MAPE</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.667071</td>\n",
       "      <td>-0.035826</td>\n",
       "      <td>0.031448</td>\n",
       "      <td>0.022623</td>\n",
       "      <td>0.158407</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      Scale        R2      RMSE       MAE      MAPE\n",
       "0  0.667071 -0.035826  0.031448  0.022623  0.158407"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.linear_model import Ridge\n",
    "\n",
    "# 创建岭回归模型实例\n",
    "ridge = Ridge(alpha=1.0)\n",
    "\n",
    "def objective(trial):\n",
    "    alpha = trial.suggest_float('alpha', 0, 1) \n",
    "    model = Lasso(alpha=alpha).fit(X_train, y_train)\n",
    "    score = cross_val_score(model, X_train, y_train, cv=5, scoring='r2')\n",
    "    return score.mean() \n",
    "study = optuna.create_study(study_name='Ridge alpha', direction='maximize') # TPESampler is used\n",
    "optuna.logging.set_verbosity(optuna.logging.WARNING) # not to print\n",
    "study.optimize(objective, n_trials=100, n_jobs=-1, gc_after_trial=True)  # number of iterations\n",
    "best_alpha = study.best_params['alpha']\n",
    "\n",
    "model = Ridge(alpha=best_alpha)\n",
    "model.fit(X_train, y_train)\n",
    "y_pred = model.predict(X_test)\n",
    "eval_result(y_test, y_pred) \n",
    "# all 3.797127\t0.045885\t0.141985\t0.099644\t2.411082\n",
    "# lr 0.796758\t0.045913\t0.029793\t0.020908\t0.143407"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "59ea9e35",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['TBR_5Y', 'RG', 'ADY', 'ATURNOVER', 'CINT', 'ITAX', 'TBR_10Y', 'APE',\n",
       "       'FATI', 'CATI'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "c = pd.DataFrame(model.coef_)\n",
    "c.columns = df_all.columns[1:]\n",
    "c = c.T.sort_values(by=[0])\n",
    "c = c.abs().sort_values(by=[0])[-10:][::-1]\n",
    "a = c.index\n",
    "a"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b9fcbdf6",
   "metadata": {},
   "source": [
    "#### 神经网络环境检测"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "c168061a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/device:GPU:0\n",
      "GPU [PhysicalDevice(name='/physical_device:GPU:0', device_type='GPU')]\n",
      "CUDA True\n"
     ]
    }
   ],
   "source": [
    "# Check tensorflow GPU CUDA\n",
    "import tensorflow as tf\n",
    "print(tf.test.gpu_device_name())\n",
    "print('GPU', tf.config.list_physical_devices('GPU'))\n",
    "print('CUDA', tf.test.is_built_with_cuda())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "54069637",
   "metadata": {},
   "source": [
    "#### GRU"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "707ded0c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/100\n",
      "19083/19083 [==============================] - 107s 5ms/step - loss: 9.0760e-04 - val_loss: 8.5568e-04 - lr: 0.0010\n",
      "Epoch 2/100\n",
      "19083/19083 [==============================] - 98s 5ms/step - loss: 8.5334e-04 - val_loss: 8.3438e-04 - lr: 0.0010\n",
      "Epoch 3/100\n",
      "19083/19083 [==============================] - 167s 9ms/step - loss: 8.3115e-04 - val_loss: 8.0353e-04 - lr: 0.0010\n",
      "Epoch 4/100\n",
      "19083/19083 [==============================] - 232s 12ms/step - loss: 8.0951e-04 - val_loss: 7.7494e-04 - lr: 0.0010\n",
      "Epoch 5/100\n",
      "19083/19083 [==============================] - 205s 11ms/step - loss: 7.8923e-04 - val_loss: 7.5212e-04 - lr: 0.0010\n",
      "Epoch 6/100\n",
      "19083/19083 [==============================] - 145s 8ms/step - loss: 7.6837e-04 - val_loss: 7.4586e-04 - lr: 0.0010\n",
      "Epoch 7/100\n",
      "19083/19083 [==============================] - 146s 8ms/step - loss: 7.5121e-04 - val_loss: 7.1014e-04 - lr: 0.0010\n",
      "Epoch 8/100\n",
      "19083/19083 [==============================] - 146s 8ms/step - loss: 7.3226e-04 - val_loss: 7.0256e-04 - lr: 0.0010\n",
      "Epoch 9/100\n",
      "19083/19083 [==============================] - 146s 8ms/step - loss: 7.1566e-04 - val_loss: 6.6801e-04 - lr: 0.0010\n",
      "Epoch 10/100\n",
      "19083/19083 [==============================] - 234s 12ms/step - loss: 7.0532e-04 - val_loss: 6.6559e-04 - lr: 0.0010\n",
      "Epoch 11/100\n",
      "19083/19083 [==============================] - 174s 9ms/step - loss: 6.9401e-04 - val_loss: 6.4933e-04 - lr: 0.0010\n",
      "Epoch 12/100\n",
      "19083/19083 [==============================] - 173s 9ms/step - loss: 6.8411e-04 - val_loss: 6.4435e-04 - lr: 0.0010\n",
      "Epoch 13/100\n",
      "19083/19083 [==============================] - 714s 37ms/step - loss: 6.7658e-04 - val_loss: 6.3112e-04 - lr: 0.0010\n",
      "Epoch 14/100\n",
      "19083/19083 [==============================] - 445s 23ms/step - loss: 6.6912e-04 - val_loss: 6.1891e-04 - lr: 0.0010\n",
      "Epoch 15/100\n",
      "19083/19083 [==============================] - 704s 37ms/step - loss: 6.5927e-04 - val_loss: 6.1683e-04 - lr: 0.0010\n",
      "Epoch 16/100\n",
      "19083/19083 [==============================] - 106s 6ms/step - loss: 6.5127e-04 - val_loss: 6.2607e-04 - lr: 0.0010\n",
      "Epoch 17/100\n",
      "19083/19083 [==============================] - 106s 6ms/step - loss: 6.4651e-04 - val_loss: 6.0097e-04 - lr: 0.0010\n",
      "Epoch 18/100\n",
      "19083/19083 [==============================] - 106s 6ms/step - loss: 6.4107e-04 - val_loss: 5.9522e-04 - lr: 0.0010\n",
      "Epoch 19/100\n",
      "19083/19083 [==============================] - 106s 6ms/step - loss: 6.3770e-04 - val_loss: 6.0064e-04 - lr: 0.0010\n",
      "Epoch 20/100\n",
      "19083/19083 [==============================] - 106s 6ms/step - loss: 6.3419e-04 - val_loss: 5.9124e-04 - lr: 0.0010\n",
      "Epoch 21/100\n",
      "19083/19083 [==============================] - 135s 7ms/step - loss: 6.3064e-04 - val_loss: 5.8592e-04 - lr: 0.0010\n",
      "Epoch 22/100\n",
      "19083/19083 [==============================] - 148s 8ms/step - loss: 6.1169e-04 - val_loss: 5.7502e-04 - lr: 1.0000e-04\n",
      "Epoch 23/100\n",
      "19083/19083 [==============================] - 147s 8ms/step - loss: 6.0957e-04 - val_loss: 5.7326e-04 - lr: 1.0000e-04\n",
      "Epoch 24/100\n",
      "19083/19083 [==============================] - 148s 8ms/step - loss: 6.0833e-04 - val_loss: 5.7243e-04 - lr: 1.0000e-04\n",
      "Epoch 25/100\n",
      "19083/19083 [==============================] - 118s 6ms/step - loss: 6.0773e-04 - val_loss: 5.7127e-04 - lr: 1.0000e-04\n",
      "Epoch 26/100\n",
      "19083/19083 [==============================] - 98s 5ms/step - loss: 6.0793e-04 - val_loss: 5.7171e-04 - lr: 1.0000e-04\n",
      "Epoch 27/100\n",
      "19083/19083 [==============================] - 98s 5ms/step - loss: 6.0668e-04 - val_loss: 5.6962e-04 - lr: 1.0000e-04\n",
      "Epoch 28/100\n",
      "19083/19083 [==============================] - 98s 5ms/step - loss: 6.0668e-04 - val_loss: 5.6965e-04 - lr: 1.0000e-04\n",
      "Epoch 29/100\n",
      "19083/19083 [==============================] - 98s 5ms/step - loss: 6.0475e-04 - val_loss: 5.7140e-04 - lr: 1.0000e-04\n",
      "Epoch 30/100\n",
      "19083/19083 [==============================] - 99s 5ms/step - loss: 6.0558e-04 - val_loss: 5.6859e-04 - lr: 1.0000e-04\n",
      "Epoch 31/100\n",
      "19083/19083 [==============================] - 97s 5ms/step - loss: 6.0472e-04 - val_loss: 5.6773e-04 - lr: 1.0000e-04\n",
      "Epoch 32/100\n",
      "19083/19083 [==============================] - 97s 5ms/step - loss: 6.0312e-04 - val_loss: 5.6749e-04 - lr: 1.0000e-05\n",
      "Epoch 33/100\n",
      "19083/19083 [==============================] - 97s 5ms/step - loss: 6.0164e-04 - val_loss: 5.6705e-04 - lr: 1.0000e-05\n",
      "Epoch 34/100\n",
      "19083/19083 [==============================] - 96s 5ms/step - loss: 6.0331e-04 - val_loss: 5.6764e-04 - lr: 1.0000e-05\n",
      "Epoch 35/100\n",
      "19083/19083 [==============================] - 96s 5ms/step - loss: 6.0279e-04 - val_loss: 5.6720e-04 - lr: 1.0000e-05\n",
      "Epoch 36/100\n",
      "19083/19083 [==============================] - 97s 5ms/step - loss: 6.0185e-04 - val_loss: 5.6711e-04 - lr: 1.0000e-05\n",
      "Epoch 37/100\n",
      "19083/19083 [==============================] - 100s 5ms/step - loss: 6.0208e-04 - val_loss: 5.6740e-04 - lr: 1.0000e-05\n",
      "Epoch 38/100\n",
      "19083/19083 [==============================] - 102s 5ms/step - loss: 6.0236e-04 - val_loss: 5.6730e-04 - lr: 1.0000e-05\n",
      "Epoch 39/100\n",
      "19083/19083 [==============================] - 101s 5ms/step - loss: 6.0231e-04 - val_loss: 5.6727e-04 - lr: 1.0000e-05\n",
      "Epoch 40/100\n",
      "19083/19083 [==============================] - 100s 5ms/step - loss: 6.0160e-04 - val_loss: 5.6710e-04 - lr: 1.0000e-05\n",
      "Epoch 41/100\n",
      "19083/19083 [==============================] - 100s 5ms/step - loss: 6.0143e-04 - val_loss: 5.6702e-04 - lr: 1.0000e-05\n",
      "Epoch 42/100\n",
      "19083/19083 [==============================] - 101s 5ms/step - loss: 6.0236e-04 - val_loss: 5.6685e-04 - lr: 1.0000e-06\n",
      "Epoch 43/100\n",
      "19083/19083 [==============================] - 168s 9ms/step - loss: 6.0231e-04 - val_loss: 5.6690e-04 - lr: 1.0000e-06\n",
      "Epoch 44/100\n",
      "19083/19083 [==============================] - 148s 8ms/step - loss: 6.0208e-04 - val_loss: 5.6690e-04 - lr: 1.0000e-06\n",
      "Epoch 45/100\n",
      "19083/19083 [==============================] - 113s 6ms/step - loss: 6.0235e-04 - val_loss: 5.6689e-04 - lr: 1.0000e-06\n",
      "Epoch 46/100\n",
      "19083/19083 [==============================] - 113s 6ms/step - loss: 6.0151e-04 - val_loss: 5.6689e-04 - lr: 1.0000e-06\n",
      "Epoch 47/100\n",
      "19083/19083 [==============================] - 114s 6ms/step - loss: 6.0121e-04 - val_loss: 5.6687e-04 - lr: 1.0000e-06\n",
      "Epoch 48/100\n",
      "19083/19083 [==============================] - 110s 6ms/step - loss: 6.0283e-04 - val_loss: 5.6695e-04 - lr: 1.0000e-06\n",
      "Epoch 49/100\n",
      "19083/19083 [==============================] - 109s 6ms/step - loss: 6.0232e-04 - val_loss: 5.6695e-04 - lr: 1.0000e-06\n",
      "Epoch 50/100\n",
      "19083/19083 [==============================] - 109s 6ms/step - loss: 6.0163e-04 - val_loss: 5.6694e-04 - lr: 1.0000e-06\n",
      "Epoch 51/100\n",
      "19083/19083 [==============================] - 110s 6ms/step - loss: 6.0169e-04 - val_loss: 5.6693e-04 - lr: 1.0000e-06\n",
      "Epoch 52/100\n",
      "19083/19083 [==============================] - 109s 6ms/step - loss: 6.0135e-04 - val_loss: 5.6692e-04 - lr: 1.0000e-07\n",
      "Epoch 53/100\n",
      "19083/19083 [==============================] - 106s 6ms/step - loss: 6.0304e-04 - val_loss: 5.6693e-04 - lr: 1.0000e-07\n",
      "Epoch 54/100\n",
      "19083/19083 [==============================] - 100s 5ms/step - loss: 6.0166e-04 - val_loss: 5.6693e-04 - lr: 1.0000e-07\n",
      "Epoch 55/100\n",
      "19083/19083 [==============================] - 100s 5ms/step - loss: 6.0263e-04 - val_loss: 5.6693e-04 - lr: 1.0000e-07\n",
      "Epoch 56/100\n",
      "19083/19083 [==============================] - 112s 6ms/step - loss: 6.0145e-04 - val_loss: 5.6693e-04 - lr: 1.0000e-07\n",
      "Epoch 57/100\n",
      "19083/19083 [==============================] - 123s 6ms/step - loss: 6.0101e-04 - val_loss: 5.6693e-04 - lr: 1.0000e-07\n",
      "Epoch 58/100\n",
      "19083/19083 [==============================] - 115s 6ms/step - loss: 6.0220e-04 - val_loss: 5.6693e-04 - lr: 1.0000e-07\n",
      "Epoch 59/100\n",
      "19083/19083 [==============================] - 102s 5ms/step - loss: 6.0184e-04 - val_loss: 5.6693e-04 - lr: 1.0000e-07\n",
      "Epoch 60/100\n",
      "19083/19083 [==============================] - 105s 5ms/step - loss: 6.0218e-04 - val_loss: 5.6693e-04 - lr: 1.0000e-07\n",
      "Epoch 61/100\n",
      "19083/19083 [==============================] - 104s 5ms/step - loss: 6.0112e-04 - val_loss: 5.6693e-04 - lr: 1.0000e-07\n",
      "Epoch 62/100\n",
      "19083/19083 [==============================] - 104s 5ms/step - loss: 6.0197e-04 - val_loss: 5.6693e-04 - lr: 1.0000e-08\n",
      "Epoch 63/100\n",
      "19083/19083 [==============================] - 104s 5ms/step - loss: 6.0118e-04 - val_loss: 5.6693e-04 - lr: 1.0000e-08\n",
      "Epoch 64/100\n",
      "19083/19083 [==============================] - 104s 5ms/step - loss: 6.0265e-04 - val_loss: 5.6693e-04 - lr: 1.0000e-08\n",
      "Epoch 65/100\n",
      "19083/19083 [==============================] - 105s 5ms/step - loss: 6.0238e-04 - val_loss: 5.6693e-04 - lr: 1.0000e-08\n",
      "Epoch 66/100\n",
      "19083/19083 [==============================] - 104s 5ms/step - loss: 6.0096e-04 - val_loss: 5.6693e-04 - lr: 1.0000e-08\n",
      "Epoch 67/100\n",
      "19083/19083 [==============================] - 102s 5ms/step - loss: 6.0099e-04 - val_loss: 5.6693e-04 - lr: 1.0000e-08\n",
      "Epoch 68/100\n",
      "19083/19083 [==============================] - 102s 5ms/step - loss: 6.0151e-04 - val_loss: 5.6693e-04 - lr: 1.0000e-08\n",
      "Epoch 69/100\n",
      "19083/19083 [==============================] - 101s 5ms/step - loss: 6.0212e-04 - val_loss: 5.6693e-04 - lr: 1.0000e-08\n",
      "Epoch 70/100\n",
      "19083/19083 [==============================] - 102s 5ms/step - loss: 6.0208e-04 - val_loss: 5.6693e-04 - lr: 1.0000e-08\n",
      "Epoch 71/100\n",
      "19083/19083 [==============================] - 102s 5ms/step - loss: 6.0150e-04 - val_loss: 5.6693e-04 - lr: 1.0000e-08\n",
      "Epoch 72/100\n",
      "19083/19083 [==============================] - 100s 5ms/step - loss: 6.0278e-04 - val_loss: 5.6693e-04 - lr: 1.0000e-09\n",
      "Epoch 73/100\n",
      "19083/19083 [==============================] - 104s 5ms/step - loss: 6.0192e-04 - val_loss: 5.6693e-04 - lr: 1.0000e-09\n",
      "Epoch 74/100\n",
      "19083/19083 [==============================] - 103s 5ms/step - loss: 6.0075e-04 - val_loss: 5.6693e-04 - lr: 1.0000e-09\n",
      "Epoch 75/100\n",
      "19083/19083 [==============================] - 102s 5ms/step - loss: 6.0120e-04 - val_loss: 5.6693e-04 - lr: 1.0000e-09\n",
      "Epoch 76/100\n",
      "19083/19083 [==============================] - 104s 5ms/step - loss: 6.0190e-04 - val_loss: 5.6693e-04 - lr: 1.0000e-09\n",
      "Epoch 77/100\n",
      "19083/19083 [==============================] - 118s 6ms/step - loss: 6.0138e-04 - val_loss: 5.6693e-04 - lr: 1.0000e-09\n",
      "Epoch 78/100\n",
      "19083/19083 [==============================] - 143s 7ms/step - loss: 6.0179e-04 - val_loss: 5.6693e-04 - lr: 1.0000e-09\n",
      "Epoch 79/100\n",
      "19083/19083 [==============================] - 150s 8ms/step - loss: 6.0241e-04 - val_loss: 5.6693e-04 - lr: 1.0000e-09\n",
      "Epoch 80/100\n",
      "19083/19083 [==============================] - 145s 8ms/step - loss: 6.0142e-04 - val_loss: 5.6693e-04 - lr: 1.0000e-09\n",
      "Epoch 81/100\n",
      "19083/19083 [==============================] - 143s 8ms/step - loss: 6.0054e-04 - val_loss: 5.6693e-04 - lr: 1.0000e-09\n",
      "Epoch 82/100\n",
      "19083/19083 [==============================] - 147s 8ms/step - loss: 6.0210e-04 - val_loss: 5.6693e-04 - lr: 1.0000e-10\n",
      "Epoch 83/100\n",
      "19083/19083 [==============================] - 133s 7ms/step - loss: 6.0140e-04 - val_loss: 5.6693e-04 - lr: 1.0000e-10\n",
      "Epoch 84/100\n",
      "19083/19083 [==============================] - 130s 7ms/step - loss: 6.0168e-04 - val_loss: 5.6693e-04 - lr: 1.0000e-10\n",
      "Epoch 85/100\n",
      "19083/19083 [==============================] - 137s 7ms/step - loss: 6.0053e-04 - val_loss: 5.6693e-04 - lr: 1.0000e-10\n",
      "Epoch 86/100\n",
      "19083/19083 [==============================] - 144s 8ms/step - loss: 6.0151e-04 - val_loss: 5.6693e-04 - lr: 1.0000e-10\n",
      "Epoch 87/100\n",
      "19083/19083 [==============================] - 145s 8ms/step - loss: 6.0183e-04 - val_loss: 5.6693e-04 - lr: 1.0000e-10\n",
      "Epoch 88/100\n",
      "19083/19083 [==============================] - 139s 7ms/step - loss: 6.0118e-04 - val_loss: 5.6693e-04 - lr: 1.0000e-10\n",
      "Epoch 89/100\n",
      "19083/19083 [==============================] - 137s 7ms/step - loss: 6.0106e-04 - val_loss: 5.6693e-04 - lr: 1.0000e-10\n",
      "Epoch 90/100\n",
      "19083/19083 [==============================] - 129s 7ms/step - loss: 6.0196e-04 - val_loss: 5.6693e-04 - lr: 1.0000e-10\n",
      "Epoch 91/100\n",
      "19083/19083 [==============================] - 126s 7ms/step - loss: 6.0097e-04 - val_loss: 5.6693e-04 - lr: 1.0000e-10\n",
      "Epoch 92/100\n",
      "19083/19083 [==============================] - 113s 6ms/step - loss: 6.0251e-04 - val_loss: 5.6693e-04 - lr: 1.0000e-11\n"
     ]
    },
    {
     "ename": "ValueError",
     "evalue": "in user code:\n\n    File \"c:\\Software\\Anaconda\\envs\\GPU\\lib\\site-packages\\keras\\engine\\training.py\", line 2041, in predict_function  *\n        return step_function(self, iterator)\n    File \"c:\\Software\\Anaconda\\envs\\GPU\\lib\\site-packages\\keras\\engine\\training.py\", line 2027, in step_function  **\n        outputs = model.distribute_strategy.run(run_step, args=(data,))\n    File \"c:\\Software\\Anaconda\\envs\\GPU\\lib\\site-packages\\keras\\engine\\training.py\", line 2015, in run_step  **\n        outputs = model.predict_step(data)\n    File \"c:\\Software\\Anaconda\\envs\\GPU\\lib\\site-packages\\keras\\engine\\training.py\", line 1983, in predict_step\n        return self(x, training=False)\n    File \"c:\\Software\\Anaconda\\envs\\GPU\\lib\\site-packages\\keras\\utils\\traceback_utils.py\", line 70, in error_handler\n        raise e.with_traceback(filtered_tb) from None\n    File \"c:\\Software\\Anaconda\\envs\\GPU\\lib\\site-packages\\keras\\engine\\input_spec.py\", line 295, in assert_input_compatibility\n        raise ValueError(\n\n    ValueError: Input 0 of layer \"sequential\" is incompatible with the layer: expected shape=(None, 1, 72), found shape=(None, 72)\n",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[16], line 30\u001b[0m\n\u001b[0;32m     27\u001b[0m Reduce \u001b[38;5;241m=\u001b[39m ReduceLROnPlateau(monitor\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mval_loss\u001b[39m\u001b[38;5;124m'\u001b[39m, patience\u001b[38;5;241m=\u001b[39mpatience, verbose\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m0\u001b[39m, mode\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mauto\u001b[39m\u001b[38;5;124m'\u001b[39m) \u001b[38;5;66;03m# Adaptive learning rate\u001b[39;00m\n\u001b[0;32m     28\u001b[0m history \u001b[38;5;241m=\u001b[39m model\u001b[38;5;241m.\u001b[39mfit(train_X, y_train, epochs\u001b[38;5;241m=\u001b[39mEpochs, batch_size\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m16\u001b[39m, validation_split\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m0.1\u001b[39m, verbose\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m1\u001b[39m, shuffle\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m, callbacks\u001b[38;5;241m=\u001b[39m[EarlyStop,Reduce]) \u001b[38;5;66;03m# Train the model\u001b[39;00m\n\u001b[1;32m---> 30\u001b[0m y_pred \u001b[38;5;241m=\u001b[39m \u001b[43mmodel\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mpredict\u001b[49m\u001b[43m(\u001b[49m\u001b[43mX_test\u001b[49m\u001b[43m)\u001b[49m \u001b[38;5;66;03m# Predict\u001b[39;00m\n\u001b[0;32m     31\u001b[0m eval_result(y_test, y_pred) \u001b[38;5;66;03m# Evaluate model\u001b[39;00m\n",
      "File \u001b[1;32mc:\\Software\\Anaconda\\envs\\GPU\\lib\\site-packages\\keras\\utils\\traceback_utils.py:70\u001b[0m, in \u001b[0;36mfilter_traceback.<locals>.error_handler\u001b[1;34m(*args, **kwargs)\u001b[0m\n\u001b[0;32m     67\u001b[0m     filtered_tb \u001b[38;5;241m=\u001b[39m _process_traceback_frames(e\u001b[38;5;241m.\u001b[39m__traceback__)\n\u001b[0;32m     68\u001b[0m     \u001b[38;5;66;03m# To get the full stack trace, call:\u001b[39;00m\n\u001b[0;32m     69\u001b[0m     \u001b[38;5;66;03m# `tf.debugging.disable_traceback_filtering()`\u001b[39;00m\n\u001b[1;32m---> 70\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m e\u001b[38;5;241m.\u001b[39mwith_traceback(filtered_tb) \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[0;32m     71\u001b[0m \u001b[38;5;28;01mfinally\u001b[39;00m:\n\u001b[0;32m     72\u001b[0m     \u001b[38;5;28;01mdel\u001b[39;00m filtered_tb\n",
      "File \u001b[1;32m~\\AppData\\Local\\Temp\\__autograph_generated_file5m09gh97.py:15\u001b[0m, in \u001b[0;36mouter_factory.<locals>.inner_factory.<locals>.tf__predict_function\u001b[1;34m(iterator)\u001b[0m\n\u001b[0;32m     13\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m     14\u001b[0m     do_return \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mTrue\u001b[39;00m\n\u001b[1;32m---> 15\u001b[0m     retval_ \u001b[38;5;241m=\u001b[39m ag__\u001b[38;5;241m.\u001b[39mconverted_call(ag__\u001b[38;5;241m.\u001b[39mld(step_function), (ag__\u001b[38;5;241m.\u001b[39mld(\u001b[38;5;28mself\u001b[39m), ag__\u001b[38;5;241m.\u001b[39mld(iterator)), \u001b[38;5;28;01mNone\u001b[39;00m, fscope)\n\u001b[0;32m     16\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m:\n\u001b[0;32m     17\u001b[0m     do_return \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mFalse\u001b[39;00m\n",
      "\u001b[1;31mValueError\u001b[0m: in user code:\n\n    File \"c:\\Software\\Anaconda\\envs\\GPU\\lib\\site-packages\\keras\\engine\\training.py\", line 2041, in predict_function  *\n        return step_function(self, iterator)\n    File \"c:\\Software\\Anaconda\\envs\\GPU\\lib\\site-packages\\keras\\engine\\training.py\", line 2027, in step_function  **\n        outputs = model.distribute_strategy.run(run_step, args=(data,))\n    File \"c:\\Software\\Anaconda\\envs\\GPU\\lib\\site-packages\\keras\\engine\\training.py\", line 2015, in run_step  **\n        outputs = model.predict_step(data)\n    File \"c:\\Software\\Anaconda\\envs\\GPU\\lib\\site-packages\\keras\\engine\\training.py\", line 1983, in predict_step\n        return self(x, training=False)\n    File \"c:\\Software\\Anaconda\\envs\\GPU\\lib\\site-packages\\keras\\utils\\traceback_utils.py\", line 70, in error_handler\n        raise e.with_traceback(filtered_tb) from None\n    File \"c:\\Software\\Anaconda\\envs\\GPU\\lib\\site-packages\\keras\\engine\\input_spec.py\", line 295, in assert_input_compatibility\n        raise ValueError(\n\n    ValueError: Input 0 of layer \"sequential\" is incompatible with the layer: expected shape=(None, 1, 72), found shape=(None, 72)\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.layers import Dense, Activation, Dropout, LSTM, GRU\n",
    "from tensorflow.keras.callbacks import ReduceLROnPlateau, EarlyStopping\n",
    "\n",
    "def GRU_model(trainset_shape):# Build GRU model\n",
    "    model = Sequential()\n",
    "    model.add(GRU(128, input_shape=((1,72)), activation='tanh', return_sequences=True)) # input_shape=((timestep, features))\n",
    "    model.add(Dropout(0.2))\n",
    "    model.add(GRU(64,activation='tanh',return_sequences=True))\n",
    "    model.add(Dropout(0.2))\n",
    "    model.add(GRU(32,activation='tanh',return_sequences=False))\n",
    "    model.add(Dropout(0.2))\n",
    "    model.add(Dense(1,activation='tanh'))\n",
    "    model.compile(loss='mse', optimizer='adam')\n",
    "    return model\n",
    "\n",
    "Epochs = 100\n",
    "predict_duration = 100\n",
    "\n",
    "train_X =  X_train.values.reshape((X_train.shape[0], 1, X_train.shape[1])) # Convert to tensor \n",
    "test_X = X_test.values.reshape((X_test.shape[0], 1, X_test.shape[1]))# Convert to tensor \n",
    "\n",
    "model = GRU_model(train_X.shape) # Build the model # Use model.summary() to show the model structure\n",
    "patience = Epochs//10 \n",
    "EarlyStop = EarlyStopping(monitor='val_loss', patience=5*patience, verbose=0, mode='auto') # Early stop at small learning rate\n",
    "Reduce = ReduceLROnPlateau(monitor='val_loss', patience=patience, verbose=0, mode='auto') # Adaptive learning rate\n",
    "history = model.fit(train_X, y_train, epochs=Epochs, batch_size=16, validation_split=0.1, verbose=1, shuffle=True, callbacks=[EarlyStop,Reduce]) # Train the model\n",
    "\n",
    "y_pred = model.predict(X_test) # Predict\n",
    "eval_result(y_test, y_pred) # Evaluate model"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.19"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}