deliverable3.py

# -*- coding: utf-8 -*-
"""Deliverable3.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1uKoZ9BgWMzM_TishayBeFQbY9qUfIjds
"""

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import csv
import random
import string
import re

from sklearn import svm
from sklearn import tree
from sklearn import metrics
from sklearn.svm import LinearSVC
from sklearn.metrics import mean_absolute_error
from sklearn.neural_network import MLPRegressor
from sklearn.preprocessing import StandardScaler
from sklearn.linear_model import LinearRegression
from sklearn.ensemble import RandomForestRegressor
from sklearn.model_selection import train_test_split

from keras.callbacks import ModelCheckpoint
from keras.models import Sequential
from keras.layers import Dense, Activation, Flatten

import seaborn as sb
import warnings
warnings.filterwarnings('ignore')
warnings.filterwarnings('ignore', category=DeprecationWarning)
from xgboost import XGBRegressor

#READ IN THE APPROPRIATE DATA AS YEAR SPECIFIC DATAFRAMES
datapath_2014 = "https://raw.githubusercontent.com/omarw99/MAIS202Project_StockPredictor/master/Dataset/2014_Financial_Data.csv"
datapath_2015 = "https://raw.githubusercontent.com/omarw99/MAIS202Project_StockPredictor/master/Dataset/2015_Financial_Data.csv"
datapath_2016 = "https://raw.githubusercontent.com/omarw99/MAIS202Project_StockPredictor/master/Dataset/2016_Financial_Data.csv"
datapath_2017 = "https://raw.githubusercontent.com/omarw99/MAIS202Project_StockPredictor/master/Dataset/2017_Financial_Data.csv"
datapath_2018 = "https://raw.githubusercontent.com/omarw99/MAIS202Project_StockPredictor/master/Dataset/2018_Financial_Data.csv"

df_2014 = pd.read_csv(datapath_2014)
df_2015 = pd.read_csv(datapath_2015)
df_2016 = pd.read_csv(datapath_2016)
df_2017 = pd.read_csv(datapath_2017)
df_2018 = pd.read_csv(datapath_2018)

#EXTRACT THE RETURN COLUMNS AS SERIES FROM THE YEAR SPECIFIC DATAFFRAMES
returns_2014 = df_2014['2015 PRICE VAR [%]']
returns_2015 = df_2015['2016 PRICE VAR [%]']
returns_2016 = df_2016['2017 PRICE VAR [%]']
returns_2017 = df_2017['2018 PRICE VAR [%]']
returns_2018 = df_2018['2019 PRICE VAR [%]']

returns_2014 = returns_2014.rename('Next Year Stock Return')
returns_2015 = returns_2015.rename('Next Year Stock Return')
returns_2016 = returns_2016.rename('Next Year Stock Return')
returns_2017 = returns_2017.rename('Next Year Stock Return')
returns_2018 = returns_2018.rename('Next Year Stock Return')

#IDENTIFY THE COLUMNS IN THE YEAR SPECIFIC DATAFRAMES TO KEEP AS THE ONES WITH RELEVANT FINANCIAL RATIOS
columnsToKeep = ['Revenue Growth', 'EPS', 'Dividend per Share', 'EBITDA Margin', 'Net Profit Margin', 'priceToSalesRatio', 'priceToFreeCashFlowsRatio', 
                 'dividendYield', 'grossProfitMargin', 'returnOnEquity', 'currentRatio', 'quickRatio', 'cashRatio', 'debtRatio', 'debtEquityRatio', 
                 'interestCoverage', 'PE ratio', 'Receivables Turnover', 'Payables Turnover', 'Inventory Turnover', 'Sector']

df_2014 = df_2014[columnsToKeep]
df_2015 = df_2015[columnsToKeep]
df_2016 = df_2016[columnsToKeep]
df_2017 = df_2017[columnsToKeep]
df_2018 = df_2018[columnsToKeep]

#ADD THE YEARLY RETURNS SERIES BACK TO THE YEAR SPECIFIC DATAFRAMES
df_2014 = pd.concat([df_2014, returns_2014], axis = 1)
df_2015 = pd.concat([df_2015, returns_2015], axis = 1)
df_2016 = pd.concat([df_2016, returns_2016], axis = 1)
df_2017 = pd.concat([df_2017, returns_2017], axis = 1)
df_2018 = pd.concat([df_2018, returns_2018], axis = 1)

#COMBINE ALL THE YEAR SPECIFIC DATAFRAMES INTO ONE INITIAL_COMBINED_DF, DROPNA, AND RESET INDEX
individual_dataframes = [df_2014, df_2015, df_2016, df_2017, df_2018]
initial_combined_df = pd.concat(individual_dataframes)
initial_combined_df = initial_combined_df.dropna()
initial_combined_df = initial_combined_df.reset_index(drop = True)

#ONE HOT ENCODE THE SECTOR COLUMN
combined_df = pd.get_dummies(initial_combined_df, columns = ['Sector'])

print(combined_df)

def remove_outlier(df_in, col_name):
    q1 = df_in[col_name].quantile(0.25)
    q3 = df_in[col_name].quantile(0.75)
    iqr = q3-q1 #Interquartile range
    fence_low  = q1-1.5*iqr
    fence_high = q3+1.5*iqr
    df_out = df_in.loc[(df_in[col_name] > fence_low) & (df_in[col_name] < fence_high)]
    return df_out

std = combined_df.describe().iloc[2]
sortedSTD = std.sort_values(ascending=False)

x = 0
for std in sortedSTD:
  if std > 1000:
    x += 1

print(x)

colNamesWithHighestSTD = sortedSTD.index
for i in range(x):
  combined_df = remove_outlier(combined_df, colNamesWithHighestSTD[i])

combined_df = combined_df.reset_index(drop = True)
print(combined_df)

#SPLIT THE COMBINED_DF INTO TRAIN AND TEST SETS, THEN RESET THEIR INDICES
train, test = train_test_split(combined_df, test_size=0.2)
train.reset_index(inplace = True, drop = True)
test.reset_index(inplace = True, drop = True)

print(train)
print(test)

#CREATE THE X_TRAIN, X_TEST, Y_TRAIN, Y_TEST ARRAYS
y_train = np.asarray(train['Next Year Stock Return'])
X_train = np.asarray(train.drop(columns = ['Next Year Stock Return']))

y_test = np.asarray(test['Next Year Stock Return'])
X_test = np.asarray(test.drop(columns = ['Next Year Stock Return']))

print(X_train.shape)
print(y_train.shape)
print(X_test.shape)
print(y_test.shape)

"""TESTING MODELS

Linear Regression
"""

model = LinearRegression()  
model.fit(X_train, y_train)

y_train_predict = model.predict(X_train)

print(y_train.mean())
print(y_train.min())
print(y_train.max())
print()
print(y_train_predict.mean())
print(y_train_predict.min())
print(y_train_predict.max())

plt.scatter(y_train, y_train_predict)
plt.show()

#MSE OF THE TRAINING SET
mse_train = np.mean(np.square(np.subtract(y_train, y_train_predict)))
print("Training set Mean Squared Error: {}".format(mse_train))

y_test_predict = model.predict(X_test)

print(y_test.mean())
print(y_test.min())
print(y_test.max())
print()
print(y_test_predict.mean())
print(y_test_predict.min())
print(y_test_predict.max())

plt.scatter(y_test, y_test_predict)
plt.show()

#MSE OF THE TESTING SET
mse_test = np.mean(np.square(np.subtract(y_test, y_test_predict)))
print("Testing set Mean Squared Error: {}".format(mse_test))

"""Random Forest Regression"""

model = RandomForestRegressor(random_state = 0)
model.fit(X_train, y_train)

y_train_predict = model.predict(X_train)

print(y_train.mean())
print(y_train.min())
print(y_train.max())
print()
print(y_train_predict.mean())
print(y_train_predict.min())
print(y_train_predict.max())

plt.scatter(y_train, y_train_predict)
plt.show()

#MSE OF THE TRAINING SET
mse_train = np.mean(np.square(np.subtract(y_train, y_train_predict)))
print("Training set Mean Squared Error: {}".format(mse_train))

y_test_predict = model.predict(X_test)

print(y_test.mean())
print(y_test.min())
print(y_test.max())
print()
print(y_test_predict.mean())
print(y_test_predict.min())
print(y_test_predict.max())

plt.scatter(y_test, y_test_predict)
plt.show()

#MSE OF THE TESTING SET
mse_test = np.mean(np.square(np.subtract(y_test, y_test_predict)))
print("Testing set Mean Squared Error: {}".format(mse_test))

"""SVM"""

#Call the SVR model and fit using X_train and y_train
model = svm.SVR()
model.fit(X_train, y_train)

y_train_predict = model.predict(X_train)

print(y_train.mean())
print(y_train.min())
print(y_train.max())
print()
print(y_train_predict.mean())
print(y_train_predict.min())
print(y_train_predict.max())

plt.scatter(y_train, y_train_predict)
plt.show()

#MSE OF THE TRAINING SET
mse_train = np.mean(np.square(np.subtract(y_train, y_train_predict)))
print("Training set Mean Squared Error: {}".format(mse_train))

y_test_predict = model.predict(X_test)

print(y_test.mean())
print(y_test.min())
print(y_test.max())
print()
print(y_test_predict.mean())
print(y_test_predict.min())
print(y_test_predict.max())

plt.scatter(y_test, y_test_predict)
plt.show()

#MSE OF THE TESTING SET
mse_test = np.mean(np.square(np.subtract(y_test, y_test_predict)))
print("Testing set Mean Squared Error: {}".format(mse_test))

"""Decision Tree"""

#Call the Decision Tree model and fit using X_train and y_train
model = tree.DecisionTreeRegressor()
model.fit(X_train, y_train)

y_train_predict = model.predict(X_train)

print(y_train.mean())
print(y_train.min())
print(y_train.max())
print()
print(y_train_predict.mean())
print(y_train_predict.min())
print(y_train_predict.max())

plt.scatter(y_train, y_train_predict)
plt.show()

#MSE OF THE TRAINING SET
mse_train = np.mean(np.square(np.subtract(y_train, y_train_predict)))
print("Training set Mean Squared Error: {}".format(mse_train))

y_test_predict = model.predict(X_test)

print(y_test.mean())
print(y_test.min())
print(y_test.max())
print()
print(y_test_predict.mean())
print(y_test_predict.min())
print(y_test_predict.max())

plt.scatter(y_test, y_test_predict)
plt.show()

#MSE OF THE TESTING SET
mse_test = np.mean(np.square(np.subtract(y_test, y_test_predict)))
print("Testing set Mean Squared Error: {}".format(mse_test))

"""MLP"""

#Call the MLP model and fit using X_train and y_train
model = MLPRegressor()
model.fit(X_train, y_train)

y_train_predict = model.predict(X_train)

print(y_train.mean())
print(y_train.min())
print(y_train.max())
print()
print(y_train_predict.mean())
print(y_train_predict.min())
print(y_train_predict.max())

plt.scatter(y_train, y_train_predict)
plt.show()

#MSE OF THE TRAINING SET
mse_train = np.mean(np.square(np.subtract(y_train, y_train_predict)))
print("Training set Mean Squared Error: {}".format(mse_train))

y_test_predict = model.predict(X_test)

print(y_test.mean())
print(y_test.min())
print(y_test.max())
print()
print(y_test_predict.mean())
print(y_test_predict.min())
print(y_test_predict.max())

plt.scatter(y_test, y_test_predict)
plt.show()

#MSE OF THE TESTING SET
mse_test = np.mean(np.square(np.subtract(y_test, y_test_predict)))
print("Testing set Mean Squared Error: {}".format(mse_test))

"""CNN"""

model = Sequential()

# The Input Layer :
model.add(Dense(4096, kernel_initializer='normal',input_dim = X_train.shape[1], activation='relu'))

# The Hidden Layers :
'''
NN_model.add(Dense(2048, kernel_initializer='normal',activation='relu'))
NN_model.add(Dense(1024, kernel_initializer='normal',activation='relu'))
'''
model.add(Dense(512, kernel_initializer='normal',activation='relu'))
model.add(Dense(256, kernel_initializer='normal',activation='relu'))

# The Output Layer :
model.add(Dense(1, kernel_initializer='normal',activation='linear'))

# Compile the network :
model.compile(loss='mean_absolute_error', optimizer='adam', metrics=['mean_absolute_error'])
#NN_model.compile(optimizer='adam', loss='mean_absolute_error', metrics=['accuracy'])

model.summary()

checkpoint_name = 'Weights-{epoch:03d}--{val_loss:.5f}.hdf5' 
checkpoint = ModelCheckpoint(checkpoint_name, monitor='val_loss', verbose = 1, save_best_only = True, mode ='auto')
callbacks_list = [checkpoint]

model.fit(X_train, y_train, epochs=100, validation_split = 0.25, callbacks=callbacks_list)

y_train_predict = model.predict(X_train)

print(y_train.mean())
print(y_train.min())
print(y_train.max())
print()
print(y_train_predict.mean())
print(y_train_predict.min())
print(y_train_predict.max())

plt.scatter(y_train, y_train_predict)
plt.show()

#MSE OF THE TRAINING SET
mse_train = np.mean(np.square(np.subtract(y_train, y_train_predict)))
print("Training set Mean Squared Error: {}".format(mse_train))

y_test_predict = model.predict(X_test)

print(y_test.mean())
print(y_test.min())
print(y_test.max())
print()
print(y_test_predict.mean())
print(y_test_predict.min())
print(y_test_predict.max())

plt.scatter(y_test, y_test_predict)
plt.show()

#MSE OF THE TESTING SET
mse_test = np.mean(np.square(np.subtract(y_test, y_test_predict)))
print("Testing set Mean Squared Error: {}".format(mse_test))