audio-emotion-detection-using-autoencoders.ipynb

{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":256618,"sourceType":"datasetVersion","datasetId":107620}],"dockerImageVersionId":30213,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport re\nimport warnings\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport librosa\nimport librosa.display\nfrom sklearn.preprocessing import minmax_scale\nimport IPython.display as ipd\n\nplt.rcParams['figure.figsize'] = (20,8)\nplt.rcParams['font.size'] = 16\nsns.set_style('darkgrid')\nwarnings.filterwarnings(\"ignore\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-10-25T06:28:18.068096Z","iopub.execute_input":"2022-10-25T06:28:18.068597Z","iopub.status.idle":"2022-10-25T06:28:31.302353Z","shell.execute_reply.started":"2022-10-25T06:28:18.068487Z","shell.execute_reply":"2022-10-25T06:28:31.30132Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"markdown","source":"# Data Preparation\n- Create a dataframe containing the meta data\n- Create a list of all combinations given in the specification\n- Filter out the data using these combinations\n- Assign classes to each record (happy or sad)\n- Assign gender class to each record so that it can be leveraged during analysis","metadata":{}},{"cell_type":"code","source":"records = []\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        records.append([filename, os.path.join(dirname,filename)])\n\ndata = pd.DataFrame(records, columns=['filename','path'])\ndata['actor'] = data['path'].apply(lambda x: re.findall(\"\\w+_\\d+\",x)[0])\ndata = data[data['actor']!=\"audio_speech_actors_01\"]\ndata.reset_index(inplace=True,drop=True)\ndata['type'] = data['filename'].apply(lambda x: re.split(\"-\\d+\\.wav\",x)[0])\ndata","metadata":{"execution":{"iopub.status.busy":"2022-10-25T06:28:31.304188Z","iopub.execute_input":"2022-10-25T06:28:31.304511Z","iopub.status.idle":"2022-10-25T06:28:32.134395Z","shell.execute_reply.started":"2022-10-25T06:28:31.304479Z","shell.execute_reply":"2022-10-25T06:28:32.133217Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Number of Unique Speakers/Actors","metadata":{}},{"cell_type":"code","source":"data['actor'].nunique()","metadata":{"execution":{"iopub.status.busy":"2022-10-25T06:28:32.135639Z","iopub.execute_input":"2022-10-25T06:28:32.136025Z","iopub.status.idle":"2022-10-25T06:28:32.145772Z","shell.execute_reply.started":"2022-10-25T06:28:32.13599Z","shell.execute_reply":"2022-10-25T06:28:32.145024Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Number of Audio files for each of the Speakers/Actors","metadata":{}},{"cell_type":"code","source":"data['actor'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-10-25T06:28:32.147895Z","iopub.execute_input":"2022-10-25T06:28:32.148513Z","iopub.status.idle":"2022-10-25T06:28:32.164061Z","shell.execute_reply.started":"2022-10-25T06:28:32.14848Z","shell.execute_reply":"2022-10-25T06:28:32.163012Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"filter_list = []\nfor a in [\"03\",\"04\"]:\n    for b in [\"01\",\"02\"]:\n        for c in [\"01\",\"02\"]:\n            for d in [\"01\",\"02\"]:\n                filter_list.append(f\"03-01-{a}-{b}-{c}-{d}\")\n                \ndata = data[data['type'].isin(filter_list)]\ndata.reset_index(inplace=True,drop=True)\ndata['class'] = data['type'].apply(lambda x: 'happy' if x.startswith('03-01-03') else 'sad')\ndata['gender'] = data['actor'].apply(lambda x: 'female' if int(x.split('_')[1])%2==0 else 'male')","metadata":{"execution":{"iopub.status.busy":"2022-10-25T06:28:32.165574Z","iopub.execute_input":"2022-10-25T06:28:32.166252Z","iopub.status.idle":"2022-10-25T06:28:32.182904Z","shell.execute_reply.started":"2022-10-25T06:28:32.166218Z","shell.execute_reply":"2022-10-25T06:28:32.181984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data","metadata":{"execution":{"iopub.status.busy":"2022-10-25T06:28:32.18424Z","iopub.execute_input":"2022-10-25T06:28:32.185189Z","iopub.status.idle":"2022-10-25T06:28:32.205445Z","shell.execute_reply.started":"2022-10-25T06:28:32.185155Z","shell.execute_reply":"2022-10-25T06:28:32.204042Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Number of Audio Files for Gender and Emotion Class","metadata":{}},{"cell_type":"code","source":"data['gender'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-10-25T06:28:32.20687Z","iopub.execute_input":"2022-10-25T06:28:32.207705Z","iopub.status.idle":"2022-10-25T06:28:32.216413Z","shell.execute_reply.started":"2022-10-25T06:28:32.207672Z","shell.execute_reply":"2022-10-25T06:28:32.215059Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data['class'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-10-25T06:28:32.220103Z","iopub.execute_input":"2022-10-25T06:28:32.220722Z","iopub.status.idle":"2022-10-25T06:28:32.235034Z","shell.execute_reply.started":"2022-10-25T06:28:32.220684Z","shell.execute_reply":"2022-10-25T06:28:32.233749Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Feature Extraction \n- The acoustic features of an audio can be extracted using different methods, but in this kernel only 2 methods will be used\n1. MFCCs\n2. Mel Spectrograms\n\n<img src='https://images.deepai.org/converted-papers/2005.12779/x3.png'>","metadata":{}},{"cell_type":"code","source":"from tqdm import tqdm\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.model_selection import train_test_split, cross_val_score\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.metrics import classification_report, confusion_matrix","metadata":{"execution":{"iopub.status.busy":"2022-10-25T06:28:32.23673Z","iopub.execute_input":"2022-10-25T06:28:32.238481Z","iopub.status.idle":"2022-10-25T06:28:32.24828Z","shell.execute_reply.started":"2022-10-25T06:28:32.238441Z","shell.execute_reply":"2022-10-25T06:28:32.247376Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def feature_extraction(df, mfcc=True):\n    features = []\n    for i,record in tqdm(df.iterrows(),total=df.shape[0]):\n        x , sr = librosa.load(record['path'])\n        mean_mfcc = np.mean(librosa.feature.mfcc(y=x, sr=sr, n_mfcc=128),axis=1)\n        mean_ms = np.mean(librosa.feature.melspectrogram(y=x, sr=sr, n_mels=128),axis=1)\n        features.append(mean_mfcc if mfcc else mean_ms)\n        \n    dataf = pd.DataFrame(features)\n    dataf['class'] = df['class']\n    return dataf","metadata":{"execution":{"iopub.status.busy":"2022-10-25T06:28:32.251842Z","iopub.execute_input":"2022-10-25T06:28:32.252186Z","iopub.status.idle":"2022-10-25T06:28:32.260817Z","shell.execute_reply.started":"2022-10-25T06:28:32.252156Z","shell.execute_reply":"2022-10-25T06:28:32.259485Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\n\nencoder = LabelEncoder()","metadata":{"execution":{"iopub.status.busy":"2022-10-25T06:28:32.262374Z","iopub.execute_input":"2022-10-25T06:28:32.262715Z","iopub.status.idle":"2022-10-25T06:28:32.2873Z","shell.execute_reply.started":"2022-10-25T06:28:32.262686Z","shell.execute_reply":"2022-10-25T06:28:32.28543Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## MFCC Features","metadata":{}},{"cell_type":"code","source":"dataf = feature_extraction(data)\ndataf","metadata":{"execution":{"iopub.status.busy":"2022-10-25T06:28:32.289119Z","iopub.execute_input":"2022-10-25T06:28:32.289897Z","iopub.status.idle":"2022-10-25T06:30:31.932652Z","shell.execute_reply.started":"2022-10-25T06:28:32.289847Z","shell.execute_reply":"2022-10-25T06:30:31.931011Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X1 = dataf.iloc[:,:-1].values\ny1 = dataf.iloc[:,-1].values \ny1 = encoder.fit_transform(y1)","metadata":{"execution":{"iopub.status.busy":"2022-10-25T06:30:31.934556Z","iopub.execute_input":"2022-10-25T06:30:31.935252Z","iopub.status.idle":"2022-10-25T06:30:31.942246Z","shell.execute_reply.started":"2022-10-25T06:30:31.935214Z","shell.execute_reply":"2022-10-25T06:30:31.940917Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Mel Spectrogram Features","metadata":{}},{"cell_type":"code","source":"dataf = feature_extraction(data,mfcc=False)\ndataf","metadata":{"execution":{"iopub.status.busy":"2022-10-25T06:30:31.943912Z","iopub.execute_input":"2022-10-25T06:30:31.944258Z","iopub.status.idle":"2022-10-25T06:32:28.350709Z","shell.execute_reply.started":"2022-10-25T06:30:31.944229Z","shell.execute_reply":"2022-10-25T06:32:28.34927Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X2 = dataf.iloc[:,:-1].values\ny2 = dataf.iloc[:,-1].values\ny2 = encoder.fit_transform(y2)","metadata":{"execution":{"iopub.status.busy":"2022-10-25T06:32:28.352735Z","iopub.execute_input":"2022-10-25T06:32:28.353979Z","iopub.status.idle":"2022-10-25T06:32:28.360835Z","shell.execute_reply.started":"2022-10-25T06:32:28.35388Z","shell.execute_reply":"2022-10-25T06:32:28.359528Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Modelling\n- The following models will be used for building the models for emotion detection\n1. Logistic Regression on MFCC and Mel Spec Features\n2. CNNs on MFCC and Mel Spec Features\n3. Autoencoders and Variational Autoencoders on whichever features are giving better performance in the above models","metadata":{}},{"cell_type":"markdown","source":"## Logistic Regression\n\n<img src='https://miro.medium.com/max/1400/1*Ba7LqnrsRnhjJyJl5LPW6Q.gif'>","metadata":{}},{"cell_type":"code","source":"def LogisticRegressionPipeline(X,y):\n    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, stratify=y)\n    pipeline = Pipeline([('scaler',StandardScaler()),('LogisticRegression',LogisticRegression())])\n    pipeline.fit(X_train,y_train)\n    y_train_pred = pipeline.predict(X_train)\n    y_pred = pipeline.predict(X_test)\n    \n    cmatrix = confusion_matrix(y_test,y_pred)\n    \n    print(\"Training Performance\")\n    print(classification_report(y_train,y_train_pred))\n    print(\"-----------------------------------------\")\n    print(\"Test Performance\")\n    print(classification_report(y_test,y_pred))\n    print(\"-----------------------------------------\")\n    \n    cv_score = cross_val_score(pipeline,X,y,cv=5)\n    average = lambda x: sum(x)/len(x)\n    print(\"5-Folds Scores: \", cv_score)\n    print(\"-----------------------------------------\")\n    print(\"5-Folds Average Score: \",average(cv_score))\n    \n    return cv_score, cmatrix","metadata":{"execution":{"iopub.status.busy":"2022-10-25T06:32:28.362384Z","iopub.execute_input":"2022-10-25T06:32:28.362936Z","iopub.status.idle":"2022-10-25T06:32:28.374598Z","shell.execute_reply.started":"2022-10-25T06:32:28.3629Z","shell.execute_reply":"2022-10-25T06:32:28.373403Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Logistic Regression: MFCCs","metadata":{}},{"cell_type":"code","source":"scores, cmatrix = LogisticRegressionPipeline(X1,y1)","metadata":{"execution":{"iopub.status.busy":"2022-10-25T06:32:28.378792Z","iopub.execute_input":"2022-10-25T06:32:28.379214Z","iopub.status.idle":"2022-10-25T06:32:28.763267Z","shell.execute_reply.started":"2022-10-25T06:32:28.37918Z","shell.execute_reply":"2022-10-25T06:32:28.761333Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.heatmap(cmatrix,annot=True,fmt='.3g',cmap='viridis',xticklabels=['happy','sad'],yticklabels=['happy','sad'])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-10-25T06:32:28.76609Z","iopub.execute_input":"2022-10-25T06:32:28.767384Z","iopub.status.idle":"2022-10-25T06:32:29.138872Z","shell.execute_reply.started":"2022-10-25T06:32:28.767318Z","shell.execute_reply":"2022-10-25T06:32:29.137678Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Logistic Regression: Mel Spectrogram","metadata":{}},{"cell_type":"code","source":"scores, cmatrix = LogisticRegressionPipeline(X2,y2)","metadata":{"execution":{"iopub.status.busy":"2022-10-25T06:32:29.140281Z","iopub.execute_input":"2022-10-25T06:32:29.140639Z","iopub.status.idle":"2022-10-25T06:32:29.452188Z","shell.execute_reply.started":"2022-10-25T06:32:29.140607Z","shell.execute_reply":"2022-10-25T06:32:29.450537Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.heatmap(cmatrix,annot=True,fmt='.3g',cmap='viridis',xticklabels=['happy','sad'],yticklabels=['happy','sad'])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-10-25T06:32:29.459849Z","iopub.execute_input":"2022-10-25T06:32:29.461122Z","iopub.status.idle":"2022-10-25T06:32:29.821134Z","shell.execute_reply.started":"2022-10-25T06:32:29.46105Z","shell.execute_reply":"2022-10-25T06:32:29.819952Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## CNNs (Convolutional Neural Networks)\n\n<img src='https://miro.medium.com/max/1400/1*ciDgQEjViWLnCbmX-EeSrA.gif'>","metadata":{}},{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow.keras.models import Sequential, Model\nfrom tensorflow.keras.layers import Input, Conv2D, MaxPooling2D, Dense, Flatten, InputLayer, UpSampling2D, Layer, Reshape\nfrom tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint","metadata":{"execution":{"iopub.status.busy":"2022-10-25T06:32:29.822698Z","iopub.execute_input":"2022-10-25T06:32:29.824052Z","iopub.status.idle":"2022-10-25T06:32:36.077773Z","shell.execute_reply.started":"2022-10-25T06:32:29.824002Z","shell.execute_reply":"2022-10-25T06:32:36.076388Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## CNNs: MFCCs","metadata":{}},{"cell_type":"code","source":"X_train, X_test, y_train, y_test = train_test_split(X1, y1, test_size=0.2, stratify=y1)\nX_train = X_train.reshape(-1,16,8,1)\nX_test = X_test.reshape(-1,16,8,1)","metadata":{"execution":{"iopub.status.busy":"2022-10-25T06:32:36.079892Z","iopub.execute_input":"2022-10-25T06:32:36.081371Z","iopub.status.idle":"2022-10-25T06:32:36.088992Z","shell.execute_reply.started":"2022-10-25T06:32:36.08132Z","shell.execute_reply":"2022-10-25T06:32:36.087492Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = Sequential([\n    InputLayer(input_shape=(16, 8, 1)),\n    Conv2D(filters=32, kernel_size=(3, 3), activation='relu', padding = \"same\"),\n    MaxPooling2D(2, 2),\n    Conv2D(filters=64, kernel_size=(3, 3), activation='relu', padding = \"same\"),\n    MaxPooling2D(2, 2),\n    Flatten(),\n    Dense(32, activation='relu'),\n    Dense(1, activation='sigmoid')\n])\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2022-10-25T06:32:36.090877Z","iopub.execute_input":"2022-10-25T06:32:36.091297Z","iopub.status.idle":"2022-10-25T06:32:36.274656Z","shell.execute_reply.started":"2022-10-25T06:32:36.091262Z","shell.execute_reply":"2022-10-25T06:32:36.273434Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])","metadata":{"execution":{"iopub.status.busy":"2022-10-25T06:32:36.275991Z","iopub.execute_input":"2022-10-25T06:32:36.276314Z","iopub.status.idle":"2022-10-25T06:32:36.291719Z","shell.execute_reply.started":"2022-10-25T06:32:36.276285Z","shell.execute_reply":"2022-10-25T06:32:36.290455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_name = \"model.h5\"\ncheckpoint = ModelCheckpoint(model_name,\n                            monitor=\"val_loss\",\n                            mode=\"min\",\n                            save_best_only = True,\n                            verbose=1)\n\nearlystopping = EarlyStopping(monitor='val_loss',min_delta = 0, patience = 5, verbose = 1, restore_best_weights=True)","metadata":{"execution":{"iopub.status.busy":"2022-10-25T06:32:36.294243Z","iopub.execute_input":"2022-10-25T06:32:36.294697Z","iopub.status.idle":"2022-10-25T06:32:36.301535Z","shell.execute_reply.started":"2022-10-25T06:32:36.294652Z","shell.execute_reply":"2022-10-25T06:32:36.300297Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = model.fit(X_train, y_train, epochs=50, validation_data=(X_test,y_test), callbacks=[checkpoint,earlystopping])","metadata":{"execution":{"iopub.status.busy":"2022-10-25T06:32:36.303468Z","iopub.execute_input":"2022-10-25T06:32:36.303836Z","iopub.status.idle":"2022-10-25T06:32:39.091927Z","shell.execute_reply.started":"2022-10-25T06:32:36.303803Z","shell.execute_reply":"2022-10-25T06:32:39.089952Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot(history.history['loss'])\nplt.plot(history.history['val_loss'])\nplt.title('Model Loss')\nplt.ylabel('loss')\nplt.xlabel('epoch')\nplt.legend(['train', 'val'], loc='upper left')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-10-25T06:32:39.093559Z","iopub.execute_input":"2022-10-25T06:32:39.093913Z","iopub.status.idle":"2022-10-25T06:32:39.427871Z","shell.execute_reply.started":"2022-10-25T06:32:39.093881Z","shell.execute_reply":"2022-10-25T06:32:39.426972Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = model.predict(X_test)\ny_pred[y_pred>=0.5] = 1\ny_pred[y_pred<0.5] = 0","metadata":{"execution":{"iopub.status.busy":"2022-10-25T06:32:39.434086Z","iopub.execute_input":"2022-10-25T06:32:39.435277Z","iopub.status.idle":"2022-10-25T06:32:39.606886Z","shell.execute_reply.started":"2022-10-25T06:32:39.43522Z","shell.execute_reply":"2022-10-25T06:32:39.606014Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(classification_report(y_test,y_pred))\ncmatrix = confusion_matrix(y_test,y_pred)\nsns.heatmap(cmatrix,annot=True,fmt='.3g',cmap='viridis',xticklabels=['happy','sad'],yticklabels=['happy','sad'])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-10-25T06:32:39.608411Z","iopub.execute_input":"2022-10-25T06:32:39.609539Z","iopub.status.idle":"2022-10-25T06:32:39.896047Z","shell.execute_reply.started":"2022-10-25T06:32:39.609498Z","shell.execute_reply":"2022-10-25T06:32:39.895192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## CNNs: Mel Spectrogram","metadata":{}},{"cell_type":"code","source":"X_train, X_test, y_train, y_test = train_test_split(X2, y2, test_size=0.2, stratify=y2)\nX_train = X_train.reshape(-1,16,8,1)\nX_test = X_test.reshape(-1,16,8,1)","metadata":{"execution":{"iopub.status.busy":"2022-10-25T06:32:39.897198Z","iopub.execute_input":"2022-10-25T06:32:39.898239Z","iopub.status.idle":"2022-10-25T06:32:39.905652Z","shell.execute_reply.started":"2022-10-25T06:32:39.898203Z","shell.execute_reply":"2022-10-25T06:32:39.904741Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = Sequential([\n    InputLayer(input_shape=(16, 8, 1)),\n    Conv2D(filters=32, kernel_size=(3, 3), activation='relu', padding = \"same\"),\n    MaxPooling2D(2, 2),\n    Conv2D(filters=64, kernel_size=(3, 3), activation='relu', padding = \"same\"),\n    MaxPooling2D(2, 2),\n    Flatten(),\n    Dense(32, activation='relu'),\n    Dense(1, activation='sigmoid')\n])\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2022-10-25T06:32:39.907496Z","iopub.execute_input":"2022-10-25T06:32:39.907945Z","iopub.status.idle":"2022-10-25T06:32:40.217942Z","shell.execute_reply.started":"2022-10-25T06:32:39.907901Z","shell.execute_reply":"2022-10-25T06:32:40.21662Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])\nmodel_name = \"model_mel.h5\"\ncheckpoint = ModelCheckpoint(model_name,\n                            monitor=\"val_loss\",\n                            mode=\"min\",\n                            save_best_only = True,\n                            verbose=1)\n\nearlystopping = EarlyStopping(monitor='val_loss',min_delta = 0, patience = 5, verbose = 1, restore_best_weights=True)","metadata":{"execution":{"iopub.status.busy":"2022-10-25T06:32:40.220037Z","iopub.execute_input":"2022-10-25T06:32:40.220481Z","iopub.status.idle":"2022-10-25T06:32:40.234267Z","shell.execute_reply.started":"2022-10-25T06:32:40.220436Z","shell.execute_reply":"2022-10-25T06:32:40.233138Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = model.fit(X_train, y_train, epochs=50, validation_data=(X_test,y_test), callbacks=[checkpoint,earlystopping])","metadata":{"execution":{"iopub.status.busy":"2022-10-25T06:32:40.235585Z","iopub.execute_input":"2022-10-25T06:32:40.235995Z","iopub.status.idle":"2022-10-25T06:32:43.440202Z","shell.execute_reply.started":"2022-10-25T06:32:40.235904Z","shell.execute_reply":"2022-10-25T06:32:43.439022Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot(history.history['loss'])\nplt.plot(history.history['val_loss'])\nplt.title('Model Loss')\nplt.ylabel('loss')\nplt.xlabel('epoch')\nplt.ylim([0,2])\nplt.legend(['train', 'val'], loc='upper left')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-10-25T06:32:43.441667Z","iopub.execute_input":"2022-10-25T06:32:43.442029Z","iopub.status.idle":"2022-10-25T06:32:43.7781Z","shell.execute_reply.started":"2022-10-25T06:32:43.441995Z","shell.execute_reply":"2022-10-25T06:32:43.776802Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(classification_report(y_test,y_pred))\ncmatrix = confusion_matrix(y_test,y_pred)\nsns.heatmap(cmatrix,annot=True,fmt='.3g',cmap='viridis',xticklabels=['happy','sad'],yticklabels=['happy','sad'])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-10-25T06:32:43.779448Z","iopub.execute_input":"2022-10-25T06:32:43.779806Z","iopub.status.idle":"2022-10-25T06:32:44.089015Z","shell.execute_reply.started":"2022-10-25T06:32:43.779774Z","shell.execute_reply":"2022-10-25T06:32:44.087702Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Inference:** MFCC Features give better performance","metadata":{}},{"cell_type":"markdown","source":"## Autoencoders\n<img src='https://miro.medium.com/max/1400/1*44eDEuZBEsmG_TCAKRI3Kw@2x.png'>","metadata":{}},{"cell_type":"code","source":"dataf = feature_extraction(data)\ndataf","metadata":{"execution":{"iopub.status.busy":"2022-10-25T06:32:44.090339Z","iopub.execute_input":"2022-10-25T06:32:44.090682Z","iopub.status.idle":"2022-10-25T06:34:51.62767Z","shell.execute_reply.started":"2022-10-25T06:32:44.09065Z","shell.execute_reply":"2022-10-25T06:34:51.626192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.preprocessing import MinMaxScaler\n\nscaler = MinMaxScaler()","metadata":{"execution":{"iopub.status.busy":"2022-10-25T06:34:51.630051Z","iopub.execute_input":"2022-10-25T06:34:51.631333Z","iopub.status.idle":"2022-10-25T06:34:51.637442Z","shell.execute_reply.started":"2022-10-25T06:34:51.631269Z","shell.execute_reply":"2022-10-25T06:34:51.63621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_happy = dataf[dataf['class']=='happy'].iloc[:,:-1].values \nX_sad = dataf[dataf['class']=='sad'].iloc[:,:-1].values \n\nX_happy = scaler.fit_transform(X_happy)\nX_sad = scaler.transform(X_sad)\n\nX_happy = X_happy.reshape(-1,16,8,1)\nX_sad = X_sad.reshape(-1,16,8,1)","metadata":{"execution":{"iopub.status.busy":"2022-10-25T06:34:51.638839Z","iopub.execute_input":"2022-10-25T06:34:51.639804Z","iopub.status.idle":"2022-10-25T06:34:51.65546Z","shell.execute_reply.started":"2022-10-25T06:34:51.639757Z","shell.execute_reply":"2022-10-25T06:34:51.654169Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = Sequential([\n    Input(shape=(16,8,1)),\n    Conv2D(filters=32, kernel_size=(3,3), activation='relu', padding='same'),\n    MaxPooling2D(2,2),\n    Conv2D(filters=64, kernel_size=(3,3), activation='relu', padding='same'),\n    MaxPooling2D(2,2),\n    Conv2D(filters=64, kernel_size=(3,3), activation='relu', padding='same'),\n    UpSampling2D((2,2)),\n    Conv2D(filters=32, kernel_size=(3,3), activation='relu', padding='same'),\n    UpSampling2D((2,2)),\n    Conv2D(filters=1, kernel_size=(3,3), activation='sigmoid', padding='same')\n])\n\nmodel.compile(loss='mean_squared_error',optimizer='adam')","metadata":{"execution":{"iopub.status.busy":"2022-10-25T06:34:51.657842Z","iopub.execute_input":"2022-10-25T06:34:51.658311Z","iopub.status.idle":"2022-10-25T06:34:51.751336Z","shell.execute_reply.started":"2022-10-25T06:34:51.658275Z","shell.execute_reply":"2022-10-25T06:34:51.750129Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.summary()","metadata":{"execution":{"iopub.status.busy":"2022-10-25T06:34:51.752946Z","iopub.execute_input":"2022-10-25T06:34:51.753463Z","iopub.status.idle":"2022-10-25T06:34:51.761444Z","shell.execute_reply.started":"2022-10-25T06:34:51.753392Z","shell.execute_reply":"2022-10-25T06:34:51.760153Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = model.fit(X_happy,X_happy,epochs=100)","metadata":{"execution":{"iopub.status.busy":"2022-10-25T06:34:51.76323Z","iopub.execute_input":"2022-10-25T06:34:51.763571Z","iopub.status.idle":"2022-10-25T06:34:59.301773Z","shell.execute_reply.started":"2022-10-25T06:34:51.763542Z","shell.execute_reply":"2022-10-25T06:34:59.300868Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot(history.history['loss'])\nplt.title('Model Loss')\nplt.ylabel('loss')\nplt.xlabel('epoch')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-10-25T06:34:59.303343Z","iopub.execute_input":"2022-10-25T06:34:59.303662Z","iopub.status.idle":"2022-10-25T06:34:59.60654Z","shell.execute_reply.started":"2022-10-25T06:34:59.303633Z","shell.execute_reply":"2022-10-25T06:34:59.605516Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.evaluate(X_happy,X_happy)","metadata":{"execution":{"iopub.status.busy":"2022-10-25T06:34:59.607987Z","iopub.execute_input":"2022-10-25T06:34:59.608768Z","iopub.status.idle":"2022-10-25T06:34:59.827479Z","shell.execute_reply.started":"2022-10-25T06:34:59.608728Z","shell.execute_reply":"2022-10-25T06:34:59.826311Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.evaluate(X_sad,X_sad)","metadata":{"execution":{"iopub.status.busy":"2022-10-25T06:34:59.82924Z","iopub.execute_input":"2022-10-25T06:34:59.829761Z","iopub.status.idle":"2022-10-25T06:34:59.923693Z","shell.execute_reply.started":"2022-10-25T06:34:59.829726Z","shell.execute_reply":"2022-10-25T06:34:59.922646Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Inference:** There is not a significant difference between the loss for happy audios and sad audios","metadata":{}},{"cell_type":"markdown","source":"## Variational Autoencoders\n<img src='https://www.jeremyjordan.me/content/images/2018/03/Screen-Shot-2018-03-18-at-12.24.19-AM.png'>","metadata":{}},{"cell_type":"code","source":"class Sampling(Layer):\n    def call(self, inputs):\n        z_mean, z_log_var = inputs\n        batch = tf.shape(z_mean)[0]\n        dim = tf.shape(z_mean)[1]\n        epsilon = tf.keras.backend.random_normal(shape=(batch, dim))\n        return z_mean + tf.exp(0.5 * z_log_var) * epsilon","metadata":{"execution":{"iopub.status.busy":"2022-10-25T06:34:59.925005Z","iopub.execute_input":"2022-10-25T06:34:59.92534Z","iopub.status.idle":"2022-10-25T06:34:59.933257Z","shell.execute_reply.started":"2022-10-25T06:34:59.925311Z","shell.execute_reply":"2022-10-25T06:34:59.932047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"input_layer = Input(shape=(16,8,1))\nx = Conv2D(filters=32, kernel_size=(3,3), activation='relu', padding='same')(input_layer)\nx = MaxPooling2D(2,2)(x)\nx = Conv2D(filters=64, kernel_size=(3,3), activation='relu', padding='same')(x)\nx = MaxPooling2D(2,2)(x)\nx = Flatten()(x)\nz_mean = Dense(128, name=\"z_mean\")(x)\nz_log_var = Dense(128, name=\"z_log_var\")(x)\nz = Sampling()([z_mean, z_log_var])\nx = Dense(4*2*64, activation=\"relu\")(z)\nx = Reshape((4, 2, 64))(x)\nx = UpSampling2D()(x)\nx = Conv2D(filters=64, kernel_size=(3,3), activation='relu', padding='same')(x)\nx = UpSampling2D()(x)\nx = Conv2D(filters=32, kernel_size=(3,3), activation='relu', padding='same')(x)\noutput = Conv2D(filters=1, kernel_size=(3,3), activation='sigmoid', padding='same')(x)\n\nmodel = Model(inputs=input_layer,outputs=output)\nmodel.compile(loss='mean_squared_error',optimizer='adam')\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2022-10-25T06:34:59.934842Z","iopub.execute_input":"2022-10-25T06:34:59.935394Z","iopub.status.idle":"2022-10-25T06:35:00.147842Z","shell.execute_reply.started":"2022-10-25T06:34:59.935358Z","shell.execute_reply":"2022-10-25T06:35:00.146642Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = model.fit(X_happy,X_happy,epochs=100)","metadata":{"execution":{"iopub.status.busy":"2022-10-25T06:35:00.14938Z","iopub.execute_input":"2022-10-25T06:35:00.149827Z","iopub.status.idle":"2022-10-25T06:35:12.928053Z","shell.execute_reply.started":"2022-10-25T06:35:00.149793Z","shell.execute_reply":"2022-10-25T06:35:12.927023Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot(history.history['loss'])\nplt.title('Model Loss')\nplt.ylabel('loss')\nplt.xlabel('epoch')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-10-25T06:35:12.930012Z","iopub.execute_input":"2022-10-25T06:35:12.930354Z","iopub.status.idle":"2022-10-25T06:35:13.245799Z","shell.execute_reply.started":"2022-10-25T06:35:12.930323Z","shell.execute_reply":"2022-10-25T06:35:13.244609Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.evaluate(X_happy,X_happy)","metadata":{"execution":{"iopub.status.busy":"2022-10-25T06:35:13.247371Z","iopub.execute_input":"2022-10-25T06:35:13.247733Z","iopub.status.idle":"2022-10-25T06:35:13.537382Z","shell.execute_reply.started":"2022-10-25T06:35:13.247701Z","shell.execute_reply":"2022-10-25T06:35:13.536337Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.evaluate(X_sad,X_sad)","metadata":{"execution":{"iopub.status.busy":"2022-10-25T06:35:13.539169Z","iopub.execute_input":"2022-10-25T06:35:13.539927Z","iopub.status.idle":"2022-10-25T06:35:13.649612Z","shell.execute_reply.started":"2022-10-25T06:35:13.539882Z","shell.execute_reply":"2022-10-25T06:35:13.648769Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Inference:** There is a significant difference between the loss of happy and sad audios","metadata":{}},{"cell_type":"code","source":"X_sc = np.vstack([X_happy,X_sad])\ny = np.hstack([np.ones((192,)),np.zeros((192,))])","metadata":{"execution":{"iopub.status.busy":"2022-10-25T06:35:13.650704Z","iopub.execute_input":"2022-10-25T06:35:13.651504Z","iopub.status.idle":"2022-10-25T06:35:13.656133Z","shell.execute_reply.started":"2022-10-25T06:35:13.651469Z","shell.execute_reply":"2022-10-25T06:35:13.655268Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"var_encoder = Model(inputs=model.input,outputs=model.layers[6].output)\nvar_encoder.summary()","metadata":{"execution":{"iopub.status.busy":"2022-10-25T06:35:13.657369Z","iopub.execute_input":"2022-10-25T06:35:13.657856Z","iopub.status.idle":"2022-10-25T06:35:13.675155Z","shell.execute_reply.started":"2022-10-25T06:35:13.657826Z","shell.execute_reply":"2022-10-25T06:35:13.673746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"z_mean = var_encoder.predict(X_sc)","metadata":{"execution":{"iopub.status.busy":"2022-10-25T06:35:13.676794Z","iopub.execute_input":"2022-10-25T06:35:13.677637Z","iopub.status.idle":"2022-10-25T06:35:13.825631Z","shell.execute_reply.started":"2022-10-25T06:35:13.677589Z","shell.execute_reply":"2022-10-25T06:35:13.824356Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.scatterplot(z_mean[:, 0], z_mean[:, 1], hue=y, s=80)\nplt.title(\"Variational Autoencoder Latent Space Representation in 2D\")\nplt.xlabel(\"z[0]\")\nplt.ylabel(\"z[1]\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-10-25T06:59:42.940416Z","iopub.execute_input":"2022-10-25T06:59:42.940796Z","iopub.status.idle":"2022-10-25T06:59:43.347671Z","shell.execute_reply.started":"2022-10-25T06:59:42.940766Z","shell.execute_reply":"2022-10-25T06:59:43.346429Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<div class='alert alert-success'><strong>Conclusion:</strong>\n    <li><strong>Logistic Regression</strong> model performs well with the MFCC Features with a decent classifcation accuracy and F1-score</li>\n    <li><strong>CNNs</strong> perform well on the MFCC Features as well, therefore for autoencoders only MFCCs are taken as features</li>\n    <li><strong>Variational Autoencoders</strong> perform a better job in terms of reconstruction loss than the regular autoencoders as they are able to model the distribution of happy audios in <strong>128-Dimensional latent space</strong></li>\n</div>","metadata":{}}]}