BH_10fold_resp_model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-01-10 08:33:20.157228: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
      "To enable the following instructions: SSE4.1 SSE4.2, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.callbacks import EarlyStopping\n",
    "from tensorflow.keras import optimizers\n",
    "from tensorflow.keras import layers\n",
    "from tensorflow.keras import Input\n",
    "from tensorflow.keras import Model\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.preprocessing import normalize\n",
    "from sklearn.model_selection import KFold\n",
    "\n",
    "from scipy.signal import find_peaks\n",
    "from scipy.stats import mode\n",
    "import numpy as np\n",
    "import pickle\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Num GPUs Available:  0\n"
     ]
    }
   ],
   "source": [
    "print(\"Num GPUs Available: \", len(tf.config.list_physical_devices('GPU')))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load input/label to models\n",
    "# RESPIRATION\n",
    "with open('resp_input.pkl', 'rb') as file:\n",
    "    x = pickle.load(file)\n",
    "    \n",
    "with open('resp_label.pkl', 'rb') as file:\n",
    "    y = pickle.load(file)\n",
    "\n",
    "# CARDIAC\n",
    "# with open('cardiac_input.pkl', 'rb') as file:\n",
    "#     x = pickle.load(file)\n",
    "\n",
    "# with open('cardiac_label.pkl', 'rb') as file:\n",
    "#     y = pickle.load(file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#Nan Labels: 590\n",
      "#Remaining Samples: 3304\n"
     ]
    }
   ],
   "source": [
    "nanmask = np.isnan(y)\n",
    "print(f'#Nan Labels: {np.sum(nanmask)}\\n#Remaining Samples: {np.sum(~nanmask)}')\n",
    "y = y[~nanmask]\n",
    "x = x[~nanmask]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/25\n",
      "1189/1189 [==============================] - 17s 14ms/step - loss: 0.5120 - accuracy: 0.7733 - val_loss: 0.4805 - val_accuracy: 0.7664\n",
      "Epoch 2/25\n",
      "1189/1189 [==============================] - 16s 14ms/step - loss: 0.3850 - accuracy: 0.8272 - val_loss: 0.4558 - val_accuracy: 0.7731\n",
      "Epoch 3/25\n",
      "1189/1189 [==============================] - 16s 14ms/step - loss: 0.3227 - accuracy: 0.8654 - val_loss: 0.4526 - val_accuracy: 0.7849\n",
      "Epoch 4/25\n",
      "1189/1189 [==============================] - 16s 14ms/step - loss: 0.2478 - accuracy: 0.8957 - val_loss: 0.5329 - val_accuracy: 0.7580\n",
      "Epoch 5/25\n",
      "1189/1189 [==============================] - 16s 14ms/step - loss: 0.1902 - accuracy: 0.9188 - val_loss: 0.5782 - val_accuracy: 0.7647\n",
      "Epoch 6/25\n",
      "1189/1189 [==============================] - 16s 14ms/step - loss: 0.1618 - accuracy: 0.9441 - val_loss: 2.8049 - val_accuracy: 0.6958\n",
      "Epoch 7/25\n",
      "1189/1189 [==============================] - 16s 14ms/step - loss: 0.1211 - accuracy: 0.9613 - val_loss: 0.8458 - val_accuracy: 0.7815\n",
      "Epoch 8/25\n",
      "1186/1189 [============================>.] - ETA: 0s - loss: 0.1162 - accuracy: 0.9595Restoring model weights from the end of the best epoch: 3.\n",
      "1189/1189 [==============================] - 16s 14ms/step - loss: 0.1159 - accuracy: 0.9596 - val_loss: 1.2237 - val_accuracy: 0.7765\n",
      "Epoch 8: early stopping\n",
      "331/331 [==============================] - 1s 2ms/step - loss: 0.5817 - accuracy: 0.7432\n",
      "Fold 0 Test Accuracy: 74.32%\n",
      "Epoch 1/25\n",
      "1189/1189 [==============================] - 17s 14ms/step - loss: 0.5278 - accuracy: 0.7553 - val_loss: 0.5233 - val_accuracy: 0.7681\n",
      "Epoch 2/25\n",
      "1189/1189 [==============================] - 18s 16ms/step - loss: 0.3925 - accuracy: 0.8217 - val_loss: 0.5008 - val_accuracy: 0.7815\n",
      "Epoch 3/25\n",
      "1189/1189 [==============================] - 16s 14ms/step - loss: 0.3249 - accuracy: 0.8621 - val_loss: 0.4747 - val_accuracy: 0.7798\n",
      "Epoch 4/25\n",
      "1189/1189 [==============================] - 16s 14ms/step - loss: 0.2870 - accuracy: 0.8835 - val_loss: 0.6494 - val_accuracy: 0.7546\n",
      "Epoch 5/25\n",
      "1189/1189 [==============================] - 16s 14ms/step - loss: 0.2318 - accuracy: 0.9096 - val_loss: 0.7820 - val_accuracy: 0.7731\n",
      "Epoch 6/25\n",
      "1189/1189 [==============================] - 17s 14ms/step - loss: 0.1992 - accuracy: 0.9298 - val_loss: 0.5895 - val_accuracy: 0.7966\n",
      "Epoch 7/25\n",
      "1189/1189 [==============================] - 16s 14ms/step - loss: 0.1482 - accuracy: 0.9474 - val_loss: 0.9288 - val_accuracy: 0.7916\n",
      "Epoch 8/25\n",
      "1188/1189 [============================>.] - ETA: 0s - loss: 0.1357 - accuracy: 0.9541Restoring model weights from the end of the best epoch: 3.\n",
      "1189/1189 [==============================] - 17s 14ms/step - loss: 0.1357 - accuracy: 0.9542 - val_loss: 0.8749 - val_accuracy: 0.7782\n",
      "Epoch 8: early stopping\n",
      "331/331 [==============================] - 1s 2ms/step - loss: 0.4670 - accuracy: 0.7764\n",
      "Fold 1 Test Accuracy: 77.64%\n",
      "Epoch 1/25\n",
      "1189/1189 [==============================] - 17s 14ms/step - loss: 0.5486 - accuracy: 0.7279 - val_loss: 0.5342 - val_accuracy: 0.7479\n",
      "Epoch 2/25\n",
      "1189/1189 [==============================] - 16s 14ms/step - loss: 0.4084 - accuracy: 0.8162 - val_loss: 0.4323 - val_accuracy: 0.8050\n",
      "Epoch 3/25\n",
      "1189/1189 [==============================] - 17s 14ms/step - loss: 0.3286 - accuracy: 0.8629 - val_loss: 0.6905 - val_accuracy: 0.7630\n",
      "Epoch 4/25\n",
      "1189/1189 [==============================] - 17s 14ms/step - loss: 0.2434 - accuracy: 0.8966 - val_loss: 0.5841 - val_accuracy: 0.7748\n",
      "Epoch 5/25\n",
      "1189/1189 [==============================] - 17s 14ms/step - loss: 0.1503 - accuracy: 0.9458 - val_loss: 0.6824 - val_accuracy: 0.7782\n",
      "Epoch 6/25\n",
      "1189/1189 [==============================] - 17s 14ms/step - loss: 0.1319 - accuracy: 0.9529 - val_loss: 0.7027 - val_accuracy: 0.7731\n",
      "Epoch 7/25\n",
      "1187/1189 [============================>.] - ETA: 0s - loss: 0.0804 - accuracy: 0.9751Restoring model weights from the end of the best epoch: 2.\n",
      "1189/1189 [==============================] - 16s 14ms/step - loss: 0.0803 - accuracy: 0.9752 - val_loss: 0.9375 - val_accuracy: 0.7832\n",
      "Epoch 7: early stopping\n",
      "331/331 [==============================] - 1s 2ms/step - loss: 0.2465 - accuracy: 0.9063\n",
      "Fold 2 Test Accuracy: 90.63%\n",
      "Epoch 1/25\n",
      "1189/1189 [==============================] - 17s 14ms/step - loss: 0.5230 - accuracy: 0.7485 - val_loss: 0.4523 - val_accuracy: 0.7630\n",
      "Epoch 2/25\n",
      "1189/1189 [==============================] - 16s 14ms/step - loss: 0.3939 - accuracy: 0.8162 - val_loss: 0.5357 - val_accuracy: 0.7630\n",
      "Epoch 3/25\n",
      "1189/1189 [==============================] - 16s 14ms/step - loss: 0.3376 - accuracy: 0.8612 - val_loss: 0.4928 - val_accuracy: 0.7815\n",
      "Epoch 4/25\n",
      "1189/1189 [==============================] - 17s 14ms/step - loss: 0.2766 - accuracy: 0.8936 - val_loss: 0.5128 - val_accuracy: 0.8084\n",
      "Epoch 5/25\n",
      "1189/1189 [==============================] - 17s 14ms/step - loss: 0.1842 - accuracy: 0.9272 - val_loss: 0.6706 - val_accuracy: 0.7832\n",
      "Epoch 6/25\n",
      "1189/1189 [==============================] - ETA: 0s - loss: 0.1298 - accuracy: 0.9554Restoring model weights from the end of the best epoch: 1.\n",
      "1189/1189 [==============================] - 16s 14ms/step - loss: 0.1298 - accuracy: 0.9554 - val_loss: 0.7718 - val_accuracy: 0.7899\n",
      "Epoch 6: early stopping\n",
      "331/331 [==============================] - 1s 3ms/step - loss: 0.4294 - accuracy: 0.7915\n",
      "Fold 3 Test Accuracy: 79.15%\n",
      "Epoch 1/25\n",
      "1190/1190 [==============================] - 17s 14ms/step - loss: 0.5092 - accuracy: 0.7633 - val_loss: 0.4461 - val_accuracy: 0.7933\n",
      "Epoch 2/25\n",
      "1190/1190 [==============================] - 17s 14ms/step - loss: 0.3694 - accuracy: 0.8483 - val_loss: 0.4033 - val_accuracy: 0.8202\n",
      "Epoch 3/25\n",
      "1190/1190 [==============================] - 17s 14ms/step - loss: 0.3115 - accuracy: 0.8781 - val_loss: 0.4062 - val_accuracy: 0.8134\n",
      "Epoch 4/25\n",
      "1190/1190 [==============================] - 17s 14ms/step - loss: 0.2519 - accuracy: 0.9033 - val_loss: 1.7991 - val_accuracy: 0.6992\n",
      "Epoch 5/25\n",
      "1190/1190 [==============================] - 17s 14ms/step - loss: 0.2034 - accuracy: 0.9227 - val_loss: 0.6717 - val_accuracy: 0.7983\n",
      "Epoch 6/25\n",
      "1190/1190 [==============================] - 17s 14ms/step - loss: 0.1573 - accuracy: 0.9449 - val_loss: 0.5575 - val_accuracy: 0.8151\n",
      "Epoch 7/25\n",
      "1188/1190 [============================>.] - ETA: 0s - loss: 0.1203 - accuracy: 0.9550Restoring model weights from the end of the best epoch: 2.\n",
      "1190/1190 [==============================] - 17s 14ms/step - loss: 0.1202 - accuracy: 0.9550 - val_loss: 1.1869 - val_accuracy: 0.7731\n",
      "Epoch 7: early stopping\n",
      "330/330 [==============================] - 1s 2ms/step - loss: 0.4702 - accuracy: 0.7697\n",
      "Fold 4 Test Accuracy: 76.97%\n",
      "Epoch 1/25\n",
      "1190/1190 [==============================] - 17s 14ms/step - loss: 0.4956 - accuracy: 0.7642 - val_loss: 0.7441 - val_accuracy: 0.7059\n",
      "Epoch 2/25\n",
      "1190/1190 [==============================] - 17s 14ms/step - loss: 0.3832 - accuracy: 0.8365 - val_loss: 0.4434 - val_accuracy: 0.7815\n",
      "Epoch 3/25\n",
      "1190/1190 [==============================] - 17s 14ms/step - loss: 0.3332 - accuracy: 0.8676 - val_loss: 0.5291 - val_accuracy: 0.8067\n",
      "Epoch 4/25\n",
      "1190/1190 [==============================] - 17s 14ms/step - loss: 0.2400 - accuracy: 0.9126 - val_loss: 0.5367 - val_accuracy: 0.8101\n",
      "Epoch 5/25\n",
      "1190/1190 [==============================] - 17s 14ms/step - loss: 0.1563 - accuracy: 0.9445 - val_loss: 0.9591 - val_accuracy: 0.7647\n",
      "Epoch 6/25\n",
      "1190/1190 [==============================] - 18s 15ms/step - loss: 0.1006 - accuracy: 0.9626 - val_loss: 1.1017 - val_accuracy: 0.7697\n",
      "Epoch 7/25\n",
      "1188/1190 [============================>.] - ETA: 0s - loss: 0.0936 - accuracy: 0.9743Restoring model weights from the end of the best epoch: 2.\n",
      "1190/1190 [==============================] - 17s 14ms/step - loss: 0.0935 - accuracy: 0.9744 - val_loss: 1.0977 - val_accuracy: 0.7815\n",
      "Epoch 7: early stopping\n",
      "330/330 [==============================] - 1s 3ms/step - loss: 0.3176 - accuracy: 0.8515\n",
      "Fold 5 Test Accuracy: 85.15%\n",
      "Epoch 1/25\n",
      "1190/1190 [==============================] - 18s 14ms/step - loss: 0.5183 - accuracy: 0.7486 - val_loss: 0.6797 - val_accuracy: 0.6605\n",
      "Epoch 2/25\n",
      "1190/1190 [==============================] - 17s 14ms/step - loss: 0.4010 - accuracy: 0.8289 - val_loss: 0.4240 - val_accuracy: 0.8134\n",
      "Epoch 3/25\n",
      "1190/1190 [==============================] - 17s 14ms/step - loss: 0.3498 - accuracy: 0.8558 - val_loss: 0.4524 - val_accuracy: 0.7933\n",
      "Epoch 4/25\n",
      "1190/1190 [==============================] - 17s 14ms/step - loss: 0.2788 - accuracy: 0.8882 - val_loss: 0.7195 - val_accuracy: 0.7580\n",
      "Epoch 5/25\n",
      "1190/1190 [==============================] - 17s 14ms/step - loss: 0.2141 - accuracy: 0.9222 - val_loss: 0.5931 - val_accuracy: 0.7782\n",
      "Epoch 6/25\n",
      "1190/1190 [==============================] - 17s 14ms/step - loss: 0.1630 - accuracy: 0.9395 - val_loss: 1.0782 - val_accuracy: 0.7395\n",
      "Epoch 7/25\n",
      "1187/1190 [============================>.] - ETA: 0s - loss: 0.1159 - accuracy: 0.9621Restoring model weights from the end of the best epoch: 2.\n",
      "1190/1190 [==============================] - 17s 14ms/step - loss: 0.1157 - accuracy: 0.9622 - val_loss: 0.7991 - val_accuracy: 0.7950\n",
      "Epoch 7: early stopping\n",
      "330/330 [==============================] - 1s 2ms/step - loss: 0.3354 - accuracy: 0.8697\n",
      "Fold 6 Test Accuracy: 86.97%\n",
      "Epoch 1/25\n",
      "1190/1190 [==============================] - 18s 14ms/step - loss: 0.5518 - accuracy: 0.7356 - val_loss: 0.4963 - val_accuracy: 0.7546\n",
      "Epoch 2/25\n",
      "1190/1190 [==============================] - 17s 14ms/step - loss: 0.4022 - accuracy: 0.8319 - val_loss: 0.8164 - val_accuracy: 0.7244\n",
      "Epoch 3/25\n",
      "1190/1190 [==============================] - 17s 14ms/step - loss: 0.3597 - accuracy: 0.8579 - val_loss: 0.6364 - val_accuracy: 0.7345\n",
      "Epoch 4/25\n",
      "1190/1190 [==============================] - 17s 14ms/step - loss: 0.2709 - accuracy: 0.8983 - val_loss: 0.6018 - val_accuracy: 0.7832\n",
      "Epoch 5/25\n",
      "1190/1190 [==============================] - 17s 14ms/step - loss: 0.2198 - accuracy: 0.9243 - val_loss: 0.9263 - val_accuracy: 0.7782\n",
      "Epoch 6/25\n",
      "1188/1190 [============================>.] - ETA: 0s - loss: 0.1710 - accuracy: 0.9386Restoring model weights from the end of the best epoch: 1.\n",
      "1190/1190 [==============================] - 17s 14ms/step - loss: 0.1708 - accuracy: 0.9386 - val_loss: 0.8228 - val_accuracy: 0.7765\n",
      "Epoch 6: early stopping\n",
      "330/330 [==============================] - 1s 3ms/step - loss: 0.3370 - accuracy: 0.8364\n",
      "Fold 7 Test Accuracy: 83.64%\n",
      "Epoch 1/25\n",
      "1190/1190 [==============================] - 18s 14ms/step - loss: 0.5332 - accuracy: 0.7499 - val_loss: 0.4865 - val_accuracy: 0.8050\n",
      "Epoch 2/25\n",
      "1190/1190 [==============================] - 17s 14ms/step - loss: 0.4077 - accuracy: 0.8306 - val_loss: 0.5727 - val_accuracy: 0.7311\n",
      "Epoch 3/25\n",
      "1190/1190 [==============================] - 17s 14ms/step - loss: 0.3323 - accuracy: 0.8554 - val_loss: 0.4128 - val_accuracy: 0.8134\n",
      "Epoch 4/25\n",
      "1190/1190 [==============================] - 17s 14ms/step - loss: 0.2596 - accuracy: 0.8991 - val_loss: 1.2926 - val_accuracy: 0.6958\n",
      "Epoch 5/25\n",
      "1190/1190 [==============================] - 17s 14ms/step - loss: 0.1978 - accuracy: 0.9285 - val_loss: 0.5924 - val_accuracy: 0.8084\n",
      "Epoch 6/25\n",
      "1190/1190 [==============================] - 17s 14ms/step - loss: 0.1371 - accuracy: 0.9563 - val_loss: 0.7233 - val_accuracy: 0.7950\n",
      "Epoch 7/25\n",
      "1190/1190 [==============================] - 17s 14ms/step - loss: 0.1363 - accuracy: 0.9617 - val_loss: 0.7328 - val_accuracy: 0.8034\n",
      "Epoch 8/25\n",
      "1189/1190 [============================>.] - ETA: 0s - loss: 0.0969 - accuracy: 0.9693Restoring model weights from the end of the best epoch: 3.\n",
      "1190/1190 [==============================] - 17s 14ms/step - loss: 0.0970 - accuracy: 0.9693 - val_loss: 1.0217 - val_accuracy: 0.7950\n",
      "Epoch 8: early stopping\n",
      "330/330 [==============================] - 1s 2ms/step - loss: 0.3572 - accuracy: 0.8455\n",
      "Fold 8 Test Accuracy: 84.55%\n",
      "Epoch 1/25\n",
      "1190/1190 [==============================] - 18s 14ms/step - loss: 0.5328 - accuracy: 0.7423 - val_loss: 0.4568 - val_accuracy: 0.7613\n",
      "Epoch 2/25\n",
      "1190/1190 [==============================] - 17s 14ms/step - loss: 0.4037 - accuracy: 0.8134 - val_loss: 0.5684 - val_accuracy: 0.7866\n",
      "Epoch 3/25\n",
      "1190/1190 [==============================] - 17s 14ms/step - loss: 0.3435 - accuracy: 0.8621 - val_loss: 0.3225 - val_accuracy: 0.8655\n",
      "Epoch 4/25\n",
      "1190/1190 [==============================] - 17s 14ms/step - loss: 0.2532 - accuracy: 0.9029 - val_loss: 0.6902 - val_accuracy: 0.8168\n",
      "Epoch 5/25\n",
      "1190/1190 [==============================] - 17s 14ms/step - loss: 0.1979 - accuracy: 0.9269 - val_loss: 0.4084 - val_accuracy: 0.8655\n",
      "Epoch 6/25\n",
      "1190/1190 [==============================] - 17s 14ms/step - loss: 0.1595 - accuracy: 0.9391 - val_loss: 0.8364 - val_accuracy: 0.8185\n",
      "Epoch 7/25\n",
      "1190/1190 [==============================] - 17s 14ms/step - loss: 0.1028 - accuracy: 0.9622 - val_loss: 1.0644 - val_accuracy: 0.8185\n",
      "Epoch 8/25\n",
      "1188/1190 [============================>.] - ETA: 0s - loss: 0.0924 - accuracy: 0.9693Restoring model weights from the end of the best epoch: 3.\n",
      "1190/1190 [==============================] - 17s 14ms/step - loss: 0.0923 - accuracy: 0.9693 - val_loss: 0.7515 - val_accuracy: 0.8252\n",
      "Epoch 8: early stopping\n",
      "330/330 [==============================] - 1s 2ms/step - loss: 0.5607 - accuracy: 0.7788\n",
      "Fold 9 Test Accuracy: 77.88%\n",
      "10 Fold Accuracy: 81.69%±4.96\n"
     ]
    }
   ],
   "source": [
    "### GOAL is to achieve 95% classification accuracy on 10-fold CV\n",
    "\n",
    "# Step 1: Experiment with different param. + hyperparam.: layers, kernel, stride, filter size ( see https://keras.io/api/layers/convolution_layers/convolution1d/ )\n",
    "# Step 2: Experiment with different architectures\n",
    "\n",
    "test_loss, test_acc = [], []\n",
    "\n",
    "kernel = 5\n",
    "stride = 3\n",
    "kfold = KFold(n_splits=10) #, shuffle=True)\n",
    "i = 0\n",
    "for train, test in kfold.split(x, y):\n",
    "    # Data normalization\n",
    "    scaler = StandardScaler()\n",
    "    X_train = scaler.fit_transform(x[train])\n",
    "    X_test = scaler.transform(x[test])\n",
    "\n",
    "    y_train = y[train]\n",
    "    y_test = y[test]\n",
    "    \n",
    "    X_train = X_train.reshape((X_train.shape[0], X_train.shape[1], 1))\n",
    "    X_test = X_test.reshape((X_test.shape[0], X_test.shape[1], 1))\n",
    "\n",
    "\n",
    "    # TO DO: Experiment with different model architectures\n",
    "    model = Sequential([\n",
    "        layers.Conv1D(64, kernel, stride, activation='relu', input_shape=X_train.shape[1:]),\n",
    "        layers.Conv1D(32, kernel, stride, activation='relu'),\n",
    "        layers.Conv1D(16, kernel, stride, activation='relu'),\n",
    "        layers.Conv1D(8, kernel, stride, activation='relu'),\n",
    "        layers.Flatten(),\n",
    "        layers.Dense(1, activation='sigmoid')\n",
    "    ])\n",
    "        \n",
    "    # Define the early stopping callback\n",
    "    early_stopping = EarlyStopping(\n",
    "        monitor='val_loss',  # Metric to monitor (e.g., validation loss)\n",
    "        patience=5,           # Number of epochs with no improvement after which training will be stopped\n",
    "        restore_best_weights=True,  # Restore model weights to the best achieved during training\n",
    "        verbose=1  # Set to 1 to see messages about the early stopping process\n",
    "    )\n",
    "    \n",
    "    optimizer = optimizers.Adam(learning_rate=0.001)\n",
    "\n",
    "    model.compile(optimizer=optimizer, loss='binary_crossentropy', metrics=['accuracy'])\n",
    "    \n",
    "    history = model.fit(\n",
    "        X_train,\n",
    "        y_train,\n",
    "        epochs=25,      \n",
    "        validation_split=0.2,\n",
    "        batch_size=2,\n",
    "        callbacks=[early_stopping]  # Add the early stopping callback\n",
    "    )\n",
    "    \n",
    "    loss, accuracy = model.evaluate(X_test, y_test, batch_size=1)\n",
    "    print(f\"Fold {i} Test Accuracy: {accuracy * 100:.2f}%\")\n",
    "    i = i + 1\n",
    "    test_loss.append(loss)\n",
    "    test_acc.append(accuracy)\n",
    "\n",
    "print(\"{} Fold Accuracy: {:.2f}%\\u00b1{:.2f}\".format(i, np.mean(test_acc)*100, np.std(test_acc)*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0.7432023882865906,\n",
       " 0.7764350175857544,\n",
       " 0.9063444137573242,\n",
       " 0.791540801525116,\n",
       " 0.7696969509124756,\n",
       " 0.8515151739120483,\n",
       " 0.8696969747543335,\n",
       " 0.8363636136054993,\n",
       " 0.8454545736312866,\n",
       " 0.7787878513336182]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.18"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}