BH_10fold_card_model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-01-10 09:12:25.886404: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
      "To enable the following instructions: SSE4.1 SSE4.2, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.callbacks import EarlyStopping\n",
    "from tensorflow.keras import optimizers\n",
    "from tensorflow.keras import layers\n",
    "from tensorflow.keras import Input\n",
    "from tensorflow.keras import Model\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.preprocessing import normalize\n",
    "from sklearn.model_selection import KFold\n",
    "\n",
    "from scipy.signal import find_peaks\n",
    "from scipy.stats import mode\n",
    "import numpy as np\n",
    "import pickle\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Num GPUs Available:  0\n"
     ]
    }
   ],
   "source": [
    "print(\"Num GPUs Available: \", len(tf.config.list_physical_devices('GPU')))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load input/label to models\n",
    "# RESPIRATION\n",
    "# with open('resp_input.pkl', 'rb') as file:\n",
    "#     x = pickle.load(file)\n",
    "    \n",
    "# with open('resp_label.pkl', 'rb') as file:\n",
    "#     y = pickle.load(file)\n",
    "\n",
    "# CARDIAC\n",
    "with open('cardiac_input.pkl', 'rb') as file:\n",
    "    x = pickle.load(file)\n",
    "\n",
    "with open('cardiac_label.pkl', 'rb') as file:\n",
    "    y = pickle.load(file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#Nan Labels: 192\n",
      "#Remaining Samples: 3702\n"
     ]
    }
   ],
   "source": [
    "nanmask = np.isnan(y)\n",
    "print(f'#Nan Labels: {np.sum(nanmask)}\\n#Remaining Samples: {np.sum(~nanmask)}')\n",
    "y = y[~nanmask]\n",
    "x = x[~nanmask]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/25\n",
      "1332/1332 [==============================] - 19s 14ms/step - loss: 0.3781 - accuracy: 0.8806 - val_loss: 0.3983 - val_accuracy: 0.8711\n",
      "Epoch 2/25\n",
      "1332/1332 [==============================] - 18s 14ms/step - loss: 0.3213 - accuracy: 0.8821 - val_loss: 0.3267 - val_accuracy: 0.8876\n",
      "Epoch 3/25\n",
      "1332/1332 [==============================] - 18s 14ms/step - loss: 0.1945 - accuracy: 0.9219 - val_loss: 0.4900 - val_accuracy: 0.8876\n",
      "Epoch 4/25\n",
      "1332/1332 [==============================] - 18s 14ms/step - loss: 0.0839 - accuracy: 0.9718 - val_loss: 0.5930 - val_accuracy: 0.8816\n",
      "Epoch 5/25\n",
      "1332/1332 [==============================] - 18s 14ms/step - loss: 0.0511 - accuracy: 0.9809 - val_loss: 0.8855 - val_accuracy: 0.8591\n",
      "Epoch 6/25\n",
      "1332/1332 [==============================] - 18s 14ms/step - loss: 0.0261 - accuracy: 0.9929 - val_loss: 1.0538 - val_accuracy: 0.8861\n",
      "Epoch 7/25\n",
      "1329/1332 [============================>.] - ETA: 0s - loss: 0.0482 - accuracy: 0.9865Restoring model weights from the end of the best epoch: 2.\n",
      "1332/1332 [==============================] - 18s 14ms/step - loss: 0.0481 - accuracy: 0.9865 - val_loss: 0.9594 - val_accuracy: 0.8726\n",
      "Epoch 7: early stopping\n",
      "371/371 [==============================] - 1s 2ms/step - loss: 0.3251 - accuracy: 0.9003\n",
      "Fold 0 Test Accuracy: 90.03%\n",
      "Epoch 1/25\n",
      "1332/1332 [==============================] - 19s 14ms/step - loss: 0.3907 - accuracy: 0.8806 - val_loss: 0.3462 - val_accuracy: 0.8816\n",
      "Epoch 2/25\n",
      "1332/1332 [==============================] - 18s 14ms/step - loss: 0.3287 - accuracy: 0.8930 - val_loss: 0.3005 - val_accuracy: 0.9025\n",
      "Epoch 3/25\n",
      "1332/1332 [==============================] - 20s 15ms/step - loss: 0.2647 - accuracy: 0.9095 - val_loss: 0.2713 - val_accuracy: 0.9055\n",
      "Epoch 4/25\n",
      "1332/1332 [==============================] - 18s 14ms/step - loss: 0.1538 - accuracy: 0.9452 - val_loss: 0.3405 - val_accuracy: 0.9010\n",
      "Epoch 5/25\n",
      "1332/1332 [==============================] - 18s 14ms/step - loss: 0.0749 - accuracy: 0.9726 - val_loss: 0.4138 - val_accuracy: 0.9070\n",
      "Epoch 6/25\n",
      "1332/1332 [==============================] - 18s 14ms/step - loss: 0.0425 - accuracy: 0.9857 - val_loss: 0.8053 - val_accuracy: 0.8996\n",
      "Epoch 7/25\n",
      "1332/1332 [==============================] - 19s 14ms/step - loss: 0.0295 - accuracy: 0.9906 - val_loss: 0.6503 - val_accuracy: 0.9100\n",
      "Epoch 8/25\n",
      "1331/1332 [============================>.] - ETA: 0s - loss: 0.0248 - accuracy: 0.9921Restoring model weights from the end of the best epoch: 3.\n",
      "1332/1332 [==============================] - 18s 14ms/step - loss: 0.0248 - accuracy: 0.9921 - val_loss: 0.5580 - val_accuracy: 0.9205\n",
      "Epoch 8: early stopping\n",
      "371/371 [==============================] - 1s 2ms/step - loss: 0.4074 - accuracy: 0.8733\n",
      "Fold 1 Test Accuracy: 87.33%\n",
      "Epoch 1/25\n",
      "1333/1333 [==============================] - 20s 14ms/step - loss: 0.3770 - accuracy: 0.8841 - val_loss: 0.3757 - val_accuracy: 0.8711\n",
      "Epoch 2/25\n",
      "1333/1333 [==============================] - 19s 14ms/step - loss: 0.3240 - accuracy: 0.8844 - val_loss: 0.3453 - val_accuracy: 0.8711\n",
      "Epoch 3/25\n",
      "1333/1333 [==============================] - 19s 14ms/step - loss: 0.2152 - accuracy: 0.9054 - val_loss: 0.4839 - val_accuracy: 0.8561\n",
      "Epoch 4/25\n",
      "1333/1333 [==============================] - 19s 14ms/step - loss: 0.0848 - accuracy: 0.9666 - val_loss: 0.5458 - val_accuracy: 0.8696\n",
      "Epoch 5/25\n",
      "1333/1333 [==============================] - 19s 14ms/step - loss: 0.0426 - accuracy: 0.9794 - val_loss: 1.1895 - val_accuracy: 0.8966\n",
      "Epoch 6/25\n",
      "1333/1333 [==============================] - 19s 15ms/step - loss: 0.0377 - accuracy: 0.9887 - val_loss: 1.3599 - val_accuracy: 0.8906\n",
      "Epoch 7/25\n",
      "1332/1333 [============================>.] - ETA: 0s - loss: 0.0414 - accuracy: 0.9872Restoring model weights from the end of the best epoch: 2.\n",
      "1333/1333 [==============================] - 19s 14ms/step - loss: 0.0414 - accuracy: 0.9872 - val_loss: 1.0360 - val_accuracy: 0.8516\n",
      "Epoch 7: early stopping\n",
      "370/370 [==============================] - 1s 2ms/step - loss: 0.3874 - accuracy: 0.8757\n",
      "Fold 2 Test Accuracy: 87.57%\n",
      "Epoch 1/25\n",
      "1333/1333 [==============================] - 19s 14ms/step - loss: 0.3698 - accuracy: 0.8852 - val_loss: 0.3436 - val_accuracy: 0.8711\n",
      "Epoch 2/25\n",
      "1333/1333 [==============================] - 19s 14ms/step - loss: 0.3225 - accuracy: 0.8852 - val_loss: 0.3388 - val_accuracy: 0.8711\n",
      "Epoch 3/25\n",
      "1333/1333 [==============================] - 19s 14ms/step - loss: 0.2063 - accuracy: 0.9156 - val_loss: 0.4202 - val_accuracy: 0.8711\n",
      "Epoch 4/25\n",
      "1333/1333 [==============================] - 19s 14ms/step - loss: 0.0930 - accuracy: 0.9595 - val_loss: 0.5846 - val_accuracy: 0.8381\n",
      "Epoch 5/25\n",
      "1333/1333 [==============================] - 19s 14ms/step - loss: 0.0387 - accuracy: 0.9865 - val_loss: 1.4592 - val_accuracy: 0.8636\n",
      "Epoch 6/25\n",
      "1333/1333 [==============================] - 19s 14ms/step - loss: 0.0455 - accuracy: 0.9854 - val_loss: 1.2650 - val_accuracy: 0.8846\n",
      "Epoch 7/25\n",
      "1330/1333 [============================>.] - ETA: 0s - loss: 0.0468 - accuracy: 0.9835Restoring model weights from the end of the best epoch: 2.\n",
      "1333/1333 [==============================] - 19s 14ms/step - loss: 0.0467 - accuracy: 0.9835 - val_loss: 1.4413 - val_accuracy: 0.8246\n",
      "Epoch 7: early stopping\n",
      "370/370 [==============================] - 1s 2ms/step - loss: 0.4006 - accuracy: 0.8676\n",
      "Fold 3 Test Accuracy: 86.76%\n",
      "Epoch 1/25\n",
      "1333/1333 [==============================] - 20s 14ms/step - loss: 0.3597 - accuracy: 0.8938 - val_loss: 0.4042 - val_accuracy: 0.8711\n",
      "Epoch 2/25\n",
      "1333/1333 [==============================] - 19s 14ms/step - loss: 0.3306 - accuracy: 0.8946 - val_loss: 0.4300 - val_accuracy: 0.8711\n",
      "Epoch 3/25\n",
      "1333/1333 [==============================] - 19s 14ms/step - loss: 0.2709 - accuracy: 0.9069 - val_loss: 0.3393 - val_accuracy: 0.8921\n",
      "Epoch 4/25\n",
      "1333/1333 [==============================] - 19s 14ms/step - loss: 0.1548 - accuracy: 0.9392 - val_loss: 0.4219 - val_accuracy: 0.8936\n",
      "Epoch 5/25\n",
      "1333/1333 [==============================] - 19s 14ms/step - loss: 0.0682 - accuracy: 0.9760 - val_loss: 0.7568 - val_accuracy: 0.8966\n",
      "Epoch 6/25\n",
      "1333/1333 [==============================] - 19s 14ms/step - loss: 0.0311 - accuracy: 0.9906 - val_loss: 0.9523 - val_accuracy: 0.8756\n",
      "Epoch 7/25\n",
      "1333/1333 [==============================] - 19s 14ms/step - loss: 0.0264 - accuracy: 0.9914 - val_loss: 1.3624 - val_accuracy: 0.8906\n",
      "Epoch 8/25\n",
      "1331/1333 [============================>.] - ETA: 0s - loss: 0.0291 - accuracy: 0.9921Restoring model weights from the end of the best epoch: 3.\n",
      "1333/1333 [==============================] - 19s 15ms/step - loss: 0.0291 - accuracy: 0.9921 - val_loss: 1.1112 - val_accuracy: 0.8651\n",
      "Epoch 8: early stopping\n",
      "370/370 [==============================] - 1s 2ms/step - loss: 0.5252 - accuracy: 0.8135\n",
      "Fold 4 Test Accuracy: 81.35%\n",
      "Epoch 1/25\n",
      "1333/1333 [==============================] - 20s 14ms/step - loss: 0.3859 - accuracy: 0.8859 - val_loss: 0.3635 - val_accuracy: 0.8711\n",
      "Epoch 2/25\n",
      "1333/1333 [==============================] - 19s 14ms/step - loss: 0.3278 - accuracy: 0.8908 - val_loss: 0.3076 - val_accuracy: 0.9055\n",
      "Epoch 3/25\n",
      "1333/1333 [==============================] - 19s 14ms/step - loss: 0.2177 - accuracy: 0.9227 - val_loss: 0.3454 - val_accuracy: 0.8906\n",
      "Epoch 4/25\n",
      "1333/1333 [==============================] - 19s 14ms/step - loss: 0.0892 - accuracy: 0.9636 - val_loss: 0.6073 - val_accuracy: 0.9070\n",
      "Epoch 5/25\n",
      "1333/1333 [==============================] - 19s 14ms/step - loss: 0.0648 - accuracy: 0.9782 - val_loss: 0.8764 - val_accuracy: 0.8711\n",
      "Epoch 6/25\n",
      "1333/1333 [==============================] - 19s 14ms/step - loss: 0.0432 - accuracy: 0.9824 - val_loss: 0.8879 - val_accuracy: 0.8861\n",
      "Epoch 7/25\n",
      "1329/1333 [============================>.] - ETA: 0s - loss: 0.0247 - accuracy: 0.9913Restoring model weights from the end of the best epoch: 2.\n",
      "1333/1333 [==============================] - 19s 14ms/step - loss: 0.0246 - accuracy: 0.9914 - val_loss: 0.9664 - val_accuracy: 0.8786\n",
      "Epoch 7: early stopping\n",
      "370/370 [==============================] - 1s 2ms/step - loss: 0.3852 - accuracy: 0.8703\n",
      "Fold 5 Test Accuracy: 87.03%\n",
      "Epoch 1/25\n",
      "1333/1333 [==============================] - 20s 14ms/step - loss: 0.3914 - accuracy: 0.8803 - val_loss: 0.3656 - val_accuracy: 0.8711\n",
      "Epoch 2/25\n",
      "1333/1333 [==============================] - 19s 14ms/step - loss: 0.3417 - accuracy: 0.8807 - val_loss: 0.3010 - val_accuracy: 0.8726\n",
      "Epoch 3/25\n",
      "1333/1333 [==============================] - 19s 14ms/step - loss: 0.2426 - accuracy: 0.9062 - val_loss: 0.3935 - val_accuracy: 0.8711\n",
      "Epoch 4/25\n",
      "1333/1333 [==============================] - 19s 14ms/step - loss: 0.1235 - accuracy: 0.9490 - val_loss: 0.3205 - val_accuracy: 0.8786\n",
      "Epoch 5/25\n",
      "1333/1333 [==============================] - 19s 14ms/step - loss: 0.0554 - accuracy: 0.9782 - val_loss: 0.7054 - val_accuracy: 0.8756\n",
      "Epoch 6/25\n",
      "1333/1333 [==============================] - 19s 14ms/step - loss: 0.0591 - accuracy: 0.9794 - val_loss: 0.5050 - val_accuracy: 0.8861\n",
      "Epoch 7/25\n",
      "1330/1333 [============================>.] - ETA: 0s - loss: 0.0372 - accuracy: 0.9842Restoring model weights from the end of the best epoch: 2.\n",
      "1333/1333 [==============================] - 19s 14ms/step - loss: 0.0372 - accuracy: 0.9842 - val_loss: 0.6706 - val_accuracy: 0.8756\n",
      "Epoch 7: early stopping\n",
      "370/370 [==============================] - 1s 2ms/step - loss: 0.2838 - accuracy: 0.8973\n",
      "Fold 6 Test Accuracy: 89.73%\n",
      "Epoch 1/25\n",
      "1333/1333 [==============================] - 20s 14ms/step - loss: 0.4104 - accuracy: 0.8735 - val_loss: 0.3590 - val_accuracy: 0.8711\n",
      "Epoch 2/25\n",
      "1333/1333 [==============================] - 19s 14ms/step - loss: 0.3473 - accuracy: 0.8750 - val_loss: 0.3239 - val_accuracy: 0.8711\n",
      "Epoch 3/25\n",
      "1333/1333 [==============================] - 19s 14ms/step - loss: 0.2634 - accuracy: 0.8931 - val_loss: 0.4527 - val_accuracy: 0.8726\n",
      "Epoch 4/25\n",
      "1333/1333 [==============================] - 19s 14ms/step - loss: 0.1217 - accuracy: 0.9516 - val_loss: 0.5731 - val_accuracy: 0.8936\n",
      "Epoch 5/25\n",
      "1333/1333 [==============================] - 19s 14ms/step - loss: 0.0445 - accuracy: 0.9842 - val_loss: 0.8094 - val_accuracy: 0.9010\n",
      "Epoch 6/25\n",
      "1333/1333 [==============================] - 19s 14ms/step - loss: 0.0522 - accuracy: 0.9846 - val_loss: 1.0463 - val_accuracy: 0.8906\n",
      "Epoch 7/25\n",
      "1330/1333 [============================>.] - ETA: 0s - loss: 0.0239 - accuracy: 0.9925Restoring model weights from the end of the best epoch: 2.\n",
      "1333/1333 [==============================] - 19s 14ms/step - loss: 0.0251 - accuracy: 0.9921 - val_loss: 1.2295 - val_accuracy: 0.8906\n",
      "Epoch 7: early stopping\n",
      "370/370 [==============================] - 1s 2ms/step - loss: 0.1732 - accuracy: 0.9514\n",
      "Fold 7 Test Accuracy: 95.14%\n",
      "Epoch 1/25\n",
      "1333/1333 [==============================] - 21s 15ms/step - loss: 0.4052 - accuracy: 0.8724 - val_loss: 0.3729 - val_accuracy: 0.9055\n",
      "Epoch 2/25\n",
      "1333/1333 [==============================] - 19s 14ms/step - loss: 0.3579 - accuracy: 0.8762 - val_loss: 0.3084 - val_accuracy: 0.9385\n",
      "Epoch 3/25\n",
      "1333/1333 [==============================] - 19s 14ms/step - loss: 0.2738 - accuracy: 0.9021 - val_loss: 0.2582 - val_accuracy: 0.9295\n",
      "Epoch 4/25\n",
      "1333/1333 [==============================] - 19s 14ms/step - loss: 0.1334 - accuracy: 0.9527 - val_loss: 0.2589 - val_accuracy: 0.9460\n",
      "Epoch 5/25\n",
      "1333/1333 [==============================] - 19s 14ms/step - loss: 0.0564 - accuracy: 0.9805 - val_loss: 0.5475 - val_accuracy: 0.8861\n",
      "Epoch 6/25\n",
      "1333/1333 [==============================] - 19s 14ms/step - loss: 0.0365 - accuracy: 0.9861 - val_loss: 0.4828 - val_accuracy: 0.9220\n",
      "Epoch 7/25\n",
      "1333/1333 [==============================] - 19s 14ms/step - loss: 0.0372 - accuracy: 0.9891 - val_loss: 0.7238 - val_accuracy: 0.9265\n",
      "Epoch 8/25\n",
      "1330/1333 [============================>.] - ETA: 0s - loss: 0.0194 - accuracy: 0.9947Restoring model weights from the end of the best epoch: 3.\n",
      "1333/1333 [==============================] - 19s 14ms/step - loss: 0.0194 - accuracy: 0.9947 - val_loss: 0.8312 - val_accuracy: 0.9220\n",
      "Epoch 8: early stopping\n",
      "370/370 [==============================] - 1s 2ms/step - loss: 0.3271 - accuracy: 0.9027\n",
      "Fold 8 Test Accuracy: 90.27%\n",
      "Epoch 1/25\n",
      "1333/1333 [==============================] - 20s 14ms/step - loss: 0.3937 - accuracy: 0.8728 - val_loss: 0.3189 - val_accuracy: 0.9160\n",
      "Epoch 2/25\n",
      "1333/1333 [==============================] - 19s 14ms/step - loss: 0.3547 - accuracy: 0.8739 - val_loss: 0.2457 - val_accuracy: 0.9160\n",
      "Epoch 3/25\n",
      "1333/1333 [==============================] - 19s 14ms/step - loss: 0.2591 - accuracy: 0.8994 - val_loss: 0.1713 - val_accuracy: 0.9430\n",
      "Epoch 4/25\n",
      "1333/1333 [==============================] - 19s 14ms/step - loss: 0.1468 - accuracy: 0.9452 - val_loss: 0.1357 - val_accuracy: 0.9550\n",
      "Epoch 5/25\n",
      "1333/1333 [==============================] - 19s 14ms/step - loss: 0.0819 - accuracy: 0.9707 - val_loss: 0.1637 - val_accuracy: 0.9550\n",
      "Epoch 6/25\n",
      "1333/1333 [==============================] - 19s 14ms/step - loss: 0.0432 - accuracy: 0.9846 - val_loss: 0.3343 - val_accuracy: 0.9010\n",
      "Epoch 7/25\n",
      "1333/1333 [==============================] - 19s 14ms/step - loss: 0.0389 - accuracy: 0.9902 - val_loss: 0.2117 - val_accuracy: 0.9445\n",
      "Epoch 8/25\n",
      "1333/1333 [==============================] - 62s 47ms/step - loss: 0.0268 - accuracy: 0.9921 - val_loss: 0.3168 - val_accuracy: 0.9445\n",
      "Epoch 9/25\n",
      "1331/1333 [============================>.] - ETA: 0s - loss: 0.0277 - accuracy: 0.9902Restoring model weights from the end of the best epoch: 4.\n",
      "1333/1333 [==============================] - 20s 15ms/step - loss: 0.0277 - accuracy: 0.9902 - val_loss: 0.3158 - val_accuracy: 0.9355\n",
      "Epoch 9: early stopping\n",
      "370/370 [==============================] - 1s 3ms/step - loss: 0.1902 - accuracy: 0.9405\n",
      "Fold 9 Test Accuracy: 94.05%\n",
      "10 Fold Accuracy: 88.93%±3.72\n"
     ]
    }
   ],
   "source": [
    "### GOAL is to achieve 95% classification accuracy on 10-fold CV\n",
    "\n",
    "# Step 1: Experiment with different param. + hyperparam.: layers, kernel, stride, filter size ( see https://keras.io/api/layers/convolution_layers/convolution1d/ )\n",
    "# Step 2: Experiment with different architectures\n",
    "\n",
    "test_loss, test_acc = [], []\n",
    "\n",
    "kernel = 5\n",
    "stride = 3\n",
    "kfold = KFold(n_splits=10) #, shuffle=True)\n",
    "i = 0\n",
    "for train, test in kfold.split(x, y):\n",
    "    # Data normalization\n",
    "    scaler = StandardScaler()\n",
    "    X_train = scaler.fit_transform(x[train])\n",
    "    X_test = scaler.transform(x[test])\n",
    "\n",
    "    y_train = y[train]\n",
    "    y_test = y[test]\n",
    "    \n",
    "    X_train = X_train.reshape((X_train.shape[0], X_train.shape[1], 1))\n",
    "    X_test = X_test.reshape((X_test.shape[0], X_test.shape[1], 1))\n",
    "\n",
    "\n",
    "    # TO DO: Experiment with different model architectures\n",
    "    model = Sequential([\n",
    "        layers.Conv1D(64, kernel, stride, activation='relu', input_shape=X_train.shape[1:]),\n",
    "        layers.Conv1D(32, kernel, stride, activation='relu'),\n",
    "        layers.Conv1D(16, kernel, stride, activation='relu'),\n",
    "        layers.Conv1D(8, kernel, stride, activation='relu'),\n",
    "        layers.Flatten(),\n",
    "        layers.Dense(1, activation='sigmoid')\n",
    "    ])\n",
    "        \n",
    "    # Define the early stopping callback\n",
    "    early_stopping = EarlyStopping(\n",
    "        monitor='val_loss',  # Metric to monitor (e.g., validation loss)\n",
    "        patience=5,           # Number of epochs with no improvement after which training will be stopped\n",
    "        restore_best_weights=True,  # Restore model weights to the best achieved during training\n",
    "        verbose=1  # Set to 1 to see messages about the early stopping process\n",
    "    )\n",
    "    \n",
    "    optimizer = optimizers.Adam(learning_rate=0.001)\n",
    "\n",
    "    model.compile(optimizer=optimizer, loss='binary_crossentropy', metrics=['accuracy'])\n",
    "    \n",
    "    history = model.fit(\n",
    "        X_train,\n",
    "        y_train,\n",
    "        epochs=25,      \n",
    "        validation_split=0.2,\n",
    "        batch_size=2,\n",
    "        callbacks=[early_stopping]  # Add the early stopping callback\n",
    "    )\n",
    "    \n",
    "    loss, accuracy = model.evaluate(X_test, y_test, batch_size=1)\n",
    "    print(f\"Fold {i} Test Accuracy: {accuracy * 100:.2f}%\")\n",
    "    i = i + 1\n",
    "    test_loss.append(loss)\n",
    "    test_acc.append(accuracy)\n",
    "\n",
    "print(\"{} Fold Accuracy: {:.2f}%\\u00b1{:.2f}\".format(i, np.mean(test_acc)*100, np.std(test_acc)*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0.9002695679664612,\n",
       " 0.8733153343200684,\n",
       " 0.8756756782531738,\n",
       " 0.8675675392150879,\n",
       " 0.8135135173797607,\n",
       " 0.8702702522277832,\n",
       " 0.8972973227500916,\n",
       " 0.9513513445854187,\n",
       " 0.9027026891708374,\n",
       " 0.9405405521392822]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.18"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}