numpy_neuralnet.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from matplotlib import pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Replace with your file path\n",
    "data = pd.read_csv('/Users/lucastucker/REU-2023/archive/mnist_train.csv')\n",
    "data = np.array(data)\n",
    "np.random.shuffle(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(60000, 785)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "m, n = data.shape\n",
    "test_set = data[0: 1000].T\n",
    "Y_test = test_set[0]\n",
    "X_test = test_set[1: n]"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "'We now have a 1 x 1000 Y_test and 784 x 1000 X_test for later'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_set = data[1000: m].T\n",
    "Y_train = train_set[0]\n",
    "X_train = train_set[1: n] / 255 # MUST scale data to [0,1] range!"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "'Create weight and bias matrices for a single hidden layer NN with 10 nodes'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def init_params(X, Y):\n",
    "    W1 = np.random.rand(10, 784) - 0.5\n",
    "    b1 = np.random.rand(10, 1) - 0.5\n",
    "    W2 = np.random.rand(10, 10) - 0.5\n",
    "    b2 = np.random.rand(10, 1) - 0.5\n",
    "    return W1, b1, W2, b2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def relu(Z):\n",
    "    return np.maximum(0, Z)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def softmax(Z):\n",
    "    return np.exp(Z) / sum(np.exp(Z))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def forward_prop(W1, b1, W2, b2, X):\n",
    "    Z1 = W1.dot(X) + b1\n",
    "    A1 = relu(Z1)\n",
    "    Z2 = W2.dot(A1) + b2\n",
    "    A2 = softmax(Z2)\n",
    "    return Z1, A1, A2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def one_hot(Y):\n",
    "    one_hot_Y = np.zeros((Y.size, Y.max() + 1))\n",
    "    one_hot_Y[np.arange(Y.size), Y] = 1\n",
    "    one_hot_Y = one_hot_Y.T\n",
    "    return one_hot_Y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def relu_deriv(Z):\n",
    "    return Z > 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def back_prop(Z1, A1, A2, W2, X, Y):\n",
    "    m = Y.size\n",
    "    one_hot_Y = one_hot(Y)\n",
    "    dZ2 = A2 - one_hot_Y\n",
    "    dW2 = 1/m * dZ2.dot(A1.T)\n",
    "    db2 = 1/m * np.sum(dZ2)\n",
    "    dZ1 = W2.T.dot(dZ2) * relu_deriv(Z1)\n",
    "    dW1 = 1/m * dZ1.dot(X.T)\n",
    "    db1 = 1/m * np.sum(dZ1)\n",
    "    return dW1, db1, dW2, db2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def update_params(W1, b1, W2, b2, dW1, db1, dW2, db2, rate):\n",
    "    W2 = W2 - rate * dW2\n",
    "    W1 = W1 - rate * dW1\n",
    "    b2 = b2 - rate * db2\n",
    "    b1 = b1 - rate * db1\n",
    "    return W1, b1, W2, b2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_accuracy(preds, Y):\n",
    "    return np.sum(preds == Y) / Y.size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def gradient_descent(X, Y, iterations, rate):\n",
    "    W1, b1, W2, b2 = init_params(X, Y)\n",
    "    for i in range(iterations):\n",
    "        Z1, A1, A2 = forward_prop(W1, b1, W2, b2, X)\n",
    "        dW1, db1, dW2, db2 = back_prop(Z1, A1, A2, W2, X, Y)\n",
    "        W1, b1, W2, b2 = update_params(W1, b1, W2, b2, dW1, db1, dW2, db2, rate)\n",
    "        if i % 10 == 0:\n",
    "            print(\"Epoch: \", i)\n",
    "            print(get_accuracy(np.argmax(A2, axis = 0), Y))\n",
    "    return W1, b1, W2, b2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch:  0\n",
      "0.10428813559322034\n",
      "Epoch:  10\n",
      "0.2144406779661017\n",
      "Epoch:  20\n",
      "0.32445762711864407\n",
      "Epoch:  30\n",
      "0.36871186440677967\n",
      "Epoch:  40\n",
      "0.40388135593220337\n",
      "Epoch:  50\n",
      "0.43622033898305085\n",
      "Epoch:  60\n",
      "0.4681864406779661\n",
      "Epoch:  70\n",
      "0.5003559322033898\n",
      "Epoch:  80\n",
      "0.5296610169491526\n",
      "Epoch:  90\n",
      "0.5572203389830509\n",
      "Epoch:  100\n",
      "0.583135593220339\n",
      "Epoch:  110\n",
      "0.6085084745762712\n",
      "Epoch:  120\n",
      "0.6306440677966102\n",
      "Epoch:  130\n",
      "0.6504406779661017\n",
      "Epoch:  140\n",
      "0.6692203389830509\n",
      "Epoch:  150\n",
      "0.6855423728813559\n",
      "Epoch:  160\n",
      "0.6998813559322034\n",
      "Epoch:  170\n",
      "0.713406779661017\n",
      "Epoch:  180\n",
      "0.7259152542372881\n",
      "Epoch:  190\n",
      "0.7368813559322034\n",
      "Epoch:  200\n",
      "0.7469661016949153\n",
      "Epoch:  210\n",
      "0.7557118644067796\n",
      "Epoch:  220\n",
      "0.7646101694915254\n",
      "Epoch:  230\n",
      "0.7719322033898305\n",
      "Epoch:  240\n",
      "0.7789491525423728\n",
      "Epoch:  250\n",
      "0.7844576271186441\n",
      "Epoch:  260\n",
      "0.789593220338983\n",
      "Epoch:  270\n",
      "0.7947796610169492\n",
      "Epoch:  280\n",
      "0.7990338983050848\n",
      "Epoch:  290\n",
      "0.8034576271186441\n",
      "Epoch:  300\n",
      "0.8075932203389831\n",
      "Epoch:  310\n",
      "0.8111694915254237\n",
      "Epoch:  320\n",
      "0.8142372881355933\n",
      "Epoch:  330\n",
      "0.8171694915254237\n",
      "Epoch:  340\n",
      "0.8202881355932203\n"
     ]
    }
   ],
   "source": [
    "W1, b1, W2, b2 = gradient_descent(X_train, Y_train, 350, 0.10)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/33/zckmdctn235gr36g69vrq8sc0000gn/T/ipykernel_2830/2645878079.py:2: RuntimeWarning: overflow encountered in exp\n",
      "  return np.exp(Z) / sum(np.exp(Z))\n",
      "/var/folders/33/zckmdctn235gr36g69vrq8sc0000gn/T/ipykernel_2830/2645878079.py:2: RuntimeWarning: invalid value encountered in divide\n",
      "  return np.exp(Z) / sum(np.exp(Z))\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.507"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X = X_test\n",
    "Y = Y_test\n",
    "Z1, A1, A2 = forward_prop(W1, b1, W2, b2, X)\n",
    "get_accuracy(np.argmax(A2, axis = 0), Y)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.7"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}