neural_nets.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Neural Nets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "import math, matplotlib\n",
    "import import_ipynb\n",
    "from ml_tools import *\n",
    "from linear_algebra import *"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Conceptually, the idea of a neural net is fairly straightforward.  Nodes are arranged together in a network to manipulate the representation of the input data to glean insight and make predictions.  The simplest neural net has an input layer, a hidden layer, and an output layer, but neural nets with several hidden layers allow for \"deep learning.\"  In practice, though, neural nets can take on very complex architectures depending on the task.  For instance, when convolutional layers are added, image recognition can be done.  Allowing the net to have a memory of the input in a sequence of data in a recurrent neural net enables time series analysis and NLP.  As a test case, I create a simple neural net to demonstrate simple principles of a feed-forward neural net and how you train it using backpropagation to correct the errors.  This is then used to guess digits represented by characters in a matrix."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def step_function(x):\n",
    "    return 1 if x >= 0 else 0\n",
    "\n",
    "def perceptron_output(weights, bias, x):\n",
    "    '''returns 1 if the perceptron fires, 0 if not'''\n",
    "    calculation = dot(weights, x) + bias\n",
    "    return step_function(calculation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sigmoid(t):\n",
    "    return 1./(1.+math.exp(-t))\n",
    "\n",
    "def neuron_output(weights, inputs):\n",
    "    return sigmoid(dot(weights, inputs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def feed_forward(neural_network, input_vector):\n",
    "    '''takes in neural network - represented by a list of lists - \n",
    "    and returns the output from forward-propagating the input'''\n",
    "    outputs = []\n",
    "\n",
    "    for layer in neural_network:\n",
    "\n",
    "        input_with_bias = input_vector + [1]             # add a bias input\n",
    "        output = [neuron_output(neuron, input_with_bias) # compute the output\n",
    "                  for neuron in layer]                   # for this layer\n",
    "        outputs.append(output)                           # and remember it\n",
    "\n",
    "        # the input to the next layer is the output of this one\n",
    "        input_vector = output\n",
    "\n",
    "    return outputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def backpropagate(network, input_vector, target):\n",
    "\n",
    "    hidden_outputs, outputs = feed_forward(network, input_vector)\n",
    "\n",
    "    # the output * (1 - output) is from the derivative of sigmoid\n",
    "    output_deltas = [output * (1 - output) * (output - target[i])\n",
    "                     for i, output in enumerate(outputs)]\n",
    "\n",
    "    # adjust weights for output layer (network[-1])\n",
    "    for i, output_neuron in enumerate(network[-1]):\n",
    "        for j, hidden_output in enumerate(hidden_outputs + [1]):\n",
    "            output_neuron[j] -= output_deltas[i] * hidden_output\n",
    "\n",
    "    # back-propagate errors to hidden layer\n",
    "    hidden_deltas = [hidden_output * (1 - hidden_output) *\n",
    "                      dot(output_deltas, [n[i] for n in network[-1]])\n",
    "                     for i, hidden_output in enumerate(hidden_outputs)]\n",
    "\n",
    "    # adjust weights for hidden layer (network[0])\n",
    "    for i, hidden_neuron in enumerate(network[0]):\n",
    "        for j, input in enumerate(input_vector + [1]):\n",
    "            hidden_neuron[j] -= hidden_deltas[i] * input"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def patch(x, y, hatch, color):\n",
    "    \"\"\"return a matplotlib 'patch' object with the specified\n",
    "    location, crosshatch pattern, and color\"\"\"\n",
    "    return matplotlib.patches.Rectangle((x - 0.5, y - 0.5), 1, 1,\n",
    "                                        hatch=hatch, fill=False, color=color)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def show_weights(neuron_idx):\n",
    "    weights = network[0][neuron_idx]\n",
    "    abs_weights = [abs(weight) for weight in weights]\n",
    "\n",
    "    grid = [abs_weights[row:(row+5)] # turn the weights into a 5x5 grid\n",
    "            for row in range(0,25,5)] # [weights[0:5], ..., weights[20:25]]\n",
    "\n",
    "    ax = plt.gca() # to use hatching, we'll need the axis\n",
    "\n",
    "    ax.imshow(grid, # here same as plt.imshow\n",
    "              cmap=matplotlib.cm.binary, # use white-black color scale\n",
    "              interpolation='none') # plot blocks as blocks\n",
    "\n",
    "    # cross-hatch the negative weights\n",
    "    for i in range(5): # row\n",
    "        for j in range(5): # column\n",
    "            if weights[5*i + j] < 0: # row i, column j = weights[5*i + j]\n",
    "                # add black and white hatches, so visible whether dark or light\n",
    "                ax.add_patch(patch(j, i, '/', \"white\"))\n",
    "                ax.add_patch(patch(j, i, '\\\\', \"black\"))\n",
    "    plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 [0.96, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.02, 0.03, 0.0]\n",
      "1 [0.0, 0.96, 0.03, 0.02, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]\n",
      "2 [0.0, 0.02, 0.96, 0.0, 0.0, 0.03, 0.0, 0.0, 0.0, 0.0]\n",
      "3 [0.0, 0.03, 0.0, 0.97, 0.0, 0.0, 0.0, 0.02, 0.0, 0.03]\n",
      "4 [0.0, 0.02, 0.0, 0.0, 0.98, 0.0, 0.0, 0.01, 0.0, 0.0]\n",
      "5 [0.0, 0.0, 0.02, 0.0, 0.0, 0.96, 0.01, 0.0, 0.02, 0.02]\n",
      "6 [0.0, 0.0, 0.01, 0.0, 0.01, 0.02, 0.99, 0.0, 0.01, 0.0]\n",
      "7 [0.03, 0.0, 0.0, 0.02, 0.0, 0.0, 0.0, 0.97, 0.0, 0.0]\n",
      "8 [0.03, 0.0, 0.0, 0.0, 0.0, 0.01, 0.0, 0.0, 0.96, 0.03]\n",
      "9 [0.0, 0.0, 0.0, 0.01, 0.0, 0.02, 0.0, 0.0, 0.02, 0.96]\n",
      ".@@@.\n",
      "...@@\n",
      "..@@.\n",
      "...@@\n",
      ".@@@.\n",
      "[0.0, 0.0, 0.0, 0.93, 0.0, 0.0, 0.0, 0.01, 0.0, 0.1]\n",
      "\n",
      ".@@@.\n",
      "@..@@\n",
      ".@@@.\n",
      "@..@@\n",
      ".@@@.\n",
      "[0.0, 0.0, 0.0, 0.0, 0.0, 0.54, 0.0, 0.0, 0.91, 1.0]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "raw_digits = [\n",
    "      \"\"\"11111\n",
    "         1...1\n",
    "         1...1\n",
    "         1...1\n",
    "         11111\"\"\",\n",
    "\n",
    "      \"\"\"..1..\n",
    "         ..1..\n",
    "         ..1..\n",
    "         ..1..\n",
    "         ..1..\"\"\",\n",
    "\n",
    "      \"\"\"11111\n",
    "         ....1\n",
    "         11111\n",
    "         1....\n",
    "         11111\"\"\",\n",
    "\n",
    "      \"\"\"11111\n",
    "         ....1\n",
    "         11111\n",
    "         ....1\n",
    "         11111\"\"\",\n",
    "\n",
    "      \"\"\"1...1\n",
    "         1...1\n",
    "         11111\n",
    "         ....1\n",
    "         ....1\"\"\",\n",
    "\n",
    "      \"\"\"11111\n",
    "         1....\n",
    "         11111\n",
    "         ....1\n",
    "         11111\"\"\",\n",
    "\n",
    "      \"\"\"11111\n",
    "         1....\n",
    "         11111\n",
    "         1...1\n",
    "         11111\"\"\",\n",
    "\n",
    "      \"\"\"11111\n",
    "         ....1\n",
    "         ....1\n",
    "         ....1\n",
    "         ....1\"\"\",\n",
    "\n",
    "      \"\"\"11111\n",
    "         1...1\n",
    "         11111\n",
    "         1...1\n",
    "         11111\"\"\",\n",
    "\n",
    "      \"\"\"11111\n",
    "         1...1\n",
    "         11111\n",
    "         ....1\n",
    "         11111\"\"\"]\n",
    "\n",
    "def make_digit(raw_digit):\n",
    "    return [1 if c == '1' else 0\n",
    "            for row in raw_digit.split(\"\\n\")\n",
    "            for c in row.strip()]\n",
    "\n",
    "inputs = list(map(make_digit, raw_digits))\n",
    "\n",
    "targets = [[1 if i == j else 0 for i in range(10)]\n",
    "           for j in range(10)]\n",
    "\n",
    "random.seed(0)   # to get repeatable results\n",
    "input_size = 25  # each input is a vector of length 25\n",
    "num_hidden = 5   # we'll have 5 neurons in the hidden layer\n",
    "output_size = 10 # we need 10 outputs for each input\n",
    "\n",
    "# each hidden neuron has one weight per input, plus a bias weight\n",
    "hidden_layer = [[random.random() for __ in range(input_size + 1)]\n",
    "                for __ in range(num_hidden)]\n",
    "\n",
    "# each output neuron has one weight per hidden neuron, plus a bias weight\n",
    "output_layer = [[random.random() for __ in range(num_hidden + 1)]\n",
    "                for __ in range(output_size)]\n",
    "\n",
    "# the network starts out with random weights\n",
    "network = [hidden_layer, output_layer]\n",
    "\n",
    "# 10,000 iterations seems enough to converge\n",
    "for __ in range(10000):\n",
    "    for input_vector, target_vector in zip(inputs, targets):\n",
    "        backpropagate(network, input_vector, target_vector)\n",
    "\n",
    "def predict(input):\n",
    "    return feed_forward(network, input)[-1]\n",
    "\n",
    "for i, input in enumerate(inputs):\n",
    "    outputs = predict(input)\n",
    "    print(i, [round(p,2) for p in outputs])\n",
    "\n",
    "print(\"\"\".@@@.\n",
    "...@@\n",
    "..@@.\n",
    "...@@\n",
    ".@@@.\"\"\")\n",
    "print([round(x, 2) for x in\n",
    "      predict(  [0,1,1,1,0,    # .@@@.\n",
    "                 0,0,0,1,1,    # ...@@\n",
    "                 0,0,1,1,0,    # ..@@.\n",
    "                 0,0,0,1,1,    # ...@@\n",
    "                 0,1,1,1,0])]) # .@@@.\n",
    "print()\n",
    "\n",
    "print(\"\"\".@@@.\n",
    "@..@@\n",
    ".@@@.\n",
    "@..@@\n",
    ".@@@.\"\"\")\n",
    "print([round(x, 2) for x in\n",
    "      predict(  [0,1,1,1,0,    # .@@@.\n",
    "                 1,0,0,1,1,    # @..@@\n",
    "                 0,1,1,1,0,    # .@@@.\n",
    "                 1,0,0,1,1,    # @..@@\n",
    "                 0,1,1,1,0])]) # .@@@.\n",
    "print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}