c3d.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# install memory util\n",
    "# import urllib.request\n",
    "# response = urllib.request.urlopen(\"https://raw.githubusercontent.com/yaroslavvb/memory_util/master/memory_util.py\")\n",
    "# open(\"memory_util.py\", \"wb\").write(response.read())\n",
    "\n",
    "import memory_util\n",
    "memory_util.vlog(1)\n",
    "\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import os\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.image as mpimg\n",
    "import random"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def conv3d(x, W_shape, b_shape, name, strides=1):\n",
    "    with tf.variable_scope(name) as scope:\n",
    "        # Conv3D wrapper, with bias and relu activation\n",
    "        W = tf.Variable(tf.random_normal(W_shape), name='W')\n",
    "        b = tf.Variable(tf.random_normal(b_shape), name='b')\n",
    "        x = tf.nn.conv3d(x, W, strides=[1, strides, strides, strides, 1], padding='SAME', name=scope.name)\n",
    "        x = tf.nn.bias_add(x, b)\n",
    "        tf.summary.histogram(\"weights\", W)\n",
    "        tf.summary.histogram(\"biases\", b)\n",
    "        tf.summary.histogram(\"activations\", x)\n",
    "        return tf.nn.relu(x)\n",
    "\n",
    "def maxpool3d(x, name, k=2):\n",
    "    with tf.variable_scope(name) as scope:\n",
    "        # MaxPool2D wrapper\n",
    "        if k == 1:\n",
    "            return tf.nn.max_pool3d(x, ksize=[1, k, 2, 2, 1], strides=[1, k, 2, 2, 1],\n",
    "                              padding='SAME', name=scope.name)\n",
    "        else:\n",
    "            return tf.nn.max_pool3d(x, ksize=[1, k, k, k, 1], strides=[1, k, k, k, 1],\n",
    "                              padding='SAME', name=scope.name)\n",
    "def fc(x, size_in, size_out, name, relu = True):\n",
    "    with tf.variable_scope(name) as scope:    \n",
    "        W = tf.Variable(tf.truncated_normal([size_in, size_out], stddev=0.1), name=\"W\")\n",
    "        b = tf.Variable(tf.constant(0.1, shape=[size_out]), name=\"b\")\n",
    "        # Matrix multiply weights and inputs and add bias\n",
    "        act = tf.add(tf.matmul(x, W), b)\n",
    "        tf.summary.histogram(\"weights\", W)\n",
    "        tf.summary.histogram(\"biases\", b)\n",
    "        tf.summary.histogram(\"activations\", act)\n",
    "        if relu == True:\n",
    "            # Apply ReLu non linearity\n",
    "            act = tf.nn.relu(act)\n",
    "        return act\n",
    "\n",
    "def dense_to_one_hot(labels_dense, num_classes=10):\n",
    "    \"\"\" Convert class labels from scalars to one-hot vectors. \"\"\"\n",
    "    num_labels = labels_dense.shape[0]\n",
    "    index_offset = np.arange(num_labels) * num_classes\n",
    "    labels_one_hot = np.zeros((num_labels, num_classes))\n",
    "    labels_one_hot.flat[index_offset + labels_dense.ravel()] = 1\n",
    "    return labels_one_hot\n",
    "\n",
    "def get_xy(start, end, vid, labels, data):\n",
    "    inp, output = [], []\n",
    "    for i in range(start,end+1):\n",
    "        tmp = []\n",
    "        # check if number of frame in the current video are greater than 16\n",
    "        if len(data[vid[i]]) > 16:\n",
    "            current_frame = random.randrange(0,len(data[vid[i]])-16)\n",
    "            c_start = 0\n",
    "            c_end = current_frame+16\n",
    "            \n",
    "            for frame in sorted(data[vid[i]]):\n",
    "                if (c_start > current_frame) and (c_end > current_frame):\n",
    "                    current_frame += 1\n",
    "                    tmp.append([data[vid[i]][frame]])\n",
    "                c_start += 1\n",
    "        else:\n",
    "            for frame in sorted(data[vid[i]]):\n",
    "                tmp.append([data[vid[i]][frame]])\n",
    "            last_frame_id = sorted(data[vid[i]])[-1]\n",
    "            while len(tmp) <= 15:\n",
    "                tmp.append([data[vid[i]][last_frame_id]])\n",
    "        \n",
    "        inp.append(tmp)\n",
    "        output.append(labels[vid[i]])\n",
    "    output = dense_to_one_hot(np.array(output), n_classes)\n",
    "\n",
    "    return np.array(inp, dtype=np.float32), output\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def C3D_model(x, weights, biases, n_classes):\n",
    "    \n",
    "#     x = tf.reshape(x, shape=[-1,128,128,16,1]),\n",
    "    conv1 = conv3d(x, weights['wc1'], biases['bc1'], name='conv1')\n",
    "    pool1 = maxpool3d(conv1, k=1, name='pool1')\n",
    "    \n",
    "    conv2 = conv3d(pool1, weights['wc2'], biases['bc2'], name='conv2')\n",
    "    pool2 = maxpool3d(conv2, k=2, name='pool2')\n",
    "    \n",
    "    conv3a = conv3d(pool2, weights['wc3a'], biases['bc3a'], name='conv3a')\n",
    "    conv3b = conv3d(conv3a, weights['wc3b'], biases['bc3b'], name='conv3b')\n",
    "    pool3 = maxpool3d(conv3b, k=2, name='pool3')\n",
    "    \n",
    "    conv4a = conv3d(pool3, weights['wc4a'], biases['bc4a'], name='conv4a')\n",
    "    conv4b = conv3d(conv4a, weights['wc4b'], biases['bc4b'], name='conv4b')\n",
    "    pool4 = maxpool3d(conv4b, k=2, name='pool4')\n",
    "    \n",
    "    conv5a = conv3d(pool4, weights['wc5a'], biases['bc5a'], name='conv5a')\n",
    "    conv5b = conv3d(conv5a, weights['wc5b'], biases['bc5b'], name='conv5b')\n",
    "    pool5 = maxpool3d(conv5b, k=2, name='pool5')\n",
    "    \n",
    "    dim = np.prod(pool5.get_shape().as_list()[1:])\n",
    "\n",
    "    flattened = tf.reshape(pool5, [-1, dim])\n",
    "    fc6 = fc(flattened, dim, 4096, name='fc6')\n",
    "    \n",
    "    fc7 = fc(fc6, 4096, 4096, name='fc7')\n",
    "    \n",
    "    # Output, class prediction\n",
    "    out = fc(fc7, 4096, n_classes, name='out', relu=False)\n",
    "    \n",
    "\n",
    "    return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def C3D_model2(x, weights, biases, n_classes):\n",
    "    \n",
    "    conv1 = conv3d(x, [3,3,3,1,32], [32], name='conv1')\n",
    "    pool1 = maxpool3d(conv1, k=1, name='pool1')\n",
    "    \n",
    "    dim = 10\n",
    "\n",
    "    flattened = tf.reshape(pool1, [-1, dim])\n",
    "    fc6 = fc(flattened, dim, 64, name='fc6')\n",
    "    \n",
    "    fc7 = fc(fc6, 64, 64, name='fc7')\n",
    "    \n",
    "    # Output, class prediction\n",
    "    out = fc(fc7, 64, n_classes, name='out', relu=False)\n",
    "    \n",
    "\n",
    "    return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# os.mkdir('/output/logs')\n",
    "root = 'backup/'\n",
    "LOGDIR = 'logs/'\n",
    "w_input = 128\n",
    "h_input = 128\n",
    "l_input = 16\n",
    "n_classes = 7\n",
    "learning_rate = 1e-3\n",
    "\n",
    "n_epoch = 100\n",
    "batch_size = 128\n",
    "display_step = 10\n",
    "\n",
    "# Store layers weight & bias\n",
    "weights = {\n",
    "    # 5x5x5 conv, 1 input, 64 outputs\n",
    "    'wc1': [3,3,3,1,64],\n",
    "    # 5x5x5 conv, 64 inputs, 128 outputs\n",
    "    'wc2': [3,3,3,64,128],\n",
    "    # 5x5x5 conv, 128 inputs, 256 outputs\n",
    "    'wc3a': [3,3,3,128,256],\n",
    "    # 5x5x5 conv, 256 inputs, 512 outputs\n",
    "    'wc3b': [3,3,3,256,512],\n",
    "    # 5x5x5 conv, 512 inputs, 512 outputs\n",
    "    'wc4a': [3,3,3,512,512],\n",
    "    # 5x5x5 conv, 512 inputs, 512 outputs\n",
    "    'wc4b': [3,3,3,512,512],\n",
    "    # 5x5x5 conv, 512 inputs, 512 outputs\n",
    "    'wc5a': [3,3,3,512,512],\n",
    "    # 5x5x5 conv, 512 inputs, 512 outputs\n",
    "    'wc5b': [3,3,3,512,512]\n",
    "}\n",
    "\n",
    "biases = {\n",
    "    'bc1': [64],\n",
    "    'bc2': [128],\n",
    "    'bc3a': [256],\n",
    "    'bc3b': [512],\n",
    "    'bc4a': [512],\n",
    "    'bc4b': [512],\n",
    "    'bc5a': [512],\n",
    "    'bc5b': [512]\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# train_data = np.load(root+'image_data_train_757.npy', encoding='latin1').item(0)\n",
    "val_data = np.load(root+'image_data_val_372.npy', encoding='latin1').item(0)\n",
    "\n",
    "# vid_train = np.load(root+'vid_train.npy')\n",
    "vid_val = np.load(root+'vid_val.npy')\n",
    "\n",
    "# labels_train = np.load(root+'labels_train.npy').item(0)\n",
    "labels_val = np.load(root+'labels_val.npy').item(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "t1,t2 = get_xy(1,1,vid_val, labels_val, val_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1, 16, 128, 128, 1)"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.transpose(t1, (0,1,3,4,2)).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "\n",
    "with tf.name_scope('input'):\n",
    "    x = tf.placeholder(tf.float32, [None, l_input, w_input, h_input,1])\n",
    "    y = tf.placeholder(tf.float32, [None, n_classes])\n",
    "#     x = tf.placeholder(tf.float32)\n",
    "#     y = tf.placeholder(tf.float32)\n",
    "\n",
    "pred = C3D_model2(x, weights, biases, n_classes)\n",
    "with tf.name_scope(\"cross_entropy\"):\n",
    "    cross_entropy = tf.reduce_mean(\n",
    "    tf.nn.softmax_cross_entropy_with_logits(\n",
    "    logits=pred, labels=y), name=\"cross_entropy\")\n",
    "    tf.summary.scalar(\"cross_entropy\", cross_entropy)\n",
    "\n",
    "with tf.name_scope(\"train\"):\n",
    "    minimize = tf.train.AdamOptimizer(learning_rate).minimize(cross_entropy)\n",
    "\n",
    "with tf.name_scope(\"accuracy\"):\n",
    "    correct_prediction = tf.equal(tf.argmax(pred, 1), tf.argmax(y, 1))\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "    tf.summary.scalar(\"accuracy\", accuracy)\n",
    "\n",
    "init = tf.global_variables_initializer()\n",
    "\n",
    "# merge all summaries into a single \"operation\" which we can execute in a session \n",
    "summary_op = tf.summary.merge_all()\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    sess.run(init)\n",
    "#     no_of_batches = int((len(train_data)) / batch_size)\n",
    "    no_of_batches = int((len(val_data)) / batch_size)\n",
    "    train_writer = tf.summary.FileWriter('%strain/'%(LOGDIR), sess.graph)\n",
    "    val_writer = tf.summary.FileWriter('%sval/'%(LOGDIR), sess.graph)\n",
    "\n",
    "    for i in range(n_epoch):\n",
    "        ptr = 0\n",
    "#         np.random.shuffle(vid_train)\n",
    "        np.random.shuffle(vid_val)\n",
    "        for j in range(no_of_batches):\n",
    "#             inp, out = get_xy(ptr,ptr+batch_size, vid_train, labels_train, train_data)\n",
    "            inp, out = get_xy(ptr,ptr+batch_size, vid_val, labels_val, val_data)\n",
    "            inp = np.transpose(inp, (0,1,3,4,2))\n",
    "            sess.run([minimize],{x: inp, y: out})\n",
    "            \n",
    "#         r = random.randrange(0, len(train_data)-300)\n",
    "        r = random.randrange(0, len(val_data)-300)\n",
    "#         train_input, train_output = get_xy(0,len(train_data)-1, vid_train, labels_train, train_data)\n",
    "        train_input, train_output = get_xy(0,len(val_data)-1, vid_val, labels_val, val_data)\n",
    "        val_input, val_output = get_xy(0,len(val_data)-1, vid_val, labels_val, val_data)\n",
    "                \n",
    "        loss_trian,  acc_train, summary_train, props_train = sess.run([cross_entropy, accuracy, summary_op, pred],{x: train_input, y: train_output})\n",
    "        train_writer.add_summary(summary_train,i)\n",
    "        \n",
    "        _, acc_val, summary_val, props_val = sess.run([cross_entropy, accuracy, summary_op, pred],{x: val_input, y: val_output})\n",
    "        val_writer.add_summary(summary_val,i)\n",
    "        # Save the variables to disk.\n",
    "        if i %  25 == 0:\n",
    "            save_path = saver.save(sess, \"/output/model_%d.ckpt\"%i)\n",
    "            print(\"Model saved in file: %s\" % save_path)       \n",
    "        print(\"Epoch \",str(i), ', acc_train: %.7f' % (acc_train*100), 'acc_val: %.7f' % (acc_val*100), ', loss_trian: %.7f' % loss_trian)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.reset_default_graph()\n",
    "\n",
    "tf.global_variables()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}