03-training-00-keras_2mlp_clsw0.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-28T22:31:54.442317Z",
     "start_time": "2018-09-28T22:31:52.053530Z"
    }
   },
   "outputs": [],
   "source": [
    "from pathlib import Path\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras as k\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from rennet.utils.plotting_utils import plot_speclike\n",
    "from rennet.utils.np_utils import confusion_matrix, normalize_confusion_matrix, print_prec_rec\n",
    "import feat_ext as fx\n",
    "from train_utils import *"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "# Define"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-29T02:04:26.942993Z",
     "start_time": "2018-09-29T02:04:26.939990Z"
    }
   },
   "outputs": [],
   "source": [
    "name = '03-training-00-keras_2mlp_clsw0'  # CHANGE!!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-28T22:31:54.603891Z",
     "start_time": "2018-09-28T22:31:54.442317Z"
    }
   },
   "outputs": [],
   "source": [
    "embedding_dim = (128,)\n",
    "nclasses = 2\n",
    "batchsize = 128\n",
    "n_passes_per_epoch = 5\n",
    "n_epochs = 50\n",
    "class_weight=[1.0, 1.0]\n",
    "\n",
    "# keras model\n",
    "model = k.Sequential()\n",
    "model.add(k.layers.InputLayer(embedding_dim, name='input'))  # DO NOT CHANGE\n",
    "\n",
    "model.add(k.layers.Dense(64, activation='relu', name='dense_01'))\n",
    "model.add(k.layers.Dense(32, activation='relu', name='dense_02'))\n",
    "\n",
    "model.add(k.layers.Dense(nclasses, activation='softmax', name='output'))  # DO NOT CHANGE\n",
    "\n",
    "model.compile(\n",
    "    optimizer='adam',\n",
    "    loss='categorical_crossentropy',\n",
    "    metrics=['categorical_accuracy'],\n",
    ")\n",
    "\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "# Prep"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-28T22:31:54.611893Z",
     "start_time": "2018-09-28T22:31:54.605925Z"
    }
   },
   "outputs": [],
   "source": [
    "dir_pickles_root = Path.cwd().joinpath(\"data/prepared/pickles/20190909-vggish_embedding/\")\n",
    "\n",
    "fp_trn = dir_pickles_root.joinpath(\"trn.tfrecord\")\n",
    "fp_val = dir_pickles_root.joinpath(\"val.tfrecord\")\n",
    "\n",
    "print(f'tfrecords found? (trn: {fp_trn.exists()}), (val: {fp_val.exists()})')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-28T22:31:58.965447Z",
     "start_time": "2018-09-28T22:31:54.725895Z"
    }
   },
   "outputs": [],
   "source": [
    "dset_trn = get_dataset(fp_trn, batchsize=batchsize).apply(tf.data.experimental.shuffle_and_repeat(16384, seed=9899))\n",
    "dset_val = get_dataset(fp_val, batchsize=batchsize).repeat()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-28T22:32:00.040507Z",
     "start_time": "2018-09-28T22:31:58.967444Z"
    }
   },
   "outputs": [],
   "source": [
    "trn_steps_per_pass = dataset_shape(fp_trn)\n",
    "val_steps = dataset_shape(fp_val)\n",
    "\n",
    "print(f'total num samples per pass: (trn: {trn_steps_per_pass}), (val: {val_steps})')\n",
    "\n",
    "trn_steps_per_pass = trn_steps_per_pass // batchsize\n",
    "val_steps = val_steps // batchsize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-28T22:32:00.048559Z",
     "start_time": "2018-09-28T22:32:00.042506Z"
    }
   },
   "outputs": [],
   "source": [
    "dir_outputs = Path.cwd().joinpath(f\"outputs/{name}\")\n",
    "dir_outputs.mkdir(parents=True, exist_ok=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-28T22:32:00.097663Z",
     "start_time": "2018-09-28T22:32:00.050505Z"
    }
   },
   "outputs": [],
   "source": [
    "MODEL_CHECKPOINT_PATTERN = 'w.{epoch:03d}-{val_loss:.3f}-{val_categorical_accuracy:.3f}.h5'\n",
    "callbacks = [\n",
    "    k.callbacks.ModelCheckpoint(\n",
    "        str(dir_outputs.joinpath(MODEL_CHECKPOINT_PATTERN)),\n",
    "        save_best_only=False,\n",
    "        save_weights_only=False,\n",
    "        period=1,\n",
    "        verbose=1,\n",
    "    ),\n",
    "    k.callbacks.TensorBoard(\n",
    "        log_dir=str(dir_outputs),\n",
    "        write_graph=True,\n",
    "        write_images=True,\n",
    "    ),\n",
    "    k.callbacks.EarlyStopping(\n",
    "        patience=3,\n",
    "        monitor='val_loss'\n",
    "    ),\n",
    "]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "# Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-28T22:42:54.229179Z",
     "start_time": "2018-09-28T22:32:00.099663Z"
    }
   },
   "outputs": [],
   "source": [
    "model.fit(\n",
    "    dset_trn,\n",
    "    steps_per_epoch=trn_steps_per_pass * n_passes_per_epoch,\n",
    "    epochs=n_epochs,\n",
    "    validation_data=dset_val,\n",
    "    validation_steps=val_steps,\n",
    "    callbacks=callbacks,\n",
    "    verbose=1,\n",
    "    class_weight=class_weight,\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "# Evaluate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-28T22:42:54.549179Z",
     "start_time": "2018-09-28T22:42:54.232185Z"
    }
   },
   "outputs": [],
   "source": [
    "dset_val = get_dataset(fp_val, batchsize=batchsize).repeat()\n",
    "preds = model.predict(dset_val, steps=val_steps)\n",
    "preds.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-28T22:42:54.805198Z",
     "start_time": "2018-09-28T22:42:54.552182Z"
    }
   },
   "outputs": [],
   "source": [
    "dset_val = get_dataset(fp_val, batchsize=batchsize).repeat()\n",
    "n = dset_val.make_one_shot_iterator().get_next()\n",
    "expected = []\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    for _ in range(val_steps):\n",
    "        expected.append(sess.run(n)[1])\n",
    "        \n",
    "expected = np.concatenate(expected)\n",
    "expected.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-28T22:42:55.842434Z",
     "start_time": "2018-09-28T22:42:54.996186Z"
    }
   },
   "outputs": [],
   "source": [
    "plot_speclike([preds.argmax(axis=1), \n",
    "               expected.argmax(axis=1)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-28T22:42:55.964651Z",
     "start_time": "2018-09-28T22:42:55.845691Z"
    }
   },
   "outputs": [],
   "source": [
    "s = np.s_[-1000:]  # last 1000 * 0.12 seconds\n",
    "plot_speclike([preds.argmax(axis=1)[s], \n",
    "               expected.argmax(axis=1)[s]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-28T22:42:56.163652Z",
     "start_time": "2018-09-28T22:42:55.974655Z"
    }
   },
   "outputs": [],
   "source": [
    "# true positive\n",
    "_ = plt.hist(preds[:, 0][expected[:, 0].astype(np.bool)], bins=20, alpha=0.3, label='pred=0,exp=0')\n",
    "_ = plt.hist(preds[:, 1][expected[:, 1].astype(np.bool)], bins=20, alpha=0.3, label='pred=1,exp=1')\n",
    "plt.legend()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-28T22:42:56.355687Z",
     "start_time": "2018-09-28T22:42:56.165653Z"
    }
   },
   "outputs": [],
   "source": [
    "# false positives\n",
    "_ = plt.hist(preds[:, 0][expected[:, 1].astype(np.bool)], bins=20, alpha=0.3, label='pred=0,exp=1')\n",
    "_ = plt.hist(preds[:, 1][expected[:, 0].astype(np.bool)], bins=20, alpha=0.3, label='pred=1,exp=0')\n",
    "plt.legend()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-28T22:42:56.409266Z",
     "start_time": "2018-09-28T22:42:56.364655Z"
    }
   },
   "outputs": [],
   "source": [
    "c = confusion_matrix(expected.argmax(axis=1), preds.argmax(axis=1))\n",
    "n = normalize_confusion_matrix(c)\n",
    "\n",
    "print(\"classwise precision and recall\")\n",
    "print_prec_rec(*n, onlydiag=True)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}