text-classification/51.batch-all-triplet-loss-lstm.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "/usr/local/lib/python3.5/dist-packages/sklearn/cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "from utils import *\n",
    "import tensorflow as tf\n",
    "from sklearn.cross_validation import train_test_split\n",
    "import time\n",
    "import random"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['negative', 'positive']\n",
      "10662\n",
      "10662\n"
     ]
    }
   ],
   "source": [
    "trainset = sklearn.datasets.load_files(container_path = 'data', encoding = 'UTF-8')\n",
    "trainset.data, trainset.target = separate_dataset(trainset,1.0)\n",
    "print (trainset.target_names)\n",
    "print (len(trainset.data))\n",
    "print (len(trainset.target))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_X, test_X, train_Y, test_Y = train_test_split(trainset.data, trainset.target,\n",
    "                                                    test_size = 0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "vocab from size: 20332\n",
      "Most common words [('film', 1453), ('movie', 1270), ('one', 727), ('like', 721), ('story', 477), ('much', 386)]\n",
      "Sample data [546, 2451, 3275, 16228, 36, 7573, 218, 150, 19, 3910] ['rock', 'destined', '21st', 'centurys', 'new', 'conan', 'hes', 'going', 'make', 'splash']\n"
     ]
    }
   ],
   "source": [
    "concat = ' '.join(trainset.data).split()\n",
    "vocabulary_size = len(list(set(concat)))\n",
    "data, count, dictionary, rev_dictionary = build_dataset(concat, vocabulary_size)\n",
    "print('vocab from size: %d'%(vocabulary_size))\n",
    "print('Most common words', count[4:10])\n",
    "print('Sample data', data[:10], [rev_dictionary[i] for i in data[:10]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "GO = dictionary['GO']\n",
    "PAD = dictionary['PAD']\n",
    "EOS = dictionary['EOS']\n",
    "UNK = dictionary['UNK']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def _pairwise_distances(embeddings, squared=False):\n",
    "    dot_product = tf.matmul(embeddings, tf.transpose(embeddings))\n",
    "    square_norm = tf.diag_part(dot_product)\n",
    "    distances = tf.expand_dims(square_norm, 1) - 2.0 * dot_product + tf.expand_dims(square_norm, 0)\n",
    "    distances = tf.maximum(distances, 0.0)\n",
    "\n",
    "    if not squared:\n",
    "        mask = tf.to_float(tf.equal(distances, 0.0))\n",
    "        distances = distances + mask * 1e-16\n",
    "        distances = tf.sqrt(distances)\n",
    "        distances = distances * (1.0 - mask)\n",
    "\n",
    "    return distances\n",
    "\n",
    "\n",
    "def _get_anchor_positive_triplet_mask(labels):\n",
    "    indices_equal = tf.cast(tf.eye(tf.shape(labels)[0]), tf.bool)\n",
    "    indices_not_equal = tf.logical_not(indices_equal)\n",
    "    labels_equal = tf.equal(tf.expand_dims(labels, 0), tf.expand_dims(labels, 1))\n",
    "    mask = tf.logical_and(indices_not_equal, labels_equal)\n",
    "\n",
    "    return mask\n",
    "\n",
    "\n",
    "def _get_anchor_negative_triplet_mask(labels):\n",
    "    labels_equal = tf.equal(tf.expand_dims(labels, 0), tf.expand_dims(labels, 1))\n",
    "    mask = tf.logical_not(labels_equal)\n",
    "\n",
    "    return mask\n",
    "\n",
    "\n",
    "def _get_triplet_mask(labels):\n",
    "    indices_equal = tf.cast(tf.eye(tf.shape(labels)[0]), tf.bool)\n",
    "    indices_not_equal = tf.logical_not(indices_equal)\n",
    "    i_not_equal_j = tf.expand_dims(indices_not_equal, 2)\n",
    "    i_not_equal_k = tf.expand_dims(indices_not_equal, 1)\n",
    "    j_not_equal_k = tf.expand_dims(indices_not_equal, 0)\n",
    "\n",
    "    distinct_indices = tf.logical_and(tf.logical_and(i_not_equal_j, i_not_equal_k), j_not_equal_k)\n",
    "\n",
    "    label_equal = tf.equal(tf.expand_dims(labels, 0), tf.expand_dims(labels, 1))\n",
    "    i_equal_j = tf.expand_dims(label_equal, 2)\n",
    "    i_equal_k = tf.expand_dims(label_equal, 1)\n",
    "\n",
    "    valid_labels = tf.logical_and(i_equal_j, tf.logical_not(i_equal_k))\n",
    "    mask = tf.logical_and(distinct_indices, valid_labels)\n",
    "\n",
    "    return mask\n",
    "\n",
    "\n",
    "def batch_all_triplet_loss(labels, embeddings, margin, squared=False):\n",
    "    pairwise_dist = _pairwise_distances(embeddings, squared=squared)\n",
    "\n",
    "    # shape (batch_size, batch_size, 1)\n",
    "    anchor_positive_dist = tf.expand_dims(pairwise_dist, 2)\n",
    "    assert anchor_positive_dist.shape[2] == 1, \"{}\".format(anchor_positive_dist.shape)\n",
    "    # shape (batch_size, 1, batch_size)\n",
    "    anchor_negative_dist = tf.expand_dims(pairwise_dist, 1)\n",
    "    assert anchor_negative_dist.shape[1] == 1, \"{}\".format(anchor_negative_dist.shape)\n",
    "\n",
    "    triplet_loss = anchor_positive_dist - anchor_negative_dist + margin\n",
    "\n",
    "    mask = _get_triplet_mask(labels)\n",
    "    mask = tf.to_float(mask)\n",
    "    triplet_loss = tf.multiply(mask, triplet_loss)\n",
    "\n",
    "    # Remove negative losses (i.e. the easy triplets)\n",
    "    triplet_loss = tf.maximum(triplet_loss, 0.0)\n",
    "\n",
    "    # Count number of positive triplets (where triplet_loss > 0)\n",
    "    valid_triplets = tf.to_float(tf.greater(triplet_loss, 1e-16))\n",
    "    num_positive_triplets = tf.reduce_sum(valid_triplets)\n",
    "    num_valid_triplets = tf.reduce_sum(mask)\n",
    "    fraction_positive_triplets = num_positive_triplets / (num_valid_triplets + 1e-16)\n",
    "\n",
    "    # Get final mean triplet loss over the positive valid triplets\n",
    "    triplet_loss = tf.reduce_sum(triplet_loss) / (num_positive_triplets + 1e-16)\n",
    "\n",
    "    return triplet_loss, fraction_positive_triplets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Model:\n",
    "    def __init__(self, size_layer, num_layers, embedded_size,\n",
    "                 dict_size, dimension_output):\n",
    "        \n",
    "        def cells(reuse=False):\n",
    "            return tf.nn.rnn_cell.LSTMCell(size_layer,initializer=tf.orthogonal_initializer(),reuse=reuse)\n",
    "        \n",
    "        def rnn(embedded,reuse=False):\n",
    "            with tf.variable_scope('model', reuse=reuse):\n",
    "                rnn_cells = tf.nn.rnn_cell.MultiRNNCell([cells() for _ in range(num_layers)])\n",
    "                outputs, _ = tf.nn.dynamic_rnn(rnn_cells, embedded, dtype = tf.float32)\n",
    "                W = tf.get_variable('w',shape=(size_layer, dimension_output),initializer=tf.orthogonal_initializer())\n",
    "                b = tf.get_variable('b',shape=(dimension_output),initializer=tf.zeros_initializer())\n",
    "                return tf.matmul(outputs[:, -1], W) + b\n",
    "              \n",
    "        self.X = tf.placeholder(tf.int32, [None, None])\n",
    "        self.Y = tf.placeholder(tf.int32, [None])\n",
    "        embeddings = tf.Variable(tf.random_uniform([dict_size, embedded_size], -1, 1))\n",
    "        embedded = tf.nn.embedding_lookup(embeddings, self.X)\n",
    "        self.logits = rnn(embedded,False)\n",
    "        self.cost, fraction = batch_all_triplet_loss(self.Y, self.logits, margin=0.5, squared=False)\n",
    "        self.optimizer = tf.train.AdamOptimizer(1e-3).minimize(self.cost)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "size_layer = 128\n",
    "num_layers = 2\n",
    "embedded_size = 128\n",
    "dimension_output = 20\n",
    "maxlen = 50\n",
    "batch_size = 64"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "sess = tf.InteractiveSession()\n",
    "model = Model(size_layer,num_layers,embedded_size,len(dictionary),dimension_output)\n",
    "sess.run(tf.global_variables_initializer())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_X = str_idx(train_X,dictionary,maxlen)\n",
    "test_X = str_idx(test_X,dictionary,maxlen)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 1, avg cost: 0.007900\n",
      "epoch: 2, avg cost: 0.007905\n",
      "epoch: 3, avg cost: 0.008033\n",
      "epoch: 4, avg cost: 0.008361\n",
      "epoch: 5, avg cost: 0.008460\n",
      "epoch: 6, avg cost: 0.008404\n",
      "epoch: 7, avg cost: 0.008089\n",
      "epoch: 8, avg cost: 0.007841\n",
      "epoch: 9, avg cost: 0.007266\n",
      "epoch: 10, avg cost: 0.006499\n"
     ]
    }
   ],
   "source": [
    "for i in range(10):\n",
    "    total_loss = 0\n",
    "    for index in range(0,len(train_X),batch_size):\n",
    "        batch_x = train_X[index:min(index+batch_size,len(train_X))]\n",
    "        batch_y = train_Y[index:min(index+batch_size,len(train_X))]\n",
    "        loss, _ = sess.run([model.cost,model.optimizer],feed_dict={model.X:batch_x,model.Y:batch_y})\n",
    "        total_loss += loss\n",
    "    total_loss /= len(train_X)\n",
    "    print('epoch: %d, avg cost: %f'%(i+1,total_loss))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2133, 20)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "logits_train = sess.run(model.logits,feed_dict={model.X:train_X})\n",
    "logits_test = sess.run(model.logits,feed_dict={model.X:test_X})\n",
    "logits_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "   negative       0.70      0.68      0.69      1067\n",
      "   positive       0.69      0.71      0.70      1066\n",
      "\n",
      "avg / total       0.70      0.70      0.70      2133\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from scipy.spatial.distance import cdist\n",
    "label_test = []\n",
    "for i in range(logits_test.shape[0]):\n",
    "    label_test.append(train_Y[np.argsort(cdist(logits_train, [logits_test[i,:]], 'cosine').ravel())[0]])\n",
    "print(metrics.classification_report(test_Y, label_test, target_names = trainset.target_names))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}