topic-model/3.xlnet-topic.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Download XLNET-Base model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !wget https://storage.googleapis.com/xlnet/released_models/cased_L-12_H-768_A-12.zip\n",
    "# !unzip cased_L-12_H-768_A-12.zip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "spiece.model\t   xlnet_model.ckpt.data-00000-of-00001  xlnet_model.ckpt.meta\r\n",
      "xlnet_config.json  xlnet_model.ckpt.index\r\n"
     ]
    }
   ],
   "source": [
    "!ls xlnet_cased_L-12_H-768_A-12"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Download simple dataset\n",
    "\n",
    "I want to use negative sentiment corpus to build unsupervised topic models using Attention from XLNET."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5330"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# !wget https://raw.githubusercontent.com/huseinzol05/NLP-Models-Tensorflow/master/text-classification/data/negative/negative\n",
    "\n",
    "with open('negative') as fopen:\n",
    "    negative = fopen.read().split('\\n')[:-1]\n",
    "len(negative)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import collections\n",
    "import re\n",
    "import numpy as np\n",
    "import itertools\n",
    "from prepro_utils import preprocess_text, encode_ids, encode_pieces\n",
    "\n",
    "SEG_ID_A = 0\n",
    "SEG_ID_B = 1\n",
    "SEG_ID_CLS = 2\n",
    "SEG_ID_SEP = 3\n",
    "SEG_ID_PAD = 4\n",
    "\n",
    "special_symbols = {\n",
    "    '<unk>': 0,\n",
    "    '<s>': 1,\n",
    "    '</s>': 2,\n",
    "    '<cls>': 3,\n",
    "    '<sep>': 4,\n",
    "    '<pad>': 5,\n",
    "    '<mask>': 6,\n",
    "    '<eod>': 7,\n",
    "    '<eop>': 8,\n",
    "}\n",
    "\n",
    "UNK_ID = special_symbols['<unk>']\n",
    "CLS_ID = special_symbols['<cls>']\n",
    "SEP_ID = special_symbols['<sep>']\n",
    "MASK_ID = special_symbols['<mask>']\n",
    "EOD_ID = special_symbols['<eod>']\n",
    "\n",
    "def generate_ngram(seq, ngram = (1, 3)):\n",
    "    g = []\n",
    "    for i in range(ngram[0], ngram[-1] + 1):\n",
    "        g.extend(list(ngrams_generator(seq, i)))\n",
    "    return g\n",
    "\n",
    "def _pad_sequence(\n",
    "    sequence,\n",
    "    n,\n",
    "    pad_left = False,\n",
    "    pad_right = False,\n",
    "    left_pad_symbol = None,\n",
    "    right_pad_symbol = None,\n",
    "):\n",
    "    sequence = iter(sequence)\n",
    "    if pad_left:\n",
    "        sequence = itertools.chain((left_pad_symbol,) * (n - 1), sequence)\n",
    "    if pad_right:\n",
    "        sequence = itertools.chain(sequence, (right_pad_symbol,) * (n - 1))\n",
    "    return sequence\n",
    "\n",
    "\n",
    "def ngrams_generator(\n",
    "    sequence,\n",
    "    n,\n",
    "    pad_left = False,\n",
    "    pad_right = False,\n",
    "    left_pad_symbol = None,\n",
    "    right_pad_symbol = None,\n",
    "):\n",
    "    \"\"\"\n",
    "    generate ngrams.\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    sequence : list of str\n",
    "        list of tokenize words.\n",
    "    n : int\n",
    "        ngram size\n",
    "\n",
    "    Returns\n",
    "    -------\n",
    "    ngram: list\n",
    "    \"\"\"\n",
    "    sequence = _pad_sequence(\n",
    "        sequence, n, pad_left, pad_right, left_pad_symbol, right_pad_symbol\n",
    "    )\n",
    "\n",
    "    history = []\n",
    "    while n > 1:\n",
    "        try:\n",
    "            next_item = next(sequence)\n",
    "        except StopIteration:\n",
    "            return\n",
    "        history.append(next_item)\n",
    "        n -= 1\n",
    "    for item in sequence:\n",
    "        history.append(item)\n",
    "        yield tuple(history)\n",
    "        del history[0]\n",
    "\n",
    "def tokenize_fn(text, sp_model):\n",
    "    text = preprocess_text(text, lower = False)\n",
    "    return encode_ids(sp_model, text)        \n",
    "        \n",
    "def merge_sentencepiece_tokens(paired_tokens, weighted = True):\n",
    "    new_paired_tokens = []\n",
    "    n_tokens = len(paired_tokens)\n",
    "    rejected = ['<cls>', '<sep>']\n",
    "\n",
    "    i = 0\n",
    "\n",
    "    while i < n_tokens:\n",
    "\n",
    "        current_token, current_weight = paired_tokens[i]\n",
    "        if not current_token.startswith('▁') and current_token not in rejected:\n",
    "            previous_token, previous_weight = new_paired_tokens.pop()\n",
    "            merged_token = previous_token\n",
    "            merged_weight = [previous_weight]\n",
    "            while (\n",
    "                not current_token.startswith('▁')\n",
    "                and current_token not in rejected\n",
    "            ):\n",
    "                merged_token = merged_token + current_token.replace('▁', '')\n",
    "                merged_weight.append(current_weight)\n",
    "                i = i + 1\n",
    "                current_token, current_weight = paired_tokens[i]\n",
    "            merged_weight = np.mean(merged_weight)\n",
    "            new_paired_tokens.append((merged_token, merged_weight))\n",
    "\n",
    "        else:\n",
    "            new_paired_tokens.append((current_token, current_weight))\n",
    "            i = i + 1\n",
    "\n",
    "    words = [\n",
    "        i[0].replace('▁', '')\n",
    "        for i in new_paired_tokens\n",
    "        if i[0] not in ['<cls>', '<sep>', '<pad>']\n",
    "    ]\n",
    "    weights = [\n",
    "        i[1]\n",
    "        for i in new_paired_tokens\n",
    "        if i[0] not in ['<cls>', '<sep>', '<pad>']\n",
    "    ]\n",
    "    if weighted:\n",
    "        weights = np.array(weights)\n",
    "        weights = weights / np.sum(weights)\n",
    "    return list(zip(words, weights))\n",
    "\n",
    "def xlnet_tokenization(tokenizer, texts):\n",
    "    input_ids, input_masks, segment_ids, s_tokens = [], [], [], []\n",
    "    for text in texts:\n",
    "        tokens_a = tokenize_fn(text, tokenizer)\n",
    "        tokens = []\n",
    "        segment_id = []\n",
    "        for token in tokens_a:\n",
    "            tokens.append(token)\n",
    "            segment_id.append(SEG_ID_A)\n",
    "\n",
    "        tokens.append(SEP_ID)\n",
    "        segment_id.append(SEG_ID_A)\n",
    "        tokens.append(CLS_ID)\n",
    "        segment_id.append(SEG_ID_CLS)\n",
    "\n",
    "        input_id = tokens\n",
    "        input_mask = [0] * len(input_id)\n",
    "\n",
    "        input_ids.append(input_id)\n",
    "        input_masks.append(input_mask)\n",
    "        segment_ids.append(segment_id)\n",
    "        s_tokens.append([tokenizer.IdToPiece(i) for i in tokens])\n",
    "\n",
    "    maxlen = max([len(i) for i in input_ids])\n",
    "    input_ids = padding_sequence(input_ids, maxlen, padding = 'pre')\n",
    "    input_masks = padding_sequence(\n",
    "        input_masks, maxlen, padding = 'pre', pad_int = 1\n",
    "    )\n",
    "    segment_ids = padding_sequence(\n",
    "        segment_ids, maxlen, padding = 'pre', pad_int = SEG_ID_PAD\n",
    "    )\n",
    "\n",
    "    return input_ids, input_masks, segment_ids, s_tokens\n",
    "\n",
    "def padding_sequence(seq, maxlen, padding = 'post', pad_int = 0):\n",
    "    padded_seqs = []\n",
    "    for s in seq:\n",
    "        if padding == 'post':\n",
    "            padded_seqs.append(s + [pad_int] * (maxlen - len(s)))\n",
    "        if padding == 'pre':\n",
    "            padded_seqs.append([pad_int] * (maxlen - len(s)) + s)\n",
    "    return padded_seqs\n",
    "\n",
    "def get_assignment_map_from_checkpoint(tvars, init_checkpoint):\n",
    "    assignment_map = {}\n",
    "    initialized_variable_names = {}\n",
    "\n",
    "    name_to_variable = collections.OrderedDict()\n",
    "    for var in tvars:\n",
    "        name = var.name\n",
    "        m = re.match('^(.*):\\\\d+$', name)\n",
    "        if m is not None:\n",
    "            name = m.group(1)\n",
    "        name_to_variable[name] = var\n",
    "\n",
    "    init_vars = tf.train.list_variables(init_checkpoint)\n",
    "\n",
    "    assignment_map = collections.OrderedDict()\n",
    "    for x in init_vars:\n",
    "        (name, var) = (x[0], x[1])\n",
    "        if name not in name_to_variable:\n",
    "            continue\n",
    "        assignment_map[name] = name_to_variable[name]\n",
    "        initialized_variable_names[name] = 1\n",
    "        initialized_variable_names[name + ':0'] = 1\n",
    "\n",
    "    return (assignment_map, initialized_variable_names)\n",
    "\n",
    "\n",
    "class _Model:\n",
    "    def __init__(self, xlnet_config, tokenizer, checkpoint, pool_mode = 'last'):\n",
    "\n",
    "        kwargs = dict(\n",
    "            is_training = True,\n",
    "            use_tpu = False,\n",
    "            use_bfloat16 = False,\n",
    "            dropout = 0.0,\n",
    "            dropatt = 0.0,\n",
    "            init = 'normal',\n",
    "            init_range = 0.1,\n",
    "            init_std = 0.05,\n",
    "            clamp_len = -1,\n",
    "        )\n",
    "\n",
    "        xlnet_parameters = xlnet_lib.RunConfig(**kwargs)\n",
    "\n",
    "        self._tokenizer = tokenizer\n",
    "        _graph = tf.Graph()\n",
    "        with _graph.as_default():\n",
    "            self.X = tf.placeholder(tf.int32, [None, None])\n",
    "            self.segment_ids = tf.placeholder(tf.int32, [None, None])\n",
    "            self.input_masks = tf.placeholder(tf.float32, [None, None])\n",
    "\n",
    "            xlnet_model = xlnet_lib.XLNetModel(\n",
    "                xlnet_config = xlnet_config,\n",
    "                run_config = xlnet_parameters,\n",
    "                input_ids = tf.transpose(self.X, [1, 0]),\n",
    "                seg_ids = tf.transpose(self.segment_ids, [1, 0]),\n",
    "                input_mask = tf.transpose(self.input_masks, [1, 0]),\n",
    "            )\n",
    "\n",
    "            self.logits = xlnet_model.get_pooled_out(pool_mode, True)\n",
    "            self._sess = tf.InteractiveSession()\n",
    "            self._sess.run(tf.global_variables_initializer())\n",
    "            tvars = tf.trainable_variables()\n",
    "            assignment_map, _ = get_assignment_map_from_checkpoint(\n",
    "                tvars, checkpoint\n",
    "            )\n",
    "            self._saver = tf.train.Saver(var_list = assignment_map)\n",
    "            attentions = [\n",
    "                n.name\n",
    "                for n in tf.get_default_graph().as_graph_def().node\n",
    "                if 'rel_attn/Softmax' in n.name\n",
    "            ]\n",
    "            g = tf.get_default_graph()\n",
    "            self.attention_nodes = [\n",
    "                g.get_tensor_by_name('%s:0' % (a)) for a in attentions\n",
    "            ]\n",
    "\n",
    "    def vectorize(self, strings):\n",
    "        \"\"\"\n",
    "        Vectorize string inputs using bert attention.\n",
    "\n",
    "        Parameters\n",
    "        ----------\n",
    "        strings : str / list of str\n",
    "\n",
    "        Returns\n",
    "        -------\n",
    "        array: vectorized strings\n",
    "        \"\"\"\n",
    "\n",
    "        if isinstance(strings, list):\n",
    "            if not isinstance(strings[0], str):\n",
    "                raise ValueError('input must be a list of strings or a string')\n",
    "        else:\n",
    "            if not isinstance(strings, str):\n",
    "                raise ValueError('input must be a list of strings or a string')\n",
    "        if isinstance(strings, str):\n",
    "            strings = [strings]\n",
    "\n",
    "        input_ids, input_masks, segment_ids, _ = xlnet_tokenization(\n",
    "            self._tokenizer, strings\n",
    "        )\n",
    "        return self._sess.run(\n",
    "            self.logits,\n",
    "            feed_dict = {\n",
    "                self.X: input_ids,\n",
    "                self.segment_ids: segment_ids,\n",
    "                self.input_masks: input_masks,\n",
    "            },\n",
    "        )\n",
    "\n",
    "    def attention(self, strings, method = 'last', **kwargs):\n",
    "        \"\"\"\n",
    "        Get attention string inputs from xlnet attention.\n",
    "\n",
    "        Parameters\n",
    "        ----------\n",
    "        strings : str / list of str\n",
    "        method : str, optional (default='last')\n",
    "            Attention layer supported. Allowed values:\n",
    "\n",
    "            * ``'last'`` - attention from last layer.\n",
    "            * ``'first'`` - attention from first layer.\n",
    "            * ``'mean'`` - average attentions from all layers.\n",
    "\n",
    "        Returns\n",
    "        -------\n",
    "        array: attention\n",
    "        \"\"\"\n",
    "\n",
    "        if isinstance(strings, list):\n",
    "            if not isinstance(strings[0], str):\n",
    "                raise ValueError('input must be a list of strings or a string')\n",
    "        else:\n",
    "            if not isinstance(strings, str):\n",
    "                raise ValueError('input must be a list of strings or a string')\n",
    "        if isinstance(strings, str):\n",
    "            strings = [strings]\n",
    "\n",
    "        method = method.lower()\n",
    "        if method not in ['last', 'first', 'mean']:\n",
    "            raise Exception(\n",
    "                \"method not supported, only support ['last', 'first', 'mean']\"\n",
    "            )\n",
    "\n",
    "        input_ids, input_masks, segment_ids, s_tokens = xlnet_tokenization(\n",
    "            self._tokenizer, strings\n",
    "        )\n",
    "        maxlen = max([len(s) for s in s_tokens])\n",
    "        s_tokens = padding_sequence(s_tokens, maxlen, pad_int = '<cls>')\n",
    "        attentions = self._sess.run(\n",
    "            self.attention_nodes,\n",
    "            feed_dict = {\n",
    "                self.X: input_ids,\n",
    "                self.segment_ids: segment_ids,\n",
    "                self.input_masks: input_masks,\n",
    "            },\n",
    "        )\n",
    "\n",
    "        if method == 'first':\n",
    "            cls_attn = np.transpose(attentions[0][:, 0], (1, 0, 2))\n",
    "\n",
    "        if method == 'last':\n",
    "            cls_attn = np.transpose(attentions[-1][:, 0], (1, 0, 2))\n",
    "\n",
    "        if method == 'mean':\n",
    "            cls_attn = np.transpose(\n",
    "                np.mean(attentions, axis = 0).mean(axis = 1), (1, 0, 2)\n",
    "            )\n",
    "\n",
    "        cls_attn = np.mean(cls_attn, axis = 1)\n",
    "        total_weights = np.sum(cls_attn, axis = -1, keepdims = True)\n",
    "        attn = cls_attn / total_weights\n",
    "        output = []\n",
    "        for i in range(attn.shape[0]):\n",
    "            output.append(\n",
    "                merge_sentencepiece_tokens(list(zip(s_tokens[i], attn[i])))\n",
    "            )\n",
    "        return output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/husein/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:516: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:517: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:518: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:519: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:520: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:525: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:541: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:542: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:543: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:544: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:545: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:550: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n",
      "WARNING: Logging before flag parsing goes to stderr.\n",
      "W0831 03:24:51.882980 139627729786688 deprecation_wrapper.py:119] From /home/husein/local/xlnet.py:70: The name tf.gfile.Open is deprecated. Please use tf.io.gfile.GFile instead.\n",
      "\n",
      "W0831 03:24:51.902457 139627729786688 deprecation_wrapper.py:119] From /home/husein/local/xlnet.py:253: The name tf.variable_scope is deprecated. Please use tf.compat.v1.variable_scope instead.\n",
      "\n",
      "W0831 03:24:51.903432 139627729786688 deprecation_wrapper.py:119] From /home/husein/local/xlnet.py:253: The name tf.AUTO_REUSE is deprecated. Please use tf.compat.v1.AUTO_REUSE instead.\n",
      "\n",
      "W0831 03:24:51.905369 139627729786688 deprecation_wrapper.py:119] From /home/husein/local/modeling.py:686: The name tf.logging.info is deprecated. Please use tf.compat.v1.logging.info instead.\n",
      "\n",
      "W0831 03:24:51.906327 139627729786688 deprecation_wrapper.py:119] From /home/husein/local/modeling.py:693: The name tf.get_variable is deprecated. Please use tf.compat.v1.get_variable instead.\n",
      "\n",
      "W0831 03:24:51.970893 139627729786688 deprecation.py:323] From /home/husein/local/modeling.py:797: dropout (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use keras.layers.dropout instead.\n",
      "W0831 03:24:53.032129 139627729786688 lazy_loader.py:50] \n",
      "The TensorFlow contrib module will not be included in TensorFlow 2.0.\n",
      "For more information, please see:\n",
      "  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n",
      "  * https://github.com/tensorflow/addons\n",
      "  * https://github.com/tensorflow/io (for I/O related ops)\n",
      "If you depend on functionality not listed there, please file an issue.\n",
      "\n",
      "W0831 03:24:53.051974 139627729786688 deprecation.py:323] From /home/husein/local/modeling.py:99: dense (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use keras.layers.dense instead.\n",
      "W0831 03:25:01.478370 139627729786688 deprecation.py:323] From /home/husein/.local/lib/python3.6/site-packages/tensorflow/python/training/saver.py:1276: checkpoint_exists (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use standard file APIs to check for files with this prefix.\n"
     ]
    }
   ],
   "source": [
    "import sentencepiece as spm\n",
    "import xlnet as xlnet_lib\n",
    "import tensorflow as tf\n",
    "\n",
    "sp_model = spm.SentencePieceProcessor()\n",
    "sp_model.Load('xlnet_cased_L-12_H-768_A-12/spiece.model')\n",
    "xlnet_config = xlnet_lib.XLNetConfig(\n",
    "    json_path = 'xlnet_cased_L-12_H-768_A-12/xlnet_config.json'\n",
    ")\n",
    "xlnet_checkpoint = 'xlnet_cased_L-12_H-768_A-12/xlnet_model.ckpt'\n",
    "model = _Model(\n",
    "    xlnet_config, sp_model, xlnet_checkpoint, pool_mode = 'last'\n",
    ")\n",
    "model._saver.restore(model._sess, xlnet_checkpoint)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-0.9849674 , -0.67051286,  0.99992144, ..., -0.99999577,\n",
       "        -0.99904066, -0.85338414],\n",
       "       [-0.64562905, -0.7124205 ,  0.99995923, ..., -0.9999986 ,\n",
       "        -0.9997058 , -0.9994817 ]], dtype=float32)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "v = model.vectorize(['hello nice to meet u', 'so long sucker'])\n",
    "v"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2, 768)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "v.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[('hello', 0.6345004),\n",
       "  ('nice', 0.22861008),\n",
       "  ('to', 0.04936926),\n",
       "  ('meet', 0.022651237),\n",
       "  ('u', 0.064868994)],\n",
       " [('so', 0.117449395), ('long', 0.13799533), ('sucker', 0.74455523)]]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.attention(['hello nice to meet u', 'so long sucker'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Building topic modeling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 10\n",
    "ngram = (1, 3)\n",
    "n_topics = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 533/533 [00:41<00:00, 12.94it/s]\n"
     ]
    }
   ],
   "source": [
    "from sklearn.cluster import KMeans\n",
    "from tqdm import tqdm\n",
    "\n",
    "rows, attentions = [], []\n",
    "for i in tqdm(range(0, len(negative), batch_size)):\n",
    "    index = min(i + batch_size, len(negative))\n",
    "    rows.append(model.vectorize(negative[i:index]))\n",
    "    attentions.extend(model.attention(negative[i:index]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Download simple english stopwords\n",
    "\n",
    "You might want to gather more of stopwords."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--2019-08-31 03:25:45--  https://raw.githubusercontent.com/stopwords-iso/stopwords-en/master/stopwords-en.json\n",
      "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 151.101.192.133, 151.101.128.133, 151.101.64.133, ...\n",
      "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|151.101.192.133|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 10275 (10K) [text/plain]\n",
      "Saving to: ‘stopwords-en.json’\n",
      "\n",
      "stopwords-en.json   100%[===================>]  10.03K  --.-KB/s    in 0s      \n",
      "\n",
      "2019-08-31 03:25:46 (52.9 MB/s) - ‘stopwords-en.json’ saved [10275/10275]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "!wget https://raw.githubusercontent.com/stopwords-iso/stopwords-en/master/stopwords-en.json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1298"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import json\n",
    "with open('stopwords-en.json') as fopen:\n",
    "    stopwords = json.load(fopen)\n",
    "len(stopwords)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processed 500\n",
      "processed 1000\n",
      "processed 1500\n",
      "processed 2000\n",
      "processed 2500\n",
      "processed 3000\n",
      "processed 3500\n",
      "processed 4000\n",
      "processed 4500\n",
      "processed 5000\n"
     ]
    }
   ],
   "source": [
    "concat = np.concatenate(rows, axis = 0)\n",
    "kmeans = KMeans(n_clusters = n_topics, random_state = 0).fit(concat)\n",
    "labels = kmeans.labels_\n",
    "\n",
    "overall, filtered_a = [], []\n",
    "for a in attentions:\n",
    "    f = [i for i in a if i[0] not in stopwords]\n",
    "    overall.extend(f)\n",
    "    filtered_a.append(f)\n",
    "\n",
    "o_ngram = generate_ngram(overall, ngram)\n",
    "features = []\n",
    "for i in o_ngram:\n",
    "    features.append(' '.join([w[0] for w in i]))\n",
    "features = list(set(features))\n",
    "\n",
    "components = np.zeros((n_topics, len(features)))\n",
    "for no, i in enumerate(labels):\n",
    "    if (no + 1) % 500 == 0:\n",
    "        print('processed %d'%(no + 1))\n",
    "    f = generate_ngram(filtered_a[no], ngram)\n",
    "    for w in f:\n",
    "        word = ' '.join([r[0] for r in w])\n",
    "        score = np.mean([r[1] for r in w])\n",
    "        if word in features:\n",
    "            components[i, features.index(word)] += score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def print_topics_modelling(\n",
    "    topics, feature_names, sorting, n_words = 20, return_df = True\n",
    "):\n",
    "    if return_df:\n",
    "        try:\n",
    "            import pandas as pd\n",
    "        except:\n",
    "            raise Exception(\n",
    "                'pandas not installed. Please install it and try again or set `return_df = False`'\n",
    "            )\n",
    "    df = {}\n",
    "    for i in range(topics):\n",
    "        words = []\n",
    "        for k in range(n_words):\n",
    "            words.append(feature_names[sorting[i, k]])\n",
    "        df['topic %d' % (i)] = words\n",
    "    if return_df:\n",
    "        return pd.DataFrame.from_dict(df)\n",
    "    else:\n",
    "        return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>topic 0</th>\n",
       "      <th>topic 1</th>\n",
       "      <th>topic 2</th>\n",
       "      <th>topic 3</th>\n",
       "      <th>topic 4</th>\n",
       "      <th>topic 5</th>\n",
       "      <th>topic 6</th>\n",
       "      <th>topic 7</th>\n",
       "      <th>topic 8</th>\n",
       "      <th>topic 9</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>movie</td>\n",
       "      <td>movie</td>\n",
       "      <td>film</td>\n",
       "      <td>film</td>\n",
       "      <td>movie</td>\n",
       "      <td>movie</td>\n",
       "      <td>movie</td>\n",
       "      <td>film</td>\n",
       "      <td>music</td>\n",
       "      <td>movie</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>film</td>\n",
       "      <td>film</td>\n",
       "      <td>movie</td>\n",
       "      <td>movie</td>\n",
       "      <td>film</td>\n",
       "      <td>film</td>\n",
       "      <td>film</td>\n",
       "      <td>movie</td>\n",
       "      <td>twists</td>\n",
       "      <td>film</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>comedy</td>\n",
       "      <td>bad</td>\n",
       "      <td>time</td>\n",
       "      <td>characters</td>\n",
       "      <td>characters</td>\n",
       "      <td>life</td>\n",
       "      <td>glory</td>\n",
       "      <td>action</td>\n",
       "      <td>film</td>\n",
       "      <td>comedy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>bad</td>\n",
       "      <td>pretentious</td>\n",
       "      <td>drama</td>\n",
       "      <td>comedy</td>\n",
       "      <td>movies</td>\n",
       "      <td>boring</td>\n",
       "      <td>story</td>\n",
       "      <td>bad</td>\n",
       "      <td>respectable</td>\n",
       "      <td>bad</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>horrible</td>\n",
       "      <td>comedy</td>\n",
       "      <td>films</td>\n",
       "      <td>intentions</td>\n",
       "      <td>story</td>\n",
       "      <td>bad</td>\n",
       "      <td>tension</td>\n",
       "      <td>bike</td>\n",
       "      <td>paced</td>\n",
       "      <td>clue</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>love</td>\n",
       "      <td>silly</td>\n",
       "      <td>bad</td>\n",
       "      <td>films</td>\n",
       "      <td>comedy</td>\n",
       "      <td>minutes</td>\n",
       "      <td>bmovie</td>\n",
       "      <td>dr</td>\n",
       "      <td>rohypnol</td>\n",
       "      <td>story</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>junk</td>\n",
       "      <td>dull</td>\n",
       "      <td>story</td>\n",
       "      <td>story</td>\n",
       "      <td>feels</td>\n",
       "      <td>waste</td>\n",
       "      <td>distressingly</td>\n",
       "      <td>subject</td>\n",
       "      <td>tiresome</td>\n",
       "      <td>video</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>time</td>\n",
       "      <td>boring</td>\n",
       "      <td>set</td>\n",
       "      <td>time</td>\n",
       "      <td>audience</td>\n",
       "      <td>performances</td>\n",
       "      <td>hollywood</td>\n",
       "      <td>films</td>\n",
       "      <td>teengang</td>\n",
       "      <td>plot</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>effort</td>\n",
       "      <td>mess</td>\n",
       "      <td>regan</td>\n",
       "      <td>bibbidybobbidibland</td>\n",
       "      <td>bad</td>\n",
       "      <td>story</td>\n",
       "      <td>guilty</td>\n",
       "      <td>hard</td>\n",
       "      <td>imposter</td>\n",
       "      <td>script</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>exercise</td>\n",
       "      <td>plain</td>\n",
       "      <td>minutes</td>\n",
       "      <td>boring</td>\n",
       "      <td>black</td>\n",
       "      <td>thriller</td>\n",
       "      <td>material</td>\n",
       "      <td>mother</td>\n",
       "      <td>thriller</td>\n",
       "      <td>scenes</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    topic 0      topic 1  topic 2              topic 3     topic 4  \\\n",
       "0     movie        movie     film                 film       movie   \n",
       "1      film         film    movie                movie        film   \n",
       "2    comedy          bad     time           characters  characters   \n",
       "3       bad  pretentious    drama               comedy      movies   \n",
       "4  horrible       comedy    films           intentions       story   \n",
       "5      love        silly      bad                films      comedy   \n",
       "6      junk         dull    story                story       feels   \n",
       "7      time       boring      set                 time    audience   \n",
       "8    effort         mess    regan  bibbidybobbidibland         bad   \n",
       "9  exercise        plain  minutes               boring       black   \n",
       "\n",
       "        topic 5        topic 6  topic 7      topic 8 topic 9  \n",
       "0         movie          movie     film        music   movie  \n",
       "1          film           film    movie       twists    film  \n",
       "2          life          glory   action         film  comedy  \n",
       "3        boring          story      bad  respectable     bad  \n",
       "4           bad        tension     bike        paced    clue  \n",
       "5       minutes         bmovie       dr     rohypnol   story  \n",
       "6         waste  distressingly  subject     tiresome   video  \n",
       "7  performances      hollywood    films     teengang    plot  \n",
       "8         story         guilty     hard     imposter  script  \n",
       "9      thriller       material   mother     thriller  scenes  "
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print_topics_modelling(\n",
    "    10,\n",
    "    feature_names = np.array(features),\n",
    "    sorting = np.argsort(components)[:, ::-1],\n",
    "    n_words = 10,\n",
    "    return_df = True,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}