spelling-correction/4.bert-accurate.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data from https://github.com/cbaziotis/ekphrasis/blob/master/ekphrasis/utils/helpers.py\n",
    "# reuploaded to husein's S3\n",
    "# !wget https://malaya-dataset.s3-ap-southeast-1.amazonaws.com/counts_1grams.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ['CUDA_VISIBLE_DEVICES'] = ''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('counts_1grams.txt') as fopen:\n",
    "    f = fopen.read().split('\\n')[:-1]\n",
    "    \n",
    "words = {}\n",
    "for l in f:\n",
    "    w, c = l.split('\\t')\n",
    "    c = int(c)\n",
    "    words[w] = c + words.get(w, 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# original from https://github.com/cbaziotis/ekphrasis/blob/master/ekphrasis/classes/spellcorrect.py\n",
    "# improved it\n",
    "\n",
    "import re\n",
    "from collections import Counter\n",
    "\n",
    "class SpellCorrector:\n",
    "    \"\"\"\n",
    "    The SpellCorrector extends the functionality of the Peter Norvig's\n",
    "    spell-corrector in http://norvig.com/spell-correct.html\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self):\n",
    "        \"\"\"\n",
    "        :param corpus: the statistics from which corpus to use for the spell correction.\n",
    "        \"\"\"\n",
    "        super().__init__()\n",
    "        self.WORDS = words\n",
    "        self.N = sum(self.WORDS.values())\n",
    "        \n",
    "    @staticmethod\n",
    "    def tokens(text):\n",
    "        return REGEX_TOKEN.findall(text.lower())\n",
    "\n",
    "    def P(self, word):\n",
    "        \"\"\"\n",
    "        Probability of `word`.\n",
    "        \"\"\"\n",
    "        return self.WORDS[word] / self.N\n",
    "\n",
    "    def most_probable(self, words):\n",
    "        _known = self.known(words)\n",
    "        if _known:\n",
    "            return max(_known, key=self.P)\n",
    "        else:\n",
    "            return []\n",
    "\n",
    "    @staticmethod\n",
    "    def edit_step(word):\n",
    "        \"\"\"\n",
    "        All edits that are one edit away from `word`.\n",
    "        \"\"\"\n",
    "        letters = 'abcdefghijklmnopqrstuvwxyz'\n",
    "        splits = [(word[:i], word[i:]) for i in range(len(word) + 1)]\n",
    "        deletes = [L + R[1:] for L, R in splits if R]\n",
    "        transposes = [L + R[1] + R[0] + R[2:] for L, R in splits if len(R) > 1]\n",
    "        replaces = [L + c + R[1:] for L, R in splits if R for c in letters]\n",
    "        inserts = [L + c + R for L, R in splits for c in letters]\n",
    "        return set(deletes + transposes + replaces + inserts)\n",
    "\n",
    "    def edits2(self, word):\n",
    "        \"\"\"\n",
    "        All edits that are two edits away from `word`.\n",
    "        \"\"\"\n",
    "        return (e2 for e1 in self.edit_step(word)\n",
    "                for e2 in self.edit_step(e1))\n",
    "\n",
    "    def known(self, words):\n",
    "        \"\"\"\n",
    "        The subset of `words` that appear in the dictionary of WORDS.\n",
    "        \"\"\"\n",
    "        return set(w for w in words if w in self.WORDS)\n",
    "\n",
    "    def edit_candidates(self, word, assume_wrong=False, fast=True):\n",
    "        \"\"\"\n",
    "        Generate possible spelling corrections for word.\n",
    "        \"\"\"\n",
    "\n",
    "        if fast:\n",
    "            ttt = self.known(self.edit_step(word)) or {word}\n",
    "        else:\n",
    "            ttt = self.known(self.edit_step(word)) or self.known(self.edits2(word)) or {word}\n",
    "        \n",
    "        ttt = self.known([word]) | ttt\n",
    "        return list(ttt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "corrector = SpellCorrector()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['edting',\n",
       " 'reting',\n",
       " 'etang',\n",
       " 'eling',\n",
       " 'beting',\n",
       " 'eating',\n",
       " 'ering',\n",
       " 'eking',\n",
       " 'ebing',\n",
       " 'eting',\n",
       " 'geting',\n",
       " 'etting',\n",
       " 'ating',\n",
       " 'enting',\n",
       " 'eying',\n",
       " 'meting',\n",
       " 'epting',\n",
       " 'etling',\n",
       " 'ting',\n",
       " 'sting',\n",
       " 'elting',\n",
       " 'eing',\n",
       " 'etin',\n",
       " 'kting',\n",
       " 'ewing']"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "possible_states = corrector.edit_candidates('eting')\n",
    "possible_states"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !wget https://storage.googleapis.com/bert_models/2018_10_18/uncased_L-12_H-768_A-12.zip\n",
    "# !unzip uncased_L-12_H-768_A-12.zip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "BERT_VOCAB = 'uncased_L-12_H-768_A-12/vocab.txt'\n",
    "BERT_INIT_CHKPNT = 'uncased_L-12_H-768_A-12/bert_model.ckpt'\n",
    "BERT_CONFIG = 'uncased_L-12_H-768_A-12/bert_config.json'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/husein/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:516: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:517: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:518: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:519: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:520: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:525: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/bert/optimization.py:87: The name tf.train.Optimizer is deprecated. Please use tf.compat.v1.train.Optimizer instead.\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/husein/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:541: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:542: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:543: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:544: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:545: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:550: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n"
     ]
    }
   ],
   "source": [
    "import bert\n",
    "from bert import run_classifier\n",
    "from bert import optimization\n",
    "from bert import tokenization\n",
    "from bert import modeling\n",
    "import tensorflow as tf\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/bert/tokenization.py:125: The name tf.gfile.GFile is deprecated. Please use tf.io.gfile.GFile instead.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "tokenization.validate_case_matches_checkpoint(True,BERT_INIT_CHKPNT)\n",
    "tokenizer = tokenization.FullTokenizer(\n",
    "      vocab_file=BERT_VOCAB, do_lower_case=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'scientist suggests **mask** berger can lead to obesity'"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text = 'scientist suggests eting berger can lead to obesity'\n",
    "text_mask = text.replace('eting', '**mask**')\n",
    "text_mask"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def tokens_to_masked_ids(tokens, mask_ind):\n",
    "    masked_tokens = tokens[:]\n",
    "    masked_tokens[mask_ind] = \"[MASK]\"\n",
    "    masked_tokens = [\"[CLS]\"] + masked_tokens + [\"[SEP]\"]\n",
    "    masked_ids = tokenizer.convert_tokens_to_ids(masked_tokens)\n",
    "    return masked_ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "bert_config = modeling.BertConfig.from_json_file(BERT_CONFIG)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Model:\n",
    "    def __init__(\n",
    "        self,\n",
    "    ):\n",
    "        self.X = tf.placeholder(tf.int32, [None, None])\n",
    "        \n",
    "        model = modeling.BertModel(\n",
    "            config=bert_config,\n",
    "            is_training=False,\n",
    "            input_ids=self.X,\n",
    "            use_one_hot_embeddings=False)\n",
    "        \n",
    "        output_layer = model.get_sequence_output()\n",
    "        embedding = model.get_embedding_table()\n",
    "        \n",
    "        with tf.variable_scope('cls/predictions'):\n",
    "            with tf.variable_scope('transform'):\n",
    "                input_tensor = tf.layers.dense(\n",
    "                    output_layer,\n",
    "                    units = bert_config.hidden_size,\n",
    "                    activation = modeling.get_activation(bert_config.hidden_act),\n",
    "                    kernel_initializer = modeling.create_initializer(\n",
    "                        bert_config.initializer_range\n",
    "                    ),\n",
    "                )\n",
    "                input_tensor = modeling.layer_norm(input_tensor)\n",
    "            \n",
    "            output_bias = tf.get_variable(\n",
    "            'output_bias',\n",
    "            shape = [bert_config.vocab_size],\n",
    "            initializer = tf.zeros_initializer(),\n",
    "            )\n",
    "            logits = tf.matmul(input_tensor, embedding, transpose_b = True)\n",
    "            self.logits = tf.nn.bias_add(logits, output_bias)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/bert/modeling.py:171: The name tf.variable_scope is deprecated. Please use tf.compat.v1.variable_scope instead.\n",
      "\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/bert/modeling.py:409: The name tf.get_variable is deprecated. Please use tf.compat.v1.get_variable instead.\n",
      "\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/bert/modeling.py:490: The name tf.assert_less_equal is deprecated. Please use tf.compat.v1.assert_less_equal instead.\n",
      "\n",
      "WARNING:tensorflow:\n",
      "The TensorFlow contrib module will not be included in TensorFlow 2.0.\n",
      "For more information, please see:\n",
      "  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n",
      "  * https://github.com/tensorflow/addons\n",
      "  * https://github.com/tensorflow/io (for I/O related ops)\n",
      "If you depend on functionality not listed there, please file an issue.\n",
      "\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/bert/modeling.py:671: dense (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use keras.layers.dense instead.\n",
      "WARNING:tensorflow:Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fde7ac828d0>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fde7ac828d0>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fde7ac828d0>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fde7ac828d0>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fde7ac828d0>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fde7ac828d0>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fde7ac828d0>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fde7ac828d0>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fde7ac828d0>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fde7ac828d0>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fde7ac828d0>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fde7ac828d0>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fde7ac82a58>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fde7ac82a58>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fde7ac82a58>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fde7ac82a58>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fde7ac91358>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fde7ac91358>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fde7ac91358>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fde7ac91358>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fde7ac912e8>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fde7ac912e8>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fde7ac912e8>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fde7ac912e8>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fde7ac82ac8>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fde7ac82ac8>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fde7ac82ac8>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fde7ac82ac8>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fde7ac82ac8>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fde7ac82ac8>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fde7ac82ac8>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fde7ac82ac8>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fde7ac82ac8>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fde7ac82ac8>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fde7ac82ac8>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fde7ac82ac8>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc72154d68>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc72154d68>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc72154d68>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc72154d68>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fde7ac91278>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fde7ac91278>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fde7ac91278>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fde7ac91278>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc720bed30>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc720bed30>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc720bed30>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc720bed30>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc71ea6240>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc71ea6240>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc71ea6240>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc71ea6240>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc71ea6240>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc71ea6240>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc71ea6240>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc71ea6240>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc71ea6240>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc71ea6240>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc71ea6240>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc71ea6240>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc71f716d8>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc71f716d8>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc71f716d8>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc71f716d8>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc720beac8>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc720beac8>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc720beac8>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc720beac8>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fde7ac91358>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fde7ac91358>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fde7ac91358>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fde7ac91358>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc71cc3f98>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc71cc3f98>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc71cc3f98>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc71cc3f98>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc71cc3f98>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc71cc3f98>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc71cc3f98>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc71cc3f98>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc71cc3f98>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc71cc3f98>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc71cc3f98>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc71cc3f98>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc71f716d8>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc71f716d8>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc71f716d8>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc71f716d8>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fde7ac91128>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fde7ac91128>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fde7ac91128>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fde7ac91128>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc720be0b8>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc720be0b8>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc720be0b8>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc720be0b8>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc721789e8>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc721789e8>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc721789e8>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc721789e8>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc721789e8>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc721789e8>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc721789e8>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc721789e8>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc721789e8>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc721789e8>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc721789e8>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc721789e8>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc71b3ecf8>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc71b3ecf8>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc71b3ecf8>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc71b3ecf8>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc720be278>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc720be278>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc720be278>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc720be278>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fde7ac91278>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fde7ac91278>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fde7ac91278>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fde7ac91278>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc71b1d208>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc71b1d208>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc71b1d208>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc71b1d208>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc71b1d208>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc71b1d208>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc71b1d208>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc71b1d208>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc71b1d208>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc71b1d208>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc71b1d208>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc71b1d208>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc719b1940>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc719b1940>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc719b1940>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc719b1940>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fde7ac91198>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fde7ac91198>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fde7ac91198>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fde7ac91198>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc720be550>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc720be550>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc720be550>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc720be550>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc71998cf8>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc71998cf8>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc71998cf8>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc71998cf8>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc71998cf8>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc71998cf8>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc71998cf8>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc71998cf8>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc71998cf8>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc71998cf8>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc71998cf8>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc71998cf8>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc717efda0>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc717efda0>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc717efda0>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc717efda0>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc720be0b8>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc720be0b8>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc720be0b8>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc720be0b8>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fde7ac91128>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fde7ac91128>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fde7ac91128>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fde7ac91128>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc7195f780>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc7195f780>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc7195f780>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc7195f780>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc7195f780>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc7195f780>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc7195f780>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc7195f780>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc7195f780>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc7195f780>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc7195f780>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc7195f780>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc716ecd68>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc716ecd68>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc716ecd68>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc716ecd68>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fde7ac91198>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fde7ac91198>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fde7ac91198>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fde7ac91198>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc720beac8>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc720beac8>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc720beac8>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc720beac8>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc714e8d30>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc714e8d30>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc714e8d30>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc714e8d30>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc714e8d30>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc714e8d30>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc714e8d30>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc714e8d30>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc71388f60>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc71388f60>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc71388f60>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc71388f60>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc71406e48>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc71406e48>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc71406e48>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc71406e48>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc720be0b8>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc720be0b8>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc720be0b8>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc720be0b8>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc71638630>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc71638630>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc71638630>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc71638630>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc7217a128>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc7217a128>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc7217a128>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc7217a128>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc711ed3c8>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc711ed3c8>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc711ed3c8>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc711ed3c8>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc71a40080>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc71a40080>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc71a40080>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc71a40080>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc713c2a58>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc713c2a58>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc713c2a58>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc713c2a58>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc71638630>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc71638630>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc71638630>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc71638630>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc720be278>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc720be278>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc720be278>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc720be278>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc710f0be0>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc710f0be0>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc710f0be0>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc710f0be0>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc710294a8>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc710294a8>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc710294a8>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc710294a8>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc710294a8>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc710294a8>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc710294a8>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc710294a8>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc710ea160>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc710ea160>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc710ea160>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc710ea160>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc720be550>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc720be550>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc720be550>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc720be550>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc71638630>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc71638630>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc71638630>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc71638630>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc72113a20>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc72113a20>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc72113a20>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc72113a20>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc72113a20>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc72113a20>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc72113a20>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc72113a20>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc72113a20>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc72113a20>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc72113a20>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc72113a20>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc70f0dda0>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc70f0dda0>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc70f0dda0>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc70f0dda0>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc71638630>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc71638630>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc71638630>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc71638630>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc720be550>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc720be550>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc720be550>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc720be550>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fde7307f128>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fde7307f128>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fde7307f128>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fde7307f128>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc7259f9e8>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc7259f9e8>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc7259f9e8>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fdc7259f9e8>>: AssertionError: Bad argument number for Name: 3, expecting 4\n"
     ]
    }
   ],
   "source": [
    "tf.reset_default_graph()\n",
    "sess = tf.InteractiveSession()\n",
    "model = Model()\n",
    "\n",
    "sess.run(tf.global_variables_initializer())\n",
    "var_lists = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope = 'bert')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Variable 'cls/predictions/transform/dense/kernel:0' shape=(768, 768) dtype=float32_ref>,\n",
       " <tf.Variable 'cls/predictions/transform/dense/bias:0' shape=(768,) dtype=float32_ref>,\n",
       " <tf.Variable 'cls/predictions/transform/LayerNorm/beta:0' shape=(768,) dtype=float32_ref>,\n",
       " <tf.Variable 'cls/predictions/transform/LayerNorm/gamma:0' shape=(768,) dtype=float32_ref>,\n",
       " <tf.Variable 'cls/predictions/output_bias:0' shape=(30522,) dtype=float32_ref>]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cls = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope = 'cls')\n",
    "cls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow/python/training/saver.py:1276: checkpoint_exists (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use standard file APIs to check for files with this prefix.\n",
      "INFO:tensorflow:Restoring parameters from uncased_L-12_H-768_A-12/bert_model.ckpt\n"
     ]
    }
   ],
   "source": [
    "saver = tf.train.Saver(var_list = var_lists + cls)\n",
    "saver.restore(sess, BERT_INIT_CHKPNT)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['scientist suggests edting berger can lead to obesity',\n",
       " 'scientist suggests reting berger can lead to obesity',\n",
       " 'scientist suggests etang berger can lead to obesity',\n",
       " 'scientist suggests eling berger can lead to obesity',\n",
       " 'scientist suggests beting berger can lead to obesity',\n",
       " 'scientist suggests eating berger can lead to obesity',\n",
       " 'scientist suggests ering berger can lead to obesity',\n",
       " 'scientist suggests eking berger can lead to obesity',\n",
       " 'scientist suggests ebing berger can lead to obesity',\n",
       " 'scientist suggests eting berger can lead to obesity',\n",
       " 'scientist suggests geting berger can lead to obesity',\n",
       " 'scientist suggests etting berger can lead to obesity',\n",
       " 'scientist suggests ating berger can lead to obesity',\n",
       " 'scientist suggests enting berger can lead to obesity',\n",
       " 'scientist suggests eying berger can lead to obesity',\n",
       " 'scientist suggests meting berger can lead to obesity',\n",
       " 'scientist suggests epting berger can lead to obesity',\n",
       " 'scientist suggests etling berger can lead to obesity',\n",
       " 'scientist suggests ting berger can lead to obesity',\n",
       " 'scientist suggests sting berger can lead to obesity',\n",
       " 'scientist suggests elting berger can lead to obesity',\n",
       " 'scientist suggests eing berger can lead to obesity',\n",
       " 'scientist suggests etin berger can lead to obesity',\n",
       " 'scientist suggests kting berger can lead to obesity',\n",
       " 'scientist suggests ewing berger can lead to obesity']"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "replaced_masks = [text_mask.replace('**mask**', state) for state in possible_states]\n",
    "replaced_masks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[101, 103, 6083, 3968, 3436, 16758, 2064, 2599, 2000, 24552, 102],\n",
       " [101, 7155, 103, 3968, 3436, 16758, 2064, 2599, 2000, 24552, 102],\n",
       " [101, 7155, 6083, 103, 3436, 16758, 2064, 2599, 2000, 24552, 102],\n",
       " [101, 7155, 6083, 3968, 103, 16758, 2064, 2599, 2000, 24552, 102],\n",
       " [101, 7155, 6083, 3968, 3436, 103, 2064, 2599, 2000, 24552, 102],\n",
       " [101, 7155, 6083, 3968, 3436, 16758, 103, 2599, 2000, 24552, 102],\n",
       " [101, 7155, 6083, 3968, 3436, 16758, 2064, 103, 2000, 24552, 102],\n",
       " [101, 7155, 6083, 3968, 3436, 16758, 2064, 2599, 103, 24552, 102],\n",
       " [101, 7155, 6083, 3968, 3436, 16758, 2064, 2599, 2000, 103, 102]]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokens = tokenizer.tokenize(replaced_masks[0])\n",
    "input_ids = [tokens_to_masked_ids(tokens, i) for i in range(len(tokens))]\n",
    "input_ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[7155, 6083, 3968, 3436, 16758, 2064, 2599, 2000, 24552]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokens_ids = tokenizer.convert_tokens_to_ids(tokens)\n",
    "tokens_ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_ids(mask):\n",
    "    tokens = tokenizer.tokenize(mask)\n",
    "    input_ids = [tokens_to_masked_ids(tokens, i) for i in range(len(tokens))]\n",
    "    tokens_ids = tokenizer.convert_tokens_to_ids(tokens)\n",
    "    return tokens, input_ids, tokens_ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "ids = [get_score(mask) for mask in replaced_masks]\n",
    "tokens, input_ids, tokens_ids = list(zip(*ids))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "indices, ids = [], []\n",
    "for i in range(len(input_ids)):\n",
    "    indices.extend([i] * len(input_ids[i]))\n",
    "    ids.extend(input_ids[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[101, 103, 6083, 3968, 3436, 16758, 2064, 2599, 2000, 24552, 102]"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ids[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(221, 11)"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "masked_padded = tf.keras.preprocessing.sequence.pad_sequences(ids,padding='post')\n",
    "masked_padded.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(221, 11, 30522)"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preds = sess.run(tf.nn.log_softmax(model.logits), feed_dict = {model.X: masked_padded})\n",
    "preds.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[-70.87423,\n",
       " -63.164944,\n",
       " -62.3369,\n",
       " -63.397655,\n",
       " -69.86493,\n",
       " -45.841267,\n",
       " -62.576523,\n",
       " -57.582092,\n",
       " -73.42107,\n",
       " -71.33391,\n",
       " -70.08537,\n",
       " -67.14623,\n",
       " -67.53539,\n",
       " -62.374245,\n",
       " -61.71485,\n",
       " -60.225086,\n",
       " -73.1943,\n",
       " -73.97394,\n",
       " -67.466835,\n",
       " -63.56203,\n",
       " -67.8916,\n",
       " -65.7337,\n",
       " -67.74832,\n",
       " -73.778435,\n",
       " -62.557587]"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "indices = np.array(indices)\n",
    "scores = []\n",
    "\n",
    "for i in range(len(tokens)):\n",
    "    filter_preds = preds[indices == i]\n",
    "    total = np.sum([filter_preds[k, k + 1, x] for k, x in enumerate(tokens_ids[i])])\n",
    "    scores.append(total)\n",
    "    \n",
    "scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.04307465, 0.03838924, 0.03788599, 0.03853067, 0.04246124,\n",
       "       0.02786057, 0.03803162, 0.0349962 , 0.04462252, 0.04335402,\n",
       "       0.04259521, 0.04080892, 0.04104543, 0.03790868, 0.03750793,\n",
       "       0.03660251, 0.0444847 , 0.04495853, 0.04100376, 0.03863057,\n",
       "       0.04126192, 0.03995043, 0.04117484, 0.04483971, 0.03802011],\n",
       "      dtype=float32)"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "prob_scores = np.array(scores) / np.sum(scores)\n",
    "prob_scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('eating', 0.02786057),\n",
       " ('eking', 0.034996197),\n",
       " ('meting', 0.03660251),\n",
       " ('eying', 0.03750793),\n",
       " ('etang', 0.037885986),\n",
       " ('enting', 0.037908684),\n",
       " ('ewing', 0.03802011),\n",
       " ('ering', 0.03803162),\n",
       " ('reting', 0.03838924),\n",
       " ('eling', 0.038530674),\n",
       " ('sting', 0.038630575),\n",
       " ('eing', 0.039950434),\n",
       " ('etting', 0.040808916),\n",
       " ('ting', 0.041003764),\n",
       " ('ating', 0.04104543),\n",
       " ('etin', 0.04117484),\n",
       " ('elting', 0.041261923),\n",
       " ('beting', 0.042461235),\n",
       " ('geting', 0.04259521),\n",
       " ('edting', 0.04307465),\n",
       " ('eting', 0.043354023),\n",
       " ('epting', 0.044484697),\n",
       " ('ebing', 0.044622518),\n",
       " ('kting', 0.044839714),\n",
       " ('etling', 0.04495853)]"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "probs = list(zip(possible_states, prob_scores))\n",
    "probs.sort(key = lambda x: x[1])  \n",
    "probs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}