ace_cnn_model_multi_task_w_pos_w2v.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# coding=utf-8\n",
    "import tensorflow as tf\n",
    "\"\"\"\n",
    "    中文事件提取 卷积神经网络模型 多任务分类 词+词性+word2vec 主要考虑不同的角色主要针对几种特定的词性\n",
    "    john.zhang 2016-12-20\n",
    "\"\"\"\n",
    "class ace_cnn_model():\n",
    "    \"\"\"\n",
    "    中文事件的发现 由以后候选词和一个触发词是否构成事件 这里是另外一种比较简单的模型\n",
    "    Relation Classification via Convolutional Deep Neural Network\n",
    "    这篇文章句子特征用的是每一个词的上下文(包含当前词)作为整体的输入\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self, sentence_length=30, event_num_labels=10, role_num_labels=6, windows=3, vocab_size=2048,\n",
    "                 word_embedding_size=100, pos_tag_embedding_size=5, pos_tag_size = 32,\n",
    "                 pos_embedding_size=10, filter_sizes=[3, 4, 5], filter_num=50, batch_size=2, word2vec_embedding_size=20):\n",
    "        \"\"\"\n",
    "        :param sentence_length: 输入句子的长度\n",
    "        :param event_num_labels: 事件类别数目\n",
    "        :param role_num_labels: 角色类别数目\n",
    "        :param windows: 窗口的大小\n",
    "        :param vocab_size: 训练集中词的数目\n",
    "        :param word_embedding_size: 词嵌入维数\n",
    "        :param pos_tag_embedding_size: 词性嵌入维数\n",
    "        :param pos_tag_size: 词性数目\n",
    "        :param pos_embedding_size: 位置嵌入维数\n",
    "        :param trigger_vec: 触发词向量\n",
    "        :param candidate_vec: 候选词向量\n",
    "        :param filter_sizes: 滤波器尺寸\n",
    "        :param filter_num: 滤波器数目\n",
    "        :param word2vec_embedding_size: pretrain得到的word2vec特征\n",
    "        \"\"\"\n",
    "        #  输入句子特征\n",
    "        input_x = tf.placeholder(tf.int32, shape=[batch_size, sentence_length], name=\"input_x\")\n",
    "        self.input_x = input_x\n",
    "        #  输入词性特征\n",
    "        input_pos_tag = tf.placeholder(tf.int32, shape=[batch_size, sentence_length], name=\"input_pos\")\n",
    "        self.input_pos_tag = input_pos_tag\n",
    "        #  输入的w2v特征 [batch_size, sentence_length, word2vec_embedding_size]\n",
    "        input_x_w2v = tf.placeholder(tf.float32, shape=[batch_size, sentence_length, word2vec_embedding_size], name=\"input_x_w2v\")\n",
    "        self.input_x_w2v = input_x_w2v\n",
    "        # 输入事件标签占位符\n",
    "        # [batch_size, event_num_labels]\n",
    "        input_event_y = tf.placeholder(tf.float32, shape=[batch_size, event_num_labels], name=\"input_event_y\")\n",
    "        self.input_event_y = input_event_y\n",
    "        # 输入角色标签占位符\n",
    "        # [batch_size, role_num_labels]\n",
    "        input_role_y = tf.placeholder(tf.float32, shape=[batch_size, role_num_labels], name=\"input_role_y\")\n",
    "        self.input_role_y = input_role_y\n",
    "\n",
    "        # 触发词位置向量\n",
    "        input_t_pos = tf.placeholder(tf.int32, shape=[batch_size, sentence_length], name=\"input_t_pos\")\n",
    "        self.input_t_pos = input_t_pos\n",
    "        # 候选词位置向量\n",
    "        input_c_pos = tf.placeholder(tf.int32, shape=[batch_size, sentence_length], name=\"input_c_pos\")\n",
    "        self.input_c_pos = input_c_pos\n",
    "        # dropout参数\n",
    "        dropout_keep_prob = tf.placeholder(tf.float32, name=\"dropout_keep_prob\")\n",
    "        self.dropout_keep_prob = dropout_keep_prob\n",
    "        \n",
    "        # 候选词的上下文组成的特征向量\n",
    "        input_c_context = tf.placeholder(tf.int32, shape=[batch_size, windows], name=\"input_c_context\")\n",
    "        self.input_c_context = input_c_context\n",
    "        # 候选词的上下文词性组成的特征向量\n",
    "        input_c_context_pos_tag = tf.placeholder(tf.int32, shape=[batch_size, windows], name=\"input_c_context_pos_tag\")\n",
    "        self.input_c_context_pos_tag = input_c_context_pos_tag\n",
    "        # 候选词的上下文word2vec组成的特征向量\n",
    "        input_c_context_w2v = tf.placeholder(tf.float32, shape=[batch_size, windows, word2vec_embedding_size], name=\"input_c_context_w2v\")\n",
    "        self.input_c_context_w2v = input_c_context_w2v\n",
    "        # 触发词的上下文组成的特征向量\n",
    "        input_t_context = tf.placeholder(tf.int32, shape=[batch_size, windows], name=\"input_t_context\")\n",
    "        self.input_t_context = input_t_context\n",
    "        # 触发词的上下文组成的词性向量\n",
    "        input_t_context_pos_tag = tf.placeholder(tf.int32, shape=[batch_size, windows], name=\"input_t_context_pos_tag\")\n",
    "        self.input_t_context_pos_tag = input_t_context_pos_tag\n",
    "        # 触发词的上下文word2vec组成的特征向量\n",
    "        input_t_context_w2v = tf.placeholder(tf.float32, shape=[batch_size, windows, word2vec_embedding_size], name=\"input_t_context_w2v\")\n",
    "        self.input_t_context_w2v = input_t_context_w2v\n",
    "        # 生成word_embedding 这部分操作相对比较简单 建议用cpu做\n",
    "        with tf.name_scope(\"word_embedding_layer\"):\n",
    "            # 词性表 [pos_tag_size, pos_tag_embedding_size]\n",
    "            W_pos_tag = tf.Variable(tf.random_normal(shape=[pos_tag_size, pos_tag_embedding_size], mean=0.0, stddev=0.5),\n",
    "                            name=\"pos_tag_table\") \n",
    "            # 词性特征向量 [batch_size, sentence_length, pos_tag_embedding_size]\n",
    "            input_pos_tag_vec = tf.nn.embedding_lookup(W_pos_tag, input_pos_tag)\n",
    "            \n",
    "            # 候选词以及其上下文词性特征向量\n",
    "            input_c_context_pos_tag_vec = tf.nn.embedding_lookup(W_pos_tag, input_c_context_pos_tag)\n",
    "            # 触发词以及其上下文词性特征向量\n",
    "            input_t_context_pos_tag_vec = tf.nn.embedding_lookup(W_pos_tag, input_t_context_pos_tag)\n",
    "            \n",
    "            # 词表 [vocab_size, embedding_size]\n",
    "            W = tf.Variable(tf.random_normal(shape=[vocab_size, word_embedding_size], mean=0.0, stddev=0.5),\n",
    "                            name=\"word_table\")\n",
    "            # 句子特征向量 [batch_size, sentence_length, word_embedding]\n",
    "            input_word_vec = tf.nn.embedding_lookup(W, input_x)\n",
    "            #             print sentence_features\n",
    "            #             print sentences_features_vec\n",
    "            #             根据 候选词位置 触发词位置 选取位置向量\n",
    "            #             在(-sentence_length+1,sentence_length-1)之间一共2*(sentence_length-1)+1个数\n",
    "            #             look_up 变成合适的正整数\n",
    "            input_t_pos_t = input_t_pos + (sentence_length - 1)\n",
    "            Tri_pos = tf.Variable(\n",
    "                tf.random_normal(shape=[2 * (sentence_length - 1) + 1, pos_embedding_size], mean=0.0, stddev=0.5),\n",
    "                name=\"tri_pos_table\")\n",
    "            input_t_pos_vec = tf.nn.embedding_lookup(Tri_pos, input_t_pos_t)\n",
    "            #             look_up 变成合适的正整数\n",
    "            input_c_pos_c = input_c_pos + (sentence_length - 1)\n",
    "            Can_pos = tf.Variable(\n",
    "                tf.random_normal(shape=[2 * (sentence_length - 1) + 1, pos_embedding_size], mean=0.0, stddev=0.5),\n",
    "                name=\"candidate_pos_table\")\n",
    "            input_c_pos_vec = tf.nn.embedding_lookup(Can_pos, input_c_pos_c)\n",
    "            #             print input_t_pos_vec\n",
    "            #             print input_c_pos_vec\n",
    "            # 将距离特征, 句子的词特征, 词性特征, word2vec特征 构成一个整体的特征 作为卷积神经网络的输入\n",
    "            # [batch_size, sentence_length, word_embedding_size+2*pos_size+pos_tag_embedding_size+word2vec_embedding_size]\n",
    "            input_sentence_vec = tf.concat(2, [input_word_vec, input_t_pos_vec, input_c_pos_vec, input_pos_tag_vec, input_x_w2v])\n",
    "            # CNN支持4d输入 因此增加一维向量 用于表示输入通道数目\n",
    "            intput_sentence_vec_expanded = tf.expand_dims(input_sentence_vec, -1)\n",
    "            # 词汇特征 lexical leval features\n",
    "            # 候选词极其上下文 [batch_size, windows, word_embedding_size]\n",
    "            input_c_context_vec = tf.nn.embedding_lookup(W, input_c_context)\n",
    "            # 触发词及其上下文 [batch_size, windows, word_embedding_size]\n",
    "            input_t_context_vec = tf.nn.embedding_lookup(W, input_t_context)\n",
    "            # print input_sentence_vec\n",
    "            # print input_c_context_vec\n",
    "            # print input_t_context_vec\n",
    "        pooled_outputs = []\n",
    "        for i, filter_size in enumerate(filter_sizes):\n",
    "            with tf.name_scope('conv-maxpool-%s' % filter_size):\n",
    "                # 这里的句子特征考虑的上当前词\n",
    "                # word_embedding_size 自身训练的词向量 \n",
    "                # 2*pos_embedding_size 触发词和候选词位置向量\n",
    "                # pos_tag_embedding_size 词性向量\n",
    "                # word2vec_embedding_size 事先训练的word2vec向量\n",
    "                filter_shape = [filter_size, word_embedding_size + 2 * pos_embedding_size+pos_tag_embedding_size+word2vec_embedding_size, 1, filter_num]\n",
    "                W = tf.Variable(tf.truncated_normal(filter_shape, stddev=0.1), name=\"W\")\n",
    "                b = tf.Variable(tf.constant(0.1, shape=[filter_num]), name=\"b\")\n",
    "                # 卷积运算\n",
    "                conv = tf.nn.conv2d(\n",
    "                    intput_sentence_vec_expanded,\n",
    "                    W,\n",
    "                    strides=[1, 1, 1, 1],\n",
    "                    padding=\"VALID\",\n",
    "                    name=\"conv\")\n",
    "                h = tf.nn.relu(tf.nn.bias_add(conv, b), name=\"relu\")\n",
    "                # 最大化池化 暂时不用动态池化\n",
    "                pooled = tf.nn.max_pool(\n",
    "                    h,\n",
    "                    ksize=[1, sentence_length - filter_size + 1, 1, 1],\n",
    "                    strides=[1, 1, 1, 1],\n",
    "                    padding='VALID',\n",
    "                    name=\"pool\")\n",
    "                pooled_outputs.append(pooled)\n",
    "        # 使用到的所有滤波器数目(输出的通道数目)\n",
    "        num_filters_total = filter_num * len(filter_sizes)\n",
    "        # 多通道的数据合并\n",
    "        h_pool = tf.concat(3, pooled_outputs)\n",
    "        #         print sentences_features_vec_flat\n",
    "        #         print h_pool\n",
    "        # 展开送入下一层分类器\n",
    "        # [batch_size, num_filters_total]\n",
    "        h_pool_flat = tf.reshape(h_pool, [-1, num_filters_total])\n",
    "        # print h_pool_flat\n",
    "        # 候选词极其上下文 触发词极其上下文 [batch_size, windows, word_embedding_size] => [batch_size, windows*word_embedding_size]\n",
    "        # [batch_size, windows * word_embedding_size]\n",
    "        input_c_context_vec_flat = tf.reshape(input_c_context_vec, [-1, windows * word_embedding_size])\n",
    "        # print input_c_context_vec_flat\n",
    "        # [batch_size, windows * word_embedding_size]\n",
    "        # 候选词极其词性\n",
    "        input_c_context_pos_tag_vec_flat = tf.reshape(input_c_context_pos_tag_vec, [-1, windows*pos_tag_embedding_size])\n",
    "        # 候选词的word2vec特征展开\n",
    "        input_c_context_w2v_flat = tf.reshape(input_c_context_w2v, [-1, windows * word2vec_embedding_size])\n",
    "        # 触发词的word2vec特征展开\n",
    "        input_t_context_vec_flat = tf.reshape(input_t_context_vec, [-1, windows * word_embedding_size])        \n",
    "        # print input_t_context_vec_flat\n",
    "        # print h_pool_flat\n",
    "        # 触发词极其词性 \n",
    "        input_t_context_pos_tag_vec_flat = tf.reshape(input_t_context_pos_tag_vec, [-1, windows*pos_tag_embedding_size])\n",
    "        # 触发词的word2vec特征展开\n",
    "        input_t_context_w2v_flat = tf.reshape(input_t_context_w2v, [-1, windows * word2vec_embedding_size])\n",
    "        # 原本的论文将lexical leval features和sentence leval features组合\n",
    "        # 这里采用multi-task的思路 sentence leval features主要应用于事件的发现\n",
    "        # lexical leval features主要应用于角色的分类\n",
    "        input_sentence_features = h_pool_flat\n",
    "        # 候选词极其触发词组成的特征\n",
    "        # input_c_context_vec_flat 候选词及其上下文\n",
    "        # input_t_context_vec_flat 触发词及其上下文\n",
    "        # input_c_context_w2v_flat 候选词及其上下文word2vec特征\n",
    "        # input_t_context_w2v_flat 触发词及其上下文word2vec特征\n",
    "        # input_c_context_pos_tag_vec_flat 候选词词性极其上下文 \n",
    "        # input_t_context_pos_tag_vec_flat 候选词词性极其上下文\n",
    "#         print input_c_context_vec_flat\n",
    "#         print input_c_context_w2v_flat\n",
    "#         print input_t_context_vec_flat\n",
    "#         print input_t_context_w2v_flat\n",
    "        input_lexical_features = tf.concat(1, [input_c_context_vec_flat, input_t_context_vec_flat, \n",
    "                                               input_c_context_w2v_flat, input_t_context_w2v_flat,\n",
    "                                              input_c_context_pos_tag_vec_flat, input_t_context_pos_tag_vec_flat])\n",
    "#         print input_lexical_features\n",
    "        # 总体的分类器 经过一层dropout 然后再送入softmax\n",
    "        with tf.name_scope('dropout'):\n",
    "            input_sentence_features_dropout = tf.nn.dropout(input_sentence_features, dropout_keep_prob)\n",
    "            input_lexical_features_dropout = tf.nn.dropout(input_lexical_features, dropout_keep_prob)\n",
    "        # 分类器\n",
    "        with tf.name_scope('softmax'):\n",
    "            # num_filters_total是卷积之后的结果 sentence level features\n",
    "            # 事件类别分类\n",
    "            W1 = tf.Variable(tf.truncated_normal([num_filters_total, event_num_labels], stddev=0.1), name=\"W1\")\n",
    "            b1 = tf.Variable(tf.constant(0.1, shape=[event_num_labels]), name=\"b1\")\n",
    "            xw1 = tf.nn.xw_plus_b(input_sentence_features_dropout, W1, b1)\n",
    "            scores_event = tf.nn.softmax(xw1, name=\"scores_event\")\n",
    "            # 2*windows*word_embedding_size是lexical leval features 具体的请看论文当中的详细介绍\n",
    "            predicts_event = tf.arg_max(scores_event, dimension=1, name=\"predicts_event\")\n",
    "            self.scores_event = scores_event\n",
    "            self.predicts_event = predicts_event\n",
    "            # 角色类别分类\n",
    "            # num_filters_total sentence特征\n",
    "            # event_num_labels 事件特征\n",
    "            \n",
    "            W2 = tf.Variable(tf.truncated_normal(\n",
    "                [num_filters_total + 2 * windows * (word_embedding_size+pos_tag_embedding_size+word2vec_embedding_size) + event_num_labels, role_num_labels],\n",
    "                stddev=0.1), name=\"W2\")\n",
    "#             print num_filters_total + 2 * windows * (word_embedding_size+pos_tag_embedding_size+word2vec_embedding_size) + event_num_labels\n",
    "            b2 = tf.Variable(tf.constant(0.1, shape=[role_num_labels]), name=\"b2\")\n",
    "            # 将句子特征 候选词和词性以及触发词和词性特征合并用于角色分类\n",
    "            # input_sentence_features_dropout  sentence features\n",
    "            # input_lexical_features_dropout  lexical_features\n",
    "            # scores_event  事件类型分类的输出得分结果\n",
    "            all_input_fatures = tf.concat(1, [input_sentence_features_dropout, input_lexical_features_dropout, scores_event])\n",
    "            xw2 = tf.nn.xw_plus_b(all_input_fatures, W2, b2)\n",
    "            scores_role = tf.nn.softmax(xw2, name=\"scores_role\")\n",
    "            predicts_role = tf.arg_max(scores_role, dimension=1, name=\"predicts_role\")\n",
    "            self.scores_role = scores_role\n",
    "            self.predicts_role = predicts_role\n",
    "        # 模型的代价函数 交叉熵代价函数\n",
    "        with tf.name_scope('loss'):\n",
    "            # 事件的交叉熵代价函数\n",
    "            entropy_event = tf.nn.softmax_cross_entropy_with_logits(scores_event, input_event_y)\n",
    "            loss_event = tf.reduce_mean(entropy_event)\n",
    "            self.loss_event = loss_event\n",
    "            # 角色的交叉熵代价函数\n",
    "            entropy_role = tf.nn.softmax_cross_entropy_with_logits(scores_role, input_role_y)\n",
    "            loss_role = tf.reduce_mean(entropy_role)\n",
    "            self.loss_role = loss_role\n",
    "        # 准确度 用于每一次训练时调用\n",
    "        with tf.name_scope(\"accuracy\"):\n",
    "            # 事件的准确度\n",
    "            correct_event = tf.equal(predicts_event, tf.argmax(input_event_y, 1))\n",
    "            accuracy_event = tf.reduce_mean(tf.cast(correct_event, \"float\"), name=\"accuracy_event\")\n",
    "            self.accuracy_event = accuracy_event\n",
    "            # 角色的准确度\n",
    "            correct_role = tf.equal(predicts_role, tf.argmax(input_role_y, 1))\n",
    "            accuracy_role = tf.reduce_mean(tf.cast(correct_role, \"float\"), name=\"accuracy_role\")\n",
    "            self.accuracy_role = accuracy_role"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    ""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    ""
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2.0
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}