english-hindi-NMT-Pytorch-small.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#from __future__ import unicode_literals, print_function, division\n",
    "from __future__ import unicode_literals, print_function, division\n",
    "from io import open\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch import optim\n",
    "import torch.nn.functional as F\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import string\n",
    "from string import digits\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "from sklearn.utils import shuffle\n",
    "from sklearn.model_selection import train_test_split\n",
    "from keras.layers import Input, LSTM, Embedding, Dense, CuDNNLSTM, TimeDistributed, Flatten, Dropout, Bidirectional\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.models import Model, Sequential, load_model\n",
    "from keras.callbacks import ModelCheckpoint\n",
    "from numpy import array, argmax\n",
    "from numpy.random import rand, shuffle\n",
    "from nltk.translate.bleu_score import corpus_bleu\n",
    "from nltk.translate.bleu_score import sentence_bleu\n",
    "from keras.utils import multi_gpu_model\n",
    "import scipy\n",
    "import statsmodels\n",
    "import sklearn\n",
    "import tensorflow\n",
    "import keras\n",
    "from io import open\n",
    "import unicodedata\n",
    "import random\n",
    "import math\n",
    "import time\n",
    "import os\n",
    "import time\n",
    "import re\n",
    "import sys\n",
    "from unicodedata import normalize\n",
    "from numpy import array\n",
    "from pickle import dump, load\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.utils import to_categorical\n",
    "from keras.utils.vis_utils import plot_model\n",
    "from keras.layers import RepeatVector, TimeDistributed\n",
    "from keras.callbacks import ModelCheckpoint\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "INDIC_NLP_LIB_HOME=r\"/home/arushi/Neural-Machine-Translation/anoopkunchukuttan-indic_nlp_library-eccde81/src\"\n",
    "INDIC_NLP_RESOURCES=r\"/home/arushi/Neural-Machine-Translation/indic_nlp_resources-master\"\n",
    "sys.path.append(r'{}'.format(INDIC_NLP_LIB_HOME))\n",
    "from indicnlp import common\n",
    "common.set_resources_path(INDIC_NLP_RESOURCES)\n",
    "from indicnlp import loader\n",
    "loader.load()\n",
    "from indicnlp.normalize.indic_normalize import IndicNormalizerFactory\n",
    "from indicnlp.tokenize import indic_tokenize\n",
    "\n",
    "MAX_LENGTH = 100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "SOS_token = 0\n",
    "EOS_token = 1\n",
    "\n",
    "class Lang:\n",
    "    def __init__(self, name):\n",
    "        self.name = name\n",
    "        self.word2index = {}\n",
    "        self.word2count = {}\n",
    "        self.index2word = {0: \"SOS\", 1: \"EOS\"}\n",
    "        self.n_words = 2  # Count SOS and EOS\n",
    "\n",
    "    def addSentence(self, sentence):\n",
    "        for word in sentence.split(' '):\n",
    "            self.addWord(word)\n",
    "\n",
    "    def addWord(self, word):\n",
    "        if word not in self.word2index:\n",
    "            self.word2index[word] = self.n_words\n",
    "            self.word2count[word] = 1\n",
    "            self.index2word[self.n_words] = word\n",
    "            self.n_words += 1\n",
    "        else:\n",
    "            self.word2count[word] += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_doc(filename):\n",
    "    file = open(filename, mode='rt', encoding='utf-8')\n",
    "    text = file.read()\n",
    "    file.close()\n",
    "    return text\n",
    "\n",
    "def to_pairs(english_text, hindi_text):\n",
    "    english_lines = english_text.strip().split('\\n')\n",
    "    hindi_lines = hindi_text.strip().split('\\n')\n",
    "    pairs = []\n",
    "    for i in range(len(hindi_lines)):\n",
    "        pairs.append([])\n",
    "        pairs[i].append(pre_process_english_sentence(english_lines[i]))\n",
    "        pairs[i].append(pre_process_hindi_sentence(hindi_lines[i]))\n",
    "    return pairs\n",
    "\n",
    "def clean_text(text):\n",
    "    text = text.replace(u',','')\n",
    "    text = text.replace(u'\"','')\n",
    "    text = text.replace(u'\"','')\n",
    "    text = text.replace(u\"‘‘\",'')\n",
    "    text = text.replace(u\"’’\",'')\n",
    "    text = text.replace(u\"''\",'')\n",
    "    text = text.replace(u\"।\",'')\n",
    "    text=text.replace(u',','')\n",
    "    text=text.replace(u'\"','')\n",
    "    text=text.replace(u'(','')\n",
    "    text=text.replace(u')','')\n",
    "    text=text.replace(u'\"','')\n",
    "    text=text.replace(u':','')\n",
    "    text=text.replace(u\"'\",'')\n",
    "    text=text.replace(u\"‘‘\",'')\n",
    "    text=text.replace(u\"’’\",'')\n",
    "    text=text.replace(u\"''\",'')\n",
    "    text=text.replace(u\".\",'')\n",
    "    text=text.replace(u\"-\",'')\n",
    "    text=text.replace(u\"।\",'')\n",
    "    text=text.replace(u\"?\",'')\n",
    "    text=text.replace(u\"\\\\\",'')\n",
    "    text=text.replace(u\"_\",'')\n",
    "    text=text.replace(\"'\", \"\")\n",
    "    text=text.replace('\"', \"\")\n",
    "    text= re.sub(\"'\", '', text)\n",
    "    text= re.sub(\"’\", '', text)\n",
    "    text=re.sub('[0-9+\\-*/.%]', '', text)\n",
    "    text=text.strip()\n",
    "    text=re.sub(' +', ' ',text)\n",
    "    exclude = set(string.punctuation)\n",
    "    text= ''.join(ch for ch in text if ch not in exclude)\n",
    "    return text\n",
    "\n",
    "def pre_process_english_sentence(line):\n",
    "    line = line.lower()\n",
    "    line = clean_text(line)\n",
    "    re_print = re.compile('[^%s]' % re.escape(string.printable))\n",
    "    line = normalize('NFD', line).encode('ascii', 'ignore')\n",
    "    line = line.decode('UTF-8')\n",
    "    line = line.split()\n",
    "    line = [re_print.sub('', w) for w in line]\n",
    "    line = [word for word in line if word.isalpha()]\n",
    "    line = ' '.join(line)\n",
    "    return line\n",
    "\n",
    "def pre_process_hindi_sentence(line):\n",
    "    line=re.sub('[a-zA-Z]', '', line)\n",
    "    line = clean_text(line)\n",
    "    remove_nuktas = False\n",
    "    factory = IndicNormalizerFactory()\n",
    "    normalizer = factory.get_normalizer(\"hi\",remove_nuktas)\n",
    "    line = normalizer.normalize(line)\n",
    "    tokens = list()\n",
    "    for t in indic_tokenize.trivial_tokenize(line):\n",
    "        tokens.append(t)\n",
    "    line = tokens\n",
    "    line = [word for word in line if not re.search(r'\\d', word)]\n",
    "    line = ' '.join(line)\n",
    "    return (line)\n",
    "\n",
    "def prepareData(pairs):\n",
    "    input_lang = Lang('eng')\n",
    "    output_lang = Lang('hin')\n",
    "    for pair in pairs:\n",
    "        input_lang.addSentence(pair[0])\n",
    "        output_lang.addSentence(pair[1])\n",
    "    return input_lang, output_lang\n",
    "\n",
    "english_text = load_doc('a')\n",
    "hindi_text = load_doc('b')\n",
    "pairs_all = to_pairs(english_text, hindi_text)\n",
    "input_lang, output_lang = prepareData(pairs_all)\n",
    "\n",
    "df = pd.DataFrame(pairs_all)\n",
    "df.columns = [\"eng\", \"mar\"]\n",
    "lines = df\n",
    "lenght_list=[]\n",
    "for l in lines.eng:\n",
    "    lenght_list.append(len(l.split(' ')))\n",
    "max_length_src = np.max(lenght_list)\n",
    "print (max_length_src)\n",
    "lenght_list=[]\n",
    "for l in lines.mar:\n",
    "    lenght_list.append(len(l.split(' ')))\n",
    "max_length_tar = np.max(lenght_list)\n",
    "print (max_length_tar)\n",
    "\n",
    "print(random.choice(pairs_all))\n",
    "print(random.choice(pairs_all))\n",
    "print(random.choice(pairs_all))\n",
    "print(random.choice(pairs_all))\n",
    "\n",
    "\n",
    "english_text = load_doc('a_test')\n",
    "hindi_text = load_doc('b_test')\n",
    "pairs_testing = to_pairs(english_text, hindi_text)\n",
    "\n",
    "english_text = load_doc('a_train')\n",
    "hindi_text = load_doc('b_train')\n",
    "pairs = to_pairs(english_text, hindi_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class EncoderRNN(nn.Module):\n",
    "    def __init__(self, input_size, embedding_size, hidden_size,n_layers=1, bidirectional=True, batch_size=1):\n",
    "        super(EncoderRNN, self).__init__()\n",
    "        self.bidirectional = bidirectional\n",
    "        self.hidden_size = int(hidden_size/2)\n",
    "        self.hidden_dim = self.hidden_size\n",
    "        self.num_layers = n_layers\n",
    "        self.batch_size = batch_size\n",
    "        self.embedding = nn.Embedding(input_size, embedding_size)\n",
    "        self.gru = nn.GRU(embedding_size, self.hidden_size, n_layers*2, bidirectional=bidirectional)\n",
    "        self.linear = nn.Linear(hidden_size * 2, 1)\n",
    "\n",
    "    def forward(self, input, hidden=None):\n",
    "        embedded = self.embedding(input).view(1, 1, -1)\n",
    "        output, hidden = self.gru(embedded, hidden)\n",
    "        fwd_final = hidden[0:hidden.size(0):2]\n",
    "        bwd_final = hidden[1:hidden.size(0):2]\n",
    "        final = torch.cat([fwd_final, bwd_final], dim=2)\n",
    "        return output, final\n",
    "\n",
    "    def initHidden(self):\n",
    "        directions = 2 if self.bidirectional else 1\n",
    "        return torch.zeros(\n",
    "            self.num_layers*2,\n",
    "            self.batch_size,\n",
    "            self.hidden_size*2,\n",
    "            device=device\n",
    "        )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class AttnDecoderRNN(nn.Module):\n",
    "    def __init__(self, hidden_size, output_size, dropout_p=0.2, max_length=output_lang.n_words,n_layers=1):\n",
    "        super(AttnDecoderRNN, self).__init__()\n",
    "        self.hidden_size = hidden_size\n",
    "        self.output_size = output_size\n",
    "        self.dropout_p = dropout_p/5\n",
    "        self.max_length = max_length\n",
    "\n",
    "        self.embedding = nn.Embedding(self.output_size, self.hidden_size)\n",
    "        self.attn1 = nn.Linear(self.hidden_size * 2, self.max_length)\n",
    "        self.attn_combine = nn.Linear(self.hidden_size, self.hidden_size)\n",
    "        self.dropout = nn.Dropout(self.dropout_p)\n",
    "        self.gru = nn.GRU(self.hidden_size, self.hidden_size, n_layers*2)\n",
    "        self.out = nn.Linear(self.hidden_size, self.output_size)\n",
    "\n",
    "    def forward(self, input, hidden, encoder_outputs):\n",
    "        embedded = self.embedding(input).view(1, 1, -1)\n",
    "        embedded = self.dropout(embedded)\n",
    "\n",
    "        intmd_torch = torch.cat((embedded[0],hidden[0]),1)\n",
    "        attn_out= self.attn1(intmd_torch)\n",
    "        attn_weights = F.softmax(attn_out)\n",
    "        attn_applied = torch.bmm(attn_weights.unsqueeze(0),\n",
    "                                 encoder_outputs.unsqueeze(0))\n",
    "       \n",
    "        output = torch.cat((embedded, attn_applied), 0)\n",
    "        output = self.attn_combine(output).unsqueeze(0)\n",
    "        output = F.relu(output)\n",
    "        output = torch.squeeze(output,0)\n",
    "        dec_hidden = self.initHidden()\n",
    "        output1, dec_hidden = self.gru(output, hidden)\n",
    "        output2 = F.log_softmax(self.out(output1[0]), dim=1)\n",
    "        return output2, dec_hidden, attn_weights\n",
    "\n",
    "    def initHidden(self):\n",
    "        return torch.zeros(1*2, 1, self.hidden_size, device=device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def indexesFromSentence(lang, sentence):\n",
    "    return [lang.word2index[word] for word in sentence.split(' ')]\n",
    "\n",
    "def tensorFromSentence(lang, sentence):\n",
    "    indexes = indexesFromSentence(lang, sentence)\n",
    "    indexes.append(EOS_token)\n",
    "    return torch.tensor(indexes, dtype=torch.long, device=device).view(-1, 1)\n",
    "\n",
    "def tensorsFromPair(pair):\n",
    "    input_tensor = tensorFromSentence(input_lang, pair[0])\n",
    "    target_tensor = tensorFromSentence(output_lang, pair[1])\n",
    "    return (input_tensor, target_tensor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "teacher_forcing_ratio = 1\n",
    "\n",
    "def train(input_tensor, target_tensor, encoder, decoder, encoder_optimizer, decoder_optimizer, criterion, max_length=output_lang.n_words):\n",
    "    \n",
    "    encoder_hidden = encoder.initHidden()\n",
    "    \n",
    "    encoder_optimizer.zero_grad()\n",
    "    decoder_optimizer.zero_grad()\n",
    "\n",
    "    input_length = input_tensor.size(0)\n",
    "    target_length = target_tensor.size(0)\n",
    "    \n",
    "    encoder_outputs = torch.zeros(max_length, encoder.hidden_dim*2, device=device)\n",
    "    \n",
    "    loss = 0\n",
    "\n",
    "    for ei in range(input_length):\n",
    "        encoder_output, encoder_hidden = encoder(input_tensor[ei])\n",
    "        encoder_outputs[ei] = encoder_output[0, 0]\n",
    "\n",
    "    decoder_input = torch.tensor([[SOS_token]], device=device)\n",
    "    \n",
    "    decoder_hidden = encoder_hidden\n",
    "    \n",
    "    use_teacher_forcing = True if random.random() < teacher_forcing_ratio else False\n",
    "\n",
    "    if use_teacher_forcing:\n",
    "        # Teacher forcing: Feed the target as the next input\n",
    "        for di in range(target_length):\n",
    "            decoder_output, decoder_hidden, decoder_attention = decoder(decoder_input, decoder_hidden, encoder_outputs)\n",
    "            loss += criterion(decoder_output, target_tensor[di])\n",
    "            decoder_input = target_tensor[di]  # Teacher forcing\n",
    "\n",
    "    else:\n",
    "        # Without teacher forcing: use its own predictions as the next input\n",
    "        for di in range(target_length):\n",
    "            decoder_output, decoder_hidden, decoder_attention = decoder(decoder_input, decoder_hidden, encoder_outputs)\n",
    "            topv, topi = decoder_output.topk(1)\n",
    "            decoder_input = topi.squeeze().detach()  # detach from history as input\n",
    "\n",
    "            loss += criterion(decoder_output, target_tensor[di])\n",
    "            if decoder_input.item() == EOS_token:\n",
    "                break\n",
    "\n",
    "    loss.backward()\n",
    "\n",
    "    encoder_optimizer.step()\n",
    "    decoder_optimizer.step()\n",
    "\n",
    "    return loss.item() / target_length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def asMinutes(s):\n",
    "    m = math.floor(s / 60)\n",
    "    s -= m * 60\n",
    "    return '%dm %ds' % (m, s)\n",
    "\n",
    "\n",
    "def timeSince(since, percent):\n",
    "    now = time.time()\n",
    "    s = now - since\n",
    "    es = s / (percent)\n",
    "    rs = es - s\n",
    "    return '%s (- %s)' % (asMinutes(s), asMinutes(rs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def trainIters(encoder, decoder, n_iters, print_every=1000, plot_every=100, learning_rate=0.01):\n",
    "    start = time.time()\n",
    "    plot_losses = []\n",
    "    print_loss_total = 0  # Reset every print_every\n",
    "    plot_loss_total = 0  # Reset every plot_every\n",
    "    \n",
    "    encoder_optimizer = optim.SGD(encoder.parameters(), lr=learning_rate)\n",
    "    decoder_optimizer = optim.SGD(decoder.parameters(), lr=learning_rate)\n",
    "    \n",
    "    random.shuffle(pairs)\n",
    "    \n",
    "    criterion = nn.NLLLoss()\n",
    "    \n",
    "    for iter in range(0, len(pairs)*30 + 1):\n",
    "        training_pairs = tensorsFromPair(pairs[(iter)%len(pairs)])\n",
    "        training_pair = training_pairs\n",
    "        input_tensor = training_pair[0]\n",
    "        target_tensor = training_pair[1]\n",
    "        \n",
    "        loss = train(input_tensor, target_tensor, encoder, decoder, encoder_optimizer, decoder_optimizer, criterion)\n",
    "        print_loss_total += loss\n",
    "        plot_loss_total += loss\n",
    "\n",
    "        if iter % print_every == 0:\n",
    "            print_loss_avg = print_loss_total / print_every\n",
    "            print_loss_total = 0\n",
    "            print('%.4f' % (print_loss_avg))\n",
    "\n",
    "        if iter % plot_every == 0:\n",
    "            plot_loss_avg = plot_loss_total / plot_every\n",
    "            plot_losses.append(plot_loss_avg)\n",
    "            plot_loss_total = 0\n",
    "\n",
    "    showPlot(plot_losses)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "plt.switch_backend('agg')\n",
    "import matplotlib.ticker as ticker\n",
    "import numpy as np\n",
    "\n",
    "def showPlot(points):\n",
    "    plt.figure()\n",
    "    fig, ax = plt.subplots()\n",
    "    # this locator puts ticks at regular intervals\n",
    "    loc = ticker.MultipleLocator(base=0.2)\n",
    "    ax.yaxis.set_major_locator(loc)\n",
    "    plt.plot(points)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate(encoder, decoder, sentence, max_length=output_lang.n_words):\n",
    "    with torch.no_grad():\n",
    "        input_tensor = tensorFromSentence(input_lang, sentence)\n",
    "        input_length = input_tensor.size()[0]\n",
    "        encoder_outputs = torch.zeros(max_length, encoder.hidden_dim*2, device=device)\n",
    "        \n",
    "        for ei in range(input_length):\n",
    "            encoder_output, encoder_hidden = encoder(input_tensor[ei])\n",
    "            encoder_outputs[ei] += encoder_output[0, 0]\n",
    "\n",
    "        decoder_input = torch.tensor([[SOS_token]], device=device)\n",
    "        \n",
    "        decoder_hidden = encoder_hidden\n",
    "\n",
    "        decoded_words = []\n",
    "        decoder_attentions = torch.zeros(max_length, max_length)\n",
    "        \n",
    "        for di in range(max_length):\n",
    "            decoder_output, decoder_hidden, decoder_attention = decoder(\n",
    "                decoder_input, decoder_hidden, encoder_outputs)\n",
    "            decoder_attentions[di] = decoder_attention.data\n",
    "            topv, topi = decoder_output.data.topk(1)\n",
    "            if topi.item() == EOS_token:\n",
    "                decoded_words.append('<EOS>')\n",
    "                break\n",
    "            else:\n",
    "                decoded_words.append(output_lang.index2word[topi.item()])\n",
    "\n",
    "            decoder_input = topi.squeeze().detach()\n",
    "\n",
    "        return decoded_words, decoder_attentions[:di + 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluateRandomly(encoder, decoder, n=20):\n",
    "    for i in range(n):\n",
    "        pair = random.choice(pairs)\n",
    "        print('>', pair[0])\n",
    "        print('=', pair[1])\n",
    "        output_words, attentions = evaluate(encoder, decoder, pair[0])\n",
    "        output_sentence = ' '.join(output_words)\n",
    "        print('<', output_sentence)\n",
    "        print('')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "embed_dim = 300\n",
    "hidden_size = embed_dim\n",
    "hidden_dim = embed_dim\n",
    "n_layers = 1\n",
    "\n",
    "encoder1 = EncoderRNN(input_lang.n_words, embed_dim, hidden_dim, n_layers).to(device)\n",
    "attn_decoder1 = AttnDecoderRNN(embed_dim, output_lang.n_words, n_layers).to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "trainIters(encoder1, attn_decoder1, 75000, print_every=5000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "evaluateRandomly(encoder1, attn_decoder1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def storetraintest(encoder, decoder, n):\n",
    "    actual, predicted = list(), list()\n",
    "    filename = 'sad'\n",
    "    pairs1 = pairs\n",
    "    if (n==1):\n",
    "        filename = 'training_result_pytorch.txt'\n",
    "    else:\n",
    "        filename = 'testing_result_pytorch.txt'\n",
    "        pairs1 = pairs_testing\n",
    "    sample = open(filename, 'w')\n",
    "    for i in range(len(pairs1)):\n",
    "        pair = pairs1[i]\n",
    "        print('Input: {}'.format(pair[0]), file = sample)\n",
    "        print('Actual_translation: {}'.format(pair[1]), file = sample)\n",
    "        output_words, attentions = evaluate(encoder, decoder, pair[0])\n",
    "        output_sentence = ' '.join(output_words)\n",
    "        print('Predicted translation: {}'.format(output_sentence), file = sample)\n",
    "        print (\"\", file=sample)\n",
    "        hindi_list = pair[1].split()\n",
    "        predicted_sen = output_sentence.split()\n",
    "        if (hindi_list[-1] == \"<EOS>\"):\n",
    "            hindi_list = hindi_list[:-1]\n",
    "        if (hindi_list[0] == \"<SOS>\"):\n",
    "            hindi_list.pop(0)\n",
    "        if (predicted_sen[-1] == \"<EOS>\"):\n",
    "            predicted_sen = predicted_sen[:-1]\n",
    "        actual.append(hindi_list)\n",
    "        predicted.append(predicted_sen)\n",
    "        \n",
    "    sample.close()\n",
    "    print('BLEU-1: %f' % corpus_bleu(actual, predicted, weights=(1.0, 0, 0, 0)))\n",
    "    print('BLEU-2: %f' % corpus_bleu(actual, predicted, weights=(0.5, 0.5, 0, 0)))\n",
    "    print('BLEU-3: %f' % corpus_bleu(actual, predicted, weights=(0.33, 0.33, 0.33, 0)))\n",
    "    print('BLEU-4: %f' % corpus_bleu(actual, predicted, weights=(0.25, 0.25, 0.25, 0.25)))\n",
    "    \n",
    "    print('Individual 1-gram: %f' % corpus_bleu(actual, predicted, weights=(1, 0, 0, 0)))\n",
    "    print('Individual 2-gram: %f' % corpus_bleu(actual, predicted, weights=(0, 1, 0, 0)))\n",
    "    print('Individual 3-gram: %f' % corpus_bleu(actual, predicted, weights=(0, 0, 1, 0)))\n",
    "    print('Individual 4-gram: %f' % corpus_bleu(actual, predicted, weights=(0, 0, 0, 1)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "storetraintest(encoder1, attn_decoder1,1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "storetraintest(encoder1, attn_decoder1,0)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}