cnns2s.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "from __future__ import unicode_literals, print_function, division\n",
    "from io import open\n",
    "import unicodedata\n",
    "import string\n",
    "import re\n",
    "import random\n",
    "import numpy as np\n",
    "import pickle\n",
    "import time\n",
    "import math\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.ticker as ticker\n",
    "from html.parser import HTMLParser\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.autograd import Variable\n",
    "from torch import optim\n",
    "import torch.nn.functional as F"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "use_cuda = torch.cuda.is_available()\n",
    "print(use_cuda)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def readWord2vecDict(file_dir):\n",
    "    with open(file_dir, 'rb') as f:\n",
    "        word2vecDict = pickle.load(f)\n",
    "    f.close()\n",
    "    return word2vecDict\n",
    "\n",
    "\n",
    "en_word2vecDict = readWord2vecDict(\"en_dict_512.bin\")\n",
    "zh_word2vecDict = readWord2vecDict(\"zh_dict_512.bin\")\n",
    "\n",
    "# print(en_word2vecDict[\"premiumization\"])\n",
    "# print(zh_word2vecDict[\"我\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ls = list(zh_word2vecDict.values())\n",
    "\n",
    "# for key in zh_word2vecDict.keys():\n",
    "#     zh_word2vecDict[key] = zh_word2vecDict[key] * 0.01\n",
    "# ls = list(zh_word2vecDict.values())\n",
    "# # ls = np.array(ls)\n",
    "# min_=1\n",
    "# max_=-1\n",
    "# for i in range(len(ls)):\n",
    "#     if min(ls[i]) < min_:\n",
    "#         min_ = min(ls[i])\n",
    "#     if max(ls[i]) > max_:\n",
    "#         max_ = max(ls[i])\n",
    "# print(\"after:\")\n",
    "# print(min_, max_)\n",
    "    \n",
    "    \n",
    "\n",
    "# for key in en_word2vecDict.keys():\n",
    "#     en_word2vecDict[key] = en_word2vecDict[key] * 0.01\n",
    "# ls = list(en_word2vecDict.values())\n",
    "# # ls = np.array(ls)\n",
    "# min_=1\n",
    "# max_=-1\n",
    "# for i in range(len(ls)):\n",
    "#     if min(ls[i]) < min_:\n",
    "#         min_ = min(ls[i])\n",
    "#     if max(ls[i]) > max_:\n",
    "#         max_ = max(ls[i])\n",
    "# print(\"before\")\n",
    "# print(min_, max_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "en_word2vecDict[\"SOS\"] = -np.ones(512)\n",
    "en_word2vecDict[\"UNK\"] = np.zeros(512)\n",
    "en_word2vecDict[\"EOS\"] = np.ones(512)\n",
    "zh_word2vecDict[\"SOS\"] = -np.ones(512)\n",
    "zh_word2vecDict[\"UNK\"] = np.zeros(512)\n",
    "zh_word2vecDict[\"EOS\"] = np.ones(512)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "SOS_token = -1\n",
    "EOS_token = 1\n",
    "\n",
    "class Lang:\n",
    "    def __init__(self, name, word2vecDict):\n",
    "        self.name = name\n",
    "        self.word2index = {}\n",
    "        self.word2count = {}\n",
    "        self.index2word = {-1: \"SOS\", 0:\"UNK\", 1:\"EOS\"}\n",
    "        self.n_words = 3  # Count SOS, EOS\n",
    "        self.word2vecDict = word2vecDict\n",
    "\n",
    "    def addSentence(self, sentence):\n",
    "        for word in sentence.strip().split(' '):\n",
    "            self.addWord(word)\n",
    "\n",
    "    def addWord(self, word):\n",
    "        if word in self.word2vecDict:\n",
    "            if word not in self.word2index:\n",
    "                self.word2index[word] = self.n_words\n",
    "                self.word2count[word] = 1\n",
    "                self.index2word[self.n_words] = word\n",
    "                self.n_words += 1\n",
    "            else:\n",
    "                self.word2count[word] += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class read_Langs(object):\n",
    "    def __init__(self,dir):\n",
    "        self.dir = dir\n",
    "    def __iter__(self):\n",
    "        for line in open(self.dir, encoding = 'utf-8'):\n",
    "            if len(line.strip().split('\\t'))==2:\n",
    "                yield line.strip().split('\\t')\n",
    "            else:\n",
    "                pass\n",
    "def readLangs():\n",
    "    pairs = read_Langs('train_pair')\n",
    "    pairs = [pair for pair in pairs]\n",
    "    input_lang = Lang(\"en\", en_word2vecDict)\n",
    "    output_lang = Lang(\"zh\", zh_word2vecDict)\n",
    "    return input_lang, output_lang, pairs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "MAX_LENGTH = 20\n",
    "\n",
    "def filterPair(p):\n",
    "#     print(str(p[0]) + str(p[1]), end = \"\\r\")\n",
    "    return len(p[0].split(' ')) < MAX_LENGTH and len(p[1].split(' ')) < MAX_LENGTH\n",
    "\n",
    "def filterPairs(pairs):\n",
    "    return [pair for pair in pairs if filterPair(pair)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def prepareData():\n",
    "    input_lang, output_lang, pairs = readLangs()\n",
    "    pairs = filterPairs(pairs)\n",
    "    print(\"Read %s sentence pairs\" % len(pairs))\n",
    "#     i=0\n",
    "    for pair in pairs:\n",
    "#         print(i)\n",
    "#         i+=1\n",
    "#         print(pair)\n",
    "        input_lang.addSentence(pair[0].strip())\n",
    "        output_lang.addSentence(pair[1].strip())\n",
    "    print(\"Counted words:\")\n",
    "    print(input_lang.name, input_lang.n_words)\n",
    "    print(output_lang.name, output_lang.n_words)\n",
    "    print(\"Count of sentence pairs:\")\n",
    "    print(len(pairs))\n",
    "    return input_lang, output_lang, pairs\n",
    "\n",
    "input_lang, output_lang, pairs = prepareData()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# with open(\"input_lang.lang\", 'wb') as f:\n",
    "#     word2vecDict = pickle.dump(input_lang, f)\n",
    "# f.close()\n",
    "# with open(\"output_lang.lang\", 'wb') as f:\n",
    "#     word2vecDict = pickle.dump(output_lang, f)\n",
    "# f.close()\n",
    "# with open(\"pairs.lang\", 'wb') as f:\n",
    "#     word2vecDict = pickle.dump(pairs, f)\n",
    "# f.close()\n",
    "        \n",
    "# with open(\"input_lang.lang\", 'rb') as f:\n",
    "#     input_lang = pickle.load(f)\n",
    "# f.close()\n",
    "# with open(\"output_lang.lang\", 'rb') as f:\n",
    "#     output_lang = pickle.load(f)\n",
    "# f.close()\n",
    "# with open(\"pairs.lang\", 'rb') as f:\n",
    "#     pairs = pickle.load(f)\n",
    "# f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def embedding(input, lang, word2vecDict):  #input(1, seq_len)\n",
    "#     print(input)\n",
    "    input = input.cpu().data.numpy()\n",
    "#         print(\"input[i][0]\")\n",
    "#         print(input[i][0])\n",
    "#         print(\"lang.index2word[input[i][0]]\")\n",
    "#         print(lang.index2word[input[i][0]])\n",
    "    embeded = torch.FloatTensor(1, len(input[0]), 512)\n",
    "    for i in range(len(input[0])):\n",
    "        word = lang.index2word[input[0][i]]\n",
    "        if word in word2vecDict:\n",
    "            embeded[0, i] = torch.FloatTensor(word2vecDict[word])\n",
    "        else:\n",
    "            embeded[0, i] = torch.FloatTensor(word2vecDict[\"UNK\"])\n",
    "#         print(\"embedding[i]\")\n",
    "#         print(embedding[i])\n",
    "    return Variable(embeded)  #embedding(1, seq_len, embedding_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def indexesFromSentence(lang, sentence):\n",
    "    return [lang.word2index[word] if word in lang.word2index else 0 for word in sentence.strip().split(' ')]\n",
    "\n",
    "\n",
    "def variableFromSentence(lang, sentence):\n",
    "    indexes = indexesFromSentence(lang, sentence)\n",
    "    indexes.insert(0, SOS_token)\n",
    "    indexes.append(EOS_token)\n",
    "    result = Variable(torch.LongTensor(indexes))\n",
    "    return result.cuda() if use_cuda else result\n",
    "\n",
    "\n",
    "def variablesFromPair(pair):\n",
    "    input_variable = variableFromSentence(input_lang, pair[0])\n",
    "    target_variable = variableFromSentence(output_lang, pair[1])\n",
    "    return (input_variable, target_variable)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def grad_multiply(x, scale):\n",
    "    return GradMultiply.apply(x, scale)\n",
    "\n",
    "\n",
    "class GradMultiply(torch.autograd.Function):\n",
    "    @staticmethod\n",
    "    def forward(ctx, x, scale):\n",
    "        ctx.scale = scale\n",
    "        res = x.new(x)\n",
    "        ctx.mark_shared_storage((x, res))\n",
    "        return res\n",
    "\n",
    "    @staticmethod\n",
    "    def backward(ctx, grad):\n",
    "        return grad * ctx.scale, None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Encoder(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(Encoder,self).__init__()\n",
    "        self.embedding_size = 512\n",
    "        self.hidden_size = 512\n",
    "        self.n_layers = 13\n",
    "        self.linear1 = nn.Linear(self.embedding_size, self.hidden_size)\n",
    "#         self.linear2 = nn.Linear(self.hidden_size, 2*self.hidden_size)\n",
    "#         self.linear3 = nn.Linear(2*self.hidden_size, 2*self.hidden_size)\n",
    "        self.out_channels = 2*self.hidden_size\n",
    "        self.kernel = (3, self.hidden_size)\n",
    "        self.padding = (int((self.kernel[0]-1)/2), 0)\n",
    "        #conv input(N_batches, Channels_in, Height_in, Width_in)\n",
    "        self.conv = nn.Conv2d(1, self.out_channels, self.kernel, (1,1), self.padding)\n",
    "        \n",
    "        \n",
    "    def forward(self, input):  #input is only a sequence of indexes of words\n",
    "        input = input[1:input.size(0)]  #delete the SOS_token\n",
    "        input = input.unsqueeze(0)  #in input(seq_len), out input(1, seq_len) where 1 means batch size is 1\n",
    "        global en_word2vecDict, input_lang\n",
    "        input = embedding(input, input_lang, en_word2vecDict).cuda()*0.1  #out input(1, seq_len, embedding_size)\n",
    "        input = F.dropout(input, p=0.1, training=self.training)\n",
    "        input = input.unsqueeze(1) #out input(1, 1, seq_len, hidden_size)\n",
    "        input_temp = input\n",
    "#         input = self.linear1(input)  #out input(1, 1, seq_len, hidden_size)\n",
    "        for _ in range(self.n_layers):\n",
    "            input_ = input\n",
    "            input = F.dropout(input, p=0.1, training=self.training)\n",
    "            input = self.conv(input).transpose(1,3)  #out input(1, 1, seq_len, 2*hidden_size) \n",
    "            input = F.glu(input, 3)  #out input(1, 1, seq_len, hidden_size)\n",
    "            input = (input + input_)*math.sqrt(0.5)\n",
    "#             input = F.relu(input)\n",
    "#         attn = input\n",
    "#         out = input + input_temp\n",
    "        input = grad_multiply(input, 1.0 / (2.0 * self.n_layers))  # scale gradients (this only affects backward, not forward)\n",
    "        return input, input + input_temp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Decoder(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(Decoder,self).__init__()\n",
    "        self.embedding_size = 512\n",
    "        self.hidden_size = 512\n",
    "        global output_lang\n",
    "        self.dict_size = output_lang.n_words\n",
    "        print(self.dict_size)\n",
    "        self.n_layers = 13\n",
    "        self.linear1 = nn.Linear(self.embedding_size, self.hidden_size)\n",
    "        self.linear2 = nn.Linear(self.hidden_size, 2*self.hidden_size)\n",
    "        self.linear3 = nn.Linear(self.hidden_size, self.dict_size)\n",
    "        self.linear4 = nn.Linear(2*self.hidden_size, 2*self.hidden_size)\n",
    "        self.in_channels = 1\n",
    "        self.out_channels = 2*self.hidden_size\n",
    "        self.kernel = (3, self.hidden_size)\n",
    "        self.stride = (1, 1)\n",
    "        self.padding = (0, 0)  #mannually pad, without using its inner auto padding\n",
    "        self.conv = nn.Conv2d(self.in_channels, self.out_channels, self.kernel, self.stride, self.padding)\n",
    "        \n",
    "        \n",
    "    def forward(self, target, enc_attn, enc_out):  #target is only a sequence of indexes of words\n",
    "        input = target[0:-1]  #delete the EOS_token\n",
    "        input = input.unsqueeze(0)  #target(tar_seq_len), input(1, tar_seq_len) where 1 means batch size is 1\n",
    "        global zh_word2vecDict, output_lang\n",
    "        input = embedding(input, output_lang, zh_word2vecDict).cuda()*0.1 #out input(1, tar_seq_len, embedding_size)\n",
    "        input = F.dropout(input, p=0.1, training=self.training)\n",
    "        input = input.unsqueeze(1) #out input(1, 1, tar_seq_len, hidden_size)\n",
    "        input_temp = input\n",
    "#         input = self.linear1(input)  #out input(1, 1, tar_seq_len, hidden_size)\n",
    "        for _ in range(self.n_layers):\n",
    "            input_ = input\n",
    "            input = F.dropout(input, p=0.1, training=self.training)\n",
    "            cat_temp = Variable(torch.FloatTensor(torch.zeros(1, 1, self.kernel[0]-1, self.hidden_size))).cuda()\n",
    "            input = torch.cat((cat_temp, input), 2)  #pad left with kernel_size-1 elements\n",
    "            input = self.conv(input)  #out input(1, 2*hidden_size, tar_seq_len, 1) \n",
    "                                        #where the second 1 means 2*hidden_size changes into is 1\n",
    "            input = input.transpose(1, 3)  #out input(1, 1, tar_seq_len, 2*hidden_size)\n",
    "            input = F.glu(input, 3)  #out input(1, 1, tar_seq_len, hidden_size)\n",
    "            input__ = input\n",
    "            input = (input + input_temp)*math.sqrt(0.5)\n",
    "            input = torch.bmm(input.squeeze(0), enc_attn.squeeze(0).transpose(1, 2))  #out input(1, tar_seq_len, sor_seq_len)\n",
    "            #softmax:result_i_j = exp(x_i_j)/sum(exp(x_j)), input should be in two directions\n",
    "            input = F.softmax(input.squeeze(0)).unsqueeze(0) #out input(1, tar_seq_len, sor_seq_len)\n",
    "            input = torch.bmm(input, enc_out.squeeze(0))  #out input(1, tar_seq_len, hidden_size)\n",
    "            input = input*(enc_out.size(2) * math.sqrt(1.0/enc_out.size(2)))\n",
    "            input = (input.unsqueeze(0) + input__)*math.sqrt(0.5) #out input(1, 1, tar_seq_len, hidden_size)\n",
    "            input = (input + input_)*math.sqrt(0.5)\n",
    "#             input = F.relu(input)\n",
    "        input = input.squeeze(0).squeeze(0)  #out input(tar_seq_len, hidden_size)\n",
    "        input = F.dropout(input, p=0.1, training=self.training)\n",
    "        input = F.log_softmax(self.linear3(input))  #out input(tar_seq_len, dict_size) has tar_seq_len choices of words\n",
    "        return input"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class NMTModel(nn.Module):\n",
    "    \"\"\"\n",
    "    NMTModel:\n",
    "    Input:\n",
    "        encoder:\n",
    "        decoder:\n",
    "        attention:\n",
    "        generator:\n",
    "    return:\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self, encoder, decocer):\n",
    "        super(NMTModel, self).__init__()\n",
    "        self.encoder = encoder\n",
    "        self.decocer = decocer\n",
    "\n",
    "    def forward(self, source, target):\n",
    "        # attn(1, 1, seq_len, 2*hidden_size)\n",
    "        # out(1, 1, seq_len, 2*hidden_size)\n",
    "        attn, source_seq_out = self.encoder(source)\n",
    "\n",
    "        # out(tar_seq_len, dict_size)\n",
    "        out = self.decocer(target, attn, source_seq_out)\n",
    "\n",
    "        return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(input_variable, target_variable, NMTModel, NMTModel_optimizer, criterion):\n",
    "\n",
    "    NMTModel_optimizer.zero_grad()\n",
    "   \n",
    "    loss = 0\n",
    "    \n",
    "    for i in range(len(input_variable)):\n",
    "        out = NMTModel(input_variable[i], target_variable[i])\n",
    "#         print(\"out\")\n",
    "#         print(out)\n",
    "#         print(\"target_to_compare\")\n",
    "#         print(target_variable[i][1:target_variable[i].size(0)])\n",
    "        loss += criterion(out, target_variable[i][1:target_variable[i].size(0)])\n",
    "    loss.backward()\n",
    "    \n",
    "    NMTModel_optimizer.step()\n",
    "\n",
    "    return loss.data[0]/(len(input_variable))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def asMinutes(s):\n",
    "    m = math.floor(s / 60)\n",
    "    s -= m * 60\n",
    "    return '%dm %ds' % (m, s)\n",
    "\n",
    "\n",
    "def timeSince(since, percent):\n",
    "    now = time.time()\n",
    "    s = now - since\n",
    "    es = s / (percent)\n",
    "    rs = es - s\n",
    "    return '%s (- %s)' % (asMinutes(s), asMinutes(rs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def showPlot(points):\n",
    "    plt.figure()\n",
    "    fig, ax = plt.subplots()\n",
    "    # this locator puts ticks at regular intervals\n",
    "    loc = ticker.MultipleLocator(base=0.2)\n",
    "    ax.yaxis.set_major_locator(loc)\n",
    "    plt.plot(points)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def saveModel(model, model_path):\n",
    "    torch.save(model.state_dict(), model_path)\n",
    "\n",
    "def loadModel(model, model_path):\n",
    "    model.load_state_dict(torch.load(model_path))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def trainIters(NMTModel, n_iters, print_every=1000, plot_every=100, save_every = 5000, learning_rate=0.01):\n",
    "    NMTModel.train(True)\n",
    "    start = time.time()\n",
    "    plot_losses = []\n",
    "    print_loss_total = 0  # Reset every print_every\n",
    "    plot_loss_total = 0  # Reset every plot_every\n",
    "    \n",
    "    NMTModel_optimizer = optim.SGD(NMTModel.parameters(), lr=learning_rate, momentum=0.99,nesterov=True)\n",
    "    \n",
    "    criterion = nn.NLLLoss()\n",
    "    \n",
    "    batch_size = 64\n",
    "    \n",
    "    for iter in range(1, n_iters+1):\n",
    "        training_pairs = [variablesFromPair(random.choice(pairs)) for _ in range(batch_size)]\n",
    "        input_variable_batch = [training_pairs[i][0] for i in range(batch_size)]\n",
    "        target_variable_batch = [training_pairs[i][1] for i in range(batch_size)]\n",
    "        \n",
    "        loss = train(input_variable_batch, target_variable_batch, NMTModel, NMTModel_optimizer, criterion)\n",
    "    \n",
    "        print_loss_total += loss\n",
    "        plot_loss_total += loss\n",
    "\n",
    "        if iter % print_every == 0:\n",
    "            print_loss_avg = print_loss_total / print_every\n",
    "            print_loss_total = 0\n",
    "            print(\"%s  (%d  %d%%)  %f\" %(timeSince(start, iter/n_iters), iter, iter/n_iters*100, print_loss_avg))\n",
    "\n",
    "        if iter % plot_every == 0:\n",
    "            plot_loss_avg = plot_loss_total / plot_every\n",
    "            plot_losses.append(plot_loss_avg)\n",
    "            plot_loss_total = 0\n",
    "            \n",
    "        if iter % save_every == 0:\n",
    "            saveModel(NMTModel, \"./NMTModel_\" + str(time.strftime('%d-%H',time.localtime(time.time()))) + \".mod\")\n",
    "    showPlot(plot_losses)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def weight_init(model):\n",
    "    print(model.parameters())\n",
    "    for param in model.parameters():\n",
    "        print(type(param.data))\n",
    "        torch.nn.init.normal(param.data, 0, 0.01)\n",
    "#     print(model.weight)\n",
    "    #torch.nn.init.normal(model.weight.data, mean=0, std=0.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "encoder = Encoder()\n",
    "decoder = Decoder()\n",
    "nmtmodel = NMTModel(encoder, decoder)\n",
    "# weight_init(nmtmodel)\n",
    "\n",
    "if True:\n",
    "    \n",
    "    loadModel(nmtmodel, \"./NMTModel_23-04.mod\")\n",
    "\n",
    "    \n",
    "if use_cuda:\n",
    "    nmtmodel = nmtmodel.cuda()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# trainIters(nmtmodel, n_iters = 40000, print_every=100, plot_every = 10, save_every = 5000, learning_rate = 0.0001)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate(model, input_sentence):\n",
    "    model.train(False)\n",
    "    output_sentence = \"\"\n",
    "    pair = [input_sentence, output_sentence]\n",
    "    input_variable, _= variablesFromPair(pair)\n",
    "    output_variable = Variable(torch.LongTensor([-1,1])).cuda()\n",
    "    end_flag = False\n",
    "    while(end_flag == False):\n",
    "#         print(input_variable)\n",
    "#         print(output_variable)\n",
    "        out = model(input_variable, output_variable)\n",
    "        topi = out.data.topk(1, dim = 1)\n",
    "        indexes = topi[1].transpose(0,1).cpu().numpy()[0]\n",
    "#         print(indexes)\n",
    "        output_sentence = \"\"\n",
    "        for i in range(len(indexes)):\n",
    "            if i != len(indexes)-1 :\n",
    "#                 print(str(i) + \": \"  + str(output_lang.index2word[indexes[i]]))\n",
    "                output_sentence += output_lang.index2word[indexes[i]] + \" \"\n",
    "            else:\n",
    "#                 print(str(i) + \": \"  + str(output_lang.index2word[indexes[i]]))\n",
    "                if output_lang.index2word[indexes[i]] == \"EOS\":\n",
    "                    end_flag = True\n",
    "                else:\n",
    "                    output_sentence += output_lang.index2word[indexes[i]]\n",
    "        pair = [input_sentence, output_sentence]\n",
    "        input_variable, output_variable= variablesFromPair(pair)\n",
    "    print(output_sentence)\n",
    "    return output_sentence\n",
    "#     print(topi)\n",
    "#     print(indexes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_sentence = \"For Access To Guest the floors 23 and 24 , Use Key Card Is Located on floor 24 .\"\n",
    "evaluate(nmtmodel, input_sentence.lower())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MyHTMLParser(HTMLParser):\n",
    "    def __init__(self):\n",
    "        HTMLParser.__init__(self)\n",
    "        self.sentence_dict = {}\n",
    "        self.getdataflag = False\n",
    "        self.sentence_dict_key = \"\"\n",
    "\n",
    "    def handle_starttag(self, tag, attrs):\n",
    "        if tag == \"seg\":\n",
    "            self.getdataflag = True\n",
    "            self.sentence_dict_key = attrs[0][1]\n",
    "            #print(attrs)\n",
    "            \n",
    "    def handle_data(self, data):\n",
    "        if self.getdataflag == True:\n",
    "            self.sentence_dict[self.sentence_dict_key] = data\n",
    "        self.getdataflag = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sgmfile = open(\"test_b.sgm\",'r')\n",
    "text = sgmfile.read()\n",
    "parser = MyHTMLParser()\n",
    "parser.feed(text)\n",
    "en_sentences_dict = parser.sentence_dict.copy()\n",
    "\n",
    "print(en_sentences_dict[\"999\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def normalizeString(s):\n",
    "    #s=jieba.cut(s)\n",
    "    #正则表达式，标点符号前加空格 去掉特殊标点符号 \n",
    "    s = re.sub(r\"(\\,)\", r\" \\1 \", s)\n",
    "    s = re.sub(r\"(\\.)\", r\" \\1 \", s)\n",
    "    s = re.sub(r\"(\\?)\", r\" \\1 \", s)\n",
    "    s = re.sub(r\"(\\!)\", r\" \\1 \", s)\n",
    "    s = re.sub(r\"(\\:)\", r\" \\1 \", s)\n",
    "\n",
    "#     s = re.sub(r\"([,!?])\", r\" \\1\", s)\n",
    "#     s = re.sub(r\"(\\d),(\\d)\",r\"\\1,\\2\", s)\n",
    "    s = re.sub(r\"[^\\da-zA-Z.!,'?]+\", r\" \", s)\n",
    "    #s = re.sub(r\"[a-zA-Z] - [a-zA-Z]\", r\"\\1-\\2 \", s)\n",
    "    return s.lower().strip()\n",
    "ss = \"It's (df) df.,fdf@ !\"\n",
    "print(normalizeString(ss))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(len(en_sentences_dict)):\n",
    "    en_sentences_dict[str(i+1)] = normalizeString(en_sentences_dict[str(i+1)].strip()).strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluateAll(model, test_sentences_dict):\n",
    "    print(len(test_sentences_dict))\n",
    "    i=0\n",
    "#     j=0\n",
    "    test_file = open('test_result','w')\n",
    "    for i in range(len(test_sentences_dict)):\n",
    "        print(i)\n",
    "        out_sentence = evaluate(model, test_sentences_dict[str(i+1)]).strip() + \"\\n\"\n",
    "        out_sentence = re.sub('UNK', '', out_sentence)\n",
    "        out_sentence = re.sub(' ', '', out_sentence)\n",
    "        test_file.write(out_sentence)\n",
    "    test_file.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "evaluateAll(nmtmodel, en_sentences_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}