{"nbformat":4,"nbformat_minor":0,"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.13"},"colab":{"name":"finished seq2seq.ipynb","provenance":[{"file_id":"https://github.com/pytorch/tutorials/blob/gh-pages/_downloads/a60617788061539b5449701ae76aee56/seq2seq_translation_tutorial.ipynb","timestamp":1629456076187}],"collapsed_sections":["gS2JayUWYOlv","DqNQ5JNlXUQK"]}},"cells":[{"cell_type":"code","metadata":{"id":"Q8mxVcojYOlM"},"source":["%matplotlib inline"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"oWH31pB5XG_s"},"source":["# Setup"]},{"cell_type":"code","metadata":{"id":"rBZ6DhNobA1c","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1630162442217,"user_tz":-330,"elapsed":21222,"user":{"displayName":"Gihan Weeraprameshwara","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiCuBWPBFW3AvbkkMkY_XlfhSKS9YjMV2DdXXxL=s64","userId":"07090364678417433377"}},"outputId":"18663e58-01ac-47f0-c1b7-b6e6c16309be"},"source":["from google.colab import drive\n","drive.mount('/content/drive')"],"execution_count":null,"outputs":[{"output_type":"stream","text":["Mounted at /content/drive\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"2kjq0WSRH7PT"},"source":["from __future__ import unicode_literals, print_function, division\n","from io import open\n","import unicodedata\n","import string\n","import re\n","import random\n","\n","import torch\n","import torch.nn as nn\n","from torch import optim\n","import torch.nn.functional as F\n","\n","torch.set_printoptions(precision=8)\n","\n","import pandas as pd\n","import numpy as np\n","\n","import time\n","import math\n","\n","import matplotlib.pyplot as plt\n","plt.switch_backend('agg')\n","import matplotlib.ticker as ticker"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"9UWY_uSiKur2"},"source":["embedding_size = 300 #@param [50, 150, 200, 250, 300, 350, 400, 450, 500]\n","embedding_type = \"glove\" #@param [\"fastText\",\"word2vec\",\"glove\"]\n","model_type = \"GRU\" #@param [\"GRU\", \"LSTM\", \"Transformer\" ] \n","hidden_size = \"300\" #@param [] {allow-input: true}\n","apply_Attention_decoder = True #@param {type:\"boolean\"}\n","hidden_size = int(hidden_size)\n","embedding_size= int(embedding_size)\n","if (apply_Attention_decoder):\n","  decodr='attention'\n","else:\n","  decodr='simple'"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"rKHGWcypb8Rm"},"source":["folder_path = '/content/drive/MyDrive/17 Batch FYP - Facebook Sinhala Data/'\n","word_embedding_path= folder_path + \"Word embedding/fastText/\"+embedding_type+str(embedding_size)\n","encoder_path = folder_path + 'Word embedding/seq2seq/'+embedding_type+'_'+model_type+'_'+decodr+'encoder.dict'\n","decoder_path = folder_path + 'Word embedding/seq2seq/'+embedding_type+'_'+model_type+'_'+decodr+'decoder.dict'"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"Hoai0MCmHk0x","executionInfo":{"status":"ok","timestamp":1630163713397,"user_tz":-330,"elapsed":94903,"user":{"displayName":"Gihan Weeraprameshwara","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiCuBWPBFW3AvbkkMkY_XlfhSKS9YjMV2DdXXxL=s64","userId":"07090364678417433377"}},"outputId":"cba85c5a-3526-434a-c431-fedc8ee795bc"},"source":["data = pd.read_csv(cleaned_data_path, header=None)\n","messages = [[y for y in x if pd.notna(y)] for x in data.values.tolist()]"],"execution_count":null,"outputs":[{"output_type":"stream","text":["/usr/local/lib/python3.7/dist-packages/IPython/core/interactiveshell.py:2718: DtypeWarning: Columns (68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162,163) have mixed types.Specify dtype option on import or set low_memory=False.\n","  interactivity=interactivity, compiler=compiler, result=result)\n"],"name":"stderr"}]},{"cell_type":"code","metadata":{"id":"4mRGkNJuOJo2"},"source":["window_size = 40\n","min_word = 5\n","down_sampling = 1e-2"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"ELuVIj2tcFXF"},"source":["from gensim.models.fasttext import FastText\n","embedding_model = FastText(size=embedding_size,\n","                      window=window_size,\n","                      min_count=min_word,\n","                      sample=down_sampling,\n","                      sg=1)\n","embedding_model = FastText.load(word_embedding_path)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"06a5w1YzPayV"},"source":["SOS_token=np.zeros(300)\n","EOS_token =SOS_token+1"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"5xAVCaOqjY1Z"},"source":["# count=[]\n","# for i in range(0,len(messages)):\n","#       for j in range(len(messages[i])):\n","#         try:\n","#           messages[i][j]=embedding_model.wv[messages[i][j]]\n","#         except KeyError:\n","#           count+=[i]\n","#       messages[i].append(EOS_token)\n","\n","# for i in range(len(messages)):\n","#   for j in range(len(messages[i])):\n","#     try:\n","#       messages[i][j] = embedding_model.wv[messages[i][j]]\n","#     except KeyError:\n","#       k = messages[i].pop(j)\n","#       print ('KeyError', i, j, k)\n","#       i -= 1\n","#       break"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"o9hT8oBqRBMW"},"source":["unwanted=[]\n","for i in range(len(messages)):\n","  keyerr = False\n","  line = []\n","  for j in range(len(messages[i])):\n","    try:\n","      line.append(embedding_model.wv[messages[i][j]])\n","    except KeyError:\n","      # print ('KeyError', i, j, messages[i][j])\n","      continue\n","\n","  messages[i] = line\n","  if (len(messages[i])==0):\n","    unwanted+=[i]\n","  else:\n","    messages[i].append(EOS_token)\n"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"-UhyL3QIPQv1"},"source":["for ele in sorted(unwanted, reverse = True):\n","    del messages[ele]"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"MzJ060nwulYf"},"source":["np.random.shuffle(messages)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"qFFQKp5pv2N0"},"source":["SOS_token=torch.zeros(300)\n","EOS_token =SOS_token+1"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"yiAWFVAQRpz_"},"source":["Max_Length=165"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"AMVFESiLYOlu"},"source":["# The Encoder"]},{"cell_type":"code","metadata":{"id":"aWU7gfNqYOlv"},"source":["class EncoderRNN(nn.Module):\n","    def __init__(self, hidden_size):\n","        super(EncoderRNN, self).__init__()\n","        self.hidden_size = hidden_size\n","        #loook up FT\n","        #self.embedding = nn.Embedding(input_size, hidden_size)\n","        self.gru = nn.GRU(hidden_size, hidden_size)\n","\n","    def forward(self, inp, hidden):\n","        # embedded = self.embedding(input).view(1, 1, -1)\n","        # output = embedded\n","        input=(inp).view(1,1,-1)\n","        # input=input.float()\n","        output, hidden = self.gru(input, hidden)\n","        return output, hidden\n","\n","    def initHidden(self):\n","        return torch.zeros(1, 1, self.hidden_size)"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"gS2JayUWYOlv"},"source":["# The Decoder\n","-----------\n","\n","The decoder is another RNN that takes the encoder output vector(s) and\n","outputs a sequence of words to create the translation.\n","\n","\n"]},{"cell_type":"code","metadata":{"id":"KSFyLPjnYOlw"},"source":["# class DecoderRNN(nn.Module):\n","#     def __init__(self, hidden_size):\n","#         super(DecoderRNN, self).__init__()\n","#         self.hidden_size = hidden_size\n","\n","#         # self.embedding = nn.Embedding(output_size, hidden_size)\n","#         self.gru = nn.GRU(hidden_size, hidden_size)\n","#         # self.out = nn.Linear(hidden_size, hidden_size)\n","#         # self.softmax = nn.LogSoftmax(dim=1)\n","\n","#     def forward(self, input, hidden):\n","#         # output = self.embedding(input).view(1, 1, -1)\n","#         output=(input).view(1,1,-1)\n","#         # output = F.relu(output)\n","#         output, hidden = self.gru(output, hidden)\n","#         # output = self.softmax(self.out(output[0]))\n","#         return output, hidden\n","\n","#     def initHidden(self):\n","#         return torch.zeros(1, 1, self.hidden_size, device=device)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"K46gFci1QOrG"},"source":["class AttnDecoderRNN(nn.Module):\n","    def __init__(self, hidden_size, dropout_p=0.1, max_length=Max_Length):\n","        super(AttnDecoderRNN, self).__init__()\n","        self.hidden_size = hidden_size\n","        self.dropout_p = dropout_p\n","        self.max_length = max_length\n","\n","        # self.embedding = nn.Embedding(self.output_size, self.hidden_size)\n","        self.attn = nn.Linear(self.hidden_size * 2, self.max_length)\n","        self.attn_combine = nn.Linear(self.hidden_size * 2, self.hidden_size)\n","        self.dropout = nn.Dropout(self.dropout_p)\n","        self.gru = nn.GRU(self.hidden_size, self.hidden_size)\n","        # self.out = nn.Linear(self.hidden_size, self.hidden_size)\n","\n","    def forward(self, inp, hidden, encoder_outputs):\n","        # embedded = self.embedding(input).view(1, 1, -1)\n","        input=(inp).view(1,1,-1)\n","        embedded = self.dropout(input)\n","        # print(encoder_outputs.size())\n","        # print(encoder_outputs)\n","        # print(encoder_outputs.unsqueeze(0))\n","        attn_weights = F.softmax(\n","            self.attn(torch.cat((embedded[0], hidden[0]), 1)), dim=1)\n","        # print(attn_weights.unsqueeze(0).size(),encoder_outputs.unsqueeze(0).size())\n","        attn_applied = torch.bmm(attn_weights.unsqueeze(0),\n","                                 encoder_outputs.unsqueeze(0))\n","\n","        output = torch.cat((embedded[0], attn_applied[0]), 1)\n","        output = self.attn_combine(output).unsqueeze(0)\n","\n","        # output = F.relu(output)\n","        output, hidden = self.gru(output, hidden)\n","\n","        # output = F.log_softmax(self.out(output[0]), dim=1)\n","        return output, hidden, attn_weights\n","\n","    def initHidden(self):\n","        return torch.zeros(1, 1, self.hidden_size)"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"iTO6eaxI7SSG"},"source":["# Train functions"]},{"cell_type":"code","metadata":{"id":"hFQisQ7hYOly"},"source":["teacher_forcing_ratio = 0.5\n","\n","\n","def train(input_tensor, target_tensor, encoder, decoder, encoder_optimizer, decoder_optimizer, criterion, max_length):\n","    encoder_hidden = encoder.initHidden()\n","    encoder_optimizer.zero_grad()\n","    decoder_optimizer.zero_grad()\n","\n","    length = input_tensor.size(0)\n","\n","    encoder_outputs = torch.zeros(max_length, encoder.hidden_size)\n","\n","    loss = 0\n","    for ei in range(length):\n","        encoder_output, encoder_hidden = encoder(\n","            input_tensor[ei], encoder_hidden)\n","        encoder_outputs[ei] = encoder_output[0, 0]\n","\n","    decoder_input = torch.tensor(SOS_token.view(1,-1))\n","    decoder_hidden = encoder_hidden\n","    \n","    use_teacher_forcing = True if random.random() < teacher_forcing_ratio else False\n","\n","    if use_teacher_forcing:\n","        # Teacher forcing: Feed the target as the next input\n","        for di in range(length):\n","            # decoder_output, decoder_hidden = decoder(\n","            #     decoder_input, decoder_hidden)\n","            decoder_output, decoder_hidden, decoder_attention = decoder(\n","                decoder_input, decoder_hidden, encoder_outputs)\n","            loss += criterion(decoder_output, target_tensor[di])\n","            decoder_input = target_tensor[di]  # Teacher forcing\n","\n","    else:\n","        # Without teacher forcing: use its own predictions as the next input\n","        for di in range(length):\n","            # decoder_output, decoder_hidden = decoder(\n","            #     decoder_input, decoder_hidden)\n","            decoder_output, decoder_hidden, decoder_attention = decoder(\n","                decoder_input, decoder_hidden, encoder_outputs)\n","            # topv, topi = decoder_output.topk(1)\n","            decoder_input = decoder_output.squeeze().detach()  # detach from history as input\n","            loss += criterion(decoder_output, target_tensor[di])\n","            if (torch.equal(decoder_input,torch.tensor(EOS_token))):\n","                break\n","\n","    loss.backward()\n","\n","    encoder_optimizer.step()\n","    decoder_optimizer.step()\n","\n","    return loss.item() / length"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"DqNQ5JNlXUQK"},"source":["## Time"]},{"cell_type":"code","metadata":{"id":"Vbcx3xzkYOlz"},"source":["def asMinutes(s):\n","    m = math.floor(s / 60)\n","    s -= m * 60\n","    return '%dm %ds' % (m, s)\n","\n","\n","def timeSince(since, percent):\n","    now = time.time()\n","    s = now - since\n","    es = s / (percent)\n","    rs = es - s\n","    return '%s (- %s)' % (asMinutes(s), asMinutes(rs))"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"0PBEbWvlYOl0"},"source":["def showPlot(points):\n","    plt.figure()\n","    fig, ax = plt.subplots()\n","    # this locator puts ticks at regular intervals\n","    loc = ticker.MultipleLocator(base=0.2)\n","    ax.yaxis.set_major_locator(loc)\n","    plt.plot(points)"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"QdB0rQ2HXZNn"},"source":["## Train Iterations"]},{"cell_type":"code","metadata":{"id":"gP-jtXOyYOlz"},"source":["def trainIters(encoder, decoder,train_set , max_length, print_every=1000, plot_every=100, learning_rate=0.01):\n","    start = time.time()\n","    n_iters = len(train_set)\n","    plot_losses = []\n","    print_loss_total = 0  # Reset every print_every\n","    plot_loss_total = 0  # Reset every plot_every\n","    encoder_optimizer = optim.SGD(encoder.parameters(), lr=learning_rate)\n","    decoder_optimizer = optim.SGD(decoder.parameters(), lr=learning_rate)\n","    criterion = nn.MSELoss()\n","\n","    for i in range(1,n_iters+1):\n","      input_tensor = torch.tensor(train_set[i-1],dtype=torch.float32)\n","      target_tensor =input_tensor\n","\n","      loss = train(input_tensor, target_tensor, encoder,\n","                  decoder, encoder_optimizer, decoder_optimizer, criterion, max_length)\n","      print_loss_total += loss\n","      plot_loss_total += loss\n","\n","      if i % print_every == 0:\n","          print_loss_avg = print_loss_total / print_every\n","          print_loss_total = 0\n","          print('%s (%d %d%%) %.4f' % (timeSince(start, i / n_iters),\n","                                      i, i / n_iters * 100, print_loss_avg))\n","          torch.save(encoder.state_dict(), encoder_path)\n","          torch.save(decoder.state_dict(), decoder_path)\n","      if i % plot_every == 0:\n","          plot_loss_avg = plot_loss_total / plot_every\n","          plot_losses.append(plot_loss_avg)\n","          plot_loss_total = 0\n","    showPlot(plot_losses)"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"00WXCUsB7BRT"},"source":["# Visualizing"]},{"cell_type":"code","metadata":{"id":"BdupBEXnQiBb"},"source":["def evaluate(encoder, decoder, test_set, max_length=Max_Length,criterion = nn.MSELoss()):\n","    with torch.no_grad():\n","      for i in range(len(test_set)):\n","        input_tensor = torch.tensor(test_set[i-1],dtype=torch.float32)\n","        target_tensor=input_tensor\n","        length = input_tensor.size(0)\n","        encoder_hidden = encoder.initHidden()\n","        encoder_outputs = torch.zeros(max_length, encoder.hidden_size)\n","\n","        for ei in range(length):\n","            encoder_output, encoder_hidden = encoder(input_tensor[ei],\n","                                                    encoder_hidden)\n","            encoder_outputs[ei] += encoder_output[0, 0]\n","        print(encoder_outputs.size())\n","        decoder_input = torch.tensor(SOS_token.view(1,-1))  # SOS\n","\n","        decoder_hidden = encoder_hidden\n","\n","        # decoded_words = []\n","        decoder_attentions = torch.zeros(max_length, max_length)\n","        loss=0\n","        for di in range(length):\n","            decoder_output, decoder_hidden, decoder_attention = decoder(\n","                decoder_input, decoder_hidden, encoder_outputs)\n","            loss += criterion(decoder_output, target_tensor[di])\n","            decoder_attentions[di] = decoder_attention.data\n","            # topv, topi = decoder_output.data.topk(1)\n","            # print(di)\n","            # print(decoder_output)\n","            decoder_input = decoder_output.squeeze()\n","            if (torch.equal(decoder_input,torch.tensor(EOS_token))):\n","                break     \n","            # else:\n","            #     decoded_words.append(decoder_input)\n","\n","        loss.backward()\n","        loss=loss.item() / length\n","        print_loss_total += loss\n","        plot_loss_total += loss\n","\n","        if i % print_every == 0:\n","            print_loss_avg = print_loss_total / print_every\n","            print_loss_total = 0\n","            print('%s (%d %d%%) %.4f' % (timeSince(start, i / n_iters),\n","                                        i, i / n_iters * 100, print_loss_avg))\n","            torch.save(encoder1.state_dict(), encoder_path)\n","            torch.save(decoder1.state_dict(), decoder_path)\n","        if i % plot_every == 0:\n","            plot_loss_avg = plot_loss_total / plot_every\n","            plot_losses.append(plot_loss_avg)\n","            plot_loss_total = 0\n","\n","      showPlot(plot_losses)"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"tKJRAfGoYOmA"},"source":["Training and Evaluating\n","=====================\n","\n",".. Note::\n","   If you run this notebook you can train, interrupt the kernel,\n","   evaluate, and continue training later. Comment out the lines where the\n","   encoder and decoder are initialized and run ``trainIters`` again.\n","\n","\n"]},{"cell_type":"code","metadata":{"id":"bThSSSgycuO0"},"source":["encoder1 = EncoderRNN(hidden_size)   #hidden_size=length of FT embedding\n","decoder1 = AttnDecoderRNN(hidden_size, dropout_p=0.1)\n","# decoder1 = DecoderRNN(hidden_size)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"6j4eHJupVyFb"},"source":["train_set,test_set=messages[:20],messages[20:25]"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"BAe2kl4SYOmB","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1630163920688,"user_tz":-330,"elapsed":2612,"user":{"displayName":"Gihan Weeraprameshwara","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiCuBWPBFW3AvbkkMkY_XlfhSKS9YjMV2DdXXxL=s64","userId":"07090364678417433377"}},"outputId":"3cd13e5c-384f-4dc3-f229-90086071753a"},"source":["trainIters(encoder1, attn_decoder1,train_set, Max_Length, 5000, 5000)"],"execution_count":null,"outputs":[{"output_type":"stream","text":["/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:19: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n","/usr/local/lib/python3.7/dist-packages/torch/nn/modules/loss.py:528: UserWarning: Using a target size (torch.Size([300])) that is different to the input size (torch.Size([1, 1, 300])). This will likely lead to incorrect results due to broadcasting. Please ensure they have the same size.\n","  return F.mse_loss(input, target, reduction=self.reduction)\n","/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:44: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n"],"name":"stderr"}]},{"cell_type":"code","metadata":{"id":"jdhLII6-W44I"},"source":["torch.save(encoder1.state_dict(), encoder_path)\n","torch.save(decoder1.state_dict(), decoder_path)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"o1ZUlzqzZxYH","executionInfo":{"status":"ok","timestamp":1630165369297,"user_tz":-330,"elapsed":401,"user":{"displayName":"Gihan Weeraprameshwara","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiCuBWPBFW3AvbkkMkY_XlfhSKS9YjMV2DdXXxL=s64","userId":"07090364678417433377"}},"outputId":"4eaae90e-a35a-404b-dea4-02fabfcc1bce"},"source":["encoder1 = EncoderRNN(hidden_size)   #hidden_size=length of FT embedding\n","decoder1 = AttnDecoderRNN(hidden_size, dropout_p=0.1)\n","encoder1.load_state_dict(torch.load(encoder_path))\n","decoder1.load_state_dict(torch.load(decoder_path))"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["<All keys matched successfully>"]},"metadata":{},"execution_count":49}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":478},"id":"shHjKtBIdZmn","executionInfo":{"status":"error","timestamp":1630152636836,"user_tz":-330,"elapsed":489,"user":{"displayName":"Gihan Weeraprameshwara","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiCuBWPBFW3AvbkkMkY_XlfhSKS9YjMV2DdXXxL=s64","userId":"07090364678417433377"}},"outputId":"59427068-67d1-490e-b88f-aa120b0752b6"},"source":["evaluate(encoder1, decoder1, test_set)"],"execution_count":null,"outputs":[{"output_type":"stream","text":["torch.Size([165, 300])\n"],"name":"stdout"},{"output_type":"stream","text":["/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:15: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n","  from ipykernel import kernelapp as app\n","/usr/local/lib/python3.7/dist-packages/torch/nn/modules/loss.py:528: UserWarning: Using a target size (torch.Size([300])) that is different to the input size (torch.Size([1, 1, 300])). This will likely lead to incorrect results due to broadcasting. Please ensure they have the same size.\n","  return F.mse_loss(input, target, reduction=self.reduction)\n","/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:31: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n"],"name":"stderr"},{"output_type":"error","ename":"RuntimeError","evalue":"ignored","traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m","\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)","\u001b[0;32m<ipython-input-54-b70f525228a2>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mevaluate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mencoder1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mattn_decoder1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtest_set\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m","\u001b[0;32m<ipython-input-52-d707420fb913>\u001b[0m in \u001b[0;36mevaluate\u001b[0;34m(encoder, decoder, test_set, max_length, criterion)\u001b[0m\n\u001b[1;32m     34\u001b[0m             \u001b[0;31m#     decoded_words.append(decoder_input)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     35\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 36\u001b[0;31m         \u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     37\u001b[0m         \u001b[0mloss\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitem\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0mlength\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     38\u001b[0m         \u001b[0mprint_loss_total\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mloss\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/usr/local/lib/python3.7/dist-packages/torch/_tensor.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(self, gradient, retain_graph, create_graph, inputs)\u001b[0m\n\u001b[1;32m    253\u001b[0m                 \u001b[0mcreate_graph\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcreate_graph\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    254\u001b[0m                 inputs=inputs)\n\u001b[0;32m--> 255\u001b[0;31m         \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mautograd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgradient\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    256\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    257\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mregister_hook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/usr/local/lib/python3.7/dist-packages/torch/autograd/__init__.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables, inputs)\u001b[0m\n\u001b[1;32m    147\u001b[0m     Variable._execution_engine.run_backward(\n\u001b[1;32m    148\u001b[0m         \u001b[0mtensors\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgrad_tensors_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 149\u001b[0;31m         allow_unreachable=True, accumulate_grad=True)  # allow_unreachable flag\n\u001b[0m\u001b[1;32m    150\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    151\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;31mRuntimeError\u001b[0m: element 0 of tensors does not require grad and does not have a grad_fn"]}]},{"cell_type":"markdown","metadata":{"id":"50QrYx50YOmC"},"source":["Visualizing Attention\n","---------------------\n","\n","A useful property of the attention mechanism is its highly interpretable\n","outputs. Because it is used to weight specific encoder outputs of the\n","input sequence, we can imagine looking where the network is focused most\n","at each time step.\n","\n","You could simply run ``plt.matshow(attentions)`` to see attention output\n","displayed as a matrix, with the columns being input steps and rows being\n","output steps:\n","\n","\n"]},{"cell_type":"markdown","metadata":{"id":"i0yurZ4oYOmD"},"source":["For a better viewing experience we will do the extra work of adding axes\n","and labels:\n","\n","\n"]},{"cell_type":"code","metadata":{"id":"ToT6ggTNYOmD"},"source":["def showAttention(input_sentence, output_words, attentions):\n","    # Set up figure with colorbar\n","    fig = plt.figure()\n","    ax = fig.add_subplot(111)\n","    cax = ax.matshow(attentions.numpy(), cmap='bone')\n","    fig.colorbar(cax)\n","\n","    # Set up axes\n","    ax.set_xticklabels([''] + input_sentence.split(' ') +\n","                       ['<EOS>'], rotation=90)\n","    ax.set_yticklabels([''] + output_words)\n","\n","    # Show label at every tick\n","    ax.xaxis.set_major_locator(ticker.MultipleLocator(1))\n","    ax.yaxis.set_major_locator(ticker.MultipleLocator(1))\n","\n","    plt.show()\n","\n","\n","def evaluateAndShowAttention(input_sentence):\n","    output_words, attentions = evaluate(\n","        encoder1, attn_decoder1, input_sentence)\n","    print('input =', input_sentence)\n","    print('output =', ' '.join(output_words))\n","    showAttention(input_sentence, output_words, attentions)\n","\n","\n","evaluateAndShowAttention(\"elle a cinq ans de moins que moi .\")"],"execution_count":null,"outputs":[]}]}