Add files using upload-large-folder tool

stanza/stanza/models/common/beam.py

@@ -0,0 +1,146 @@
+from __future__ import division
+import torch
+
+import stanza.models.common.seq2seq_constant as constant
+
+r"""
+ Adapted and modified from the OpenNMT project.
+
+ Class for managing the internals of the beam search process.
+
+
+         hyp1-hyp1---hyp1 -hyp1
+                 \             /
+         hyp2 \-hyp2 /-hyp2hyp2
+                               /      \
+         hyp3-hyp3---hyp3 -hyp3
+         ========================
+
+ Takes care of beams, back pointers, and scores.
+"""
+
+
+# TORCH COMPATIBILITY
+#
+# Here we special case trunc division
+# torch < 1.8.0 has no rounding_model='trunc' argument for torch.div
+# however, there were several versions in a row where // would loudly
+# proclaim it was buggy, and users complained about that
+# this hopefully maintains compatibility for torch
+try:
+    a = torch.tensor([1.])
+    b = torch.tensor([2.])
+    c = torch.div(a, b, rounding_mode='trunc')
+    def trunc_division(a, b):
+        return torch.div(a, b, rounding_mode='trunc')
+except TypeError:
+    def trunc_division(a, b):
+        return a // b
+
+class Beam(object):
+    def __init__(self, size, device=None):
+        self.size = size
+        self.done = False
+
+        # The score for each translation on the beam.
+        self.scores = torch.zeros(size, dtype=torch.float32, device=device)
+        self.allScores = []
+
+        # The backpointers at each time-step.
+        self.prevKs = []
+
+        # The outputs at each time-step.
+        self.nextYs = [torch.zeros(size, dtype=torch.int64, device=device).fill_(constant.PAD_ID)]
+        self.nextYs[0][0] = constant.SOS_ID
+
+        # The copy indices for each time
+        self.copy = []
+
+    def get_current_state(self):
+        "Get the outputs for the current timestep."
+        return self.nextYs[-1]
+
+    def get_current_origin(self):
+        "Get the backpointers for the current timestep."
+        return self.prevKs[-1]
+
+    def advance(self, wordLk, copy_indices=None):
+        """
+        Given prob over words for every last beam `wordLk` and attention
+        `attnOut`: Compute and update the beam search.
+
+        Parameters:
+
+        * `wordLk`- probs of advancing from the last step (K x words)
+        * `copy_indices` - copy indices (K x ctx_len)
+
+        Returns: True if beam search is complete.
+        """
+        if self.done:
+            return True
+        numWords = wordLk.size(1)
+
+        # Sum the previous scores.
+        if len(self.prevKs) > 0:
+            beamLk = wordLk + self.scores.unsqueeze(1).expand_as(wordLk)
+        else:
+            # first step, expand from the first position
+            beamLk = wordLk[0]
+
+        flatBeamLk = beamLk.view(-1)
+
+        bestScores, bestScoresId = flatBeamLk.topk(self.size, 0, True, True)
+        self.allScores.append(self.scores)
+        self.scores = bestScores
+
+        # bestScoresId is flattened beam x word array, so calculate which
+        # word and beam each score came from
+        # bestScoreId is the integer ids, and numWords is the integer length.
+        # Need to do integer division
+        prevK = trunc_division(bestScoresId, numWords)
+        self.prevKs.append(prevK)
+        self.nextYs.append(bestScoresId - prevK * numWords)
+        if copy_indices is not None:
+            self.copy.append(copy_indices.index_select(0, prevK))
+
+        # End condition is when top-of-beam is EOS.
+        if self.nextYs[-1][0] == constant.EOS_ID:
+            self.done = True
+            self.allScores.append(self.scores)
+
+        return self.done
+
+    def sort_best(self):
+        return torch.sort(self.scores, 0, True)
+
+    def get_best(self):
+        "Get the score of the best in the beam."
+        scores, ids = self.sortBest()
+        return scores[1], ids[1]
+
+    def get_hyp(self, k):
+        """
+        Walk back to construct the full hypothesis.
+
+        Parameters:
+
+             * `k` - the position in the beam to construct.
+
+         Returns: The hypothesis
+        """
+        hyp = []
+        cpy = []
+        for j in range(len(self.prevKs) - 1, -1, -1):
+            hyp.append(self.nextYs[j+1][k])
+            if len(self.copy) > 0:
+                cpy.append(self.copy[j][k])
+            k = self.prevKs[j][k]
+
+        hyp = hyp[::-1]
+        cpy = cpy[::-1]
+        # postprocess: if cpy index is not -1, use cpy index instead of hyp word
+        for i,cidx in enumerate(cpy):
+            if cidx >= 0:
+                hyp[i] = -(cidx+1) # make index 1-based and flip it for token generation
+
+        return hyp

stanza/stanza/models/common/convert_pretrain.py

@@ -0,0 +1,42 @@
+"""
+A utility script to load a word embedding file from a text file and save it as a .pt
+
+Run it as follows:
+  python stanza/models/common/convert_pretrain.py <.pt file> <text file> <# vectors>
+
+Note that -1 for # of vectors will keep all the vectors.
+You probably want to keep fewer than that for most publicly released
+embeddings, though, as they can get quite large.
+
+As a concrete example, you can convert a newly downloaded Faroese WV file as follows:
+  python3 stanza/models/common/convert_pretrain.py ~/stanza/saved_models/pos/fo_farpahc.pretrain.pt ~/extern_data/wordvec/fasttext/faroese.txt -1
+or save part of an Icelandic WV file:
+  python3 stanza/models/common/convert_pretrain.py ~/stanza/saved_models/pos/is_icepahc.pretrain.pt ~/extern_data/wordvec/fasttext/icelandic.cc.is.300.vec 150000
+Note that if the pretrain already exists, nothing will be changed.  It will not overwrite an existing .pt file.
+
+"""
+
+import argparse
+import os
+import sys
+
+from stanza.models.common import pretrain
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("output_pt", default=None, help="Where to write the converted PT file")
+    parser.add_argument("input_vec", default=None, help="Unconverted vectors file")
+    parser.add_argument("max_vocab", type=int, default=-1, nargs="?", help="How many vectors to convert.  -1 means convert them all")
+    args = parser.parse_args()
+
+    if os.path.exists(args.output_pt):
+        print("Not overwriting existing pretrain file in %s" % args.output_pt)
+
+    if args.input_vec.endswith(".csv"):
+        pt = pretrain.Pretrain(args.output_pt, max_vocab=args.max_vocab, csv_filename=args.input_vec)
+    else:
+        pt = pretrain.Pretrain(args.output_pt, args.input_vec, max_vocab=args.max_vocab)
+    print("Pretrain is of size {}".format(len(pt.vocab)))
+
+if __name__ == '__main__':
+    main()

stanza/stanza/models/common/pretrain.py

@@ -0,0 +1,289 @@
+"""
+Supports for pretrained data.
+"""
+import csv
+import os
+import re
+
+import lzma
+import logging
+import numpy as np
+import torch
+
+from .vocab import BaseVocab, VOCAB_PREFIX, UNK_ID
+
+from stanza.models.common.utils import open_read_binary, open_read_text
+from stanza.resources.common import DEFAULT_MODEL_DIR
+
+from pickle import UnpicklingError
+import warnings
+
+logger = logging.getLogger('stanza')
+
+class PretrainedWordVocab(BaseVocab):
+    def build_vocab(self):
+        self._id2unit = VOCAB_PREFIX + self.data
+        self._unit2id = {w:i for i, w in enumerate(self._id2unit)}
+
+    def normalize_unit(self, unit):
+        unit = super().normalize_unit(unit)
+        if unit:
+            unit = unit.replace(" ","\xa0")
+        return unit
+
+class Pretrain:
+    """ A loader and saver for pretrained embeddings. """
+
+    def __init__(self, filename=None, vec_filename=None, max_vocab=-1, save_to_file=True, csv_filename=None):
+        self.filename = filename
+        self._vec_filename = vec_filename
+        self._csv_filename = csv_filename
+        self._max_vocab = max_vocab
+        self._save_to_file = save_to_file
+
+    @property
+    def vocab(self):
+        if not hasattr(self, '_vocab'):
+            self.load()
+        return self._vocab
+
+    @property
+    def emb(self):
+        if not hasattr(self, '_emb'):
+            self.load()
+        return self._emb
+
+    def load(self):
+        if self.filename is not None and os.path.exists(self.filename):
+            try:
+                # TODO: after making the next release, remove the weights_only=False version
+                try:
+                    data = torch.load(self.filename, lambda storage, loc: storage, weights_only=True)
+                except UnpicklingError:
+                    data = torch.load(self.filename, lambda storage, loc: storage, weights_only=False)
+                    warnings.warn("The saved pretrain has an old format using numpy.ndarray instead of torch to store weights.  This version of Stanza can support reading both the new and the old formats.  Future versions will only allow loading with weights_only=True.  Please resave the pretrained embedding using this version ASAP.")
+                logger.debug("Loaded pretrain from {}".format(self.filename))
+                if not isinstance(data, dict):
+                    raise RuntimeError("File {} exists but is not a stanza pretrain file.  It is not a dict, whereas a Stanza pretrain should have a dict with 'emb' and 'vocab'".format(self.filename))
+                if 'emb' not in data or 'vocab' not in data:
+                    raise RuntimeError("File {} exists but is not a stanza pretrain file.  A Stanza pretrain file should have 'emb' and 'vocab' fields in its state dict".format(self.filename))
+                self._vocab = PretrainedWordVocab.load_state_dict(data['vocab'])
+                self._emb = data['emb']
+                if isinstance(self._emb, np.ndarray):
+                    self._emb = torch.from_numpy(self._emb)
+                return
+            except (KeyboardInterrupt, SystemExit):
+                raise
+            except BaseException as e:
+                if not self._vec_filename and not self._csv_filename:
+                    raise
+                logger.warning("Pretrained file exists but cannot be loaded from {}, due to the following exception:\n\t{}".format(self.filename, e))
+                vocab, emb = self.read_pretrain()
+        else:
+            if not self._vec_filename and not self._csv_filename:
+                raise FileNotFoundError("Pretrained file {} does not exist, and no text/xz file was provided".format(self.filename))
+            if self.filename is not None:
+                logger.info("Pretrained filename %s specified, but file does not exist.  Attempting to load from text file" % self.filename)
+            vocab, emb = self.read_pretrain()
+
+        self._vocab = vocab
+        self._emb = emb
+
+        if self._save_to_file:
+            # save to file
+            assert self.filename is not None, "Filename must be provided to save pretrained vector to file."
+            self.save(self.filename)
+
+    def save(self, filename):
+        directory, _ = os.path.split(filename)
+        if directory:
+            os.makedirs(directory, exist_ok=True)
+        # should not infinite loop since the load function sets _vocab and _emb before trying to save
+        data = {'vocab': self.vocab.state_dict(), 'emb': self.emb}
+        try:
+            torch.save(data, filename, _use_new_zipfile_serialization=False)
+            logger.info("Saved pretrained vocab and vectors to {}".format(filename))
+        except (KeyboardInterrupt, SystemExit):
+            raise
+        except BaseException as e:
+            logger.warning("Saving pretrained data failed due to the following exception... continuing anyway.\n\t{}".format(e))
+
+
+    def write_text(self, filename, header=False):
+        """
+        Write the vocab & values to a text file
+        """
+        with open(filename, "w") as fout:
+            if header:
+                word_dim = self.emb[0].shape[0]
+                fout.write("%d %d\n" % (len(self.vocab), word_dim))
+            for word_idx, word in enumerate(self.vocab):
+                row = self.emb[word_idx].to("cpu")
+                fout.write(word)
+                fout.write(" ")
+                fout.write(" ".join(["%.6f" % x.item() for x in row]))
+                fout.write("\n")
+
+
+    def read_pretrain(self):
+        # load from pretrained filename
+        if self._vec_filename is not None:
+            words, emb, failed = self.read_from_file(self._vec_filename, self._max_vocab)
+        elif self._csv_filename is not None:
+            words, emb = self.read_from_csv(self._csv_filename)
+        else:
+            raise RuntimeError("Vector file is not provided.")
+
+        if len(emb) - len(VOCAB_PREFIX) != len(words):
+            raise RuntimeError("Loaded number of vectors does not match number of words.")
+
+        # Use a fixed vocab size
+        if self._max_vocab > len(VOCAB_PREFIX) and self._max_vocab < len(words) + len(VOCAB_PREFIX):
+            words = words[:self._max_vocab - len(VOCAB_PREFIX)]
+            emb = emb[:self._max_vocab]
+
+        vocab = PretrainedWordVocab(words)
+        
+        return vocab, emb
+
+    @staticmethod
+    def read_from_csv(filename):
+        """
+        Read vectors from CSV
+
+        Skips the first row
+        """
+        logger.info("Reading pretrained vectors from csv file %s ...", filename)
+        with open_read_text(filename) as fin:
+            csv_reader = csv.reader(fin)
+            # the header of the thai csv vector file we have is just the number of columns
+            # so we read past the first line
+            for line in csv_reader:
+                break
+            lines = [line for line in csv_reader]
+
+        rows = len(lines)
+        cols = len(lines[0]) - 1
+
+        emb = torch.zeros((rows + len(VOCAB_PREFIX), cols), dtype=torch.float32)
+        for i, line in enumerate(lines):
+            emb[i+len(VOCAB_PREFIX)] = torch.tensor([float(x) for x in line[-cols:]], dtype=torch.float32)
+        words = [line[0].replace(' ', '\xa0') for line in lines]
+        return words, emb
+
+    @staticmethod
+    def read_from_file(filename, max_vocab=None):
+        """
+        Open a vector file using the provided function and read from it.
+        """
+        logger.info("Reading pretrained vectors from %s ...", filename)
+
+        # some vector files, such as Google News, use tabs
+        tab_space_pattern = re.compile(r"[ \t]+")
+        first = True
+        cols = None
+        lines = []
+        failed = 0
+        unk_line = None
+        with open_read_binary(filename) as f:
+            for i, line in enumerate(f):
+                try:
+                    line = line.decode()
+                except UnicodeDecodeError:
+                    failed += 1
+                    continue
+                line = line.rstrip()
+                if not line:
+                    continue
+                pieces = tab_space_pattern.split(line)
+                if first:
+                    # the first line contains the number of word vectors and the dimensionality
+                    # note that a 1d embedding with a number as the first entry
+                    # will fail to read properly.  we ignore that case
+                    first = False
+                    if len(pieces) == 2:
+                        cols = int(pieces[1])
+                        continue
+
+                if pieces[0] == '<unk>':
+                    if unk_line is not None:
+                        logger.error("More than one <unk> line in the pretrain!  Keeping the most recent one")
+                    else:
+                        logger.debug("Found an unk line while reading the pretrain")
+                    unk_line = pieces
+                else:
+                    if not max_vocab or max_vocab < 0 or len(lines) < max_vocab:
+                        lines.append(pieces)
+
+        if cols is None:
+            # another failure case: all words have spaces in them
+            cols = min(len(x) for x in lines) - 1
+        rows = len(lines)
+        emb = torch.zeros((rows + len(VOCAB_PREFIX), cols), dtype=torch.float32)
+        if unk_line is not None:
+            emb[UNK_ID] = torch.tensor([float(x) for x in unk_line[-cols:]], dtype=torch.float32)
+        for i, line in enumerate(lines):
+            emb[i+len(VOCAB_PREFIX)] = torch.tensor([float(x) for x in line[-cols:]], dtype=torch.float32)
+
+        # if there were word pieces separated with spaces, rejoin them with nbsp instead
+        # this way, the normalize_unit method in vocab.py can find the word at test time
+        words = ['\xa0'.join(line[:-cols]) for line in lines]
+        if failed > 0:
+            logger.info("Failed to read %d lines from embedding", failed)
+        return words, emb, failed
+
+
+def find_pretrain_file(wordvec_pretrain_file, save_dir, shorthand, lang):
+    """
+    When training a model, look in a few different places for a .pt file
+
+    If a specific argument was passsed in, prefer that location
+    Otherwise, check in a few places:
+      saved_models/{model}/{shorthand}.pretrain.pt
+      saved_models/{model}/{shorthand}_pretrain.pt
+      ~/stanza_resources/{language}/pretrain/{shorthand}_pretrain.pt
+    """
+    if wordvec_pretrain_file:
+        return wordvec_pretrain_file
+
+    default_pretrain_file = os.path.join(save_dir, '{}.pretrain.pt'.format(shorthand))
+    if os.path.exists(default_pretrain_file):
+        logger.debug("Found existing .pt file in %s" % default_pretrain_file)
+        return default_pretrain_file
+    else:
+        logger.debug("Cannot find pretrained vectors in %s" % default_pretrain_file)
+
+    pretrain_file = os.path.join(save_dir, '{}_pretrain.pt'.format(shorthand))
+    if os.path.exists(pretrain_file):
+        logger.debug("Found existing .pt file in %s" % pretrain_file)
+        return pretrain_file
+    else:
+        logger.debug("Cannot find pretrained vectors in %s" % pretrain_file)
+
+    if shorthand.find("_") >= 0:
+        # try to assemble /home/user/stanza_resources/vi/pretrain/vtb.pt for example
+        pretrain_file = os.path.join(DEFAULT_MODEL_DIR, lang, 'pretrain', '{}.pt'.format(shorthand.split('_', 1)[1]))
+        if os.path.exists(pretrain_file):
+            logger.debug("Found existing .pt file in %s" % pretrain_file)
+            return pretrain_file
+        else:
+            logger.debug("Cannot find pretrained vectors in %s" % pretrain_file)
+
+    # if we can't find it anywhere, just return the first location searched...
+    # maybe we'll get lucky and the original .txt file can be found
+    return default_pretrain_file
+
+
+if __name__ == '__main__':
+    with open('test.txt', 'w') as fout:
+        fout.write('3 2\na 1 1\nb -1 -1\nc 0 0\n')
+    # 1st load: save to pt file
+    pretrain = Pretrain('test.pt', 'test.txt')
+    print(pretrain.emb)
+    # verify pt file
+    x = torch.load('test.pt', weights_only=True)
+    print(x)
+    # 2nd load: load saved pt file
+    pretrain = Pretrain('test.pt', 'test.txt')
+    print(pretrain.emb)
+

stanza/stanza/models/common/seq2seq_modules.py

@@ -0,0 +1,244 @@
+"""
+Pytorch implementation of basic sequence to Sequence modules.
+"""
+
+import logging
+import torch
+import torch.nn as nn
+import math
+import numpy as np
+
+import stanza.models.common.seq2seq_constant as constant
+
+logger = logging.getLogger('stanza')
+
+class BasicAttention(nn.Module):
+    """
+    A basic MLP attention layer.
+    """
+    def __init__(self, dim):
+        super(BasicAttention, self).__init__()
+        self.linear_in = nn.Linear(dim, dim, bias=False)
+        self.linear_c = nn.Linear(dim, dim)
+        self.linear_v = nn.Linear(dim, 1, bias=False)
+        self.linear_out = nn.Linear(dim * 2, dim, bias=False)
+        self.tanh = nn.Tanh()
+        self.sm = nn.Softmax(dim=1)
+
+    def forward(self, input, context, mask=None, attn_only=False):
+        """
+        input: batch x dim
+        context: batch x sourceL x dim
+        """
+        batch_size = context.size(0)
+        source_len = context.size(1)
+        dim = context.size(2)
+        target = self.linear_in(input) # batch x dim
+        source = self.linear_c(context.contiguous().view(-1, dim)).view(batch_size, source_len, dim)
+        attn = target.unsqueeze(1).expand_as(context) + source
+        attn = self.tanh(attn) # batch x sourceL x dim
+        attn = self.linear_v(attn.view(-1, dim)).view(batch_size, source_len)
+
+        if mask is not None:
+            attn.masked_fill_(mask, -constant.INFINITY_NUMBER)
+
+        attn = self.sm(attn)
+        if attn_only:
+            return attn
+
+        weighted_context = torch.bmm(attn.unsqueeze(1), context).squeeze(1)
+        h_tilde = torch.cat((weighted_context, input), 1)
+        h_tilde = self.tanh(self.linear_out(h_tilde))
+
+        return h_tilde, attn
+
+class SoftDotAttention(nn.Module):
+    """Soft Dot Attention.
+
+    Ref: http://www.aclweb.org/anthology/D15-1166
+    Adapted from PyTorch OPEN NMT.
+    """
+
+    def __init__(self, dim):
+        """Initialize layer."""
+        super(SoftDotAttention, self).__init__()
+        self.linear_in = nn.Linear(dim, dim, bias=False)
+        self.sm = nn.Softmax(dim=1)
+        self.linear_out = nn.Linear(dim * 2, dim, bias=False)
+        self.tanh = nn.Tanh()
+        self.mask = None
+
+    def forward(self, input, context, mask=None, attn_only=False, return_logattn=False):
+        """Propagate input through the network.
+
+        input: batch x dim
+        context: batch x sourceL x dim
+        """
+        target = self.linear_in(input).unsqueeze(2)  # batch x dim x 1
+
+        # Get attention
+        attn = torch.bmm(context, target).squeeze(2)  # batch x sourceL
+
+        if mask is not None:
+            # sett the padding attention logits to -inf
+            assert mask.size() == attn.size(), "Mask size must match the attention size!"
+            attn.masked_fill_(mask, -constant.INFINITY_NUMBER)
+
+        if return_logattn:
+            attn = torch.log_softmax(attn, 1)
+            attn_w = torch.exp(attn)
+        else:
+            attn = self.sm(attn)
+            attn_w = attn
+        if attn_only:
+            return attn
+
+        attn3 = attn_w.view(attn_w.size(0), 1, attn_w.size(1))  # batch x 1 x sourceL
+
+        weighted_context = torch.bmm(attn3, context).squeeze(1)  # batch x dim
+        h_tilde = torch.cat((weighted_context, input), 1)
+
+        h_tilde = self.tanh(self.linear_out(h_tilde))
+
+        return h_tilde, attn
+
+
+class LinearAttention(nn.Module):
+    """ A linear attention form, inspired by BiDAF:
+        a = W (u; v; u o v)
+    """
+
+    def __init__(self, dim):
+        super(LinearAttention, self).__init__()
+        self.linear = nn.Linear(dim*3, 1, bias=False)
+        self.linear_out = nn.Linear(dim * 2, dim, bias=False)
+        self.sm = nn.Softmax(dim=1)
+        self.tanh = nn.Tanh()
+        self.mask = None
+
+    def forward(self, input, context, mask=None, attn_only=False):
+        """
+        input: batch x dim
+        context: batch x sourceL x dim
+        """
+        batch_size = context.size(0)
+        source_len = context.size(1)
+        dim = context.size(2)
+        u = input.unsqueeze(1).expand_as(context).contiguous().view(-1, dim)  # batch*sourceL x dim
+        v = context.contiguous().view(-1, dim)
+        attn_in = torch.cat((u, v, u.mul(v)), 1)
+        attn = self.linear(attn_in).view(batch_size, source_len)
+
+        if mask is not None:
+            # sett the padding attention logits to -inf
+            assert mask.size() == attn.size(), "Mask size must match the attention size!"
+            attn.masked_fill_(mask, -constant.INFINITY_NUMBER)
+
+        attn = self.sm(attn)
+        if attn_only:
+            return attn
+
+        attn3 = attn.view(batch_size, 1, source_len)  # batch x 1 x sourceL
+
+        weighted_context = torch.bmm(attn3, context).squeeze(1)  # batch x dim
+        h_tilde = torch.cat((weighted_context, input), 1)
+        h_tilde = self.tanh(self.linear_out(h_tilde))
+        return h_tilde, attn
+
+class DeepAttention(nn.Module):
+    """ A deep attention form, invented by Robert:
+        u = ReLU(Wx)
+        v = ReLU(Wy)
+        a = V.(u o v)
+    """
+
+    def __init__(self, dim):
+        super(DeepAttention, self).__init__()
+        self.linear_in = nn.Linear(dim, dim, bias=False)
+        self.linear_v = nn.Linear(dim, 1, bias=False)
+        self.linear_out = nn.Linear(dim * 2, dim, bias=False)
+        self.relu = nn.ReLU()
+        self.sm = nn.Softmax(dim=1)
+        self.tanh = nn.Tanh()
+        self.mask = None
+
+    def forward(self, input, context, mask=None, attn_only=False):
+        """
+        input: batch x dim
+        context: batch x sourceL x dim
+        """
+        batch_size = context.size(0)
+        source_len = context.size(1)
+        dim = context.size(2)
+        u = input.unsqueeze(1).expand_as(context).contiguous().view(-1, dim)  # batch*sourceL x dim
+        u = self.relu(self.linear_in(u))
+        v = self.relu(self.linear_in(context.contiguous().view(-1, dim)))
+        attn = self.linear_v(u.mul(v)).view(batch_size, source_len)
+
+        if mask is not None:
+            # sett the padding attention logits to -inf
+            assert mask.size() == attn.size(), "Mask size must match the attention size!"
+            attn.masked_fill_(mask, -constant.INFINITY_NUMBER)
+
+        attn = self.sm(attn)
+        if attn_only:
+            return attn
+
+        attn3 = attn.view(batch_size, 1, source_len)  # batch x 1 x sourceL
+
+        weighted_context = torch.bmm(attn3, context).squeeze(1)  # batch x dim
+        h_tilde = torch.cat((weighted_context, input), 1)
+        h_tilde = self.tanh(self.linear_out(h_tilde))
+        return h_tilde, attn
+
+class LSTMAttention(nn.Module):
+    r"""A long short-term memory (LSTM) cell with attention."""
+
+    def __init__(self, input_size, hidden_size, batch_first=True, attn_type='soft'):
+        """Initialize params."""
+        super(LSTMAttention, self).__init__()
+        self.input_size = input_size
+        self.hidden_size = hidden_size
+        self.batch_first = batch_first
+
+        self.lstm_cell = nn.LSTMCell(input_size, hidden_size)
+
+        if attn_type == 'soft':
+            self.attention_layer = SoftDotAttention(hidden_size)
+        elif attn_type == 'mlp':
+            self.attention_layer = BasicAttention(hidden_size)
+        elif attn_type == 'linear':
+            self.attention_layer = LinearAttention(hidden_size)
+        elif attn_type == 'deep':
+            self.attention_layer = DeepAttention(hidden_size)
+        else:
+            raise Exception("Unsupported LSTM attention type: {}".format(attn_type))
+        logger.debug("Using {} attention for LSTM.".format(attn_type))
+
+    def forward(self, input, hidden, ctx, ctx_mask=None, return_logattn=False):
+        """Propagate input through the network."""
+        if self.batch_first:
+            input = input.transpose(0,1)
+
+        output = []
+        attn = []
+        steps = range(input.size(0))
+        for i in steps:
+            hidden = self.lstm_cell(input[i], hidden)
+            hy, cy = hidden
+            h_tilde, alpha = self.attention_layer(hy, ctx, mask=ctx_mask, return_logattn=return_logattn)
+            output.append(h_tilde)
+            attn.append(alpha)
+        output = torch.cat(output, 0).view(input.size(0), *output[0].size())
+
+        if self.batch_first:
+            output = output.transpose(0,1)
+
+        if return_logattn:
+            attn = torch.stack(attn, 0)
+            if self.batch_first:
+                attn = attn.transpose(0, 1)
+            return output, hidden, attn
+
+        return output, hidden
+

stanza/stanza/models/common/stanza_object.py

@@ -0,0 +1,31 @@
+def _readonly_setter(self, name):
+    full_classname = self.__class__.__module__
+    if full_classname is None:
+        full_classname = self.__class__.__qualname__
+    else:
+        full_classname += '.' + self.__class__.__qualname__
+    raise ValueError(f'Property "{name}" of "{full_classname}" is read-only.')
+
+class StanzaObject(object):
+    """
+    Base class for all Stanza data objects that allows for some flexibility handling annotations
+    """
+
+    @classmethod
+    def add_property(cls, name, default=None, getter=None, setter=None):
+        """
+        Add a property accessible through self.{name} with underlying variable self._{name}.
+        Optionally setup a setter as well.
+        """
+
+        if hasattr(cls, name):
+            raise ValueError(f'Property by the name of {name} already exists in {cls}. Maybe you want to find another name?')
+
+        setattr(cls, f'_{name}', default)
+        if getter is None:
+            getter = lambda self: getattr(self, f'_{name}')
+        if setter is None:
+            setter = lambda self, value: _readonly_setter(self, name)
+
+        setattr(cls, name, property(getter, setter))
+