Add files using upload-large-folder tool

fairseq-0.10.2/examples/backtranslation/prepare-wmt18en2de.sh

@@ -0,0 +1,135 @@
+#!/bin/bash
+# Adapted from https://github.com/facebookresearch/MIXER/blob/master/prepareData.sh
+
+echo 'Cloning Moses github repository (for tokenization scripts)...'
+git clone https://github.com/moses-smt/mosesdecoder.git
+
+echo 'Cloning Subword NMT repository (for BPE pre-processing)...'
+git clone https://github.com/rsennrich/subword-nmt.git
+
+SCRIPTS=mosesdecoder/scripts
+TOKENIZER=$SCRIPTS/tokenizer/tokenizer.perl
+CLEAN=$SCRIPTS/training/clean-corpus-n.perl
+NORM_PUNC=$SCRIPTS/tokenizer/normalize-punctuation.perl
+REM_NON_PRINT_CHAR=$SCRIPTS/tokenizer/remove-non-printing-char.perl
+BPEROOT=subword-nmt/subword_nmt
+BPE_TOKENS=32000
+
+URLS=(
+    "http://statmt.org/wmt13/training-parallel-europarl-v7.tgz"
+    "http://statmt.org/wmt13/training-parallel-commoncrawl.tgz"
+    "http://data.statmt.org/wmt18/translation-task/training-parallel-nc-v13.tgz"
+    "http://data.statmt.org/wmt18/translation-task/rapid2016.tgz"
+    "http://data.statmt.org/wmt17/translation-task/dev.tgz"
+    "http://statmt.org/wmt14/test-full.tgz"
+)
+FILES=(
+    "training-parallel-europarl-v7.tgz"
+    "training-parallel-commoncrawl.tgz"
+    "training-parallel-nc-v13.tgz"
+    "rapid2016.tgz"
+    "dev.tgz"
+    "test-full.tgz"
+)
+CORPORA=(
+    "training/europarl-v7.de-en"
+    "commoncrawl.de-en"
+    "training-parallel-nc-v13/news-commentary-v13.de-en"
+    "rapid2016.de-en"
+)
+
+if [ ! -d "$SCRIPTS" ]; then
+    echo "Please set SCRIPTS variable correctly to point to Moses scripts."
+    exit 1
+fi
+
+OUTDIR=wmt18_en_de
+
+src=en
+tgt=de
+lang=en-de
+prep=$OUTDIR
+tmp=$prep/tmp
+orig=orig
+
+mkdir -p $orig $tmp $prep
+
+cd $orig
+
+for ((i=0;i<${#URLS[@]};++i)); do
+    file=${FILES[i]}
+    if [ -f $file ]; then
+        echo "$file already exists, skipping download"
+    else
+        url=${URLS[i]}
+        wget "$url"
+        if [ -f $file ]; then
+            echo "$url successfully downloaded."
+        else
+            echo "$url not successfully downloaded."
+            exit 1
+        fi
+        if [ ${file: -4} == ".tgz" ]; then
+            tar zxvf $file
+        elif [ ${file: -4} == ".tar" ]; then
+            tar xvf $file
+        fi
+    fi
+done
+cd ..
+
+echo "pre-processing train data..."
+for l in $src $tgt; do
+    rm $tmp/train.tags.$lang.tok.$l
+    for f in "${CORPORA[@]}"; do
+        cat $orig/$f.$l | \
+            perl $NORM_PUNC $l | \
+            perl $REM_NON_PRINT_CHAR | \
+            perl $TOKENIZER -threads 8 -a -l $l >> $tmp/train.tags.$lang.tok.$l
+    done
+done
+
+echo "pre-processing test data..."
+for l in $src $tgt; do
+    if [ "$l" == "$src" ]; then
+        t="src"
+    else
+        t="ref"
+    fi
+    grep '<seg id' $orig/test-full/newstest2014-deen-$t.$l.sgm | \
+        sed -e 's/<seg id="[0-9]*">\s*//g' | \
+        sed -e 's/\s*<\/seg>\s*//g' | \
+        sed -e "s/\’/\'/g" | \
+    perl $TOKENIZER -threads 8 -a -l $l > $tmp/test.$l
+    echo ""
+done
+
+echo "splitting train and valid..."
+for l in $src $tgt; do
+    awk '{if (NR%100 == 0)  print $0; }' $tmp/train.tags.$lang.tok.$l > $tmp/valid.$l
+    awk '{if (NR%100 != 0)  print $0; }' $tmp/train.tags.$lang.tok.$l > $tmp/train.$l
+done
+
+TRAIN=$tmp/train.de-en
+BPE_CODE=$prep/code
+rm -f $TRAIN
+for l in $src $tgt; do
+    cat $tmp/train.$l >> $TRAIN
+done
+
+echo "learn_bpe.py on ${TRAIN}..."
+python $BPEROOT/learn_bpe.py -s $BPE_TOKENS < $TRAIN > $BPE_CODE
+
+for L in $src $tgt; do
+    for f in train.$L valid.$L test.$L; do
+        echo "apply_bpe.py to ${f}..."
+        python $BPEROOT/apply_bpe.py -c $BPE_CODE < $tmp/$f > $tmp/bpe.$f
+    done
+done
+
+perl $CLEAN -ratio 1.5 $tmp/bpe.train $src $tgt $prep/train 1 250
+perl $CLEAN -ratio 1.5 $tmp/bpe.valid $src $tgt $prep/valid 1 250
+
+for L in $src $tgt; do
+    cp $tmp/bpe.test.$L $prep/test.$L
+done

fairseq-0.10.2/examples/byte_level_bpe/README.md

@@ -0,0 +1,88 @@
+# Neural Machine Translation with Byte-Level Subwords
+
+https://arxiv.org/abs/1909.03341
+
+We provide an implementation of byte-level byte-pair encoding (BBPE), taking IWSLT 2017 Fr-En translation as
+example.
+
+## Data
+Get data and generate fairseq binary dataset:
+```bash
+bash ./get_data.sh
+```
+
+## Model Training
+Train Transformer model with Bi-GRU embedding contextualization (implemented in `gru_transformer.py`):
+```bash
+# VOCAB=bytes
+# VOCAB=chars
+VOCAB=bbpe2048
+# VOCAB=bpe2048
+# VOCAB=bbpe4096
+# VOCAB=bpe4096
+# VOCAB=bpe16384
+```
+```bash
+fairseq-train "data/bin_${VOCAB}" --task translation --user-dir examples/byte_level_bpe/gru_transformer \
+    --arch gru_transformer --encoder-layers 2 --decoder-layers 2 --dropout 0.3 --share-all-embeddings \
+    --optimizer adam --adam-betas '(0.9, 0.98)' \
+    --lr 5e-4 --lr-scheduler inverse_sqrt --warmup-updates 4000 \
+    --criterion label_smoothed_cross_entropy --label-smoothing 0.1 \
+    --log-format 'simple' --log-interval 100 --save-dir "checkpoints/${VOCAB}" \
+    --batch-size 100 --max-update 100000 --update-freq 2
+```
+
+## Generation
+`fairseq-generate` requires bytes (BBPE) decoder to convert byte-level representation back to characters:
+```bash
+# BPE=--bpe bytes
+# BPE=--bpe characters
+BPE=--bpe byte_bpe --sentencepiece-model-path data/spm_bbpe2048.model
+# BPE=--bpe sentencepiece --sentencepiece-model data/spm_bpe2048.model
+# BPE=--bpe byte_bpe --sentencepiece-model-path data/spm_bbpe4096.model
+# BPE=--bpe sentencepiece --sentencepiece-model data/spm_bpe4096.model
+# BPE=--bpe sentencepiece --sentencepiece-model data/spm_bpe16384.model
+```
+
+```bash
+fairseq-generate "data/bin_${VOCAB}" --task translation --user-dir examples/byte_level_bpe/gru_transformer \
+    --source-lang fr --gen-subset test --sacrebleu --path "checkpoints/${VOCAB}/checkpoint_last.pt" \
+    --tokenizer moses --moses-target-lang en ${BPE}
+```
+When using `fairseq-interactive`, bytes (BBPE) encoder/decoder is required to tokenize input data and detokenize model predictions:
+```bash
+fairseq-interactive "data/bin_${VOCAB}" --task translation --user-dir examples/byte_level_bpe/gru_transformer \
+    --path "checkpoints/${VOCAB}/checkpoint_last.pt" --input data/test.fr --tokenizer moses --moses-source-lang fr \
+    --moses-target-lang en ${BPE} --buffer-size 1000 --max-tokens 10000
+```
+
+## Results
+| Vocabulary    | Model  | BLEU |
+|:-------------:|:-------------:|:-------------:|
+| Joint BPE 16k ([Kudo, 2018](https://arxiv.org/abs/1804.10959)) | 512d LSTM 2+2 | 33.81 |
+| Joint BPE 16k | Transformer base 2+2 (w/ GRU) | 36.64 (36.72) |
+| Joint BPE 4k | Transformer base 2+2 (w/ GRU) | 35.49 (36.10) |
+| Joint BBPE 4k | Transformer base 2+2 (w/ GRU) | 35.61 (35.82) |
+| Joint BPE 2k | Transformer base 2+2 (w/ GRU) | 34.87 (36.13) |
+| Joint BBPE 2k | Transformer base 2+2 (w/ GRU) | 34.98 (35.43) |
+| Characters | Transformer base 2+2 (w/ GRU) | 31.78 (33.30) |
+| Bytes | Transformer base 2+2 (w/ GRU) | 31.57 (33.62) |
+
+
+## Citation
+```
+@misc{wang2019neural,
+    title={Neural Machine Translation with Byte-Level Subwords},
+    author={Changhan Wang and Kyunghyun Cho and Jiatao Gu},
+    year={2019},
+    eprint={1909.03341},
+    archivePrefix={arXiv},
+    primaryClass={cs.CL}
+}
+```
+
+
+## Contact
+Changhan Wang ([changhan@fb.com](mailto:changhan@fb.com)),
+Kyunghyun Cho ([kyunghyuncho@fb.com](mailto:kyunghyuncho@fb.com)),
+Jiatao Gu ([jgu@fb.com](mailto:jgu@fb.com))

fairseq-0.10.2/examples/byte_level_bpe/get_bitext.py

@@ -0,0 +1,254 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+
+import argparse
+import os
+import os.path as op
+from collections import namedtuple
+from multiprocessing import cpu_count
+from typing import List, Optional
+
+import sentencepiece as sp
+from fairseq.data.encoders.byte_bpe import ByteBPE
+from fairseq.data.encoders.byte_utils import byte_encode
+from fairseq.data.encoders.bytes import Bytes
+from fairseq.data.encoders.characters import Characters
+from fairseq.data.encoders.moses_tokenizer import MosesTokenizer
+from fairseq.data.encoders.sentencepiece_bpe import SentencepieceBPE
+
+
+SPLITS = ["train", "valid", "test"]
+
+
+def _convert_xml(in_path: str, out_path: str):
+    with open(in_path) as f, open(out_path, "w") as f_o:
+        for s in f:
+            ss = s.strip()
+            if not ss.startswith("<seg"):
+                continue
+            ss = ss.replace("</seg>", "").split('">')
+            assert len(ss) == 2
+            f_o.write(ss[1].strip() + "\n")
+
+
+def _convert_train(in_path: str, out_path: str):
+    with open(in_path) as f, open(out_path, "w") as f_o:
+        for s in f:
+            ss = s.strip()
+            if ss.startswith("<"):
+                continue
+            f_o.write(ss.strip() + "\n")
+
+
+def _get_bytes(in_path: str, out_path: str):
+    with open(in_path) as f, open(out_path, "w") as f_o:
+        for s in f:
+            f_o.write(Bytes.encode(s.strip()) + "\n")
+
+
+def _get_chars(in_path: str, out_path: str):
+    with open(in_path) as f, open(out_path, "w") as f_o:
+        for s in f:
+            f_o.write(Characters.encode(s.strip()) + "\n")
+
+
+def pretokenize(in_path: str, out_path: str, src: str, tgt: str):
+    Args = namedtuple(
+        "Args",
+        [
+            "moses_source_lang",
+            "moses_target_lang",
+            "moses_no_dash_splits",
+            "moses_no_escape",
+        ],
+    )
+    args = Args(
+        moses_source_lang=src,
+        moses_target_lang=tgt,
+        moses_no_dash_splits=False,
+        moses_no_escape=False,
+    )
+    pretokenizer = MosesTokenizer(args)
+    with open(in_path) as f, open(out_path, "w") as f_o:
+        for s in f:
+            f_o.write(pretokenizer.encode(s.strip()) + "\n")
+
+
+def _convert_to_bchar(in_path_prefix: str, src: str, tgt: str, out_path: str):
+    with open(out_path, "w") as f_o:
+        for lang in [src, tgt]:
+            with open(f"{in_path_prefix}.{lang}") as f:
+                for s in f:
+                    f_o.write(byte_encode(s.strip()) + "\n")
+
+
+def _get_bpe(in_path: str, model_prefix: str, vocab_size: int):
+    arguments = [
+        f"--input={in_path}",
+        f"--model_prefix={model_prefix}",
+        f"--model_type=bpe",
+        f"--vocab_size={vocab_size}",
+        "--character_coverage=1.0",
+        "--normalization_rule_name=identity",
+        f"--num_threads={cpu_count()}",
+    ]
+    sp.SentencePieceTrainer.Train(" ".join(arguments))
+
+
+def _apply_bbpe(model_path: str, in_path: str, out_path: str):
+    Args = namedtuple("Args", ["sentencepiece_model_path"])
+    args = Args(sentencepiece_model_path=model_path)
+    tokenizer = ByteBPE(args)
+    with open(in_path) as f, open(out_path, "w") as f_o:
+        for s in f:
+            f_o.write(tokenizer.encode(s.strip()) + "\n")
+
+
+def _apply_bpe(model_path: str, in_path: str, out_path: str):
+    Args = namedtuple("Args", ["sentencepiece_model"])
+    args = Args(sentencepiece_model=model_path)
+    tokenizer = SentencepieceBPE(args)
+    with open(in_path) as f, open(out_path, "w") as f_o:
+        for s in f:
+            f_o.write(tokenizer.encode(s.strip()) + "\n")
+
+
+def _concat_files(in_paths: List[str], out_path: str):
+    with open(out_path, "w") as f_o:
+        for p in in_paths:
+            with open(p) as f:
+                for r in f:
+                    f_o.write(r)
+
+
+def preprocess_iwslt17(
+    root: str,
+    src: str,
+    tgt: str,
+    bpe_size: Optional[int],
+    need_chars: bool,
+    bbpe_size: Optional[int],
+    need_bytes: bool,
+):
+    # extract bitext
+    in_root = op.join(root, f"{src}-{tgt}")
+    for lang in [src, tgt]:
+        _convert_train(
+            op.join(in_root, f"train.tags.{src}-{tgt}.{lang}"),
+            op.join(root, f"train.{lang}"),
+        )
+        _convert_xml(
+            op.join(in_root, f"IWSLT17.TED.dev2010.{src}-{tgt}.{lang}.xml"),
+            op.join(root, f"valid.{lang}"),
+        )
+        _convert_xml(
+            op.join(in_root, f"IWSLT17.TED.tst2015.{src}-{tgt}.{lang}.xml"),
+            op.join(root, f"test.{lang}"),
+        )
+    # pre-tokenize
+    for lang in [src, tgt]:
+        for split in SPLITS:
+            pretokenize(
+                op.join(root, f"{split}.{lang}"),
+                op.join(root, f"{split}.moses.{lang}"),
+                src,
+                tgt,
+            )
+    # tokenize with BPE vocabulary
+    if bpe_size is not None:
+        # learn vocabulary
+        concated_train_path = op.join(root, "train.all")
+        _concat_files(
+            [op.join(root, "train.moses.fr"), op.join(root, "train.moses.en")],
+            concated_train_path,
+        )
+        bpe_model_prefix = op.join(root, f"spm_bpe{bpe_size}")
+        _get_bpe(concated_train_path, bpe_model_prefix, bpe_size)
+        os.remove(concated_train_path)
+        # apply
+        for lang in [src, tgt]:
+            for split in SPLITS:
+                _apply_bpe(
+                    bpe_model_prefix + ".model",
+                    op.join(root, f"{split}.moses.{lang}"),
+                    op.join(root, f"{split}.moses.bpe{bpe_size}.{lang}"),
+                )
+    # tokenize with bytes vocabulary
+    if need_bytes:
+        for lang in [src, tgt]:
+            for split in SPLITS:
+                _get_bytes(
+                    op.join(root, f"{split}.moses.{lang}"),
+                    op.join(root, f"{split}.moses.bytes.{lang}"),
+                )
+    # tokenize with characters vocabulary
+    if need_chars:
+        for lang in [src, tgt]:
+            for split in SPLITS:
+                _get_chars(
+                    op.join(root, f"{split}.moses.{lang}"),
+                    op.join(root, f"{split}.moses.chars.{lang}"),
+                )
+    # tokenize with byte-level BPE vocabulary
+    if bbpe_size is not None:
+        # learn vocabulary
+        bchar_path = op.join(root, "train.bchar")
+        _convert_to_bchar(op.join(root, "train.moses"), src, tgt, bchar_path)
+        bbpe_model_prefix = op.join(root, f"spm_bbpe{bbpe_size}")
+        _get_bpe(bchar_path, bbpe_model_prefix, bbpe_size)
+        os.remove(bchar_path)
+        # apply
+        for lang in [src, tgt]:
+            for split in SPLITS:
+                _apply_bbpe(
+                    bbpe_model_prefix + ".model",
+                    op.join(root, f"{split}.moses.{lang}"),
+                    op.join(root, f"{split}.moses.bbpe{bbpe_size}.{lang}"),
+                )
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--root", type=str, default="data")
+    parser.add_argument(
+        "--bpe-vocab",
+        default=None,
+        type=int,
+        help="Generate tokenized bitext with BPE of size K."
+        "Default to None (disabled).",
+    )
+    parser.add_argument(
+        "--bbpe-vocab",
+        default=None,
+        type=int,
+        help="Generate tokenized bitext with BBPE of size K."
+        "Default to None (disabled).",
+    )
+    parser.add_argument(
+        "--byte-vocab",
+        action="store_true",
+        help="Generate tokenized bitext with bytes vocabulary",
+    )
+    parser.add_argument(
+        "--char-vocab",
+        action="store_true",
+        help="Generate tokenized bitext with chars vocabulary",
+    )
+    args = parser.parse_args()
+
+    preprocess_iwslt17(
+        args.root,
+        "fr",
+        "en",
+        args.bpe_vocab,
+        args.char_vocab,
+        args.bbpe_vocab,
+        args.byte_vocab,
+    )
+
+
+if __name__ == "__main__":
+    main()

fairseq-0.10.2/examples/byte_level_bpe/get_data.sh

@@ -0,0 +1,47 @@
+#!/bin/bash
+
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+PY_BIN_ROOT=
+
+# PyPI dependency
+${PY_BIN_ROOT}pip install sentencepiece sacremoses
+
+# Get data
+if [ ! -d "data" ]; then
+  mkdir data
+fi
+
+if [ ! -f "data/fr-en.tgz" ]; then
+  wget https://wit3.fbk.eu/archive/2017-01-trnted/texts/fr/en/fr-en.tgz -P data
+  tar xvf data/fr-en.tgz -C data
+fi
+${PY_BIN_ROOT}python get_bitext.py --bpe-vocab 16384 --byte-vocab --char-vocab
+for VOCAB_SIZE in 2048 4096; do
+  ${PY_BIN_ROOT}python get_bitext.py --bpe-vocab ${VOCAB_SIZE} --bbpe-vocab ${VOCAB_SIZE}
+done
+rm -r data/fr-en data/fr-en.tgz
+
+# Generate binary dataset
+${PY_BIN_ROOT}/fairseq-preprocess --source-lang fr --target-lang en --destdir data/bin_bpe16384 --joined-dictionary \
+  --workers "$(nproc)" --trainpref data/train.moses.bpe16384 --validpref data/valid.moses.bpe16384 \
+  --testpref data/test.moses.bpe16384
+
+${PY_BIN_ROOT}/fairseq-preprocess --source-lang fr --target-lang en --destdir data/bin_bytes --joined-dictionary \
+  --workers "$(nproc)" --trainpref data/train.moses.bytes --validpref data/valid.moses.bytes \
+  --testpref data/test.moses.bytes
+
+${PY_BIN_ROOT}/fairseq-preprocess --source-lang fr --target-lang en --destdir data/bin_chars --joined-dictionary \
+  --workers "$(nproc)" --trainpref data/train.moses.chars --validpref data/valid.moses.chars \
+  --testpref data/test.moses.chars
+
+for VOCAB_SIZE in 2048 4096; do
+  for TYPE in bbpe bpe; do
+    ${PY_BIN_ROOT}/fairseq-preprocess --source-lang fr --target-lang en --destdir "data/bin_${TYPE}${VOCAB_SIZE}" \
+      --joined-dictionary --workers "$(nproc)" --trainpref "data/train.moses.${TYPE}${VOCAB_SIZE}" \
+      --validpref "data/valid.moses.${TYPE}${VOCAB_SIZE}" --testpref "data/test.moses.${TYPE}${VOCAB_SIZE}"
+  done
+done

fairseq-0.10.2/examples/byte_level_bpe/gru_transformer.py

@@ -0,0 +1,107 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+import torch.nn as nn
+import torch.nn.functional as F
+from fairseq.models import register_model, register_model_architecture
+from fairseq.models.transformer import TransformerEncoder, TransformerModel
+
+
+@register_model("gru_transformer")
+class GRUTransformerModel(TransformerModel):
+    @classmethod
+    def build_encoder(cls, args, src_dict, embed_tokens):
+        return GRUTransformerEncoder(args, src_dict, embed_tokens)
+
+
+class GRUTransformerEncoder(TransformerEncoder):
+    def __init__(self, args, dictionary, embed_tokens):
+        super().__init__(args, dictionary, embed_tokens)
+        self.emb_ctx = nn.GRU(
+            input_size=embed_tokens.embedding_dim,
+            hidden_size=embed_tokens.embedding_dim // 2,
+            num_layers=1,
+            bidirectional=True,
+        )
+
+    def forward_embedding(self, src_tokens):
+        # embed tokens and positions
+        x = embed = self.embed_scale * self.embed_tokens(src_tokens)
+        if self.embed_positions is not None:
+            x = embed + self.embed_positions(src_tokens)
+
+        # contextualize embeddings
+        x = x.transpose(0, 1)
+        x = self.dropout_module(x)
+        x, _ = self.emb_ctx.forward(x)
+        x = x.transpose(0, 1)
+
+        if self.layernorm_embedding is not None:
+            x = self.layernorm_embedding(x)
+        x = self.dropout_module(x)
+        return x, embed
+
+
+@register_model_architecture("gru_transformer", "gru_transformer")
+def gru_transformer_base_architecture(args):
+    args.encoder_embed_path = getattr(args, "encoder_embed_path", None)
+    args.encoder_embed_dim = getattr(args, "encoder_embed_dim", 512)
+    args.encoder_ffn_embed_dim = getattr(args, "encoder_ffn_embed_dim", 2048)
+    args.encoder_layers = getattr(args, "encoder_layers", 6)
+    args.encoder_attention_heads = getattr(args, "encoder_attention_heads", 8)
+    args.encoder_normalize_before = getattr(args, "encoder_normalize_before", False)
+    args.encoder_learned_pos = getattr(args, "encoder_learned_pos", False)
+    args.decoder_embed_path = getattr(args, "decoder_embed_path", None)
+    args.decoder_embed_dim = getattr(args, "decoder_embed_dim", args.encoder_embed_dim)
+    args.decoder_ffn_embed_dim = getattr(
+        args, "decoder_ffn_embed_dim", args.encoder_ffn_embed_dim
+    )
+    args.decoder_layers = getattr(args, "decoder_layers", 6)
+    args.decoder_attention_heads = getattr(args, "decoder_attention_heads", 8)
+    args.decoder_normalize_before = getattr(args, "decoder_normalize_before", False)
+    args.decoder_learned_pos = getattr(args, "decoder_learned_pos", False)
+    args.attention_dropout = getattr(args, "attention_dropout", 0.0)
+    args.activation_dropout = getattr(args, "activation_dropout", 0.0)
+    args.activation_fn = getattr(args, "activation_fn", "relu")
+    args.dropout = getattr(args, "dropout", 0.1)
+    args.adaptive_softmax_cutoff = getattr(args, "adaptive_softmax_cutoff", None)
+    args.adaptive_softmax_dropout = getattr(args, "adaptive_softmax_dropout", 0)
+    args.share_decoder_input_output_embed = getattr(
+        args, "share_decoder_input_output_embed", False
+    )
+    args.share_all_embeddings = getattr(args, "share_all_embeddings", False)
+    args.no_token_positional_embeddings = getattr(
+        args, "no_token_positional_embeddings", False
+    )
+    args.adaptive_input = getattr(args, "adaptive_input", False)
+    args.no_cross_attention = getattr(args, "no_cross_attention", False)
+    args.cross_self_attention = getattr(args, "cross_self_attention", False)
+    args.layer_wise_attention = getattr(args, "layer_wise_attention", False)
+
+    args.decoder_output_dim = getattr(
+        args, "decoder_output_dim", args.decoder_embed_dim
+    )
+    args.decoder_input_dim = getattr(args, "decoder_input_dim", args.decoder_embed_dim)
+
+    args.no_scale_embedding = getattr(args, "no_scale_embedding", False)
+    args.layernorm_embedding = getattr(args, "layernorm_embedding", False)
+
+
+@register_model_architecture("gru_transformer", "gru_transformer_big")
+def gru_transformer_big(args):
+    args.encoder_embed_dim = getattr(args, "encoder_embed_dim", 1024)
+    args.encoder_ffn_embed_dim = getattr(args, "encoder_ffn_embed_dim", 4096)
+    args.encoder_attention_heads = getattr(args, "encoder_attention_heads", 16)
+    args.encoder_normalize_before = getattr(args, "encoder_normalize_before", False)
+    args.decoder_embed_dim = getattr(args, "decoder_embed_dim", 1024)
+    args.decoder_ffn_embed_dim = getattr(args, "decoder_ffn_embed_dim", 4096)
+    args.decoder_attention_heads = getattr(args, "decoder_attention_heads", 16)
+    args.dropout = getattr(args, "dropout", 0.3)
+    gru_transformer_base_architecture(args)

fairseq-0.10.2/examples/conv_seq2seq/README.md

@@ -0,0 +1,25 @@
+# Convolutional Sequence to Sequence Learning (Gehring et al., 2017)
+
+## Pre-trained models
+
+Description | Dataset | Model | Test set(s)
+---|---|---|---
+Convolutional <br> ([Gehring et al., 2017](https://arxiv.org/abs/1705.03122)) | [WMT14 English-French](http://statmt.org/wmt14/translation-task.html#Download) | [download (.tar.bz2)](https://dl.fbaipublicfiles.com/fairseq/models/wmt14.v2.en-fr.fconv-py.tar.bz2) | newstest2014: <br> [download (.tar.bz2)](https://dl.fbaipublicfiles.com/fairseq/data/wmt14.v2.en-fr.newstest2014.tar.bz2) <br> newstest2012/2013: <br> [download (.tar.bz2)](https://dl.fbaipublicfiles.com/fairseq/data/wmt14.v2.en-fr.ntst1213.tar.bz2)
+Convolutional <br> ([Gehring et al., 2017](https://arxiv.org/abs/1705.03122)) | [WMT14 English-German](http://statmt.org/wmt14/translation-task.html#Download) | [download (.tar.bz2)](https://dl.fbaipublicfiles.com/fairseq/models/wmt14.en-de.fconv-py.tar.bz2) | newstest2014: <br> [download (.tar.bz2)](https://dl.fbaipublicfiles.com/fairseq/data/wmt14.en-de.newstest2014.tar.bz2)
+Convolutional <br> ([Gehring et al., 2017](https://arxiv.org/abs/1705.03122)) | [WMT17 English-German](http://statmt.org/wmt17/translation-task.html#Download) | [download (.tar.bz2)](https://dl.fbaipublicfiles.com/fairseq/models/wmt17.v2.en-de.fconv-py.tar.bz2) | newstest2014: <br> [download (.tar.bz2)](https://dl.fbaipublicfiles.com/fairseq/data/wmt17.v2.en-de.newstest2014.tar.bz2)
+
+## Example usage
+
+See the [translation README](../translation/README.md) for instructions on reproducing results for WMT'14 En-De and
+WMT'14 En-Fr using the `fconv_wmt_en_de` and `fconv_wmt_en_fr` model architectures.
+
+## Citation
+
+```bibtex
+@inproceedings{gehring2017convs2s,
+  title = {Convolutional Sequence to Sequence Learning},
+  author = {Gehring, Jonas, and Auli, Michael and Grangier, David and Yarats, Denis and Dauphin, Yann N},
+  booktitle = {Proc. of ICML},
+  year = 2017,
+}
+```

fairseq-0.10.2/examples/layerdrop/README.md

@@ -0,0 +1,154 @@
+# Reducing Transformer Depth on Demand with Structured Dropout (Fan et al., 2019)
+This page contains information for how to train models with LayerDrop, based on this [paper](https://arxiv.org/abs/1909.11556).
+
+## Citation:
+If you found this technique useful, please cite our paper:
+```bibtex
+@article{fan2019reducing,
+  title={Reducing Transformer Depth on Demand with Structured Dropout},
+  author={Fan, Angela and Grave, Edouard and Joulin, Armand},
+  journal={arXiv preprint arXiv:1909.11556},
+  year={2019}
+}
+```
+
+## Pre-trained models
+
+Model | Description | Download
+---|---|---
+`layerdrop_wmt_en_de_12_6` | Transformer + LayerDrop 0.2 trained on WMT16 en-de with 12 encoder and 6 decoder layers | [layerdrop_wmt_en_de_12_6.tar.gz](https://dl.fbaipublicfiles.com/fairseq/models/layerdrop_wmt_en_de_12_6.tar.gz)
+`roberta_layerdrop.base` | RoBERTa Base + LayerDrop 0.2 | [roberta_layerdrop.base.tar.gz](https://dl.fbaipublicfiles.com/fairseq/models/roberta_layerdrop.base.qnli.tar.gz)
+`roberta_layerdrop.large` | RoBERTa Large + LayerDrop 0.2 | [roberta_layerdrop.large.tar.gz](https://dl.fbaipublicfiles.com/fairseq/models/roberta_layerdrop.large.tar.gz)
+`roberta_layerdrop.large.mnli` | `roberta_layerdrop.large` finetuned on [MNLI](http://www.nyu.edu/projects/bowman/multinli) | [roberta_layerdrop.large.mnli.tar.gz](https://dl.fbaipublicfiles.com/fairseq/models/roberta_layerdrop.large.mnli.tar.gz)
+`roberta_layerdrop.large.qnli` | `roberta_layerdrop.large` finetuned on [QNLI](https://arxiv.org/abs/1804.07461) | [roberta_layerdrop.large.mnli.tar.gz](https://dl.fbaipublicfiles.com/fairseq/models/roberta_layerdrop.large.qnli.tar.gz)
+
+
+Evaluate performance of these pre-trained models:
+```bash
+# Example for Machine Translation
+fairseq-generate /path/to/bped/wmt/data --path nmt_checkpoint.pt \
+  --beam 8 --lenpen 0.4 \
+  --batch-size 64 \
+  --remove-bpe \
+  --gen-subset test > wmt16_gen.txt
+bash scripts/compound_split_bleu.sh wmt16_gen.txt
+# prints BLEU4 = 30.17
+```
+
+```python
+# Example for RoBERTa + LayerDrop finetuned on MNLI:
+from fairseq.models.roberta import RobertaModel
+
+roberta_layerdrop = RobertaModel.from_pretrained(
+    '/path/to/MNLI/model',
+    checkpoint_file='mnli_checkpoint.pt',
+    data_name_or_path='/path/to/MNLI/data/MNLI-bin'
+)
+label_map = {0: 'contradiction', 2: 'neutral', 1: 'entailment'}
+ncorrect, nsamples = 0, 0
+roberta_layerdrop.cuda()
+roberta_layerdrop.eval()
+with open('/path/to/MNLI/data/dev_matched.tsv') as fin:
+    fin.readline()
+    for index, line in enumerate(fin):
+        tokens = line.strip().split('\t')
+        sent1, sent2, target = tokens[8], tokens[9], tokens[-1]
+        tokens = roberta_layerdrop.encode(sent1, sent2)
+        prediction = roberta_layerdrop.predict('sentence_classification_head', tokens).argmax().item()
+        prediction_label = label_map[prediction]
+        ncorrect += int(prediction_label == target)
+        nsamples += 1
+print('| Accuracy: ', float(ncorrect)/float(nsamples))
+# prints | Accuracy:  0.9026999490575649
+
+
+# Example for RoBERTa + LayerDrop finetuned on QNLI:
+roberta = RobertaModel.from_pretrained(
+    '/path/to/QNLI/model',
+    checkpoint_file='qnli_checkpoint.pt',
+    data_name_or_path='/path/to/QNLI/data/QNLI-bin'
+)
+
+label_fn = lambda label: roberta.task.label_dictionary.string(
+    [label + roberta.task.target_dictionary.nspecial]
+)
+ncorrect, nsamples = 0, 0
+roberta.cuda()
+roberta.eval()
+with open('/path/to/QNLI/data/dev.tsv') as fin:
+    fin.readline()
+    for index, line in enumerate(fin):
+        tokens = line.strip().split('\t')
+        sent1, sent2, target = tokens[1], tokens[2], tokens[3]
+        tokens = roberta.encode(sent1, sent2)
+        prediction = roberta.predict('sentence_classification_head', tokens).argmax().item()
+        prediction_label = label_fn(prediction)
+        ncorrect += int(prediction_label == target)
+        nsamples += 1
+print('| Accuracy: ', float(ncorrect)/float(nsamples))
+# prints | Accuracy:  0.9480139117700896
+```
+
+
+## Example usage
+
+To train a model with LayerDrop, add the following flags. We recommend 0.2, a value that worked well in our experiments. For Language Models that are decoder-only, you need only the decoder flag. For RoBERTa, an encoder, you need only the encoder flag. The encoder and decoder LayerDrop values can be set differently.
+```
+--encoder-layerdrop 0.2 --decoder-layerdrop 0.2
+```
+
+To prune a model that has been trained with LayerDrop, add the following flags followed by a comma separated list of which layers you would like to keep.
+```
+--encoder-layers-to-keep 0,2,4,6,8,10,12,14 --decoder-layers-to-keep 0,2,4,6,8,10,12,14
+```
+Setting these flags should print a message such as:
+```
+| Pruning model to specified layer configuration
+```
+You should also see a smaller number of parameters in the model, for example the 16-Layer Transformer Language Model prints:
+```
+num. model params: 246933504
+```
+while a model pruned to 8 Layers prints:
+```
+num. model params: 146163712
+```
+
+If you would like to pick up training with a model that has been pruned, simply adding these flags is sufficient. If you would like to use a script that only does evaluation (no training), you may need to pass an override command. A specific example would be for language modeling:
+```bash
+fairseq-eval-lm /path/to/wikitext-103 \
+  --path /path/to/model/checkpoint.pt \
+  --model-overrides "{'decoder_layers_to_keep':'0,2,4,6,8,10,12,14'}"
+```
+This model override command overrides the training parameters and updates the model arguments so that the pruned model is run instead of the full model.
+
+## Reproduce Paper Results
+
+Looking to reproduce the results in the paper?
+
+1. For Translation on WMT16 en-de, we followed this setting [here](https://github.com/pytorch/fairseq/blob/master/examples/scaling_nmt/README.md)
+2. To train RoBERTa, we followed this setting [here](https://github.com/pytorch/fairseq/tree/master/examples/roberta)
+3. To train Language Models on Wikitext-103, we followed this setting [here](https://github.com/pytorch/fairseq/tree/master/examples/language_model)
+
+
+## Tips
+
+1. If you would like to train large models with better performance, LayerDrop should be set to a smaller value such as 0.1 or 0.2. Too much LayerDrop will mean the model has too much regularization, so may not reach the best performance. Since LayerDrop adds regularization, you may achieve the best performance by slightly reducing the amount of standard dropout (for example, reduce by 0.1).
+
+2. If you would like to train large models to be pruned and made smaller, LayerDrop should be set to a larger value such as 0.5 if you want to prune very aggressively (such as removing half the network or more). If you would like to prune fewer layers away, LayerDrop can be set to a smaller value such as 0.2. Our experiments were conducted with low values of LayerDrop (such as 0.1 and 0.2), for reference.
+
+3. When pruning layers at inference time, it is best to spread out the layers remaining so they are evenly spaced throughout the network. For example, if you want to remove 50% of the network, keeping every other layer is good.
+
+
+## FAQ
+
+1. How did the sharing layers experiment work? In an appendix (https://openreview.net/pdf?id=SylO2yStDr) we added an experiment on Wikitext-103 language modeling that combined LayerDrop with Weight Sharing. We shared chunks of 2 layers such that every other layer had shared weights. For example, if our network has layers 1 through 6, then layer 1 and 2 are shared, layer 3 and 4 are shared, and layer 5 and 6 are shared.
+
+2. LayerDrop hasn't been helping in my setting? During training time, LayerDrop can help regularize your network. This is most important if your network is already overfitting - if your network is underfitting, it is possible LayerDrop is adding too much regularization. We recommend using smaller values (such as 0.1 or 0.2) and also decreasing the quantity of standard dropout (for example, reduce by 0.1).
+
+3. Can you train a model without LayerDrop and finetune with LayerDrop (e.g. for BERT)? In our experiments, we did not see great performance. Models such as RoBERTa have trained for a long time in the pre-training setting, so only finetuning with LayerDrop for a few epochs on a downstream task such as MNLI does not achieve the robustness required for successful pruning.
+
+
+## Having an issue or have a question?
+
+Please open an issue in this repository with the details of your question. Thanks!

fairseq-0.10.2/examples/noisychannel/rerank_score_lm.py

@@ -0,0 +1,81 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+import os
+
+from fairseq import options
+
+from . import rerank_options, rerank_utils
+
+
+def score_lm(args):
+    using_nbest = args.nbest_list is not None
+    (
+        pre_gen,
+        left_to_right_preprocessed_dir,
+        right_to_left_preprocessed_dir,
+        backwards_preprocessed_dir,
+        lm_preprocessed_dir,
+    ) = rerank_utils.get_directories(
+        args.data_dir_name,
+        args.num_rescore,
+        args.gen_subset,
+        args.gen_model_name,
+        args.shard_id,
+        args.num_shards,
+        args.sampling,
+        args.prefix_len,
+        args.target_prefix_frac,
+        args.source_prefix_frac,
+    )
+
+    predictions_bpe_file = pre_gen + "/generate_output_bpe.txt"
+    if using_nbest:
+        print("Using predefined n-best list from interactive.py")
+        predictions_bpe_file = args.nbest_list
+
+    gen_output = rerank_utils.BitextOutputFromGen(
+        predictions_bpe_file, bpe_symbol=args.remove_bpe, nbest=using_nbest
+    )
+
+    if args.language_model is not None:
+        lm_score_file = rerank_utils.rescore_file_name(
+            pre_gen, args.prefix_len, args.lm_name, lm_file=True
+        )
+
+    if args.language_model is not None and not os.path.isfile(lm_score_file):
+        print("STEP 4.5: language modeling for P(T)")
+        if args.lm_bpe_code is None:
+            bpe_status = "no bpe"
+        elif args.lm_bpe_code == "shared":
+            bpe_status = "shared"
+        else:
+            bpe_status = "different"
+
+        rerank_utils.lm_scoring(
+            lm_preprocessed_dir,
+            bpe_status,
+            gen_output,
+            pre_gen,
+            args.lm_dict,
+            args.lm_name,
+            args.language_model,
+            args.lm_bpe_code,
+            128,
+            lm_score_file,
+            args.target_lang,
+            args.source_lang,
+            prefix_len=args.prefix_len,
+        )
+
+
+def cli_main():
+    parser = rerank_options.get_reranking_parser()
+    args = options.parse_args_and_arch(parser)
+    score_lm(args)
+
+
+if __name__ == "__main__":
+    cli_main()

fairseq-0.10.2/examples/quant_noise/README.md

@@ -0,0 +1,298 @@
+# Training with Quantization Noise for Extreme Model Compression ({Fan\*, Stock\*} *et al.*, 2020)
+This page contains information for how to train and quantize models with Quantization Noise, for both scalar quantization like `int8` and Iterative Product Quantization.
+Check out our paper [here](https://arxiv.org/abs/2004.07320).
+
+Looking for pretrained models? They will be added shortly.
+Looking for code to train vision models? We are working on open sourcing our code as part of ClassyVision. Please check back, but note that both the Scalar and Iterative Product Quantization counterparts of the `nn.Conv2d` module are already included in this release.
+
+**Contents**:
+- [Walk through of code](#walk-through-the-code)
+- [Reproduce NLP Results](#looking-to-reproduce-the-nlp-results-in-the-paper)
+- [Reproduce Vision Results](#looking-to-reproduce-the-vision-results-in-the-paper)
+
+
+## Citation
+```bibtex
+@article{fan2020training,
+    title={Training with Quantization Noise for Extreme Model Compression},
+    author={Angela Fan* and Pierre Stock* and and Benjamin Graham and Edouard Grave and Remi Gribonval and Herve Jegou and Armand Joulin},
+    year={2020},
+    eprint={2004.07320},
+    archivePrefix={arXiv},
+    primaryClass={cs.ML}
+}
+```
+
+## Walk through the code
+
+Training a model with Quant-Noise improves the performance in subsequent inference-time quantization by training models to be robust to quantization. This technique is useful for both scalar and product quantization methods, as well as multiple domains. We detail below our approach to train, quantize models and integrate our code to quantize your favorite models.
+
+### Scalar Quantization
+
+Unlike the section [Iterative Product Quantization](#iterative-product-quantization) which gives state-of-the-art compression, this section showcases the usefulness of our approach for simple scalar quantization baselines such as int8 using on-GPU Fake Quantization.
+
+#### Training
+
+Scalar quantization with Quant-Noise consists in randomly quantizing a proportion `p` of the weights during training. Scalar quantization is implemented [here](https://github.com/pytorch/fairseq/tree/master/fairseq/modules/quantization/scalar) under the form of Fake Quantization, meaning that we emulate int8 on GPU by quantizing and de-quantizing both the weights and the activations. We rely on PyTorch's [quantization primitives](https://github.com/pytorch/pytorch/tree/master/torch/quantization).
+
+To train a model with Quant-Noise, add the following flag:
+```
+--quant-noise-scalar 0.5
+```
+Large values of noise make the network easier to quantize but may result in higher non-quantized test and validation perplexities.
+
+#### Quantization
+
+When evaluating a network, all quantized modules and activation hooks automatically switch to `p=1` so the validation accuracy reported by Fairseq is actually the quantized one, nothing more to do.
+
+
+#### Integration with your own code
+
+Looking to quantize your own models with Quant-Noise + Scalar Quantization?
+- Use the function `quantize_model_` implemented [here](https://github.com/pytorch/fairseq/tree/master/fairseq/modules/quantization/scalar/utils.py) to (1) replace all your modules by their quantized counterparts and (2) add hooks to those modules to quantize the activations.
+- Then, perform your training as usual. Note that in `eval()` mode, the network is always fully quantized (weights and activations) by default (`p=1`).
+
+
+
+### Iterative Product Quantization
+
+
+Iterative Product Quantization with Quant-Noise proceeds in two steps. First, a model must be trained uncompressed with Quant-Noise. Second, the model must be quantized with iPQ. Note that we implement here the simplest form of noise, which consists in randomly dropping a proportion `p` of blocks, and that worked as well as assigning those blocks to their current centroid.
+
+#### Training
+
+To train a model with Quant-Noise, add the following flags:
+```
+--quant-noise-pq 0.1 --quant-noise-pq-block-size 8
+```
+`quant-noise-pq` controls how much dropout is applied to the blocks of the weight matrix. `quant-noise-pq-block-size` controls the size of the weight matrix blocks.
+We recommend training with 0.05 to 0.2 Quant-Noise, a value that worked well in our experiments. For the block-size, we recommend training with block-size of 8. Note that the block size must be a multiple of `input_features`, see the size checks [here](https://github.com/pytorch/fairseq/tree/master/fairseq/modules/quant_noise.py). Large block sizes result in higher compression ratio but may induce a loss in accuracy.
+
+We currently support training Transformer based models, such as sequence-to-sequence, language models, and BERT architectures. The `quant_noise` function [here](https://github.com/pytorch/fairseq/tree/master/fairseq/modules/quant_noise.py) wraps a module. It splits a weight matrix into blocks and applies random dropout to these blocks.
+In the Transformer architectures, quant-noise is applied to the input and output embeddings, the attention, and the FFN.
+
+Quant-Noise can also be combined with **LayerDrop** (see [here](https://github.com/pytorch/fairseq/tree/master/examples/layerdrop)) to add its pruning effect to the quantized model and make the model even smaller. We recommend training with LayerDrop 0.1 or 0.2.
+
+#### Quantization
+
+We implement an improved version of product quantization from Stock et al, **iPQ**, described [here](https://arxiv.org/abs/1907.05686), see code with old API [here](https://github.com/facebookresearch/kill-the-bits). Note that we improved the iPQ API in terms of both compute speed and usability as described below.
+
+For the particular case of PQ, quantization is made sequentially. We recommend first quantizing the FFNs, then the EMBs, and finally the ATTNs. Quantization is done in two sub-steps:
+- First, perform `n` steps of Product Quantization (generally `n=20` is enough).
+- Then, finetune the obtained centroids.
+
+#### Integration with your own code
+
+Looking to quantize your own models with Quant-Noise + iPQ?
+- First wrap your modules with the `quant_noise` function [here](https://github.com/pytorch/fairseq/tree/master/fairseq/modules/quant_noise.py), which is module-agnostic and train your favorite model.
+- Then, quantize your trained model using the code [here](https://github.com/pytorch/fairseq/tree/master/fairseq/modules/quantization/pq). This can be done *without any changes to your training loop*. Below is an example code for integration.
+Note that we tried our approach only on Transformers and various Convolutional Models such as EfficientNets.
+
+```python
+from fairseq.modules.quantization.pq import quantize_model_, SizeTracker
+
+# get configuration parameters
+n_centroids_config = config["n_centroids"]
+block_sizes_config = config["block_sizes"]
+layers_to_quantize = config["layers_to_quantize"]
+
+# size tracker for keeping track of assignments, centroids and non-compressed sizes
+size_tracker = SizeTracker(model)
+
+# Quantize model by stages
+for step in range(len(layers_to_quantize)):
+
+    # quantize model in-place
+    quantized_layers = quantize_model_(
+        model,
+        size_tracker,
+        layers_to_quantize,
+        block_sizes_config,
+        n_centroids_config,
+        step=step,
+    )
+    logger.info(f"Finetuning stage {step}, quantized layers: {quantized_layers}")
+    logger.info(f"{size_tracker}")
+
+    # Don't forget to re-create/update trainer/optimizer since model parameters have changed
+    optimizer = ...
+
+    # Finetune the centroids with your usual training loop for a few epochs
+    trainer.train_epoch()
+```
+
+
+## Looking to reproduce the NLP results in the paper?
+
+We detail below how to reproduce the state-of-the-art results in reported in the paper for Quant-Noise + Iterative Product Quantization.
+
+### Training with Quant-Noise
+
+To **train** RoBERTa + QuantNoise, we followed this setting [here](https://github.com/pytorch/fairseq/tree/master/examples/roberta).
+The following command can be used to train a RoBERTa Base + QuantNoise model:
+
+```bash
+TOTAL_UPDATES=125000
+WARMUP_UPDATES=10000
+PEAK_LR=0.0005
+TOKENS_PER_SAMPLE=512
+MAX_POSITIONS=512
+MAX_SENTENCES=16
+UPDATE_FREQ=2
+DATA_DIR=/path/to/data/here
+
+fairseq-train $DATA_DIR \
+    --task masked_lm --criterion masked_lm --arch roberta_base \
+    --sample-break-mode complete \
+    --tokens-per-sample $TOKENS_PER_SAMPLE --max-positions $MAX_POSITIONS \
+    --optimizer adam --adam-betas '(0.9, 0.98)' --adam-eps 1e-6 \
+    --clip-norm 0.0 \
+    --lr-scheduler polynomial_decay --lr $PEAK_LR \
+    --warmup-updates $WARMUP_UPDATES --total-num-update $TOTAL_UPDATES \
+    --dropout 0.1 --attention-dropout 0.1 \
+    --weight-decay 0.01 \
+    --batch-size $MAX_SENTENCES \
+    --update-freq $UPDATE_FREQ --max-update $TOTAL_UPDATES \
+    --save-dir checkpoint/roberta \
+    --ddp-backend no_c10d --encoder-layerdrop 0.2 \
+    --quant-noise-pq 0.2 --quant-noise-pq-block-size 8 --untie-weights-roberta
+```
+
+To **finetune** RoBERTa + QuantNoise, we followed this setting [here](https://github.com/pytorch/fairseq/blob/master/examples/roberta/README.glue.md).
+The following command can be used to finetune a RoBERTa Base + QuantNoise model on the RTE dataset:
+
+```bash
+TOTAL_NUM_UPDATES=2036
+WARMUP_UPDATES=122
+LR=2e-05
+NUM_CLASSES=2
+MAX_SENTENCES=16
+ROBERTA_PATH=/path/to/roberta_quantnoise/model.pt
+
+fairseq-train /path/to/rte/data/ \
+    --restore-file $ROBERTA_PATH \
+    --max-positions 512 \
+    --batch-size $MAX_SENTENCES \
+    --max-tokens 4400 \
+    --task sentence_prediction \
+    --reset-optimizer --reset-dataloader --reset-meters \
+    --required-batch-size-multiple 1 \
+    --init-token 0 --separator-token 2 \
+    --arch roberta_large \
+    --criterion sentence_prediction \
+    --num-classes $NUM_CLASSES \
+    --dropout 0.1 --attention-dropout 0.1 \
+    --weight-decay 0.1 --optimizer adam --adam-betas "(0.9, 0.98)" --adam-eps 1e-06 \
+    --clip-norm 0.0 \
+    --lr-scheduler polynomial_decay --lr $LR --total-num-update $TOTAL_NUM_UPDATES --warmup-updates $WARMUP_UPDATES \
+    --fp16 --fp16-init-scale 4 --threshold-loss-scale 1 --fp16-scale-window 128 \
+    --max-epoch 10 \
+    --find-unused-parameters \
+    --best-checkpoint-metric accuracy --maximize-best-checkpoint-metric \
+    --ddp-backend no_c10d \
+    --quant-noise-pq 0.2 --quant-noise-pq-block-size 8
+```
+
+To **train** Language Models on Wikitext-103, we followed this setting [here](https://github.com/pytorch/fairseq/tree/master/examples/language_model).
+The following command can be used to train a Transformer + QuantNoise model on Wikitext-103:
+
+```bash
+fairseq-train --task language_modeling /path/to/wikitext-103/data \
+    --save-dir checkpoints/transformer_wikitext-103 \
+    --adaptive-input --adaptive-input-cutoff 20000,60000 --adaptive-input-factor 4 \
+    --adaptive-softmax-cutoff 20000,60000 --adaptive-softmax-dropout 0.2 --adaptive-softmax-factor 4.0 \
+    --tie-adaptive-proj --tie-adaptive-weights \
+    --arch transformer_lm_gbw \
+    --attention-dropout 0.1 --dropout 0.2 --relu-dropout 0.1 \
+    --clip-norm 0.1 --criterion adaptive_loss \
+    --ddp-backend no_c10d \
+    --decoder-attention-heads 8 --decoder-embed-dim 1024 --decoder-ffn-embed-dim 4096 --decoder-input-dim 1024 \
+    --decoder-layers 16 --decoder-normalize-before --decoder-output-dim 1024 \
+    --lr 0.0001 --lr-period-updates 270000 --lr-scheduler cosine --lr-shrink 0.75 --max-lr 1.0 --t-mult 2.0 \
+    --max-tokens 3072 --tokens-per-sample 3072 --momentum 0.99 --optimizer nag \
+    --sample-break-mode none --update-freq 3 \
+    --warmup-init-lr 1e-07 --warmup-updates 16000 \
+    --weight-decay 0 --seed 1 --min-lr 1e-09 \
+    --quant-noise-pq 0.05 --quant-noise-pq-block-size 8
+```
+
+To **evaluate** this model, note you need to use the `eval.py` script. The following command can be used to evaluate:
+
+```bash
+fairseq-eval-lm /path/to/wikitext-103/data --path /path/to/model/checkpoint \
+    --sample-break-mode complete \
+    --max-tokens 3072 \
+    --context-window 2560 \
+    --softmax-batch 1024 \
+    --gen-subset valid
+```
+and change the `--gen-subset` to `test` if you would like to evaluate on the test set instead.
+
+
+### Iterative Product Quantization
+
+To quantize the finetuned RoBERTa model, we use this command on 1 GPU. This should run in a day.
+```bash
+TOTAL_NUM_UPDATES=6108  # 2036 updates for each iteration
+WARMUP_UPDATES=122
+LR=2e-05
+NUM_CLASSES=2
+MAX_SENTENCES=16
+fairseq-train --task sentence_prediction /path/to/data/ \
+    --restore-file $ROBERTA_PATH \
+    --save-dir checkpoints/roberta_finetuned \
+    --max-positions 512 \
+    --batch-size $MAX_SENTENCES \
+    --max-tokens 4400 \
+    --init-token 0 --separator-token 2 \
+    --arch roberta_large \
+    --criterion sentence_prediction \
+    --num-classes $NUM_CLASSES \
+    --dropout 0.1 --attention-dropout 0.1 \
+    --weight-decay 0.1 --optimizer adam --adam-betas "(0.9, 0.98)" --adam-eps 1e-06 \
+    --clip-norm 0.0 --lr-scheduler polynomial_decay \
+    --fp16 --fp16-init-scale 4 --threshold-loss-scale 1 --fp16-scale-window 128 \
+    --no-progress-bar --skip-invalid-size-inputs-valid-test --ddp-backend no_c10d \
+    --quantization-config-path /path/to/config/yaml
+```
+
+To quantize the trained Language Model, we use this command on 8 V100 23GB GPUs. This should run in a couple of hours.
+```bash
+fairseq-train --task language_modeling /path/to/wikitext-103/data \
+    --save-dir checkpoints/transformer_wikitext-103 \
+    --adaptive-input --adaptive-input-cutoff 20000,60000 --adaptive-input-factor 4 \
+    --adaptive-softmax-cutoff 20000,60000 --adaptive-softmax-dropout 0.2 --adaptive-softmax-factor 4.0 \
+    --arch transformer_lm_gbw \
+    --attention-dropout 0.1 --dropout 0.2 --relu-dropout 0.1  \
+    --bucket-cap-mb 25 --char-embedder-highway-layers 2 --character-embedding-dim 4 \
+    --clip-norm 0.1 --criterion adaptive_loss \
+    --ddp-backend no_c10d \
+    --decoder-attention-heads 8 --decoder-embed-dim 1024 --decoder-ffn-embed-dim 4096 --decoder-input-dim 1024 --decoder-layers 16 --decoder-normalize-before --decoder-output-dim 1024 \
+    --fp16 --keep-last-epochs -1 \
+    --lr 0.0001 --lr-period-updates 270000 --lr-scheduler cosine --lr-shrink 0.75 --max-lr 0.05 --min-lr 1e-09 \
+    --max-tokens 2944  --tokens-per-sample 2944\
+    --momentum 0.99 --no-epoch-checkpoints --no-progress-bar --optimizer nag --required-batch-size-multiple 8 \
+    --sample-break-mode none --t-mult 2.0 --skip-invalid-size-inputs-valid-test \
+    --tie-adaptive-proj --tie-adaptive-weights --update-freq 3 --weight-decay 0 --seed 1  \
+    --log-interval 100 --no-progress-bar --skip-invalid-size-inputs-valid-test \
+    --restore-file path/to/trained/lm/with/quant/noise \
+    --max-update 13500 --quantization-config-path /path/to/config/yaml
+```
+If you have less capacity or if your distributed training freezes, try reducing  `--max-tokens` and  `--tokens-per-sample` (this may reduce the quantized accuracy a bit).
+
+### Remarks
+
+We try to keep the open-sourced code as readable and as easy-to-plug as possible. Therefore, we did not test it for the following cases:
+- Scalar quantization with RoBERTa.
+- Quantization with iPQ and `int8` combined.
+
+If you have trouble adapting it, we will be more than happy to help!
+
+## Looking to reproduce the Vision results in the paper?
+
+We are working on open sourcing our code as part of ClassyVision. Please check back.
+
+
+## Having an issue or have a question?
+
+Please open an issue in this repository with the details of your question. Thanks!

fairseq-0.10.2/examples/quant_noise/transformer_quantization_config.yaml

@@ -0,0 +1,33 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+# This file defines example configuration arguments for quantizing
+# a transformer model with product quantization
+
+# Number of Centroids for Product Quantization, by default 256 (byte-aligned)
+n_centroids:
+    Linear:
+        key: in_features
+        value: {"*": 256}
+    Embedding:
+        key: embedding_dim
+        value: {"*": 256}
+
+# Block Sizes for Product Quantization
+# We suggest: 8 for FFN, 4 for ATTN, 4 for embedding projections, 8 for embeddings
+block_sizes:
+  Linear:
+      key: fuzzy_name
+      value: {fc: 8, attn: 4, emb: 4}
+  Embedding:
+      key: fuzzy_name
+      value: {emb: 8}
+
+# Layers to Quantize Sequentially
+# We suggest: first FFN, then EMB, then ATTN
+layers_to_quantize:
+    - decoder\\.layers\\.\d+\\.fc[12]
+    - decoder\\.embed_tokens\\.embeddings\\.[012]\\.[01]
+    - decoder\\.layers\\.\d+\\.self_attn\\.(k_proj|v_proj|q_proj|out_proj)

mosesdecoder/moses/FF/BleuScoreFeature.h

@@ -0,0 +1,191 @@
+#ifndef BLUESCOREFEATURE_H
+#define BLUESCOREFEATURE_H
+
+#include <utility>
+#include <string>
+#include <vector>
+
+#include <boost/unordered_map.hpp>
+
+#include "StatefulFeatureFunction.h"
+
+#include "moses/FF/FFState.h"
+#include "moses/Phrase.h"
+#include "moses/ChartHypothesis.h"
+
+namespace Moses
+{
+
+class BleuScoreFeature;
+
+class BleuScoreState : public FFState
+{
+public:
+  friend class BleuScoreFeature;
+  static size_t bleu_order;
+
+  BleuScoreState(bool is_syntax);
+  size_t hash() const;
+  virtual bool operator==(const FFState& other) const;
+
+  void print(std::ostream& out) const;
+
+private:
+  Phrase m_words;
+  size_t m_source_length;
+  size_t m_target_length;
+  bool m_is_syntax;
+  // scaled reference length is needed for scoring incomplete hypotheses against reference translation
+  float m_scaled_ref_length;
+
+  std::vector< size_t > m_ngram_counts;
+  std::vector< size_t > m_ngram_matches;
+
+  void AddNgramCountAndMatches(std::vector< size_t >& counts, std::vector< size_t >& matches);
+};
+
+
+std::ostream& operator<<(std::ostream& out, const BleuScoreState& state);
+
+typedef boost::unordered_map< Phrase, size_t > NGrams;
+
+class RefValue : public  std::pair<std::vector<size_t>,NGrams>
+{
+public:
+  RefValue& operator=( const RefValue& rhs ) {
+    first = rhs.first;
+    second = rhs.second;
+    return *this;
+  }
+};
+
+
+class BleuScoreFeature : public StatefulFeatureFunction
+{
+public:
+  static const std::vector<BleuScoreFeature*>& GetColl() {
+    return s_staticColl;
+  }
+
+  typedef boost::unordered_map<size_t, RefValue > RefCounts;
+  typedef boost::unordered_map<size_t, NGrams> Matches;
+
+  BleuScoreFeature(const std::string &line);
+
+  void SetParameter(const std::string& key, const std::string& value);
+
+  std::vector<float> DefaultWeights() const;
+
+  void PrintHistory(std::ostream& out) const;
+  void LoadReferences(const std::vector< std::vector< std::string > > &);
+  void SetCurrSourceLength(size_t);
+  void SetCurrNormSourceLength(size_t);
+  void SetCurrShortestRefLength(size_t);
+  void SetCurrAvgRefLength(size_t sent_id);
+  void SetAvgInputLength (float l) {
+    m_avg_input_length = l;
+  }
+  void SetCurrReferenceNgrams(size_t sent_id);
+  size_t GetShortestRefIndex(size_t ref_id);
+  size_t GetClosestRefLength(size_t ref_id, int hypoLength);
+  void UpdateHistory(const std::vector< const Word* >&);
+  void UpdateHistory(const std::vector< std::vector< const Word* > >& hypos, std::vector<size_t>& sourceLengths, std::vector<size_t>& ref_ids, size_t rank, size_t epoch);
+  void PrintRefLength(const std::vector<size_t>& ref_ids);
+  void SetBleuParameters(bool disable, bool sentenceBleu, bool scaleByInputLength, bool scaleByAvgInputLength,
+                         bool scaleByInverseLength, bool scaleByAvgInverseLength,
+                         float scaleByX, float historySmoothing, size_t scheme, bool simpleHistoryBleu);
+
+  void GetNgramMatchCounts(Phrase&,
+                           const NGrams&,
+                           std::vector< size_t >&,
+                           std::vector< size_t >&,
+                           size_t skip = 0) const;
+  void GetNgramMatchCounts_prefix(Phrase&,
+                                  const NGrams&,
+                                  std::vector< size_t >&,
+                                  std::vector< size_t >&,
+                                  size_t new_start_indices,
+                                  size_t last_end_index) const;
+  void GetNgramMatchCounts_overlap(Phrase& phrase,
+                                   const NGrams& ref_ngram_counts,
+                                   std::vector< size_t >& ret_counts,
+                                   std::vector< size_t >& ret_matches,
+                                   size_t overlap_index) const;
+  void GetClippedNgramMatchesAndCounts(Phrase&,
+                                       const NGrams&,
+                                       std::vector< size_t >&,
+                                       std::vector< size_t >&,
+                                       size_t skip = 0) const;
+
+  FFState* EvaluateWhenApplied( const Hypothesis& cur_hypo,
+                                const FFState* prev_state,
+                                ScoreComponentCollection* accumulator) const;
+  FFState* EvaluateWhenApplied(const ChartHypothesis& cur_hypo,
+                               int featureID,
+                               ScoreComponentCollection* accumulator) const;
+
+  bool Enabled() const {
+    return m_enabled;
+  }
+
+  bool IsUseable(const FactorMask &mask) const;
+
+  float CalculateBleu(BleuScoreState*) const;
+  float CalculateBleu(Phrase translation) const;
+  const FFState* EmptyHypothesisState(const InputType&) const;
+
+  float GetSourceLengthHistory() {
+    return m_source_length_history;
+  }
+  float GetTargetLengthHistory() {
+    return m_target_length_history;
+  }
+  float GetAverageInputLength() {
+    return m_avg_input_length;
+  }
+
+  void Load(AllOptions::ptr const& opts);
+
+private:
+  static std::vector<BleuScoreFeature*> s_staticColl;
+
+  bool m_enabled;
+  bool m_sentence_bleu;
+  bool m_simple_history_bleu;
+  bool m_is_syntax;
+  // counts for pseudo-document
+  std::vector< float > m_count_history;
+  std::vector< float > m_match_history;
+  float m_source_length_history;
+  float m_target_length_history;
+  float m_ref_length_history;
+
+  size_t m_cur_source_length;
+  size_t m_cur_norm_source_length; // length without <s>, </s>
+  RefCounts m_refs;
+  NGrams m_cur_ref_ngrams;
+  float m_cur_ref_length;
+
+  // scale BLEU score by history of input length
+  bool m_scale_by_input_length;
+  bool m_scale_by_avg_input_length;
+
+  // scale by the inverse of the input length * 100
+  bool m_scale_by_inverse_length;
+  bool m_scale_by_avg_inverse_length;
+
+  float m_avg_input_length;
+
+  float m_scale_by_x;
+
+  // smoothing factor for history counts
+  float m_historySmoothing;
+
+  enum SmoothingScheme { PLUS_ONE = 1, PLUS_POINT_ONE = 2, PAPINENI = 3 };
+  SmoothingScheme m_smoothing_scheme;
+};
+
+} // Namespace.
+
+#endif //BLUESCOREFEATURE_H
+

mosesdecoder/moses/FF/ConstrainedDecoding.cpp

@@ -0,0 +1,212 @@
+#include "ConstrainedDecoding.h"
+#include "moses/Hypothesis.h"
+#include "moses/Manager.h"
+#include "moses/ChartHypothesis.h"
+#include "moses/ChartManager.h"
+#include "moses/StaticData.h"
+#include "moses/InputFileStream.h"
+#include "moses/Util.h"
+#include "util/exception.hh"
+
+using namespace std;
+
+namespace Moses
+{
+ConstrainedDecodingState::ConstrainedDecodingState(const Hypothesis &hypo)
+{
+  hypo.GetOutputPhrase(m_outputPhrase);
+}
+
+ConstrainedDecodingState::ConstrainedDecodingState(const ChartHypothesis &hypo)
+{
+  hypo.GetOutputPhrase(m_outputPhrase);
+}
+
+size_t ConstrainedDecodingState::hash() const
+{
+  size_t ret = hash_value(m_outputPhrase);
+  return ret;
+}
+
+bool ConstrainedDecodingState::operator==(const FFState& other) const
+{
+  const ConstrainedDecodingState &otherFF = static_cast<const ConstrainedDecodingState&>(other);
+  bool ret = m_outputPhrase == otherFF.m_outputPhrase;
+  return ret;
+}
+
+//////////////////////////////////////////////////////////////////
+ConstrainedDecoding::ConstrainedDecoding(const std::string &line)
+  :StatefulFeatureFunction(1, line)
+  ,m_maxUnknowns(0)
+  ,m_negate(false)
+  ,m_soft(false)
+{
+  m_tuneable = false;
+  ReadParameters();
+}
+
+void ConstrainedDecoding::Load(AllOptions::ptr const& opts)
+{
+  m_options = opts;
+  const StaticData &staticData = StaticData::Instance();
+  bool addBeginEndWord
+  = ((opts->search.algo == CYKPlus) || (opts->search.algo == ChartIncremental));
+
+  for(size_t i = 0; i < m_paths.size(); ++i) {
+    InputFileStream constraintFile(m_paths[i]);
+    std::string line;
+    long sentenceID = opts->output.start_translation_id - 1 ;
+    while (getline(constraintFile, line)) {
+      vector<string> vecStr = Tokenize(line, "\t");
+
+      Phrase phrase(0);
+      if (vecStr.size() == 1) {
+        sentenceID++;
+        phrase.CreateFromString(Output, opts->output.factor_order, vecStr[0], NULL);
+      } else if (vecStr.size() == 2) {
+        sentenceID = Scan<long>(vecStr[0]);
+        phrase.CreateFromString(Output, opts->output.factor_order, vecStr[1], NULL);
+      } else {
+        UTIL_THROW(util::Exception, "Reference file not loaded");
+      }
+
+      if (addBeginEndWord) {
+        phrase.InitStartEndWord();
+      }
+      m_constraints[sentenceID].push_back(phrase);
+    }
+  }
+}
+
+std::vector<float> ConstrainedDecoding::DefaultWeights() const
+{
+  UTIL_THROW_IF2(m_numScoreComponents != 1,
+                 "ConstrainedDecoding must only have 1 score");
+  vector<float> ret(1, 1);
+  return ret;
+}
+
+template <class H, class M>
+const std::vector<Phrase> *GetConstraint(const std::map<long,std::vector<Phrase> > &constraints, const H &hypo)
+{
+  const M &mgr = hypo.GetManager();
+  const InputType &input = mgr.GetSource();
+  long id = input.GetTranslationId();
+
+  map<long,std::vector<Phrase> >::const_iterator iter;
+  iter = constraints.find(id);
+
+  if (iter == constraints.end()) {
+    UTIL_THROW(util::Exception, "Couldn't find reference " << id);
+
+    return NULL;
+  } else {
+    return &iter->second;
+  }
+}
+
+FFState* ConstrainedDecoding::EvaluateWhenApplied(
+  const Hypothesis& hypo,
+  const FFState* prev_state,
+  ScoreComponentCollection* accumulator) const
+{
+  const std::vector<Phrase> *ref = GetConstraint<Hypothesis, Manager>(m_constraints, hypo);
+  assert(ref);
+
+  ConstrainedDecodingState *ret = new ConstrainedDecodingState(hypo);
+  const Phrase &outputPhrase = ret->GetPhrase();
+
+  size_t searchPos = NOT_FOUND;
+  size_t i = 0;
+  size_t size = 0;
+  while(searchPos == NOT_FOUND && i < ref->size()) {
+    searchPos = (*ref)[i].Find(outputPhrase, m_maxUnknowns);
+    size = (*ref)[i].GetSize();
+    i++;
+  }
+
+  float score;
+  if (hypo.IsSourceCompleted()) {
+    // translated entire sentence.
+    bool match = (searchPos == 0) && (size == outputPhrase.GetSize());
+    if (!m_negate) {
+      score = match ? 0 : - ( m_soft ? 1 : std::numeric_limits<float>::infinity());
+    } else {
+      score = !match ? 0 : - ( m_soft ? 1 : std::numeric_limits<float>::infinity());
+    }
+  } else if (m_negate) {
+    // keep all derivations
+    score = 0;
+  } else {
+    score = (searchPos != NOT_FOUND) ? 0 : - ( m_soft ? 1 : std::numeric_limits<float>::infinity());
+  }
+
+  accumulator->PlusEquals(this, score);
+
+  return ret;
+}
+
+FFState* ConstrainedDecoding::EvaluateWhenApplied(
+  const ChartHypothesis &hypo,
+  int /* featureID - used to index the state in the previous hypotheses */,
+  ScoreComponentCollection* accumulator) const
+{
+  const std::vector<Phrase> *ref = GetConstraint<ChartHypothesis, ChartManager>(m_constraints, hypo);
+  assert(ref);
+
+  const ChartManager &mgr = hypo.GetManager();
+  const Sentence &source = static_cast<const Sentence&>(mgr.GetSource());
+
+  ConstrainedDecodingState *ret = new ConstrainedDecodingState(hypo);
+  const Phrase &outputPhrase = ret->GetPhrase();
+
+  size_t searchPos = NOT_FOUND;
+  size_t i = 0;
+  size_t size = 0;
+  while(searchPos == NOT_FOUND && i < ref->size()) {
+    searchPos = (*ref)[i].Find(outputPhrase, m_maxUnknowns);
+    size = (*ref)[i].GetSize();
+    i++;
+  }
+
+  float score;
+  if (hypo.GetCurrSourceRange().GetStartPos() == 0 &&
+      hypo.GetCurrSourceRange().GetEndPos() == source.GetSize() - 1) {
+    // translated entire sentence.
+    bool match = (searchPos == 0) && (size == outputPhrase.GetSize());
+
+    if (!m_negate) {
+      score = match ? 0 : - ( m_soft ? 1 : std::numeric_limits<float>::infinity());
+    } else {
+      score = !match ? 0 : - ( m_soft ? 1 : std::numeric_limits<float>::infinity());
+    }
+  } else if (m_negate) {
+    // keep all derivations
+    score = 0;
+  } else {
+    score = (searchPos != NOT_FOUND) ? 0 : - ( m_soft ? 1 : std::numeric_limits<float>::infinity());
+  }
+
+  accumulator->PlusEquals(this, score);
+
+  return ret;
+}
+
+void ConstrainedDecoding::SetParameter(const std::string& key, const std::string& value)
+{
+  if (key == "path") {
+    m_paths = Tokenize(value, ",");
+  } else if (key == "max-unknowns") {
+    m_maxUnknowns = Scan<int>(value);
+  } else if (key == "negate") {
+    m_negate = Scan<bool>(value);
+  } else if (key == "soft") {
+    m_soft = Scan<bool>(value);
+  } else {
+    StatefulFeatureFunction::SetParameter(key, value);
+  }
+}
+
+}
+

mosesdecoder/moses/FF/ControlRecombination.cpp

@@ -0,0 +1,96 @@
+#include "ControlRecombination.h"
+#include "moses/Hypothesis.h"
+#include "moses/Manager.h"
+#include "moses/ChartHypothesis.h"
+#include "moses/ChartManager.h"
+#include "moses/StaticData.h"
+#include "moses/InputFileStream.h"
+#include "moses/Util.h"
+#include "util/exception.hh"
+
+using namespace std;
+
+namespace Moses
+{
+ControlRecombinationState::ControlRecombinationState(const Hypothesis &hypo, const ControlRecombination &ff)
+  :m_ff(ff)
+{
+  if (ff.GetType() == SameOutput) {
+    //UTIL_THROW(util::Exception, "Implemented not yet completed for phrase-based model. Need to take into account the coverage");
+    hypo.GetOutputPhrase(m_outputPhrase);
+  } else {
+    m_hypo = &hypo;
+  }
+}
+
+ControlRecombinationState::ControlRecombinationState(const ChartHypothesis &hypo, const ControlRecombination &ff)
+  :m_ff(ff)
+{
+  if (ff.GetType() == SameOutput) {
+    hypo.GetOutputPhrase(m_outputPhrase);
+  } else {
+    m_hypo = &hypo;
+  }
+}
+
+size_t ControlRecombinationState::hash() const
+{
+  size_t ret;
+  if (m_ff.GetType() == SameOutput) {
+    ret = hash_value(m_outputPhrase);
+  } else {
+    // compare hypo address. Won't be equal unless they're actually the same hypo
+    ret = (size_t) m_hypo;
+  }
+  return ret;
+}
+
+bool ControlRecombinationState::operator==(const FFState& other) const
+{
+  const ControlRecombinationState &otherFF = static_cast<const ControlRecombinationState&>(other);
+
+  if (m_ff.GetType() == SameOutput) {
+    return m_outputPhrase == otherFF.m_outputPhrase;
+  } else {
+    // compare hypo address. Won't be equal unless they're actually the same hypo
+    if (m_hypo == otherFF.m_hypo)
+      return true;
+    return (m_hypo == otherFF.m_hypo);
+  }
+}
+
+std::vector<float> ControlRecombination::DefaultWeights() const
+{
+  UTIL_THROW_IF2(m_numScoreComponents,
+                 "ControlRecombination should not have any scores");
+  vector<float> ret(0);
+  return ret;
+}
+
+FFState* ControlRecombination::EvaluateWhenApplied(
+  const Hypothesis& hypo,
+  const FFState* prev_state,
+  ScoreComponentCollection* accumulator) const
+{
+  return new ControlRecombinationState(hypo, *this);
+}
+
+FFState* ControlRecombination::EvaluateWhenApplied(
+  const ChartHypothesis &hypo,
+  int /* featureID - used to index the state in the previous hypotheses */,
+  ScoreComponentCollection* accumulator) const
+{
+  return new ControlRecombinationState(hypo, *this);
+}
+
+void ControlRecombination::SetParameter(const std::string& key, const std::string& value)
+{
+  if (key == "type") {
+    m_type = (ControlRecombinationType) Scan<int>(value);
+  } else {
+    StatefulFeatureFunction::SetParameter(key, value);
+  }
+}
+
+}
+

mosesdecoder/moses/FF/ControlRecombination.h

@@ -0,0 +1,88 @@
+#pragma once
+
+#include <string>
+#include <map>
+#include "StatefulFeatureFunction.h"
+#include "FFState.h"
+#include "moses/Phrase.h"
+
+namespace Moses
+{
+enum ControlRecombinationType {
+  // when to recombine
+  SameOutput = 1,
+  Never = 2
+};
+
+class ControlRecombination;
+
+class ControlRecombinationState : public FFState
+{
+public:
+  ControlRecombinationState(const ControlRecombination &ff)
+    :m_ff(ff) {
+  }
+
+  ControlRecombinationState(const Hypothesis &hypo, const ControlRecombination &ff);
+  ControlRecombinationState(const ChartHypothesis &hypo, const ControlRecombination &ff);
+
+  virtual size_t hash() const;
+  virtual bool operator==(const FFState& other) const;
+
+  const Phrase &GetPhrase() const {
+    return m_outputPhrase;
+  }
+
+protected:
+  Phrase m_outputPhrase;
+  const ControlRecombination &m_ff;
+  const void *m_hypo;
+};
+
+//////////////////////////////////////////////////////////////////
+
+// only allow recombination for the same output
+class ControlRecombination : public StatefulFeatureFunction
+{
+public:
+  ControlRecombination(const std::string &line)
+    :StatefulFeatureFunction(0, line)
+    ,m_type(SameOutput)
+
+  {
+    m_tuneable = false;
+    ReadParameters();
+  }
+
+  bool IsUseable(const FactorMask &mask) const {
+    return true;
+  }
+
+  FFState* EvaluateWhenApplied(
+    const Hypothesis& cur_hypo,
+    const FFState* prev_state,
+    ScoreComponentCollection* accumulator) const;
+
+  FFState* EvaluateWhenApplied(
+    const ChartHypothesis& /* cur_hypo */,
+    int /* featureID - used to index the state in the previous hypotheses */,
+    ScoreComponentCollection* accumulator) const;
+
+  virtual const FFState* EmptyHypothesisState(const InputType &input) const {
+    return new ControlRecombinationState(*this);
+  }
+
+  std::vector<float> DefaultWeights() const;
+
+  void SetParameter(const std::string& key, const std::string& value);
+
+  ControlRecombinationType GetType() const {
+    return m_type;
+  }
+protected:
+  ControlRecombinationType m_type;
+};
+
+
+}
+

mosesdecoder/moses/FF/CorrectionPattern.cpp

@@ -0,0 +1,354 @@
+#include <sstream>
+#include "CorrectionPattern.h"
+#include "moses/Phrase.h"
+#include "moses/TargetPhrase.h"
+#include "moses/InputPath.h"
+#include "moses/Hypothesis.h"
+#include "moses/ChartHypothesis.h"
+#include "moses/ScoreComponentCollection.h"
+#include "moses/TranslationOption.h"
+#include "util/string_piece_hash.hh"
+#include "util/exception.hh"
+
+#include <functional>
+#include <algorithm>
+
+#include <boost/foreach.hpp>
+#include <boost/algorithm/string.hpp>
+
+#include "Diffs.h"
+
+namespace Moses
+{
+
+using namespace std;
+
+std::string MakePair(const std::string &s1, const std::string &s2, bool general)
+{
+  std::vector<std::string> sourceList;
+  std::vector<std::string> targetList;
+
+  if(general) {
+    Diffs diffs = CreateDiff(s1, s2);
+
+    size_t i = 0, j = 0;
+    char lastType = 'm';
+
+    std::string source, target;
+    std::string match;
+
+    int count = 1;
+
+    BOOST_FOREACH(Diff type, diffs) {
+      if(type == 'm') {
+        if(lastType != 'm') {
+          sourceList.push_back(source);
+          targetList.push_back(target);
+        }
+        source.clear();
+        target.clear();
+
+        if(s1[i] == '+') {
+          if(match.size() >= 3) {
+            sourceList.push_back("(\\w{3,})·");
+            std::string temp = "1";
+            sprintf((char*)temp.c_str(), "%d", count);
+            targetList.push_back("\\" + temp + "·");
+            count++;
+          } else {
+            sourceList.push_back(match + "·");
+            targetList.push_back(match + "·");
+          }
+          match.clear();
+        } else
+          match.push_back(s1[i]);
+
+        i++;
+        j++;
+      } else if(type == 'd') {
+        if(s1[i] == '+')
+          source += "·";
+        else
+          source.push_back(s1[i]);
+        i++;
+      } else if(type == 'i') {
+        if(s2[j] == '+')
+          target += "·";
+        else
+          target.push_back(s2[j]);
+        j++;
+      }
+      if(type != 'm' && !match.empty()) {
+        if(match.size() >= 3) {
+          sourceList.push_back("(\\w{3,})");
+          std::string temp = "1";
+          sprintf((char*)temp.c_str(), "%d", count);
+          targetList.push_back("\\" + temp);
+          count++;
+        } else {
+          sourceList.push_back(match);
+          targetList.push_back(match);
+        }
+
+        match.clear();
+      }
+
+      lastType = type;
+    }
+    if(lastType != 'm') {
+      sourceList.push_back(source);
+      targetList.push_back(target);
+    }
+
+    if(!match.empty()) {
+      if(match.size() >= 3) {
+        sourceList.push_back("(\\w{3,})");
+        std::string temp = "1";
+        sprintf((char*)temp.c_str(), "%d", count);
+        targetList.push_back("\\"+ temp);
+        count++;
+      } else {
+        sourceList.push_back(match);
+        targetList.push_back(match);
+      }
+    }
+    match.clear();
+  } else {
+    std::string cs1 = s1;
+    std::string cs2 = s2;
+    boost::replace_all(cs1, "+", "·");
+    boost::replace_all(cs2, "+", "·");
+
+    sourceList.push_back(cs1);
+    targetList.push_back(cs2);
+  }
+
+  std::stringstream out;
+  out << "sub(«";
+  out << boost::join(sourceList, "");
+  out << "»,«";
+  out << boost::join(targetList, "");
+  out << "»)";
+
+  return out.str();
+}
+
+std::string CorrectionPattern::CreateSinglePattern(const Tokens &s1, const Tokens &s2) const
+{
+  std::stringstream out;
+  if(s1.empty()) {
+    out << "ins(«" << boost::join(s2, "·") << "»)";
+    return out.str();
+  } else if(s2.empty()) {
+    out << "del(«" << boost::join(s1, "·") << "»)";
+    return out.str();
+  } else {
+    Tokens::value_type v1 = boost::join(s1, "+");
+    Tokens::value_type v2 = boost::join(s2, "+");
+    out << MakePair(v1, v2, m_general);
+    return out.str();
+  }
+}
+
+std::vector<std::string> GetContext(size_t pos,
+                                    size_t len,
+                                    size_t window,
+                                    const InputType &input,
+                                    const InputPath &inputPath,
+                                    const std::vector<FactorType>& factorTypes,
+                                    bool isRight)
+{
+
+  const Sentence& sentence = static_cast<const Sentence&>(input);
+  const Range& range = inputPath.GetWordsRange();
+
+  int leftPos  = range.GetStartPos() + pos - len - 1;
+  int rightPos = range.GetStartPos() + pos;
+
+  std::vector<std::string> contexts;
+
+  for(int length = 1; length <= (int)window; ++length) {
+    std::vector<std::string> current;
+    if(!isRight) {
+      for(int i = 0; i < length; i++) {
+        if(leftPos - i >= 0) {
+          current.push_back(sentence.GetWord(leftPos - i).GetString(factorTypes, false));
+        } else {
+          current.push_back("<s>");
+        }
+      }
+
+      if(current.back() == "<s>" && current.size() >= 2 && current[current.size()-2] == "<s>")
+        continue;
+
+      std::reverse(current.begin(), current.end());
+      contexts.push_back("left(«" + boost::join(current, "·") + "»)_");
+    }
+    if(isRight) {
+      for(int i = 0; i < length; i++) {
+        if(rightPos + i < (int)sentence.GetSize()) {
+          current.push_back(sentence.GetWord(rightPos + i).GetString(factorTypes, false));
+        } else {
+          current.push_back("</s>");
+        }
+      }
+
+      if(current.back() == "</s>" && current.size() >= 2 && current[current.size()-2] == "</s>")
+        continue;
+
+      contexts.push_back("_right(«" + boost::join(current, "·") + "»)");
+    }
+  }
+  return contexts;
+}
+
+std::vector<std::string>
+CorrectionPattern::CreatePattern(const Tokens &s1,
+                                 const Tokens &s2,
+                                 const InputType &input,
+                                 const InputPath &inputPath) const
+{
+
+  Diffs diffs = CreateDiff(s1, s2);
+  size_t i = 0, j = 0;
+  char lastType = 'm';
+  std::vector<std::string> patternList;
+  Tokens source, target;
+  BOOST_FOREACH(Diff type, diffs) {
+    if(type == 'm') {
+      if(lastType != 'm') {
+        std::string pattern = CreateSinglePattern(source, target);
+        patternList.push_back(pattern);
+
+        if(m_context > 0) {
+          std::vector<std::string> leftContexts =  GetContext(i, source.size(), m_context, input, inputPath, m_contextFactors, false);
+          std::vector<std::string> rightContexts = GetContext(i, source.size(), m_context, input, inputPath, m_contextFactors, true);
+
+          BOOST_FOREACH(std::string left, leftContexts)
+          patternList.push_back(left + pattern);
+
+          BOOST_FOREACH(std::string right, rightContexts)
+          patternList.push_back(pattern + right);
+
+          BOOST_FOREACH(std::string left, leftContexts)
+          BOOST_FOREACH(std::string right, rightContexts)
+          patternList.push_back(left + pattern + right);
+        }
+      }
+      source.clear();
+      target.clear();
+      if(s1[i] != s2[j]) {
+        source.push_back(s1[i]);
+        target.push_back(s2[j]);
+      }
+      i++;
+      j++;
+    } else if(type == 'd') {
+      source.push_back(s1[i]);
+      i++;
+    } else if(type == 'i') {
+      target.push_back(s2[j]);
+      j++;
+    }
+    lastType = type;
+  }
+  if(lastType != 'm') {
+    std::string pattern = CreateSinglePattern(source, target);
+    patternList.push_back(pattern);
+
+    if(m_context > 0) {
+      std::vector<std::string> leftContexts =  GetContext(i, source.size(), m_context, input, inputPath, m_contextFactors, false);
+      std::vector<std::string> rightContexts = GetContext(i, source.size(), m_context, input, inputPath, m_contextFactors, true);
+
+      BOOST_FOREACH(std::string left, leftContexts)
+      patternList.push_back(left + pattern);
+
+      BOOST_FOREACH(std::string right, rightContexts)
+      patternList.push_back(pattern + right);
+
+      BOOST_FOREACH(std::string left, leftContexts)
+      BOOST_FOREACH(std::string right, rightContexts)
+      patternList.push_back(left + pattern + right);
+    }
+  }
+
+  return patternList;
+}
+
+CorrectionPattern::CorrectionPattern(const std::string &line)
+  : StatelessFeatureFunction(0, line), m_factors(1, 0), m_general(false),
+    m_context(0), m_contextFactors(1, 0)
+{
+  std::cerr << "Initializing correction pattern feature.." << std::endl;
+  ReadParameters();
+}
+
+void CorrectionPattern::SetParameter(const std::string& key, const std::string& value)
+{
+  if (key == "factor") {
+    m_factors = std::vector<FactorType>(1, Scan<FactorType>(value));
+  } else if (key == "context-factor") {
+    m_contextFactors = std::vector<FactorType>(1, Scan<FactorType>(value));
+  } else if (key == "general") {
+    m_general = Scan<bool>(value);
+  } else if (key == "context") {
+    m_context = Scan<size_t>(value);
+  } else {
+    StatelessFeatureFunction::SetParameter(key, value);
+  }
+}
+
+void CorrectionPattern::EvaluateWithSourceContext(const InputType &input
+    , const InputPath &inputPath
+    , const TargetPhrase &targetPhrase
+    , const StackVec *stackVec
+    , ScoreComponentCollection &scoreBreakdown
+    , ScoreComponentCollection *estimatedFutureScore) const
+{
+  ComputeFeatures(input, inputPath, targetPhrase, &scoreBreakdown);
+}
+
+void CorrectionPattern::ComputeFeatures(
+  const InputType &input,
+  const InputPath &inputPath,
+  const TargetPhrase& target,
+  ScoreComponentCollection* accumulator) const
+{
+  const Phrase &source = inputPath.GetPhrase();
+
+  std::vector<std::string> sourceTokens;
+  for(size_t i = 0; i < source.GetSize(); ++i)
+    sourceTokens.push_back(source.GetWord(i).GetString(m_factors, false));
+
+  std::vector<std::string> targetTokens;
+  for(size_t i = 0; i < target.GetSize(); ++i)
+    targetTokens.push_back(target.GetWord(i).GetString(m_factors, false));
+
+  std::vector<std::string> patternList = CreatePattern(sourceTokens, targetTokens, input, inputPath);
+  for(size_t i = 0; i < patternList.size(); ++i)
+    accumulator->PlusEquals(this, patternList[i], 1);
+
+  /*
+  BOOST_FOREACH(std::string w, sourceTokens)
+    std::cerr << w << " ";
+  std::cerr << std::endl;
+  BOOST_FOREACH(std::string w, targetTokens)
+    std::cerr << w << " ";
+  std::cerr << std::endl;
+  BOOST_FOREACH(std::string w, patternList)
+    std::cerr << w << " ";
+  std::cerr << std::endl << std::endl;
+  */
+}
+
+bool CorrectionPattern::IsUseable(const FactorMask &mask) const
+{
+  bool ret = true;
+  for(size_t i = 0; i < m_factors.size(); ++i)
+    ret = ret && mask[m_factors[i]];
+  for(size_t i = 0; i < m_contextFactors.size(); ++i)
+    ret = ret && mask[m_contextFactors[i]];
+  return ret;
+}
+
+}

mosesdecoder/moses/FF/CorrectionPattern.h

@@ -0,0 +1,73 @@
+#ifndef moses_CorrectionPattern_h
+#define moses_CorrectionPattern_h
+
+#include <string>
+#include <boost/unordered_set.hpp>
+
+#include "StatelessFeatureFunction.h"
+#include "moses/FactorCollection.h"
+#include "moses/AlignmentInfo.h"
+
+namespace Moses
+{
+
+typedef std::vector<std::string> Tokens;
+
+/** Sets the features for length of source phrase, target phrase, both.
+ */
+class CorrectionPattern : public StatelessFeatureFunction
+{
+private:
+  std::vector<FactorType> m_factors;
+  bool m_general;
+  size_t m_context;
+  std::vector<FactorType> m_contextFactors;
+
+public:
+  CorrectionPattern(const std::string &line);
+
+  bool IsUseable(const FactorMask &mask) const;
+
+  void EvaluateInIsolation(const Phrase &source
+                           , const TargetPhrase &targetPhrase
+                           , ScoreComponentCollection &scoreBreakdown
+                           , ScoreComponentCollection &estimatedFutureScore) const
+  {}
+
+  virtual void EvaluateWithSourceContext(const InputType &input
+                                         , const InputPath &inputPath
+                                         , const TargetPhrase &targetPhrase
+                                         , const StackVec *stackVec
+                                         , ScoreComponentCollection &scoreBreakdown
+                                         , ScoreComponentCollection *estimatedFutureScore = NULL) const;
+
+  void EvaluateTranslationOptionListWithSourceContext(const InputType &input
+      , const TranslationOptionList &translationOptionList) const
+  {}
+
+  void EvaluateWhenApplied(const Hypothesis& hypo,
+                           ScoreComponentCollection* accumulator) const
+  {}
+  void EvaluateWhenApplied(const ChartHypothesis &hypo,
+                           ScoreComponentCollection* accumulator) const
+  {}
+
+  void ComputeFeatures(const InputType &input,
+                       const InputPath &inputPath,
+                       const TargetPhrase& targetPhrase,
+                       ScoreComponentCollection* accumulator) const;
+
+  void SetParameter(const std::string& key, const std::string& value);
+
+  std::vector<std::string> CreatePattern(const Tokens &s1,
+                                         const Tokens &s2,
+                                         const InputType &input,
+                                         const InputPath &inputPath) const;
+
+  std::string CreateSinglePattern(const Tokens &s1, const Tokens &s2) const;
+
+};
+
+}
+
+#endif // moses_CorrectionPattern_h

mosesdecoder/moses/FF/CoveredReferenceFeature.cpp

@@ -0,0 +1,129 @@
+#include <boost/functional/hash.hpp>
+#include <vector>
+#include <algorithm>
+#include <iterator>
+#include <boost/foreach.hpp>
+#include "CoveredReferenceFeature.h"
+#include "moses/ScoreComponentCollection.h"
+#include "moses/Hypothesis.h"
+#include "moses/Manager.h"
+#include "moses/ChartHypothesis.h"
+#include "moses/ChartManager.h"
+#include "moses/StaticData.h"
+#include "moses/InputFileStream.h"
+#include "moses/Util.h"
+#include "util/exception.hh"
+
+using namespace std;
+
+namespace Moses
+{
+
+size_t CoveredReferenceState::hash() const
+{
+  UTIL_THROW2("TODO:Haven't figure this out yet");
+}
+
+bool CoveredReferenceState::operator==(const FFState& other) const
+{
+  UTIL_THROW2("TODO:Haven't figure this out yet");
+}
+
+//////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+void CoveredReferenceFeature::EvaluateWithSourceContext(const InputType &input
+    , const InputPath &inputPath
+    , const TargetPhrase &targetPhrase
+    , const StackVec *stackVec
+    , ScoreComponentCollection &scoreBreakdown
+    , ScoreComponentCollection *estimatedScores) const
+{
+  long id = input.GetTranslationId();
+  boost::unordered_map<long, std::multiset<string> >::const_iterator refIt = m_refs.find(id);
+  multiset<string> wordsInPhrase = GetWordsInPhrase(targetPhrase);
+  multiset<string> covered;
+  set_intersection(wordsInPhrase.begin(), wordsInPhrase.end(),
+                   refIt->second.begin(), refIt->second.end(),
+                   inserter(covered, covered.begin()));
+  vector<float> scores;
+  scores.push_back(covered.size());
+
+  scoreBreakdown.Assign(this, scores);
+  estimatedScores->Assign(this, scores);
+}
+
+void CoveredReferenceFeature::Load(AllOptions::ptr const& opts)
+{
+  m_options = opts;
+  InputFileStream refFile(m_path);
+  std::string line;
+  const StaticData &staticData = StaticData::Instance();
+  long sentenceID = opts->output.start_translation_id;
+  while (getline(refFile, line)) {
+    vector<string> words = Tokenize(line, " ");
+    multiset<string> wordSet;
+    // TODO make Tokenize work with other containers than vector
+    copy(words.begin(), words.end(), inserter(wordSet, wordSet.begin()));
+    m_refs.insert(make_pair(sentenceID++, wordSet));
+  }
+}
+
+void CoveredReferenceFeature::SetParameter(const std::string& key, const std::string& value)
+{
+  if (key == "path") {
+    m_path = value;
+  } else {
+    StatefulFeatureFunction::SetParameter(key, value);
+  }
+}
+
+FFState* CoveredReferenceFeature::EvaluateWhenApplied(
+  const Hypothesis& cur_hypo,
+  const FFState* prev_state,
+  ScoreComponentCollection* accumulator) const
+{
+  const CoveredReferenceState &prev = static_cast<const CoveredReferenceState&>(*prev_state);
+  CoveredReferenceState *ret = new CoveredReferenceState(prev);
+
+  const Manager &mgr = cur_hypo.GetManager();
+  const InputType &input = mgr.GetSource();
+  long id = input.GetTranslationId();
+
+  // which words from the reference remain uncovered
+  multiset<string> remaining;
+  boost::unordered_map<long, std::multiset<string> >::const_iterator refIt = m_refs.find(id);
+  if (refIt == m_refs.end()) UTIL_THROW(util::Exception, "Sentence id out of range: " + SPrint<long>(id));
+  set_difference(refIt->second.begin(), refIt->second.end(),
+                 ret->m_coveredRef.begin(), ret->m_coveredRef.end(),
+                 inserter(remaining, remaining.begin()));
+
+  // which of the remaining words are present in the current phrase
+  multiset<string> wordsInPhrase = GetWordsInPhrase(cur_hypo.GetCurrTargetPhrase());
+  multiset<string> newCovered;
+  set_intersection(wordsInPhrase.begin(), wordsInPhrase.end(),
+                   remaining.begin(), remaining.end(),
+                   inserter(newCovered, newCovered.begin()));
+
+  vector<float> estimateScore =
+    cur_hypo.GetCurrTargetPhrase().GetScoreBreakdown().GetScoresForProducer(this);
+  vector<float> scores;
+  scores.push_back(newCovered.size() - estimateScore[0]);
+  accumulator->PlusEquals(this, scores);
+
+  // update feature state
+  multiset<string>::const_iterator newCoveredIt;
+  for (newCoveredIt = newCovered.begin(); newCoveredIt != newCovered.end(); newCoveredIt++) {
+    ret->m_coveredRef.insert(*newCoveredIt);
+  }
+  return ret;
+}
+
+FFState* CoveredReferenceFeature::EvaluateWhenApplied(
+  const ChartHypothesis& /* cur_hypo */,
+  int /* featureID - used to index the state in the previous hypotheses */,
+  ScoreComponentCollection* accumulator) const
+{
+  UTIL_THROW(util::Exception, "Not implemented");
+}
+
+}

mosesdecoder/moses/FF/DecodeFeature.cpp

@@ -0,0 +1,117 @@
+// $Id: PhraseDictionaryMemory.cpp 2477 2009-08-07 16:47:54Z bhaddow $
+// vim:tabstop=2
+
+/***********************************************************************
+Moses - factored phrase-based language decoder
+Copyright (C) 2010 University of Edinburgh
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Lesser General Public
+License as published by the Free Software Foundation; either
+version 2.1 of the License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public
+License along with this library; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+***********************************************************************/
+
+#include <iostream>
+
+#include "DecodeFeature.h"
+#include "moses/DecodeStep.h"
+#include "moses/StaticData.h"
+
+using namespace std;
+
+namespace Moses
+{
+
+DecodeFeature::DecodeFeature(const std::string &line, bool registerNow)
+  : StatelessFeatureFunction(line, registerNow)
+  , m_container(NULL)
+{
+  VERBOSE(2,"DecodeFeature:" << std::endl);
+}
+
+DecodeFeature::DecodeFeature(size_t numScoreComponents
+                             , const std::string &line)
+  : StatelessFeatureFunction(numScoreComponents, line)
+  , m_container(NULL)
+{
+  VERBOSE(2,"DecodeFeature: no factors yet" << std::endl);
+}
+
+DecodeFeature::DecodeFeature(size_t numScoreComponents
+                             , const std::vector<FactorType> &input
+                             , const std::vector<FactorType> &output
+                             , const std::string &line)
+  : StatelessFeatureFunction(numScoreComponents, line)
+  , m_input(input), m_output(output)
+  , m_container(NULL)
+{
+  m_inputFactors = FactorMask(input);
+  m_outputFactors = FactorMask(output);
+  VERBOSE(2,"DecodeFeature: input=" << m_inputFactors << "  output=" << m_outputFactors << std::endl);
+}
+
+void DecodeFeature::SetParameter(const std::string& key, const std::string& value)
+{
+  if (key == "input-factor") {
+    m_input =Tokenize<FactorType>(value, ",");
+    m_inputFactors = FactorMask(m_input);
+  } else if (key == "output-factor") {
+    m_output =Tokenize<FactorType>(value, ",");
+    m_outputFactors = FactorMask(m_output);
+  } else {
+    StatelessFeatureFunction::SetParameter(key, value);
+  }
+}
+
+
+const FactorMask& DecodeFeature::GetOutputFactorMask() const
+{
+  return m_outputFactors;
+}
+
+
+const FactorMask& DecodeFeature::GetInputFactorMask() const
+{
+  return m_inputFactors;
+}
+
+const std::vector<FactorType>& DecodeFeature::GetInput() const
+{
+  return m_input;
+}
+
+const std::vector<FactorType>& DecodeFeature::GetOutput() const
+{
+  return m_output;
+}
+
+bool DecodeFeature::IsUseable(const FactorMask &mask) const
+{
+  for (size_t i = 0; i < m_output.size(); ++i) {
+    const FactorType &factor = m_output[i];
+    if (!mask[factor]) {
+      return false;
+    }
+  }
+  return true;
+}
+
+const DecodeGraph &DecodeFeature::GetDecodeGraph() const
+{
+  assert(m_container);
+  const DecodeGraph *graph = m_container->GetContainer();
+  assert(graph);
+  return *graph;
+}
+
+}
+

mosesdecoder/moses/FF/DistortionScoreProducer.h

@@ -0,0 +1,57 @@
+#pragma once
+
+#include <string>
+#include "StatefulFeatureFunction.h"
+#include "moses/Range.h"
+
+namespace Moses
+{
+
+/** Calculates Distortion scores
+ */
+class DistortionScoreProducer : public StatefulFeatureFunction
+{
+protected:
+  static std::vector<const DistortionScoreProducer*> s_staticColl;
+
+  FactorType m_sparseFactorTypeSource;
+  FactorType m_sparseFactorTypeTarget;
+  bool m_useSparse;
+  bool m_sparseDistance;
+  bool m_sparseSubordinate;
+  FactorType m_sparseFactorTypeTargetSubordinate;
+  const Factor* m_subordinateConjunctionTagFactor;
+
+public:
+  static const std::vector<const DistortionScoreProducer*>& GetDistortionFeatureFunctions() {
+    return s_staticColl;
+  }
+
+  DistortionScoreProducer(const std::string &line);
+
+  void SetParameter(const std::string& key, const std::string& value);
+
+  bool IsUseable(const FactorMask &mask) const {
+    return true;
+  }
+
+  static float CalculateDistortionScore(const Hypothesis& hypo,
+                                        const Range &prev, const Range &curr, const int FirstGapPosition);
+
+  virtual const FFState* EmptyHypothesisState(const InputType &input) const;
+
+  virtual FFState* EvaluateWhenApplied(
+    const Hypothesis& cur_hypo,
+    const FFState* prev_state,
+    ScoreComponentCollection* accumulator) const;
+
+  virtual FFState* EvaluateWhenApplied(
+    const ChartHypothesis& /* cur_hypo */,
+    int /* featureID - used to index the state in the previous hypotheses */,
+    ScoreComponentCollection*) const {
+    UTIL_THROW(util::Exception, "DIstortion not implemented in chart decoder");
+  }
+
+};
+}
+

mosesdecoder/moses/FF/DynamicCacheBasedLanguageModel.cpp

@@ -0,0 +1,459 @@
+#include <utility>
+#include "moses/StaticData.h"
+#include "moses/InputFileStream.h"
+#include "DynamicCacheBasedLanguageModel.h"
+
+namespace Moses
+{
+
+std::map< const std::string, DynamicCacheBasedLanguageModel * > DynamicCacheBasedLanguageModel::s_instance_map;
+DynamicCacheBasedLanguageModel *DynamicCacheBasedLanguageModel::s_instance = NULL;
+
+DynamicCacheBasedLanguageModel::DynamicCacheBasedLanguageModel(const std::string &line)
+  : StatelessFeatureFunction(1, line)
+{
+  VERBOSE(2,"Initializing DynamicCacheBasedLanguageModel feature..." << std::endl);
+
+  m_query_type = CBLM_QUERY_TYPE_ALLSUBSTRINGS;
+  m_score_type = CBLM_SCORE_TYPE_HYPERBOLA;
+  m_maxAge = 1000;
+  m_name = "default";
+  m_constant = false;
+
+  ReadParameters();
+  UTIL_THROW_IF2(s_instance_map.find(m_name) != s_instance_map.end(), "Only 1 DynamicCacheBasedLanguageModel feature named " + m_name + " is allowed");
+  s_instance_map[m_name] = this;
+  s_instance = this; //for back compatibility
+
+  SetPreComputedScores();
+}
+
+DynamicCacheBasedLanguageModel::~DynamicCacheBasedLanguageModel() {};
+
+void DynamicCacheBasedLanguageModel::SetPreComputedScores()
+{
+#ifdef WITH_THREADS
+  boost::shared_lock<boost::shared_mutex> lock(m_cacheLock);
+#endif
+  precomputedScores.clear();
+  for (unsigned int i=0; i<m_maxAge; i++) {
+    precomputedScores.push_back(decaying_score(i));
+  }
+
+  if ( m_score_type == CBLM_SCORE_TYPE_HYPERBOLA
+       || m_score_type == CBLM_SCORE_TYPE_POWER
+       || m_score_type == CBLM_SCORE_TYPE_EXPONENTIAL
+       || m_score_type == CBLM_SCORE_TYPE_COSINE ) {
+    precomputedScores.push_back(decaying_score(m_maxAge));
+  } else { // m_score_type = CBLM_SCORE_TYPE_XXXXXXXXX_REWARD
+    precomputedScores.push_back(0.0);
+  }
+  m_lower_score = precomputedScores[m_maxAge];
+  VERBOSE(3, "SetPreComputedScores(): lower_age:|" << m_maxAge << "| lower_score:|" << m_lower_score << "|" << std::endl);
+}
+
+float DynamicCacheBasedLanguageModel::GetPreComputedScores(const unsigned int age)
+{
+  VERBOSE(2, "float DynamicCacheBasedLanguageModel::GetPreComputedScores" << std::endl);
+  VERBOSE(2, "age:|"<< age << "|" << std::endl);
+
+  if (age < m_maxAge) {
+    return precomputedScores.at(age);
+  } else {
+    VERBOSE(2, "is to big reduced to m)_maxAge:|"<< m_maxAge << "|" << std::endl);
+    return precomputedScores.at(m_maxAge);
+  }
+}
+
+void DynamicCacheBasedLanguageModel::SetParameter(const std::string& key, const std::string& value)
+{
+  VERBOSE(2, "DynamicCacheBasedLanguageModel::SetParameter key:|" << key << "| value:|" << value << "|" << std::endl);
+  if (key == "cblm-query-type") {
+    SetQueryType(Scan<size_t>(value));
+  } else if (key == "cblm-score-type") {
+    SetScoreType(Scan<size_t>(value));
+  } else if (key == "cblm-max-age") {
+    SetMaxAge(Scan<unsigned int>(value));
+  } else if (key == "cblm-file") {
+    m_initfiles = Scan<std::string>(value);
+  } else if (key == "cblm-name") {
+    m_name = Scan<std::string>(value);
+  } else if (key == "cblm-constant") {
+    m_constant = Scan<bool>(value);
+  } else {
+    StatelessFeatureFunction::SetParameter(key, value);
+  }
+}
+
+void DynamicCacheBasedLanguageModel::EvaluateInIsolation(const Phrase &sp
+    , const TargetPhrase &tp
+    , ScoreComponentCollection &scoreBreakdown
+    , ScoreComponentCollection &estimatedScores) const
+{
+  float score = m_lower_score;
+  switch(m_query_type) {
+  case CBLM_QUERY_TYPE_WHOLESTRING:
+    score = Evaluate_Whole_String(tp);
+    break;
+  case CBLM_QUERY_TYPE_ALLSUBSTRINGS:
+    score = Evaluate_All_Substrings(tp);
+    break;
+  default:
+    UTIL_THROW_IF2(false, "This score type (" << m_query_type << ") is unknown.");
+  }
+
+  scoreBreakdown.Assign(this, score);
+}
+
+float DynamicCacheBasedLanguageModel::Evaluate_Whole_String(const TargetPhrase& tp) const
+{
+  //consider all words in the TargetPhrase as one n-gram
+  // and compute the decaying_score for the whole n-gram
+  // and return this value
+
+  decaying_cache_t::const_iterator it;
+  float score = m_lower_score;
+
+  std::string w = "";
+  size_t endpos = tp.GetSize();
+  for (size_t pos = 0 ; pos < endpos ; ++pos) {
+    w += tp.GetWord(pos).GetFactor(0)->GetString().as_string();
+    if (pos < endpos - 1) {
+      w += " ";
+    }
+  }
+  it = m_cache.find(w);
+
+  VERBOSE(4,"cblm::Evaluate_Whole_String: searching w:|" << w << "|" << std::endl);
+  if (it != m_cache.end()) { //found!
+    score = ((*it).second).second;
+    VERBOSE(4,"cblm::Evaluate_Whole_String: found w:|" << w << "|" << std::endl);
+  }
+
+  VERBOSE(4,"cblm::Evaluate_Whole_String: returning score:|" << score << "|" << std::endl);
+  return score;
+}
+
+float DynamicCacheBasedLanguageModel::Evaluate_All_Substrings(const TargetPhrase& tp) const
+{
+  //loop over all n-grams in the TargetPhrase (no matter of n)
+  //and compute the decaying_score for all words
+  //and return their sum
+
+  decaying_cache_t::const_iterator it;
+  float score = 0.0;
+
+  for (size_t startpos = 0 ; startpos < tp.GetSize() ; ++startpos) {
+    std::string w = "";
+    for (size_t endpos = startpos; endpos < tp.GetSize() ; ++endpos) {
+      w += tp.GetWord(endpos).GetFactor(0)->GetString().as_string();
+      it = m_cache.find(w);
+
+      if (it != m_cache.end()) { //found!
+        score += ((*it).second).second;
+        VERBOSE(3,"cblm::Evaluate_All_Substrings: found w:|" << w << "| actual score:|" << ((*it).second).second << "| score:|" << score << "|" << std::endl);
+      } else {
+        score += m_lower_score;
+      }
+
+      if (endpos == startpos) {
+        w += " ";
+      }
+
+    }
+  }
+  VERBOSE(3,"cblm::Evaluate_All_Substrings: returning score:|" << score << "|" << std::endl);
+  return score;
+}
+
+void DynamicCacheBasedLanguageModel::Print() const
+{
+#ifdef WITH_THREADS
+  boost::shared_lock<boost::shared_mutex> read_lock(m_cacheLock);
+#endif
+  decaying_cache_t::const_iterator it;
+  std::cout << "Content of the cache of Cache-Based Language Model" << std::endl;
+  std::cout << "Size of the cache of Cache-Based Language Model:|" << m_cache.size() << "|" << std::endl;
+  for ( it=m_cache.begin() ; it != m_cache.end(); it++ ) {
+    std::cout << "word:|" << (*it).first << "| age:|" << ((*it).second).first << "| score:|" << ((*it).second).second << "|" << std::endl;
+  }
+}
+
+void DynamicCacheBasedLanguageModel::Decay()
+{
+#ifdef WITH_THREADS
+  boost::shared_lock<boost::shared_mutex> lock(m_cacheLock);
+#endif
+  decaying_cache_t::iterator it;
+
+  unsigned int age;
+  float score;
+  for ( it=m_cache.begin() ; it != m_cache.end(); it++ ) {
+    age=((*it).second).first + 1;
+    if (age > m_maxAge) {
+      m_cache.erase(it);
+      it--;
+    } else {
+      score = GetPreComputedScores(age);
+//      score = decaying_score(age);
+      decaying_cache_value_t p (age, score);
+      (*it).second = p;
+    }
+  }
+}
+
+void DynamicCacheBasedLanguageModel::Update(std::vector<std::string> words, int age)
+{
+#ifdef WITH_THREADS
+  boost::shared_lock<boost::shared_mutex> lock(m_cacheLock);
+#endif
+  VERBOSE(3,"words.size():|" << words.size() << "|" << std::endl);
+  for (size_t j=0; j<words.size(); j++) {
+    words[j] = Trim(words[j]);
+//    VERBOSE(3,"CacheBasedLanguageModel::Update   word[" << j << "]:"<< words[j] << " age:" << age << " decaying_score(age):" << decaying_score(age) << std::endl);
+//    decaying_cache_value_t p (age,decaying_score(age));
+    VERBOSE(3,"CacheBasedLanguageModel::Update   word[" << j << "]:"<< words[j] << " age:" << age << " GetPreComputedScores(age):" << GetPreComputedScores(age) << std::endl);
+    decaying_cache_value_t p (age,GetPreComputedScores(age));
+    std::pair<std::string, decaying_cache_value_t> e (words[j],p);
+    m_cache.erase(words[j]); //always erase the element (do nothing if the entry does not exist)
+    m_cache.insert(e); //insert the entry
+  }
+}
+
+void DynamicCacheBasedLanguageModel::ClearEntries(std::string &entries)
+{
+  if (entries != "") {
+    VERBOSE(3,"entries:|" << entries << "|" << std::endl);
+    std::vector<std::string> elements = TokenizeMultiCharSeparator(entries, "||");
+    VERBOSE(3,"elements.size() after:|" << elements.size() << "|" << std::endl);
+    ClearEntries(elements);
+  }
+}
+
+void DynamicCacheBasedLanguageModel::ClearEntries(std::vector<std::string> words)
+{
+#ifdef WITH_THREADS
+  boost::shared_lock<boost::shared_mutex> lock(m_cacheLock);
+#endif
+  VERBOSE(3,"words.size():|" << words.size() << "|" << std::endl);
+  for (size_t j=0; j<words.size(); j++) {
+    words[j] = Trim(words[j]);
+    VERBOSE(3,"CacheBasedLanguageModel::ClearEntries   word[" << j << "]:"<< words[j] << std::endl);
+    m_cache.erase(words[j]); //always erase the element (do nothing if the entry does not exist)
+  }
+}
+
+void DynamicCacheBasedLanguageModel::Insert(std::string &entries)
+{
+  if (entries != "") {
+    VERBOSE(3,"entries:|" << entries << "|" << std::endl);
+    std::vector<std::string> elements = TokenizeMultiCharSeparator(entries, "||");
+    VERBOSE(3,"elements.size() after:|" << elements.size() << "|" << std::endl);
+    Insert(elements);
+  }
+}
+
+void DynamicCacheBasedLanguageModel::Insert(std::vector<std::string> ngrams)
+{
+  VERBOSE(3,"DynamicCacheBasedLanguageModel Insert ngrams.size():|" << ngrams.size() << "|" << std::endl);
+  if (m_constant == false) {
+    Decay();
+  }
+  Update(ngrams,1);
+  IFVERBOSE(3) Print();
+}
+
+void DynamicCacheBasedLanguageModel::ExecuteDlt(std::map<std::string, std::string> dlt_meta)
+{
+  if (dlt_meta.find("cblm") != dlt_meta.end()) {
+    Insert(dlt_meta["cblm"]);
+  }
+  if (dlt_meta.find("cblm-command") != dlt_meta.end()) {
+    Execute(dlt_meta["cblm-command"]);
+  }
+  if (dlt_meta.find("cblm-file") != dlt_meta.end()) {
+    Load(dlt_meta["cblm-file"]);
+  }
+  if (dlt_meta.find("cblm-clear-entries") != dlt_meta.end()) {
+    ClearEntries(dlt_meta["cblm-clear-entries"]);
+  }
+  if (dlt_meta.find("cblm-clear-all") != dlt_meta.end()) {
+    Clear();
+  }
+
+}
+
+void DynamicCacheBasedLanguageModel::Execute(std::string command)
+{
+  VERBOSE(2,"DynamicCacheBasedLanguageModel::Execute(std::string command:|" << command << "|" << std::endl);
+  std::vector<std::string> commands = Tokenize(command, "||");
+  Execute(commands);
+}
+
+void DynamicCacheBasedLanguageModel::Execute(std::vector<std::string> commands)
+{
+  for (size_t j=0; j<commands.size(); j++) {
+    Execute_Single_Command(commands[j]);
+  }
+  IFVERBOSE(2) Print();
+}
+
+void DynamicCacheBasedLanguageModel::Execute_Single_Command(std::string command)
+{
+  VERBOSE(2,"CacheBasedLanguageModel::Execute_Single_Command(std::string command:|" << command << "|" << std::endl);
+  if (command == "clear") {
+    VERBOSE(2,"CacheBasedLanguageModel Execute command:|"<< command << "|. Cache cleared." << std::endl);
+    Clear();
+  } else if (command == "settype_wholestring") {
+    VERBOSE(2,"CacheBasedLanguageModel Execute command:|"<< command << "|. Query type set to " << CBLM_QUERY_TYPE_WHOLESTRING << " (CBLM_QUERY_TYPE_WHOLESTRING)." << std::endl);
+    SetQueryType(CBLM_QUERY_TYPE_WHOLESTRING);
+  } else if (command == "settype_allsubstrings") {
+    VERBOSE(2,"CacheBasedLanguageModel Execute command:|"<< command << "|. Query type set to " << CBLM_QUERY_TYPE_ALLSUBSTRINGS << " (CBLM_QUERY_TYPE_ALLSUBSTRINGS)." << std::endl);
+    SetQueryType(CBLM_QUERY_TYPE_ALLSUBSTRINGS);
+  } else {
+    VERBOSE(2,"CacheBasedLanguageModel Execute command:|"<< command << "| is unknown. Skipped." << std::endl);
+  }
+}
+
+void DynamicCacheBasedLanguageModel::Clear()
+{
+#ifdef WITH_THREADS
+  boost::shared_lock<boost::shared_mutex> lock(m_cacheLock);
+#endif
+  m_cache.clear();
+}
+
+void DynamicCacheBasedLanguageModel::Load(AllOptions::ptr const& opts)
+{
+  m_options = opts;
+//  SetPreComputedScores();
+  VERBOSE(2,"DynamicCacheBasedLanguageModel::Load()" << std::endl);
+  Load(m_initfiles);
+}
+
+void DynamicCacheBasedLanguageModel::Load(const std::string filestr)
+{
+  VERBOSE(2,"DynamicCacheBasedLanguageModel::Load(const std::string filestr)" << std::endl);
+//  std::vector<std::string> files = Tokenize(m_initfiles, "||");
+  std::vector<std::string> files = Tokenize(filestr, "||");
+  Load_Multiple_Files(files);
+}
+
+
+void DynamicCacheBasedLanguageModel::Load_Multiple_Files(std::vector<std::string> files)
+{
+  VERBOSE(2,"DynamicCacheBasedLanguageModel::Load_Multiple_Files(std::vector<std::string> files)" << std::endl);
+  for(size_t j = 0; j < files.size(); ++j) {
+    Load_Single_File(files[j]);
+  }
+}
+
+void DynamicCacheBasedLanguageModel::Load_Single_File(const std::string file)
+{
+  VERBOSE(2,"DynamicCacheBasedLanguageModel::Load_Single_File(const std::string file)" << std::endl);
+  //file format
+  //age || n-gram
+  //age || n-gram || n-gram || n-gram || ...
+  //....
+  //each n-gram is a sequence of n words (no matter of n)
+  //
+  //there is no limit on the size of n
+  //
+  //entries can be repeated, but the last entry overwrites the previous
+
+
+  VERBOSE(2,"Loading data from the cache file " << file << std::endl);
+  InputFileStream cacheFile(file);
+
+  std::string line;
+  int age;
+  std::vector<std::string> words;
+
+  while (getline(cacheFile, line)) {
+    std::vector<std::string> vecStr = TokenizeMultiCharSeparator( line , "||" );
+    if (vecStr.size() >= 2) {
+      age = Scan<int>(vecStr[0]);
+      vecStr.erase(vecStr.begin());
+      Update(vecStr,age);
+    } else {
+      UTIL_THROW_IF2(false, "The format of the loaded file is wrong: " << line);
+    }
+  }
+  IFVERBOSE(2) Print();
+}
+
+void DynamicCacheBasedLanguageModel::SetQueryType(size_t type)
+{
+#ifdef WITH_THREADS
+  boost::shared_lock<boost::shared_mutex> read_lock(m_cacheLock);
+#endif
+
+  m_query_type = type;
+  if ( m_query_type != CBLM_QUERY_TYPE_WHOLESTRING
+       && m_query_type != CBLM_QUERY_TYPE_ALLSUBSTRINGS ) {
+    VERBOSE(2, "This query type " << m_query_type << " is unknown. Instead used " << CBLM_QUERY_TYPE_ALLSUBSTRINGS << "." << std::endl);
+    m_query_type = CBLM_QUERY_TYPE_ALLSUBSTRINGS;
+  }
+  VERBOSE(2, "CacheBasedLanguageModel QueryType:  " << m_query_type << std::endl);
+
+};
+
+void DynamicCacheBasedLanguageModel::SetScoreType(size_t type)
+{
+#ifdef WITH_THREADS
+  boost::shared_lock<boost::shared_mutex> read_lock(m_cacheLock);
+#endif
+  m_score_type = type;
+  if ( m_score_type != CBLM_SCORE_TYPE_HYPERBOLA
+       && m_score_type != CBLM_SCORE_TYPE_POWER
+       && m_score_type != CBLM_SCORE_TYPE_EXPONENTIAL
+       && m_score_type != CBLM_SCORE_TYPE_COSINE
+       && m_score_type != CBLM_SCORE_TYPE_HYPERBOLA_REWARD
+       && m_score_type != CBLM_SCORE_TYPE_POWER_REWARD
+       && m_score_type != CBLM_SCORE_TYPE_EXPONENTIAL_REWARD ) {
+    VERBOSE(2, "This score type " << m_score_type << " is unknown. Instead used " << CBLM_SCORE_TYPE_HYPERBOLA << "." << std::endl);
+    m_score_type = CBLM_SCORE_TYPE_HYPERBOLA;
+  }
+  VERBOSE(2, "CacheBasedLanguageModel ScoreType:  " << m_score_type << std::endl);
+};
+
+void DynamicCacheBasedLanguageModel::SetMaxAge(unsigned int age)
+{
+#ifdef WITH_THREADS
+  boost::shared_lock<boost::shared_mutex> read_lock(m_cacheLock);
+#endif
+  m_maxAge = age;
+  VERBOSE(2, "CacheBasedLanguageModel MaxAge:  " << m_maxAge << std::endl);
+};
+
+float DynamicCacheBasedLanguageModel::decaying_score(const unsigned int age)
+{
+  float sc;
+  switch(m_score_type) {
+  case CBLM_SCORE_TYPE_HYPERBOLA:
+    sc = (float) 1.0/age - 1.0;
+    break;
+  case CBLM_SCORE_TYPE_POWER:
+    sc = (float) pow(age, -0.25) - 1.0;
+    break;
+  case CBLM_SCORE_TYPE_EXPONENTIAL:
+    sc = (age == 1) ? 0.0 : (float) exp( 1.0/age ) / exp(1.0) - 1.0;
+    break;
+  case CBLM_SCORE_TYPE_COSINE:
+    sc = (float) cos( (age-1) * (PI/2) / m_maxAge ) - 1.0;
+    break;
+  case CBLM_SCORE_TYPE_HYPERBOLA_REWARD:
+    sc = (float) 1.0/age;
+    break;
+  case CBLM_SCORE_TYPE_POWER_REWARD:
+    sc = (float) pow(age, -0.25);
+    break;
+  case CBLM_SCORE_TYPE_EXPONENTIAL_REWARD:
+    sc = (age == 1) ? 1.0 : (float) exp( 1.0/age ) / exp(1.0);
+    break;
+  default:
+    sc = -1.0;
+  }
+  return sc;
+}
+}

mosesdecoder/moses/FF/DynamicCacheBasedLanguageModel.h

@@ -0,0 +1,164 @@
+// $Id$
+
+#ifndef moses_DynamicCacheBasedLanguageModel_h
+#define moses_DynamicCacheBasedLanguageModel_h
+
+#include "moses/Util.h"
+#include "FeatureFunction.h"
+
+#ifdef WITH_THREADS
+#include <boost/thread/shared_mutex.hpp>
+#include <boost/thread/locks.hpp>
+#endif
+
+typedef std::pair<int, float> decaying_cache_value_t;
+typedef std::map<std::string, decaying_cache_value_t > decaying_cache_t;
+
+#define CBLM_QUERY_TYPE_UNDEFINED (-1)
+#define CBLM_QUERY_TYPE_ALLSUBSTRINGS 0
+#define CBLM_QUERY_TYPE_WHOLESTRING 1
+
+#define CBLM_SCORE_TYPE_UNDEFINED (-1)
+#define CBLM_SCORE_TYPE_HYPERBOLA 0
+#define CBLM_SCORE_TYPE_POWER 1
+#define CBLM_SCORE_TYPE_EXPONENTIAL 2
+#define CBLM_SCORE_TYPE_COSINE 3
+#define CBLM_SCORE_TYPE_HYPERBOLA_REWARD 10
+#define CBLM_SCORE_TYPE_POWER_REWARD 11
+#define CBLM_SCORE_TYPE_EXPONENTIAL_REWARD 12
+#define PI 3.14159265
+
+namespace Moses
+{
+
+class Range;
+
+/** Calculates score for the Dynamic Cache-Based pseudo LM
+ */
+class DynamicCacheBasedLanguageModel : public StatelessFeatureFunction
+{
+  // data structure for the cache;
+  // the key is the word and the value is the decaying score
+  decaying_cache_t m_cache;
+  size_t m_query_type; //way of querying the cache
+  size_t m_score_type; //way of scoring entries of the cache
+  std::string m_initfiles; // vector of files loaded in the initialization phase
+  std::string m_name; // internal name to identify this instance of the Cache-based pseudo LM
+  float m_lower_score; //lower_bound_score for no match
+  bool m_constant; //flag for setting a non-decaying cache
+  std::vector<float> precomputedScores;
+  unsigned int m_maxAge;
+
+#ifdef WITH_THREADS
+  //multiple readers - single writer lock
+  mutable boost::shared_mutex m_cacheLock;
+#endif
+
+  float decaying_score(unsigned int age);
+  void SetPreComputedScores();
+  float GetPreComputedScores(const unsigned int age);
+
+  float Evaluate_Whole_String( const TargetPhrase&) const;
+  float Evaluate_All_Substrings( const TargetPhrase&) const;
+
+  void Decay();
+  void Update(std::vector<std::string> words, int age);
+
+  void ClearEntries(std::vector<std::string> entries);
+
+  void Execute(std::vector<std::string> commands);
+  void Execute_Single_Command(std::string command);
+
+  void Load_Multiple_Files(std::vector<std::string> files);
+  void Load_Single_File(const std::string file);
+
+  void Insert(std::vector<std::string> ngrams);
+
+//  void EvaluateInIsolation(const Phrase&, const TargetPhrase&, ScoreComponentCollection&, ScoreComponentCollection& ) const;
+  void Print() const;
+
+protected:
+  static DynamicCacheBasedLanguageModel* s_instance;
+  static std::map< const std::string, DynamicCacheBasedLanguageModel* > s_instance_map;
+
+public:
+  DynamicCacheBasedLanguageModel(const std::string &line);
+  ~DynamicCacheBasedLanguageModel();
+
+  inline const std::string GetName() {
+    return m_name;
+  };
+  inline void SetName(const std::string name) {
+    m_name = name;
+  }
+
+  static const DynamicCacheBasedLanguageModel* Instance(const std::string& name) {
+    if (s_instance_map.find(name) == s_instance_map.end()) {
+      return NULL;
+    }
+    return s_instance_map[name];
+  }
+
+  static DynamicCacheBasedLanguageModel* InstanceNonConst(const std::string& name) {
+    if (s_instance_map.find(name) == s_instance_map.end()) {
+      return NULL;
+    }
+    return s_instance_map[name];
+  }
+
+
+
+  static const DynamicCacheBasedLanguageModel& Instance() {
+    return *s_instance;
+  }
+  static DynamicCacheBasedLanguageModel& InstanceNonConst() {
+    return *s_instance;
+  }
+
+  bool IsUseable(const FactorMask &mask) const {
+    return true;
+  }
+
+  void Load(AllOptions::ptr const& opts);
+  void Load(const std::string filestr);
+  void Execute(std::string command);
+  void SetParameter(const std::string& key, const std::string& value);
+  void ExecuteDlt(std::map<std::string, std::string> dlt_meta);
+
+  void ClearEntries(std::string &entries);
+  void Insert(std::string &entries);
+  void Clear();
+
+  virtual void EvaluateInIsolation(const Phrase &source
+                                   , const TargetPhrase &targetPhrase
+                                   , ScoreComponentCollection &scoreBreakdown
+                                   , ScoreComponentCollection &estimatedScores) const;
+
+  void EvaluateWithSourceContext(const InputType &input
+                                 , const InputPath &inputPath
+                                 , const TargetPhrase &targetPhrase
+                                 , const StackVec *stackVec
+                                 , ScoreComponentCollection &scoreBreakdown
+                                 , ScoreComponentCollection *estimatedScores = NULL) const {
+  }
+
+  void EvaluateTranslationOptionListWithSourceContext(const InputType &input
+      , const TranslationOptionList &translationOptionList) const {
+  }
+
+  void EvaluateWhenApplied(const Hypothesis& hypo,
+                           ScoreComponentCollection* accumulator) const {
+  }
+
+  void EvaluateWhenApplied(const ChartHypothesis &hypo,
+                           ScoreComponentCollection* accumulator) const {
+  }
+
+  void SetQueryType(size_t type);
+  void SetScoreType(size_t type);
+  void SetMaxAge(unsigned int age);
+};
+
+}
+
+#endif

mosesdecoder/moses/FF/EditOps.h

@@ -0,0 +1,64 @@
+#ifndef moses_EditOps_h
+#define moses_EditOps_h
+
+#include <string>
+#include <boost/unordered_set.hpp>
+
+#include "StatelessFeatureFunction.h"
+#include "moses/FactorCollection.h"
+#include "moses/AlignmentInfo.h"
+
+namespace Moses
+{
+
+typedef std::vector<std::string> Tokens;
+
+/** Calculates string edit operations that transform source phrase into target
+ * phrase using the LCS algorithm. Potentially usefule for monolingual tasks
+ * like paraphrasing, summarization, correction.
+ */
+class EditOps : public StatelessFeatureFunction
+{
+private:
+  FactorType m_factorType;
+  bool m_chars;
+  std::string m_scores;
+
+public:
+  EditOps(const std::string &line);
+
+  bool IsUseable(const FactorMask &mask) const;
+
+  void Load();
+
+  virtual void EvaluateInIsolation(const Phrase &source
+                                   , const TargetPhrase &targetPhrase
+                                   , ScoreComponentCollection &scoreBreakdown
+                                   , ScoreComponentCollection &estimatedFutureScore) const;
+
+  void EvaluateWithSourceContext(const InputType &input
+                                 , const InputPath &inputPath
+                                 , const TargetPhrase &targetPhrase
+                                 , const StackVec *stackVec
+                                 , ScoreComponentCollection &scoreBreakdown
+                                 , ScoreComponentCollection *estimatedFutureScore = NULL) const
+  {}
+  void EvaluateWhenApplied(const Hypothesis& hypo,
+                           ScoreComponentCollection* accumulator) const
+  {}
+  void EvaluateWhenApplied(const ChartHypothesis &hypo,
+                           ScoreComponentCollection* accumulator) const
+  {}
+  void EvaluateTranslationOptionListWithSourceContext(const InputType &input
+      , const TranslationOptionList &translationOptionList) const
+  {}
+
+  void ComputeFeatures(const Phrase &source,
+                       const TargetPhrase& targetPhrase,
+                       ScoreComponentCollection* accumulator) const;
+  void SetParameter(const std::string& key, const std::string& value);
+};
+
+}
+
+#endif // moses_CorrectionPattern_h

mosesdecoder/moses/FF/ExampleStatelessFF.cpp

@@ -0,0 +1,69 @@
+#include <vector>
+#include "ExampleStatelessFF.h"
+#include "moses/ScoreComponentCollection.h"
+#include "moses/TargetPhrase.h"
+
+using namespace std;
+
+namespace Moses
+{
+ExampleStatelessFF::ExampleStatelessFF(const std::string &line)
+  :StatelessFeatureFunction(2, line)
+{
+  ReadParameters();
+}
+
+void ExampleStatelessFF::EvaluateInIsolation(const Phrase &source
+    , const TargetPhrase &targetPhrase
+    , ScoreComponentCollection &scoreBreakdown
+    , ScoreComponentCollection &estimatedScores) const
+{
+  // dense scores
+  vector<float> newScores(m_numScoreComponents);
+  newScores[0] = 1.5;
+  newScores[1] = 0.3;
+  scoreBreakdown.PlusEquals(this, newScores);
+
+  // sparse scores
+  scoreBreakdown.PlusEquals(this, "sparse-name", 2.4);
+
+}
+
+void ExampleStatelessFF::EvaluateWithSourceContext(const InputType &input
+    , const InputPath &inputPath
+    , const TargetPhrase &targetPhrase
+    , const StackVec *stackVec
+    , ScoreComponentCollection &scoreBreakdown
+    , ScoreComponentCollection *estimatedScores) const
+{
+  if (targetPhrase.GetNumNonTerminals()) {
+    vector<float> newScores(m_numScoreComponents);
+    newScores[0] = - std::numeric_limits<float>::infinity();
+    scoreBreakdown.PlusEquals(this, newScores);
+  }
+}
+
+void ExampleStatelessFF::EvaluateTranslationOptionListWithSourceContext(const InputType &input
+
+    , const TranslationOptionList &translationOptionList) const
+{}
+
+void ExampleStatelessFF::EvaluateWhenApplied(const Hypothesis& hypo,
+    ScoreComponentCollection* accumulator) const
+{}
+
+void ExampleStatelessFF::EvaluateWhenApplied(const ChartHypothesis &hypo,
+    ScoreComponentCollection* accumulator) const
+{}
+
+void ExampleStatelessFF::SetParameter(const std::string& key, const std::string& value)
+{
+  if (key == "arg") {
+    // set value here
+  } else {
+    StatelessFeatureFunction::SetParameter(key, value);
+  }
+}
+
+}
+

mosesdecoder/moses/FF/ExampleTranslationOptionListFeature.h

@@ -0,0 +1,67 @@
+#pragma once
+
+#include <string>
+#include "StatelessFeatureFunction.h"
+
+namespace Moses
+{
+
+class ExampleTranslationOptionListFeature : public StatelessFeatureFunction
+{
+public:
+  ExampleTranslationOptionListFeature(const std::string &line)
+    :StatelessFeatureFunction(1, line) {
+    ReadParameters();
+  }
+
+  bool IsUseable(const FactorMask &mask) const {
+    return true;
+  }
+
+  void EvaluateInIsolation(const Phrase &source
+                           , const TargetPhrase &targetPhrase
+                           , ScoreComponentCollection &scoreBreakdown
+                           , ScoreComponentCollection &estimatedFutureScore) const {
+  }
+
+  void EvaluateWithSourceContext(const InputType &input
+                                 , const InputPath &inputPath
+                                 , const TargetPhrase &targetPhrase
+                                 , const StackVec *stackVec
+                                 , ScoreComponentCollection &scoreBreakdown
+                                 , ScoreComponentCollection *estimatedFutureScore = NULL) const {
+  }
+
+  void EvaluateTranslationOptionListWithSourceContext(const InputType &input
+      , const TranslationOptionList &translationOptionList) const {
+    std::vector<float> newScores(m_numScoreComponents);
+    newScores[0] = translationOptionList.size();
+
+    TranslationOptionList::const_iterator iterTransOpt;
+    for(iterTransOpt = translationOptionList.begin() ;
+        iterTransOpt != translationOptionList.end() ; ++iterTransOpt) {
+      TranslationOption &transOpt = **iterTransOpt;
+
+      ScoreComponentCollection &scoreBreakDown = transOpt.GetScoreBreakdown();
+      scoreBreakDown.PlusEquals(this, newScores);
+
+      transOpt.UpdateScore();
+    }
+  }
+
+  void EvaluateWhenApplied(const Hypothesis& hypo,
+                           ScoreComponentCollection* accumulator) const {
+  }
+
+  void EvaluateWhenApplied(const ChartHypothesis &hypo,
+                           ScoreComponentCollection* accumulator) const {
+  }
+
+
+  void SetParameter(const std::string& key, const std::string& value) {
+  }
+
+};
+
+}
+

mosesdecoder/moses/FF/FFState.cpp

@@ -0,0 +1,9 @@
+#include "moses/FF/FFState.h"
+
+namespace Moses
+{
+
+FFState::~FFState() {}
+
+}
+

mosesdecoder/moses/FF/FFState.h

@@ -0,0 +1,39 @@
+#ifndef moses_FFState_h
+#define moses_FFState_h
+
+#include <vector>
+#include <stddef.h>
+#include "util/exception.hh"
+
+namespace Moses
+{
+
+class FFState
+{
+public:
+  virtual ~FFState();
+  virtual size_t hash() const = 0;
+  virtual bool operator==(const FFState& other) const = 0;
+
+  virtual bool operator!=(const FFState& other) const {
+    return !(*this == other);
+  }
+};
+
+class DummyState : public FFState
+{
+public:
+  DummyState()  {}
+
+  virtual size_t hash() const {
+    return 0;
+  }
+
+  virtual bool operator==(const FFState& other) const {
+    return true;
+  }
+
+};
+
+}
+#endif

mosesdecoder/moses/FF/FeatureFunction.h

@@ -0,0 +1,200 @@
+// -*- c++ -*-
+#ifndef moses_FeatureFunction_h
+#define moses_FeatureFunction_h
+
+#include <vector>
+#include <set>
+#include <string>
+#include "moses/FeatureVector.h"
+#include "moses/TypeDef.h"
+#include "moses/parameters/AllOptions.h"
+#include <boost/shared_ptr.hpp>
+
+namespace Moses
+{
+
+class AllOptions;
+class Phrase;
+class TargetPhrase;
+class TranslationOptionList;
+class TranslationOption;
+class Hypothesis;
+class ChartHypothesis;
+class InputType;
+class ScoreComponentCollection;
+class Bitmap;
+class Range;
+class FactorMask;
+class InputPath;
+class StackVec;
+class DistortionScoreProducer;
+class TranslationTask;
+
+/** base class for all feature functions.
+ */
+class FeatureFunction
+{
+protected:
+  /**< all the score producers in this run */
+  static std::vector<FeatureFunction*> s_staticColl;
+
+  std::string m_description, m_argLine;
+  std::vector<std::vector<std::string> > m_args;
+  bool m_tuneable;
+  bool m_requireSortingAfterSourceContext;
+  size_t m_verbosity;
+  size_t m_numScoreComponents;
+  size_t m_index; // index into vector covering ALL feature function values
+  std::vector<bool> m_tuneableComponents;
+  size_t m_numTuneableComponents;
+  AllOptions::ptr m_options;
+  //In case there's multiple producers with the same description
+  static std::multiset<std::string> description_counts;
+
+public:
+  static void Register(FeatureFunction* ff);
+private:
+  // void Initialize(const std::string &line);
+  void ParseLine(const std::string &line);
+
+public:
+  static const std::vector<FeatureFunction*>& GetFeatureFunctions() {
+    return s_staticColl;
+  }
+
+  static FeatureFunction &FindFeatureFunction(const std::string& name);
+  static void Destroy();
+
+  FeatureFunction(const std::string &line, bool registerNow);
+  FeatureFunction(size_t numScoreComponents, const std::string &line, bool registerNow = true);
+  virtual bool IsStateless() const = 0;
+  virtual ~FeatureFunction();
+
+  //! override to load model files
+  virtual void Load(AllOptions::ptr const& opts) {
+    m_options = opts;
+  }
+
+  AllOptions::ptr const&
+  options() const {
+    return m_options;
+  }
+
+  static void ResetDescriptionCounts() {
+    description_counts.clear();
+  }
+
+  //! returns the number of scores that a subclass produces.
+  //! For example, a language model conventionally produces 1, a translation table some arbitrary number, etc
+  size_t GetNumScoreComponents() const {
+    return m_numScoreComponents;
+  }
+
+  //! returns a string description of this producer
+  const std::string& GetScoreProducerDescription() const {
+    return m_description;
+  }
+
+  FName GetFeatureName(const std::string& name) const {
+    return FName(GetScoreProducerDescription(), name);
+  }
+
+
+  //! if false, then this feature is not displayed in the n-best list.
+  // use with care
+  virtual bool IsTuneable() const {
+    return m_tuneable;
+  }
+
+  virtual bool HasTuneableComponents() const {
+    return m_numTuneableComponents;
+  }
+
+  virtual bool IsTuneableComponent(size_t i) const {
+    if (m_numTuneableComponents == m_numScoreComponents) {
+      return true;
+    }
+    return m_tuneableComponents[i];
+  }
+
+  virtual bool RequireSortingAfterSourceContext() const {
+    return m_requireSortingAfterSourceContext;
+  }
+
+  virtual std::vector<float> DefaultWeights() const;
+
+  size_t GetIndex() const;
+  size_t SetIndex(size_t const idx);
+
+protected:
+  virtual void
+  CleanUpAfterSentenceProcessing(InputType const& source) { }
+
+public:
+  //! Called before search and collecting of translation options
+  virtual void
+  InitializeForInput(ttasksptr const& ttask) { };
+
+  // clean up temporary memory, called after processing each sentence
+  virtual void
+  CleanUpAfterSentenceProcessing(ttasksptr const& ttask);
+
+  const std::string &
+  GetArgLine() const {
+    return m_argLine;
+  }
+
+  // given a target phrase containing only factors specified in mask
+  // return true if the feature function can be evaluated
+  virtual bool IsUseable(const FactorMask &mask) const = 0;
+
+  // used by stateless ff and stateful ff. Calculate initial score
+  // estimate during loading of phrase table
+  //
+  // source phrase is the substring that the phrase table uses to look
+  // up the target phrase,
+  //
+  // may have more factors than actually need, but not guaranteed.
+  // For SCFG decoding, the source contains non-terminals, NOT the raw
+  // source from the input sentence
+  virtual void
+  EvaluateInIsolation(const Phrase &source, const TargetPhrase &targetPhrase,
+                      ScoreComponentCollection& scoreBreakdown,
+                      ScoreComponentCollection& estimatedScores) const = 0;
+
+  // for context-dependent processing
+  static void SetupAll(TranslationTask const& task);
+  virtual void Setup(TranslationTask const& task) const { };
+
+  // This method is called once all the translation options are retrieved from the phrase table, and
+  // just before search.
+  // 'inputPath' is guaranteed to be the raw substring from the input. No factors were added or taken away
+  // 'stackVec' is a vector of chart cells that the RHS non-terms cover.
+  // It is guaranteed to be in the same order as the non-terms in the source phrase.
+  // For pb models, stackvec is NULL.
+  // No FF should set estimatedScores in both overloads!
+  virtual void EvaluateWithSourceContext(const InputType &input
+                                         , const InputPath &inputPath
+                                         , const TargetPhrase &targetPhrase
+                                         , const StackVec *stackVec
+                                         , ScoreComponentCollection &scoreBreakdown
+                                         , ScoreComponentCollection *estimatedScores = NULL) const = 0;
+
+  // This method is called once all the translation options are retrieved from the phrase table, and
+  // just before search.
+  // 'inputPath' is guaranteed to be the raw substring from the input. No factors were added or taken away
+  // 'stackVec' is a vector of chart cells that the RHS non-terms cover.
+  // It is guaranteed to be in the same order as the non-terms in the source phrase.
+  // For pb models, stackvec is NULL.
+  // No FF should set estimatedScores in both overloads!
+  virtual void EvaluateTranslationOptionListWithSourceContext(const InputType &input
+      , const TranslationOptionList &translationOptionList) const = 0;
+
+  virtual void SetParameter(const std::string& key, const std::string& value);
+  virtual void ReadParameters();
+  virtual void SetTuneableComponents(const std::string& value);
+};
+
+}
+
+#endif

mosesdecoder/moses/FF/GlobalLexicalModel.h

@@ -0,0 +1,102 @@
+#ifndef moses_GlobalLexicalModel_h
+#define moses_GlobalLexicalModel_h
+
+#include <stdexcept>
+#include <string>
+#include <vector>
+#include <memory>
+#include "StatelessFeatureFunction.h"
+#include "moses/Factor.h"
+#include "moses/Phrase.h"
+#include "moses/TypeDef.h"
+#include "moses/Util.h"
+#include "moses/Range.h"
+#include "moses/FactorTypeSet.h"
+#include "moses/Sentence.h"
+
+#ifdef WITH_THREADS
+#include <boost/thread/tss.hpp>
+#endif
+
+namespace Moses
+{
+
+class Factor;
+class Phrase;
+class Hypothesis;
+class InputType;
+
+/** Discriminatively trained global lexicon model
+ * This is a implementation of Mauser et al., 2009's model that predicts
+ * each output word from _all_ the input words. The intuition behind this
+ * feature is that it uses context words for disambiguation
+ */
+class GlobalLexicalModel : public StatelessFeatureFunction
+{
+  typedef boost::unordered_map< const Word*,
+          boost::unordered_map< const Word*, float, UnorderedComparer<Word> , UnorderedComparer<Word> >,
+          UnorderedComparer<Word>, UnorderedComparer<Word> > DoubleHash;
+  typedef boost::unordered_map< const Word*, float, UnorderedComparer<Word>, UnorderedComparer<Word> > SingleHash;
+  typedef std::map< const TargetPhrase*, float > LexiconCache;
+
+  struct ThreadLocalStorage {
+    LexiconCache cache;
+    const Sentence *input;
+  };
+
+private:
+  DoubleHash m_hash;
+#ifdef WITH_THREADS
+  boost::thread_specific_ptr<ThreadLocalStorage> m_local;
+#else
+  std::auto_ptr<ThreadLocalStorage> m_local;
+#endif
+  Word *m_bias;
+
+  FactorMask m_inputFactors, m_outputFactors;
+  std::vector<FactorType> m_inputFactorsVec, m_outputFactorsVec;
+  std::string m_filePath;
+
+  void Load(AllOptions::ptr const& opts);
+
+  float ScorePhrase( const TargetPhrase& targetPhrase ) const;
+  float GetFromCacheOrScorePhrase( const TargetPhrase& targetPhrase ) const;
+
+public:
+  GlobalLexicalModel(const std::string &line);
+  virtual ~GlobalLexicalModel();
+
+  void SetParameter(const std::string& key, const std::string& value);
+
+  void InitializeForInput(ttasksptr const& ttask);
+
+  bool IsUseable(const FactorMask &mask) const;
+
+  void EvaluateInIsolation(const Phrase &source
+                           , const TargetPhrase &targetPhrase
+                           , ScoreComponentCollection &scoreBreakdown
+                           , ScoreComponentCollection &estimatedScores) const {
+  }
+
+  void EvaluateWhenApplied(const Hypothesis& hypo,
+                           ScoreComponentCollection* accumulator) const {
+  }
+  void EvaluateWhenApplied(const ChartHypothesis &hypo,
+                           ScoreComponentCollection* accumulator) const {
+  }
+
+  void EvaluateWithSourceContext(const InputType &input
+                                 , const InputPath &inputPath
+                                 , const TargetPhrase &targetPhrase
+                                 , const StackVec *stackVec
+                                 , ScoreComponentCollection &scoreBreakdown
+                                 , ScoreComponentCollection *estimatedScores = NULL) const;
+
+  void EvaluateTranslationOptionListWithSourceContext(const InputType &input
+      , const TranslationOptionList &translationOptionList) const {
+  }
+
+};
+
+}
+#endif

mosesdecoder/moses/FF/GlobalLexicalModelUnlimited.cpp

@@ -0,0 +1,340 @@
+#include "GlobalLexicalModelUnlimited.h"
+#include <fstream>
+#include "moses/StaticData.h"
+#include "moses/InputFileStream.h"
+#include "moses/Hypothesis.h"
+#include "moses/TranslationTask.h"
+#include "util/string_piece_hash.hh"
+#include "util/string_stream.hh"
+
+using namespace std;
+
+namespace Moses
+{
+GlobalLexicalModelUnlimited::GlobalLexicalModelUnlimited(const std::string &line)
+  :StatelessFeatureFunction(0, line)
+{
+  UTIL_THROW(util::Exception,
+             "GlobalLexicalModelUnlimited hasn't been refactored for new feature function framework yet"); // TODO need to update arguments to key=value
+
+  const vector<string> modelSpec = Tokenize(line);
+
+  for (size_t i = 0; i < modelSpec.size(); i++ ) {
+    bool ignorePunctuation = true, biasFeature = false, restricted = false;
+    size_t context = 0;
+    string filenameSource, filenameTarget;
+    vector< string > factors;
+    vector< string > spec = Tokenize(modelSpec[i]," ");
+
+    // read optional punctuation and bias specifications
+    if (spec.size() > 0) {
+      if (spec.size() != 2 && spec.size() != 3 && spec.size() != 4 && spec.size() != 6) {
+        std::cerr << "Format of glm feature is <factor-src>-<factor-tgt> [ignore-punct] [use-bias] "
+                  <<  "[context-type] [filename-src filename-tgt]";
+        //return false;
+      }
+
+      factors = Tokenize(spec[0],"-");
+      if (spec.size() >= 2)
+        ignorePunctuation = Scan<size_t>(spec[1]);
+      if (spec.size() >= 3)
+        biasFeature = Scan<size_t>(spec[2]);
+      if (spec.size() >= 4)
+        context = Scan<size_t>(spec[3]);
+      if (spec.size() == 6) {
+        filenameSource = spec[4];
+        filenameTarget = spec[5];
+        restricted = true;
+      }
+    } else
+      factors = Tokenize(modelSpec[i],"-");
+
+    if ( factors.size() != 2 ) {
+      std::cerr << "Wrong factor definition for global lexical model unlimited: " << modelSpec[i];
+      //return false;
+    }
+
+    const vector<FactorType> inputFactors = Tokenize<FactorType>(factors[0],",");
+    const vector<FactorType> outputFactors = Tokenize<FactorType>(factors[1],",");
+    throw runtime_error("GlobalLexicalModelUnlimited should be reimplemented as a stateful feature");
+    GlobalLexicalModelUnlimited* glmu = NULL; // new GlobalLexicalModelUnlimited(inputFactors, outputFactors, biasFeature, ignorePunctuation, context);
+
+    if (restricted) {
+      cerr << "loading word translation word lists from " << filenameSource << " and " << filenameTarget << endl;
+      if (!glmu->Load(filenameSource, filenameTarget)) {
+        std::cerr << "Unable to load word lists for word translation feature from files "
+                  << filenameSource
+                  << " and "
+                  << filenameTarget;
+        //return false;
+      }
+    }
+  }
+}
+
+bool GlobalLexicalModelUnlimited::Load(const std::string &filePathSource,
+                                       const std::string &filePathTarget)
+{
+  // restricted source word vocabulary
+  ifstream inFileSource(filePathSource.c_str());
+  if (!inFileSource) {
+    cerr << "could not open file " << filePathSource << endl;
+    return false;
+  }
+
+  std::string line;
+  while (getline(inFileSource, line)) {
+    m_vocabSource.insert(line);
+  }
+
+  inFileSource.close();
+
+  // restricted target word vocabulary
+  ifstream inFileTarget(filePathTarget.c_str());
+  if (!inFileTarget) {
+    cerr << "could not open file " << filePathTarget << endl;
+    return false;
+  }
+
+  while (getline(inFileTarget, line)) {
+    m_vocabTarget.insert(line);
+  }
+
+  inFileTarget.close();
+
+  m_unrestricted = false;
+  return true;
+}
+
+void GlobalLexicalModelUnlimited::InitializeForInput(ttasksptr const& ttask)
+{
+  UTIL_THROW_IF2(ttask->GetSource()->GetType() != SentenceInput,
+                 "GlobalLexicalModel works only with sentence input.");
+  Sentence const* s = reinterpret_cast<Sentence const*>(ttask->GetSource().get());
+  m_local.reset(new ThreadLocalStorage);
+  m_local->input = s;
+}
+
+void GlobalLexicalModelUnlimited::EvaluateWhenApplied(const Hypothesis& cur_hypo, ScoreComponentCollection* accumulator) const
+{
+  const Sentence& input = *(m_local->input);
+  const TargetPhrase& targetPhrase = cur_hypo.GetCurrTargetPhrase();
+
+  for(size_t targetIndex = 0; targetIndex < targetPhrase.GetSize(); targetIndex++ ) {
+    StringPiece targetString = targetPhrase.GetWord(targetIndex).GetString(0); // TODO: change for other factors
+
+    if (m_ignorePunctuation) {
+      // check if first char is punctuation
+      char firstChar = targetString[0];
+      CharHash::const_iterator charIterator = m_punctuationHash.find( firstChar );
+      if(charIterator != m_punctuationHash.end())
+        continue;
+    }
+
+    if (m_biasFeature) {
+      util::StringStream feature;
+      feature << "glm_";
+      feature << targetString;
+      feature << "~";
+      feature << "**BIAS**";
+      accumulator->SparsePlusEquals(feature.str(), 1);
+    }
+
+    boost::unordered_set<uint64_t> alreadyScored;
+    for(size_t sourceIndex = 0; sourceIndex < input.GetSize(); sourceIndex++ ) {
+      const StringPiece sourceString = input.GetWord(sourceIndex).GetString(0);
+      // TODO: change for other factors
+
+      if (m_ignorePunctuation) {
+        // check if first char is punctuation
+        char firstChar = sourceString[0];
+        CharHash::const_iterator charIterator = m_punctuationHash.find( firstChar );
+        if(charIterator != m_punctuationHash.end())
+          continue;
+      }
+      const uint64_t sourceHash = util::MurmurHashNative(sourceString.data(), sourceString.size());
+
+      if ( alreadyScored.find(sourceHash) == alreadyScored.end()) {
+        bool sourceExists, targetExists;
+        if (!m_unrestricted) {
+          sourceExists = FindStringPiece(m_vocabSource, sourceString ) != m_vocabSource.end();
+          targetExists = FindStringPiece(m_vocabTarget, targetString) != m_vocabTarget.end();
+        }
+
+        // no feature if vocab is in use and both words are not in restricted vocabularies
+        if (m_unrestricted || (sourceExists && targetExists)) {
+          if (m_sourceContext) {
+            if (sourceIndex == 0) {
+              // add <s> trigger feature for source
+              util::StringStream feature;
+              feature << "glm_";
+              feature << targetString;
+              feature << "~";
+              feature << "<s>,";
+              feature << sourceString;
+              accumulator->SparsePlusEquals(feature.str(), 1);
+              alreadyScored.insert(sourceHash);
+            }
+
+            // add source words to the right of current source word as context
+            for(int contextIndex = sourceIndex+1; contextIndex < input.GetSize(); contextIndex++ ) {
+              StringPiece contextString = input.GetWord(contextIndex).GetString(0); // TODO: change for other factors
+              bool contextExists;
+              if (!m_unrestricted)
+                contextExists = FindStringPiece(m_vocabSource, contextString ) != m_vocabSource.end();
+
+              if (m_unrestricted || contextExists) {
+                util::StringStream feature;
+                feature << "glm_";
+                feature << targetString;
+                feature << "~";
+                feature << sourceString;
+                feature << ",";
+                feature << contextString;
+                accumulator->SparsePlusEquals(feature.str(), 1);
+                alreadyScored.insert(sourceHash);
+              }
+            }
+          } else if (m_biphrase) {
+            // --> look backwards for constructing context
+            int globalTargetIndex = cur_hypo.GetSize() - targetPhrase.GetSize() + targetIndex;
+
+            // 1) source-target pair, trigger source word (can be discont.) and adjacent target word (bigram)
+            StringPiece targetContext;
+            if (globalTargetIndex > 0)
+              targetContext = cur_hypo.GetWord(globalTargetIndex-1).GetString(0); // TODO: change for other factors
+            else
+              targetContext = "<s>";
+
+            if (sourceIndex == 0) {
+              StringPiece sourceTrigger = "<s>";
+              AddFeature(accumulator, sourceTrigger, sourceString,
+                         targetContext, targetString);
+            } else
+              for(int contextIndex = sourceIndex-1; contextIndex >= 0; contextIndex-- ) {
+                StringPiece sourceTrigger = input.GetWord(contextIndex).GetString(0); // TODO: change for other factors
+                bool sourceTriggerExists = false;
+                if (!m_unrestricted)
+                  sourceTriggerExists = FindStringPiece(m_vocabSource, sourceTrigger ) != m_vocabSource.end();
+
+                if (m_unrestricted || sourceTriggerExists)
+                  AddFeature(accumulator, sourceTrigger, sourceString,
+                             targetContext, targetString);
+              }
+
+            // 2) source-target pair, adjacent source word (bigram) and trigger target word (can be discont.)
+            StringPiece sourceContext;
+            if (sourceIndex-1 >= 0)
+              sourceContext = input.GetWord(sourceIndex-1).GetString(0); // TODO: change for other factors
+            else
+              sourceContext = "<s>";
+
+            if (globalTargetIndex == 0) {
+              string targetTrigger = "<s>";
+              AddFeature(accumulator, sourceContext, sourceString,
+                         targetTrigger, targetString);
+            } else
+              for(int globalContextIndex = globalTargetIndex-1; globalContextIndex >= 0; globalContextIndex-- ) {
+                StringPiece targetTrigger = cur_hypo.GetWord(globalContextIndex).GetString(0); // TODO: change for other factors
+                bool targetTriggerExists = false;
+                if (!m_unrestricted)
+                  targetTriggerExists = FindStringPiece(m_vocabTarget, targetTrigger ) != m_vocabTarget.end();
+
+                if (m_unrestricted || targetTriggerExists)
+                  AddFeature(accumulator, sourceContext, sourceString,
+                             targetTrigger, targetString);
+              }
+          } else if (m_bitrigger) {
+            // allow additional discont. triggers on both sides
+            int globalTargetIndex = cur_hypo.GetSize() - targetPhrase.GetSize() + targetIndex;
+
+            if (sourceIndex == 0) {
+              StringPiece sourceTrigger = "<s>";
+              bool sourceTriggerExists = true;
+
+              if (globalTargetIndex == 0) {
+                string targetTrigger = "<s>";
+                bool targetTriggerExists = true;
+
+                if (m_unrestricted || (sourceTriggerExists && targetTriggerExists))
+                  AddFeature(accumulator, sourceTrigger, sourceString,
+                             targetTrigger, targetString);
+              } else {
+                // iterate backwards over target
+                for(int globalContextIndex = globalTargetIndex-1; globalContextIndex >= 0; globalContextIndex-- ) {
+                  StringPiece targetTrigger = cur_hypo.GetWord(globalContextIndex).GetString(0); // TODO: change for other factors
+                  bool targetTriggerExists = false;
+                  if (!m_unrestricted)
+                    targetTriggerExists = FindStringPiece(m_vocabTarget, targetTrigger ) != m_vocabTarget.end();
+
+                  if (m_unrestricted || (sourceTriggerExists && targetTriggerExists))
+                    AddFeature(accumulator, sourceTrigger, sourceString,
+                               targetTrigger, targetString);
+                }
+              }
+            }
+            // iterate over both source and target
+            else {
+              // iterate backwards over source
+              for(int contextIndex = sourceIndex-1; contextIndex >= 0; contextIndex-- ) {
+                StringPiece sourceTrigger = input.GetWord(contextIndex).GetString(0); // TODO: change for other factors
+                bool sourceTriggerExists = false;
+                if (!m_unrestricted)
+                  sourceTriggerExists = FindStringPiece(m_vocabSource, sourceTrigger ) != m_vocabSource.end();
+
+                if (globalTargetIndex == 0) {
+                  string targetTrigger = "<s>";
+                  bool targetTriggerExists = true;
+
+                  if (m_unrestricted || (sourceTriggerExists && targetTriggerExists))
+                    AddFeature(accumulator, sourceTrigger, sourceString,
+                               targetTrigger, targetString);
+                } else {
+                  // iterate backwards over target
+                  for(int globalContextIndex = globalTargetIndex-1; globalContextIndex >= 0; globalContextIndex-- ) {
+                    StringPiece targetTrigger = cur_hypo.GetWord(globalContextIndex).GetString(0); // TODO: change for other factors
+                    bool targetTriggerExists = false;
+                    if (!m_unrestricted)
+                      targetTriggerExists = FindStringPiece(m_vocabTarget, targetTrigger ) != m_vocabTarget.end();
+
+                    if (m_unrestricted || (sourceTriggerExists && targetTriggerExists))
+                      AddFeature(accumulator, sourceTrigger, sourceString,
+                                 targetTrigger, targetString);
+                  }
+                }
+              }
+            }
+          } else {
+            util::StringStream feature;
+            feature << "glm_";
+            feature << targetString;
+            feature << "~";
+            feature << sourceString;
+            accumulator->SparsePlusEquals(feature.str(), 1);
+            alreadyScored.insert(sourceHash);
+
+          }
+        }
+      }
+    }
+  }
+}
+
+void GlobalLexicalModelUnlimited::AddFeature(ScoreComponentCollection* accumulator,
+    StringPiece sourceTrigger, StringPiece sourceWord,
+    StringPiece targetTrigger, StringPiece targetWord) const
+{
+  util::StringStream feature;
+  feature << "glm_";
+  feature << targetTrigger;
+  feature << ",";
+  feature << targetWord;
+  feature << "~";
+  feature << sourceTrigger;
+  feature << ",";
+  feature << sourceWord;
+  accumulator->SparsePlusEquals(feature.str(), 1);
+
+}
+
+}

mosesdecoder/moses/FF/HyperParameterAsWeight.cpp

@@ -0,0 +1,29 @@
+#include "HyperParameterAsWeight.h"
+#include "moses/StaticData.h"
+
+using namespace std;
+
+namespace Moses
+{
+
+HyperParameterAsWeight::HyperParameterAsWeight(const std::string &line)
+  :StatelessFeatureFunction(2, line)
+{
+  ReadParameters();
+
+  // hack into StaticData and change anything you want
+  // as an example, we have 2 weights and change
+  //   1. stack size
+  //   2. beam width
+  StaticData &staticData = StaticData::InstanceNonConst();
+
+  vector<float> weights = staticData.GetWeights(this);
+
+  staticData.m_options->search.stack_size = weights[0] * 1000;
+  staticData.m_options->search.beam_width = weights[1] * 10;
+
+}
+
+
+}
+

mosesdecoder/moses/FF/InputFeature.h

@@ -0,0 +1,70 @@
+#pragma once
+
+#include "InputFeature.h"
+#include "StatelessFeatureFunction.h"
+
+namespace Moses
+{
+
+
+class InputFeature : public StatelessFeatureFunction
+{
+protected:
+  static InputFeature *s_instance;
+
+  size_t m_numInputScores;
+  size_t m_numRealWordCount;
+  bool m_legacy;
+
+public:
+  static const InputFeature *InstancePtr() {
+    return s_instance;
+  }
+
+  InputFeature(const std::string &line);
+
+  void Load(AllOptions::ptr const& opts);
+
+  void SetParameter(const std::string& key, const std::string& value);
+
+  bool IsUseable(const FactorMask &mask) const {
+    return true;
+  }
+
+  size_t GetNumInputScores() const {
+    return m_numInputScores;
+  }
+  size_t GetNumRealWordsInInput() const {
+    return m_numRealWordCount;
+  }
+
+  void EvaluateInIsolation(const Phrase &source
+                           , const TargetPhrase &targetPhrase
+                           , ScoreComponentCollection &scoreBreakdown
+                           , ScoreComponentCollection &estimatedScores) const {
+  }
+
+  void EvaluateWithSourceContext(const InputType &input
+                                 , const InputPath &inputPath
+                                 , const TargetPhrase &targetPhrase
+                                 , const StackVec *stackVec
+                                 , ScoreComponentCollection &scoreBreakdown
+                                 , ScoreComponentCollection *estimatedScores = NULL) const;
+
+  void EvaluateTranslationOptionListWithSourceContext(const InputType &input
+      , const TranslationOptionList &translationOptionList) const {
+  }
+
+  void EvaluateWhenApplied(const Hypothesis& hypo,
+                           ScoreComponentCollection* accumulator) const {
+  }
+  void EvaluateWhenApplied(const ChartHypothesis &hypo,
+                           ScoreComponentCollection* accumulator) const {
+  }
+
+
+};
+
+
+}
+

mosesdecoder/moses/FF/Model1Feature.cpp

@@ -0,0 +1,276 @@
+#include "Model1Feature.h"
+#include "moses/StaticData.h"
+#include "moses/InputFileStream.h"
+#include "moses/ScoreComponentCollection.h"
+#include "moses/FactorCollection.h"
+
+
+using namespace std;
+
+namespace Moses
+{
+
+const std::string Model1Vocabulary::GIZANULL = "GIZANULL";
+
+Model1Vocabulary::Model1Vocabulary()
+{
+  FactorCollection &factorCollection = FactorCollection::Instance();
+  m_NULL = factorCollection.AddFactor(GIZANULL,false);
+  Store(m_NULL,0);
+}
+
+bool Model1Vocabulary::Store(const Factor* word, const unsigned id)
+{
+  boost::unordered_map<const Factor*, unsigned>::iterator iter = m_lookup.find( word );
+  if ( iter != m_lookup.end() ) {
+    return false;
+  }
+  m_lookup[ word ] = id;
+  if ( m_vocab.size() <= id ) {
+    m_vocab.resize(id+1);
+  }
+  m_vocab[id] = word;
+  return true;
+}
+
+unsigned Model1Vocabulary::StoreIfNew(const Factor* word)
+{
+  boost::unordered_map<const Factor*, unsigned>::iterator iter = m_lookup.find( word );
+
+  if ( iter != m_lookup.end() ) {
+    return iter->second;
+  }
+
+  unsigned id = m_vocab.size();
+  m_vocab.push_back( word );
+  m_lookup[ word ] = id;
+  return id;
+}
+
+unsigned Model1Vocabulary::GetWordID(const Factor* word) const
+{
+  boost::unordered_map<const Factor*, unsigned>::const_iterator iter = m_lookup.find( word );
+  if ( iter == m_lookup.end() ) {
+    return INVALID_ID;
+  }
+  return iter->second;
+}
+
+const Factor* Model1Vocabulary::GetWord(unsigned id) const
+{
+  if (id >= m_vocab.size()) {
+    return NULL;
+  }
+  return m_vocab[ id ];
+}
+
+void Model1Vocabulary::Load(const std::string& fileName)
+{
+  InputFileStream inFile(fileName);
+  FactorCollection &factorCollection = FactorCollection::Instance();
+  std::string line;
+
+  unsigned i = 0;
+  if ( getline(inFile, line) ) { // first line of MGIZA vocabulary files seems to be special : "1       UNK     0"  -- skip if it's this
+    ++i;
+    std::vector<std::string> tokens = Tokenize(line);
+    UTIL_THROW_IF2(tokens.size()!=3, "Line " << i << " in " << fileName << " has wrong number of tokens.");
+    unsigned id = atoll( tokens[0].c_str() );
+    if (! ( (id == 1) && (tokens[1] == "UNK") )) {
+      const Factor* factor = factorCollection.AddFactor(tokens[1],false); // TODO: can we assume that the vocabulary is know and filter the model on loading?
+      bool stored = Store(factor, id);
+      UTIL_THROW_IF2(!stored, "Line " << i << " in " << fileName << " overwrites existing vocabulary entry.");
+    }
+  }
+  while ( getline(inFile, line) ) {
+    ++i;
+    std::vector<std::string> tokens = Tokenize(line);
+    UTIL_THROW_IF2(tokens.size()!=3, "Line " << i << " in " << fileName << " has wrong number of tokens.");
+    unsigned id = atoll( tokens[0].c_str() );
+    const Factor* factor = factorCollection.AddFactor(tokens[1],false); // TODO: can we assume that the vocabulary is know and filter the model on loading?
+    bool stored = Store(factor, id);
+    UTIL_THROW_IF2(!stored, "Line " << i << " in " << fileName << " overwrites existing vocabulary entry.");
+  }
+  inFile.Close();
+}
+
+
+void Model1LexicalTable::Load(const std::string &fileName, const Model1Vocabulary& vcbS, const Model1Vocabulary& vcbT)
+{
+  InputFileStream inFile(fileName);
+  std::string line;
+
+  unsigned i = 0;
+  while ( getline(inFile, line) ) {
+    ++i;
+    std::vector<std::string> tokens = Tokenize(line);
+    UTIL_THROW_IF2(tokens.size()!=3, "Line " << i << " in " << fileName << " has wrong number of tokens.");
+    unsigned idS = atoll( tokens[0].c_str() );
+    unsigned idT = atoll( tokens[1].c_str() );
+    const Factor* wordS = vcbS.GetWord(idS);
+    const Factor* wordT = vcbT.GetWord(idT);
+    float prob = std::atof( tokens[2].c_str() );
+    if ( (wordS != NULL) && (wordT != NULL) ) {
+      m_ltable[ wordS ][ wordT ] = prob;
+    }
+    UTIL_THROW_IF2((wordS == NULL) || (wordT == NULL), "Line " << i << " in " << fileName << " has unknown vocabulary."); // TODO: can we assume that the vocabulary is know and filter the model on loading? Then remove this line.
+  }
+  inFile.Close();
+}
+
+// p( wordT | wordS )
+float Model1LexicalTable::GetProbability(const Factor* wordS, const Factor* wordT) const
+{
+  float prob = m_floor;
+
+  boost::unordered_map< const Factor*, boost::unordered_map< const Factor*, float > >::const_iterator iter1 = m_ltable.find( wordS );
+
+  if ( iter1 != m_ltable.end() ) {
+    boost::unordered_map< const Factor*, float >::const_iterator iter2 = iter1->second.find( wordT );
+    if ( iter2 != iter1->second.end() ) {
+      prob = iter2->second;
+      if ( prob < m_floor ) {
+        prob = m_floor;
+      }
+    }
+  }
+  return prob;
+}
+
+
+Model1Feature::Model1Feature(const std::string &line)
+  : StatelessFeatureFunction(1, line)
+  , m_skipTargetPunctuation(false)
+  , m_is_syntax(false)
+{
+  VERBOSE(1, "Initializing feature " << GetScoreProducerDescription() << " ...");
+  ReadParameters();
+  VERBOSE(1, " Done.");
+}
+
+void Model1Feature::SetParameter(const std::string& key, const std::string& value)
+{
+  if (key == "path") {
+    m_fileNameModel1 = value;
+  } else if (key == "source-vocabulary") {
+    m_fileNameVcbS = value;
+  } else if (key == "target-vocabulary") {
+    m_fileNameVcbT = value;
+  } else if (key == "skip-target-punctuation") {
+    m_skipTargetPunctuation = Scan<bool>(value);
+  } else {
+    StatelessFeatureFunction::SetParameter(key, value);
+  }
+}
+
+void Model1Feature::Load(AllOptions::ptr const& opts)
+{
+  m_options = opts;
+  m_is_syntax = is_syntax(opts->search.algo);
+
+  FEATUREVERBOSE(2, GetScoreProducerDescription() << ": Loading source vocabulary from file " << m_fileNameVcbS << " ...");
+  Model1Vocabulary vcbS;
+  vcbS.Load(m_fileNameVcbS);
+  FEATUREVERBOSE2(2, " Done." << std::endl);
+  FEATUREVERBOSE(2, GetScoreProducerDescription() << ": Loading target vocabulary from file " << m_fileNameVcbT << " ...");
+  Model1Vocabulary vcbT;
+  vcbT.Load(m_fileNameVcbT);
+  FEATUREVERBOSE2(2, " Done." << std::endl);
+  FEATUREVERBOSE(2, GetScoreProducerDescription() << ": Loading model 1 lexical translation table from file " << m_fileNameModel1 << " ...");
+  m_model1.Load(m_fileNameModel1,vcbS,vcbT);
+  FEATUREVERBOSE2(2, " Done." << std::endl);
+  FactorCollection &factorCollection = FactorCollection::Instance();
+  m_emptyWord = factorCollection.GetFactor(Model1Vocabulary::GIZANULL,false);
+  UTIL_THROW_IF2(m_emptyWord==NULL, GetScoreProducerDescription()
+                 << ": Factor for GIZA empty word does not exist.");
+
+  if (m_skipTargetPunctuation) {
+    const std::string punctuation = ",;.:!?";
+    for (size_t i=0; i<punctuation.size(); ++i) {
+      const std::string punct = punctuation.substr(i,1);
+      FactorCollection &factorCollection = FactorCollection::Instance();
+      const Factor* punctFactor = factorCollection.AddFactor(punct,false);
+      std::pair<std::set<const Factor*>::iterator,bool> inserted = m_punctuation.insert(punctFactor);
+    }
+  }
+}
+
+void Model1Feature::EvaluateWithSourceContext(const InputType &input
+    , const InputPath &inputPath
+    , const TargetPhrase &targetPhrase
+    , const StackVec *stackVec
+    , ScoreComponentCollection &scoreBreakdown
+    , ScoreComponentCollection *estimatedScores) const
+{
+  const Sentence& sentence = static_cast<const Sentence&>(input);
+  float score = 0.0;
+  float norm = TransformScore(1+sentence.GetSize());
+
+  for (size_t posT=0; posT<targetPhrase.GetSize(); ++posT) {
+    const Word &wordT = targetPhrase.GetWord(posT);
+    if (m_skipTargetPunctuation) {
+      std::set<const Factor*>::const_iterator foundPunctuation = m_punctuation.find(wordT[0]);
+      if (foundPunctuation != m_punctuation.end()) {
+        continue;
+      }
+    }
+    if ( !wordT.IsNonTerminal() ) {
+      float thisWordProb = m_model1.GetProbability(m_emptyWord,wordT[0]); // probability conditioned on empty word
+
+      // cache lookup
+      bool foundInCache = false;
+      {
+#ifdef WITH_THREADS
+        boost::shared_lock<boost::shared_mutex> read_lock(m_accessLock);
+#endif
+        boost::unordered_map<const InputType*, boost::unordered_map<const Factor*, float> >::const_iterator sentenceCache = m_cache.find(&input);
+        if (sentenceCache != m_cache.end()) {
+          boost::unordered_map<const Factor*, float>::const_iterator cacheHit = sentenceCache->second.find(wordT[0]);
+          if (cacheHit != sentenceCache->second.end()) {
+            foundInCache = true;
+            score += cacheHit->second;
+            FEATUREVERBOSE(3, "Cached score( " << wordT << " ) = " << cacheHit->second << std::endl);
+          }
+        }
+      }
+
+      if (!foundInCache) {
+        for (size_t posS=(m_is_syntax?1:0); posS<(m_is_syntax?sentence.GetSize()-1:sentence.GetSize()); ++posS) { // ignore <s> and </s>
+          const Word &wordS = sentence.GetWord(posS);
+          float modelProb = m_model1.GetProbability(wordS[0],wordT[0]);
+          FEATUREVERBOSE(4, "p( " << wordT << " | " << wordS << " ) = " << modelProb << std::endl);
+          thisWordProb += modelProb;
+        }
+        float thisWordScore = TransformScore(thisWordProb) - norm;
+        FEATUREVERBOSE(3, "score( " << wordT << " ) = " << thisWordScore << std::endl);
+        {
+#ifdef WITH_THREADS
+          // need to update cache; write lock
+          boost::unique_lock<boost::shared_mutex> lock(m_accessLock);
+#endif
+          m_cache[&input][wordT[0]] = thisWordScore;
+        }
+        score += thisWordScore;
+      }
+    }
+  }
+
+  scoreBreakdown.PlusEquals(this, score);
+}
+
+void Model1Feature::CleanUpAfterSentenceProcessing(const InputType& source)
+{
+#ifdef WITH_THREADS
+  // need to update cache; write lock
+  boost::unique_lock<boost::shared_mutex> lock(m_accessLock);
+#endif
+  // clear cache
+  boost::unordered_map<const InputType*, boost::unordered_map<const Factor*, float> >::iterator sentenceCache = m_cache.find(&source);
+  if (sentenceCache != m_cache.end()) {
+    sentenceCache->second.clear();
+    m_cache.erase(sentenceCache);
+  }
+}
+
+}
+

mosesdecoder/moses/FF/Model1Feature.h

@@ -0,0 +1,118 @@
+#pragma once
+
+#include <string>
+#include <limits>
+#include <set>
+#include <boost/unordered_map.hpp>
+#include "StatelessFeatureFunction.h"
+#include "moses/Factor.h"
+
+#ifdef WITH_THREADS
+#include <boost/thread/shared_mutex.hpp>
+#endif
+
+namespace Moses
+{
+
+class Model1Vocabulary
+{
+public:
+
+#define INVALID_ID std::numeric_limits<unsigned>::max() // UINT_MAX
+  static const std::string GIZANULL;
+
+  Model1Vocabulary();
+  bool Store(const Factor* word, const unsigned id);
+  unsigned StoreIfNew(const Factor* word);
+  unsigned GetWordID(const Factor* word) const;
+  const Factor* GetWord(unsigned id) const;
+  void Load(const std::string& fileName);
+
+protected:
+  boost::unordered_map<const Factor*, unsigned> m_lookup;
+  std::vector< const Factor* > m_vocab;
+  const Factor* m_NULL;
+};
+
+
+class Model1LexicalTable
+{
+public:
+  Model1LexicalTable(float floor=1e-7) : m_floor(floor)
+  {}
+
+  void Load(const std::string& fileName, const Model1Vocabulary& vcbS, const Model1Vocabulary& vcbT);
+
+  // p( wordT | wordS )
+  float GetProbability(const Factor* wordS, const Factor* wordT) const;
+
+protected:
+  boost::unordered_map< const Factor*, boost::unordered_map< const Factor*, float > > m_ltable;
+  const float m_floor;
+};
+
+
+
+class Model1Feature : public StatelessFeatureFunction
+{
+public:
+  Model1Feature(const std::string &line);
+
+  bool IsUseable(const FactorMask &mask) const {
+    return true;
+  }
+
+  void SetParameter(const std::string& key, const std::string& value);
+
+  void EvaluateInIsolation(const Phrase &source
+                           , const TargetPhrase &targetPhrase
+                           , ScoreComponentCollection &scoreBreakdown
+                           , ScoreComponentCollection &estimatedScores) const
+  {};
+
+  void EvaluateWithSourceContext(const InputType &input
+                                 , const InputPath &inputPath
+                                 , const TargetPhrase &targetPhrase
+                                 , const StackVec *stackVec
+                                 , ScoreComponentCollection &scoreBreakdown
+                                 , ScoreComponentCollection *estimatedScores = NULL) const;
+
+  void EvaluateTranslationOptionListWithSourceContext(const InputType &input
+      , const TranslationOptionList &translationOptionList) const
+  {}
+
+  void EvaluateWhenApplied(
+    const Hypothesis& cur_hypo,
+    ScoreComponentCollection* accumulator) const
+  {}
+
+  void EvaluateWhenApplied(
+    const ChartHypothesis& cur_hypo,
+    ScoreComponentCollection* accumulator) const
+  {}
+
+  void CleanUpAfterSentenceProcessing(const InputType& source);
+
+private:
+  std::string m_fileNameVcbS;
+  std::string m_fileNameVcbT;
+  std::string m_fileNameModel1;
+  Model1LexicalTable m_model1;
+  const Factor* m_emptyWord;
+  bool m_skipTargetPunctuation;
+  std::set<const Factor*> m_punctuation;
+  bool m_is_syntax;
+
+  void Load(AllOptions::ptr const& opts);
+
+  // cache
+  mutable boost::unordered_map<const InputType*, boost::unordered_map<const Factor*, float> > m_cache;
+#ifdef WITH_THREADS
+  // reader-writer lock
+  mutable boost::shared_mutex m_accessLock;
+#endif
+};
+
+
+}
+

mosesdecoder/moses/FF/PhraseBoundaryFeature.h

@@ -0,0 +1,70 @@
+#ifndef moses_PhraseBoundaryFeature_h
+#define moses_PhraseBoundaryFeature_h
+
+#include <stdexcept>
+#include <sstream>
+#include <string>
+
+#include "StatefulFeatureFunction.h"
+#include "moses/FF/FFState.h"
+#include "moses/Word.h"
+
+namespace Moses
+{
+
+class PhraseBoundaryState : public FFState
+{
+public:
+  PhraseBoundaryState(const Word* sourceWord, const Word* targetWord) :
+    m_sourceWord(sourceWord), m_targetWord(targetWord) {}
+  const Word* GetSourceWord() const {
+    return m_sourceWord;
+  }
+  const Word* GetTargetWord() const {
+    return m_targetWord;
+  }
+  virtual size_t hash() const;
+  virtual bool operator==(const FFState& other) const;
+
+
+private:
+  const Word* m_sourceWord;
+  const Word* m_targetWord;
+};
+
+
+/**
+ * Concatenations of factors on boundaries of phrases.
+ **/
+class PhraseBoundaryFeature : public StatefulFeatureFunction
+{
+public:
+  PhraseBoundaryFeature(const std::string &line);
+
+  bool IsUseable(const FactorMask &mask) const;
+
+  virtual const FFState* EmptyHypothesisState(const InputType &) const;
+
+  virtual FFState* EvaluateWhenApplied(const Hypothesis& cur_hypo, const FFState* prev_state,
+                                       ScoreComponentCollection* accumulator) const;
+
+  virtual FFState* EvaluateWhenApplied( const ChartHypothesis& /* cur_hypo */,
+                                        int /* featureID */,
+                                        ScoreComponentCollection* ) const {
+    throw std::logic_error("PhraseBoundaryState not supported in chart decoder, yet");
+  }
+
+  void SetParameter(const std::string& key, const std::string& value);
+
+private:
+  void AddFeatures(
+    const Word* leftWord, const Word* rightWord, const FactorList& factors,
+    const std::string& side, ScoreComponentCollection* scores) const ;
+  FactorList m_sourceFactors;
+  FactorList m_targetFactors;
+};
+
+}
+
+
+#endif

mosesdecoder/moses/FF/PhraseDistanceFeature.cpp

@@ -0,0 +1,123 @@
+#include "PhraseDistanceFeature.h"
+
+#include <vector>
+#include <boost/foreach.hpp>
+#include "moses/InputType.h"
+#include "moses/ScoreComponentCollection.h"
+#include "moses/StaticData.h"
+#include "util/exception.hh"
+
+using namespace std;
+
+namespace Moses
+{
+PhraseDistanceFeature::PhraseDistanceFeature(const string &line)
+  : StatelessFeatureFunction(2, line)
+  , m_space("")
+  , m_spaceID(0)
+  , m_measure(EuclideanDistance)
+{
+  ReadParameters();
+}
+
+void PhraseDistanceFeature::EvaluateWithSourceContext(const InputType &input
+    , const InputPath &inputPath
+    , const TargetPhrase &targetPhrase
+    , const StackVec *stackVec
+    , ScoreComponentCollection &scoreBreakdown
+    , ScoreComponentCollection *estimatedScores) const
+{
+  vector<float> scores(m_numScoreComponents, 0);
+  bool broken = false;
+  // Input coord
+  map<size_t const, vector<float> >::const_iterator ii;
+  if (input.m_coordMap) {
+    ii = input.m_coordMap->find(m_spaceID);
+  } else {
+    TRACE_ERR("No coordinates for space " << m_space << " on input (specify with coord XML tag)" << endl);
+    TRACE_ERR("Scores for " << m_description << " will be incorrect and probably all zeros" << endl);
+    broken = true;
+  }
+  if (ii == input.m_coordMap->end()) {
+    TRACE_ERR("No coordinates for space " << m_space << " on input (specify with coord XML tag)" << endl);
+    TRACE_ERR("Scores for " << m_description << " will be incorrect and probably all zeros" << endl);
+    broken = true;
+  }
+  // Target phrase coord
+  vector<SPTR<vector<float> > > const* tpp = targetPhrase.GetCoordList(m_spaceID);
+  if (tpp == NULL) {
+    TRACE_ERR("No coordinates for space " << m_space << " on target phrase (PhraseDictionary implementation needs to set)" << endl);
+    TRACE_ERR("Scores for " << m_description << " will be incorrect and probably all zeros" << endl);
+    broken = true;
+  }
+  // Compute scores
+  if (!broken) {
+    vector<float> const& inputCoord = ii->second;
+    vector<SPTR<vector<float> > > const& tpCoord = *tpp;
+    // Centroid of target phrase instances (from phrase extraction)
+    vector<float> centroid = vector<float>(inputCoord.size(), 0);
+    BOOST_FOREACH(SPTR<vector<float> > const coord, tpCoord) {
+      for (size_t i = 0; i < inputCoord.size(); ++i) {
+        centroid[i] += (*coord)[i];
+      }
+    }
+    for (size_t i = 0; i < inputCoord.size(); ++i) {
+      centroid[i] /= tpCoord.size();
+    }
+    // Average distance from the target phrase instances to (1) the input and
+    // (2) the target phrase centroid
+    float inputDistance = 0;
+    float centroidDistance = 0;
+    if (m_measure == EuclideanDistance) {
+      BOOST_FOREACH(SPTR<vector<float> > const coord, tpCoord) {
+        float pointInputDistance = 0;
+        float pointCentroidDistance = 0;
+        for (size_t i = 0; i < inputCoord.size(); ++i) {
+          pointInputDistance += pow(inputCoord[i] - (*coord)[i], 2);
+          pointCentroidDistance += pow(centroid[i] - (*coord)[i], 2);
+        }
+        inputDistance += sqrt(pointInputDistance);
+        centroidDistance += sqrt(pointCentroidDistance);
+      }
+    } else if (m_measure == TotalVariationDistance) {
+      BOOST_FOREACH(SPTR<vector<float> > const coord, tpCoord) {
+        float pointInputDistance = 0;
+        float pointCentroidDistance = 0;
+        for (size_t i = 0; i < inputCoord.size(); ++i) {
+          pointInputDistance += abs(inputCoord[i] - (*coord)[i]);
+          pointCentroidDistance += abs(centroid[i] - (*coord)[i]);
+        }
+        inputDistance += pointInputDistance / 2;
+        centroidDistance += pointCentroidDistance / 2;
+      }
+    }
+    inputDistance /= tpCoord.size();
+    centroidDistance /= tpCoord.size();
+    // Log transform scores, max with float epsilon to avoid domain error
+    scores[0] = log(max(inputDistance, Moses::FLOAT_EPSILON));
+    scores[1] = log(max(centroidDistance, Moses::FLOAT_EPSILON));
+  }
+  // Set scores
+  scoreBreakdown.Assign(this, scores);
+  return;
+}
+
+void PhraseDistanceFeature::SetParameter(const string& key, const string& value)
+{
+  if (key == "space") {
+    m_space = value;
+    m_spaceID = StaticData::InstanceNonConst().MapCoordSpace(m_space);
+  } else if (key == "measure") {
+    if (value == "euc") {
+      m_measure = EuclideanDistance;
+    } else if (value == "var") {
+      m_measure = TotalVariationDistance;
+    } else {
+      UTIL_THROW2("Unknown measure " << value << ", choices: euc var");
+    }
+  } else {
+    StatelessFeatureFunction::SetParameter(key, value);
+  }
+}
+
+} // namespace

mosesdecoder/moses/FF/PhraseDistanceFeature.h

@@ -0,0 +1,56 @@
+#pragma once
+
+#include "StatelessFeatureFunction.h"
+
+namespace Moses
+{
+
+class PhraseDistanceFeature : public StatelessFeatureFunction
+{
+  enum Measure {
+    EuclideanDistance,
+    TotalVariationDistance,
+  };
+
+public:
+  PhraseDistanceFeature(const std::string &line);
+
+  bool IsUseable(const FactorMask &mask) const {
+    return true;
+  }
+
+  virtual void EvaluateInIsolation(const Phrase &source
+                                   , const TargetPhrase &targetPhrase
+                                   , ScoreComponentCollection &scoreBreakdown
+                                   , ScoreComponentCollection &estimatedScores) const {
+  }
+
+  void EvaluateWhenApplied(const Hypothesis& hypo,
+                           ScoreComponentCollection* accumulator) const {
+  }
+  void EvaluateWhenApplied(const ChartHypothesis &hypo,
+                           ScoreComponentCollection* accumulator) const {
+  }
+  void EvaluateWhenApplied(const Syntax::SHyperedge &hyperedge,
+                           ScoreComponentCollection* accumulator) const {
+  }
+
+  void EvaluateWithSourceContext(const InputType &input
+                                 , const InputPath &inputPath
+                                 , const TargetPhrase &targetPhrase
+                                 , const StackVec *stackVec
+                                 , ScoreComponentCollection &scoreBreakdown
+                                 , ScoreComponentCollection *estimatedScores = NULL) const;
+
+  void EvaluateTranslationOptionListWithSourceContext(const InputType &input
+      , const TranslationOptionList &translationOptionList) const {
+  }
+  void SetParameter(const std::string& key, const std::string& value);
+
+protected:
+  Measure m_measure;
+  std::string m_space;
+  size_t m_spaceID;
+};
+
+} //namespace

mosesdecoder/moses/FF/PhraseLengthFeature.cpp

@@ -0,0 +1,46 @@
+#include <sstream>
+#include "PhraseLengthFeature.h"
+#include "moses/Hypothesis.h"
+#include "moses/ScoreComponentCollection.h"
+#include "moses/TranslationOption.h"
+#include "util/string_stream.hh"
+
+namespace Moses
+{
+
+using namespace std;
+
+PhraseLengthFeature::PhraseLengthFeature(const std::string &line)
+  :StatelessFeatureFunction(0, line)
+{
+  ReadParameters();
+}
+
+void PhraseLengthFeature::EvaluateInIsolation(const Phrase &source
+    , const TargetPhrase &targetPhrase
+    , ScoreComponentCollection &scoreBreakdown
+    , ScoreComponentCollection &estimatedScores) const
+{
+  // get length of source and target phrase
+  size_t targetLength = targetPhrase.GetSize();
+  size_t sourceLength = source.GetSize();
+
+  // create feature names
+  util::StringStream nameSource;
+  nameSource << "s" << sourceLength;
+
+  util::StringStream nameTarget;
+  nameTarget << "t" << targetLength;
+
+  util::StringStream nameBoth;
+  nameBoth << sourceLength << "," << targetLength;
+
+  // increase feature counts
+  scoreBreakdown.PlusEquals(this,nameSource.str(),1);
+  scoreBreakdown.PlusEquals(this,nameTarget.str(),1);
+  scoreBreakdown.PlusEquals(this,nameBoth.str(),1);
+
+  //cerr << nameSource.str() << " " << nameTarget.str() << " " << nameBoth.str() << endl;
+}
+
+}

mosesdecoder/moses/FF/PhrasePenalty.cpp

@@ -0,0 +1,51 @@
+#include <vector>
+#include "PhrasePenalty.h"
+#include "moses/ScoreComponentCollection.h"
+#include "moses/TranslationModel/PhraseDictionary.h"
+#include "util/exception.hh"
+
+using namespace std;
+
+namespace Moses
+{
+PhrasePenalty::PhrasePenalty(const std::string &line)
+  : StatelessFeatureFunction(1, line)
+  , m_perPhraseTable(false)
+{
+  ReadParameters();
+}
+
+void PhrasePenalty::EvaluateInIsolation(const Phrase &source
+                                        , const TargetPhrase &targetPhrase
+                                        , ScoreComponentCollection &scoreBreakdown
+                                        , ScoreComponentCollection &estimatedScores) const
+{
+  if (m_perPhraseTable) {
+    const PhraseDictionary *pt = targetPhrase.GetContainer();
+    if (pt) {
+      size_t ptId = pt->GetId();
+      UTIL_THROW_IF2(ptId >= m_numScoreComponents, "Wrong number of scores");
+
+      vector<float> scores(m_numScoreComponents, 0);
+      scores[ptId] = 1.0f;
+
+      scoreBreakdown.Assign(this, scores);
+    }
+
+  } else {
+    scoreBreakdown.Assign(this, 1.0f);
+  }
+}
+
+void PhrasePenalty::SetParameter(const std::string& key, const std::string& value)
+{
+  if (key == "per-phrase-table") {
+    m_perPhraseTable =Scan<bool>(value);
+  } else {
+    StatelessFeatureFunction::SetParameter(key, value);
+  }
+}
+
+
+} // namespace
+

mosesdecoder/moses/FF/PhrasePenalty.h

@@ -0,0 +1,50 @@
+#pragma once
+
+#include "StatelessFeatureFunction.h"
+
+namespace Moses
+{
+
+class PhrasePenalty : public StatelessFeatureFunction
+{
+public:
+  PhrasePenalty(const std::string &line);
+
+  bool IsUseable(const FactorMask &mask) const {
+    return true;
+  }
+
+  virtual void EvaluateInIsolation(const Phrase &source
+                                   , const TargetPhrase &targetPhrase
+                                   , ScoreComponentCollection &scoreBreakdown
+                                   , ScoreComponentCollection &estimatedScores) const;
+
+  void EvaluateWhenApplied(const Hypothesis& hypo,
+                           ScoreComponentCollection* accumulator) const {
+  }
+  void EvaluateWhenApplied(const ChartHypothesis &hypo,
+                           ScoreComponentCollection* accumulator) const {
+  }
+  void EvaluateWhenApplied(const Syntax::SHyperedge &hyperedge,
+                           ScoreComponentCollection* accumulator) const {
+  }
+
+  void EvaluateWithSourceContext(const InputType &input
+                                 , const InputPath &inputPath
+                                 , const TargetPhrase &targetPhrase
+                                 , const StackVec *stackVec
+                                 , ScoreComponentCollection &scoreBreakdown
+                                 , ScoreComponentCollection *estimatedScores = NULL) const {
+  }
+
+  void EvaluateTranslationOptionListWithSourceContext(const InputType &input
+      , const TranslationOptionList &translationOptionList) const {
+  }
+  void SetParameter(const std::string& key, const std::string& value);
+
+protected:
+  bool m_perPhraseTable;
+};
+
+} //namespace
+

mosesdecoder/moses/FF/ReferenceComparison.cpp

@@ -0,0 +1,11 @@
+#include "ReferenceComparison.h"
+
+namespace Moses
+{
+ReferenceComparison::ReferenceComparison(const std::string &line)
+  :StatelessFeatureFunction(0, line)
+{
+}
+
+}
+

mosesdecoder/moses/FF/RuleScope.h

@@ -0,0 +1,56 @@
+#pragma once
+#include <string>
+#include "StatelessFeatureFunction.h"
+
+namespace Moses
+{
+
+// Rule Scope - not quite completely implemented yet
+class RuleScope : public StatelessFeatureFunction
+{
+public:
+  RuleScope(const std::string &line);
+
+  virtual bool IsUseable(const FactorMask &mask) const {
+    return true;
+  }
+
+  virtual void EvaluateInIsolation(const Phrase &source
+                                   , const TargetPhrase &targetPhrase
+                                   , ScoreComponentCollection &scoreBreakdown
+                                   , ScoreComponentCollection &estimatedScores) const;
+
+  virtual void EvaluateWithSourceContext(const InputType &input
+                                         , const InputPath &inputPath
+                                         , const TargetPhrase &targetPhrase
+                                         , const StackVec *stackVec
+                                         , ScoreComponentCollection &scoreBreakdown
+                                         , ScoreComponentCollection *estimatedScores = NULL) const {
+  }
+
+  void EvaluateTranslationOptionListWithSourceContext(const InputType &input
+      , const TranslationOptionList &translationOptionList) const {
+  }
+
+
+  virtual void EvaluateWhenApplied(const Hypothesis& hypo,
+                                   ScoreComponentCollection* accumulator) const {
+  }
+
+  virtual void EvaluateWhenApplied(const ChartHypothesis &hypo,
+                                   ScoreComponentCollection* accumulator) const {
+  }
+
+  void SetParameter(const std::string& key, const std::string& value);
+
+protected:
+  bool m_sourceSyntax;
+  bool m_perScope;
+  bool m_futureCostOnly;
+
+  bool IsGlueRule(const Phrase &source) const;
+
+};
+
+}
+

mosesdecoder/moses/FF/SoftMatchingFeature.h

@@ -0,0 +1,68 @@
+#pragma once
+
+#include "moses/Word.h"
+#include "StatelessFeatureFunction.h"
+
+#ifdef WITH_THREADS
+#include <boost/thread/shared_mutex.hpp>
+#endif
+
+namespace Moses
+{
+
+class SoftMatchingFeature : public StatelessFeatureFunction
+{
+public:
+  SoftMatchingFeature(const std::string &line);
+
+  bool IsUseable(const FactorMask &mask) const {
+    return true;
+  }
+
+  virtual void EvaluateWhenApplied(const ChartHypothesis& hypo,
+                                   ScoreComponentCollection* accumulator) const;
+
+  void EvaluateInIsolation(const Phrase &source
+                           , const TargetPhrase &targetPhrase
+                           , ScoreComponentCollection &scoreBreakdown
+                           , ScoreComponentCollection &estimatedScores) const {};
+  void EvaluateWithSourceContext(const InputType &input
+                                 , const InputPath &inputPath
+                                 , const TargetPhrase &targetPhrase
+                                 , const StackVec *stackVec
+                                 , ScoreComponentCollection &scoreBreakdown
+                                 , ScoreComponentCollection *estimatedScores = NULL) const {};
+
+  void EvaluateTranslationOptionListWithSourceContext(const InputType &input
+      , const TranslationOptionList &translationOptionList) const {
+  }
+
+  void EvaluateWhenApplied(const Hypothesis& hypo,
+                           ScoreComponentCollection* accumulator) const {};
+
+  bool Load(const std::string &filePath);
+
+  std::vector<std::vector<Word> >& GetSoftMatches() {
+    return m_softMatches;
+  }
+
+  void ResizeCache() const;
+
+  const std::string& GetOrSetFeatureName(const Word& RHS, const Word& LHS) const;
+  void SetParameter(const std::string& key, const std::string& value);
+
+
+private:
+  mutable std::vector<std::vector<Word> > m_softMatches; // map RHS of new rule to list of possible LHS of old rule (subtree)
+  mutable std::vector<std::vector<std::string> > m_nameCache;
+  bool m_scoreIdentical;
+
+#ifdef WITH_THREADS
+  //reader-writer lock
+  mutable boost::shared_mutex m_accessLock;
+#endif
+
+};
+
+}
+

mosesdecoder/moses/FF/SoftSourceSyntacticConstraintsFeature.cpp

@@ -0,0 +1,651 @@
+#include <vector>
+#include <limits>
+#include <cassert>
+#include "SoftSourceSyntacticConstraintsFeature.h"
+#include "moses/StaticData.h"
+#include "moses/InputFileStream.h"
+#include "moses/ScoreComponentCollection.h"
+#include "moses/Hypothesis.h"
+#include "moses/ChartHypothesis.h"
+#include "moses/ChartManager.h"
+#include "moses/FactorCollection.h"
+#include "moses/TreeInput.h"
+#include "moses/PP/SourceLabelsPhraseProperty.h"
+
+
+using namespace std;
+
+namespace Moses
+{
+
+
+SoftSourceSyntacticConstraintsFeature::SoftSourceSyntacticConstraintsFeature(const std::string &line)
+  : StatelessFeatureFunction(6, line)
+  , m_useCoreSourceLabels(false)
+  , m_useLogprobs(true)
+  , m_useSparse(false)
+  , m_useSparseLabelPairs(false)
+  , m_noMismatches(false)
+  , m_floor(1e-7)
+{
+  VERBOSE(1, "Initializing feature " << GetScoreProducerDescription() << " ...");
+  ReadParameters();
+  VERBOSE(1, " Done.");
+  VERBOSE(1, " Config:");
+  VERBOSE(1, " Log probabilities");
+  if ( m_useLogprobs )         {
+    VERBOSE(1, " active.");
+  } else {
+    VERBOSE(1, " inactive.");
+  }
+  VERBOSE(1, " Sparse scores");
+  if ( m_useSparse )           {
+    VERBOSE(1, " active.");
+  } else {
+    VERBOSE(1, " inactive.");
+  }
+  VERBOSE(1, " Sparse label pair scores");
+  if ( m_useSparseLabelPairs ) {
+    VERBOSE(1, " active.");
+  } else {
+    VERBOSE(1, " inactive.");
+  }
+  VERBOSE(1, " Core labels");
+  if ( m_useCoreSourceLabels ) {
+    VERBOSE(1, " active.");
+  } else {
+    VERBOSE(1, " inactive.");
+  }
+  VERBOSE(1, " No mismatches");
+  if ( m_noMismatches )        {
+    VERBOSE(1, " active.");
+  } else {
+    VERBOSE(1, " inactive.");
+  }
+  VERBOSE(1, std::endl);
+}
+
+
+void SoftSourceSyntacticConstraintsFeature::SetParameter(const std::string& key, const std::string& value)
+{
+  if (key == "sourceLabelSetFile") {
+    m_sourceLabelSetFile = value;
+  } else if (key == "coreSourceLabelSetFile") {
+    m_coreSourceLabelSetFile = value;
+    m_useCoreSourceLabels = true;
+  } else if (key == "targetSourceLeftHandSideJointCountFile") {
+    m_targetSourceLHSJointCountFile = value;
+  } else if (key == "noMismatches") {
+    m_noMismatches = Scan<bool>(value); // for a hard constraint, allow no mismatches (also set: weights 1 0 0 0 0 0, tuneable=false)
+  } else if (key == "logProbabilities") {
+    m_useLogprobs = Scan<bool>(value);
+  } else if (key == "sparse") {
+    m_useSparse = Scan<bool>(value);
+  } else if (key == "sparseLabelPairs") {
+    m_useSparseLabelPairs = Scan<bool>(value);
+  } else {
+    StatelessFeatureFunction::SetParameter(key, value);
+  }
+}
+
+void SoftSourceSyntacticConstraintsFeature::Load(AllOptions::ptr const& opts)
+{
+  m_options = opts;
+  // don't change the loading order!
+  LoadSourceLabelSet();
+  if (!m_coreSourceLabelSetFile.empty()) {
+    LoadCoreSourceLabelSet();
+  }
+  if (!m_targetSourceLHSJointCountFile.empty()) {
+    LoadTargetSourceLeftHandSideJointCountFile();
+  }
+}
+
+void SoftSourceSyntacticConstraintsFeature::LoadSourceLabelSet()
+{
+  FEATUREVERBOSE(2, "Loading source label set from file " << m_sourceLabelSetFile << " ...");
+  InputFileStream inFile(m_sourceLabelSetFile);
+
+  FactorCollection &factorCollection = FactorCollection::Instance();
+
+  // read source label set
+  std::string line;
+  m_sourceLabels.clear();
+  m_sourceLabelsByIndex.clear();
+  m_sourceLabelsByIndex_RHS_1.clear();
+  m_sourceLabelsByIndex_RHS_0.clear();
+  m_sourceLabelsByIndex_LHS_1.clear();
+  m_sourceLabelsByIndex_LHS_0.clear();
+  m_sourceLabelIndexesByFactor.clear();
+  while (getline(inFile, line)) {
+    std::istringstream tokenizer(line);
+    std::string label;
+    size_t index;
+    try {
+      tokenizer >> label >> index;
+    } catch (const std::exception &e) {
+      UTIL_THROW2(GetScoreProducerDescription()
+                  << ": Error reading source label set file " << m_sourceLabelSetFile << " .");
+    }
+    std::pair< boost::unordered_map<std::string,size_t>::iterator, bool > inserted = m_sourceLabels.insert( std::pair<std::string,size_t>(label,index) );
+    UTIL_THROW_IF2(!inserted.second, GetScoreProducerDescription()
+                   << ": Source label set file " << m_sourceLabelSetFile << " should contain each syntactic label only once.");
+
+    if (index >= m_sourceLabelsByIndex.size()) {
+      m_sourceLabelsByIndex.resize(index+1);
+      m_sourceLabelsByIndex_RHS_1.resize(index+1);
+      m_sourceLabelsByIndex_RHS_0.resize(index+1);
+      m_sourceLabelsByIndex_LHS_1.resize(index+1);
+      m_sourceLabelsByIndex_LHS_0.resize(index+1);
+    }
+    m_sourceLabelsByIndex[index] = label;
+    m_sourceLabelsByIndex_RHS_1[index] = "RHS_1_" + label;
+    m_sourceLabelsByIndex_RHS_0[index] = "RHS_0_" + label;
+    m_sourceLabelsByIndex_LHS_1[index] = "LHS_1_" + label;
+    m_sourceLabelsByIndex_LHS_0[index] = "LHS_0_" + label;
+    const Factor* sourceLabelFactor = factorCollection.AddFactor(label,true);
+    m_sourceLabelIndexesByFactor[sourceLabelFactor] = index;
+  }
+
+  inFile.Close();
+
+  std::list<std::string> specialLabels;
+  specialLabels.push_back("GlueTop");
+  specialLabels.push_back("GlueX");
+//  specialLabels.push_back("XRHS");
+//  specialLabels.push_back("XLHS");
+  for (std::list<std::string>::const_iterator iter=specialLabels.begin();
+       iter!=specialLabels.end(); ++iter) {
+    boost::unordered_map<std::string,size_t>::iterator found = m_sourceLabels.find(*iter);
+    UTIL_THROW_IF2(found == m_sourceLabels.end(), GetScoreProducerDescription()
+                   << ": Source label set file " << m_sourceLabelSetFile << " should contain an entry for the special label \"" << *iter << "\".");
+    if (!(found->first).compare("GlueTop")) {
+      m_GlueTopLabel = found->second;
+//    } else if (!(found->first).compare("XRHS")) {
+//      m_XRHSLabel = found->second;
+//    } else if (!(found->first).compare("XLHS")) {
+//      m_XLHSLabel = found->second;
+    }
+  }
+  FEATUREVERBOSE2(2, " Done." << std::endl);
+}
+
+
+void SoftSourceSyntacticConstraintsFeature::LoadCoreSourceLabelSet()
+{
+  FEATUREVERBOSE(2, "Loading core source label set from file " << m_coreSourceLabelSetFile << " ...");
+  // read core source label set
+  LoadLabelSet(m_coreSourceLabelSetFile, m_coreSourceLabels);
+  FEATUREVERBOSE2(2, " Done." << std::endl);
+}
+
+void SoftSourceSyntacticConstraintsFeature::LoadLabelSet(std::string &filename,
+    boost::unordered_set<size_t> &labelSet)
+{
+  InputFileStream inFile(filename);
+  std::string line;
+  labelSet.clear();
+  while (getline(inFile, line)) {
+    istringstream tokenizer(line);
+    std::string label;
+    tokenizer >> label;
+    boost::unordered_map<std::string,size_t>::iterator foundSourceLabelIndex = m_sourceLabels.find( label );
+    if ( foundSourceLabelIndex != m_sourceLabels.end() ) {
+      labelSet.insert(foundSourceLabelIndex->second);
+    } else {
+      FEATUREVERBOSE(2, "Ignoring undefined source label \"" << label << "\" "
+                     << "from core source label set file " << filename << "."
+                     << std::endl);
+    }
+  }
+  inFile.Close();
+}
+
+
+void SoftSourceSyntacticConstraintsFeature::LoadTargetSourceLeftHandSideJointCountFile()
+{
+
+  FEATUREVERBOSE(2, "Loading target/source label joint counts from file " << m_targetSourceLHSJointCountFile << " ...");
+  InputFileStream inFile(m_targetSourceLHSJointCountFile);
+
+  for (boost::unordered_map<const Factor*, std::vector< std::pair<float,float> >* >::iterator iter=m_labelPairProbabilities.begin();
+       iter!=m_labelPairProbabilities.end(); ++iter) {
+    delete iter->second;
+  }
+  m_labelPairProbabilities.clear();
+
+  // read joint counts
+  std::string line;
+  FactorCollection &factorCollection = FactorCollection::Instance();
+  boost::unordered_map<const Factor*,float> targetLHSCounts;
+  std::vector<float> sourceLHSCounts(m_sourceLabels.size(),0.0);
+
+  while (getline(inFile, line)) {
+    istringstream tokenizer(line);
+    std::string targetLabel;
+    std::string sourceLabel;
+    float count;
+    tokenizer >> targetLabel;
+    tokenizer >> sourceLabel;
+    tokenizer >> count;
+
+    boost::unordered_map<std::string,size_t>::iterator foundSourceLabelIndex = m_sourceLabels.find( sourceLabel );
+    UTIL_THROW_IF2(foundSourceLabelIndex == m_sourceLabels.end(), GetScoreProducerDescription()
+                   << ": Target/source label joint count file " << m_targetSourceLHSJointCountFile
+                   << " contains undefined source label \"" << sourceLabel << "\".");
+
+    const Factor* targetLabelFactor = factorCollection.AddFactor(targetLabel,true);
+
+    sourceLHSCounts[foundSourceLabelIndex->second] += count;
+    std::pair< boost::unordered_map<const Factor*,float >::iterator, bool > insertedTargetLHSCount =
+      targetLHSCounts.insert( std::pair<const Factor*,float>(targetLabelFactor,count) );
+    if (!insertedTargetLHSCount.second) {
+      (insertedTargetLHSCount.first)->second += count;
+      boost::unordered_map<const Factor*, std::vector< std::pair<float,float> >* >::iterator jointCountIt =
+        m_labelPairProbabilities.find( targetLabelFactor );
+      assert(jointCountIt != m_labelPairProbabilities.end());
+      (jointCountIt->second)->at(foundSourceLabelIndex->second).first += count;
+      (jointCountIt->second)->at(foundSourceLabelIndex->second).second += count;
+    } else {
+      std::pair<float,float> init(0.0,0.0);
+      std::vector< std::pair<float,float> >* sourceVector = new std::vector< std::pair<float,float> >(m_sourceLabels.size(),init);
+      sourceVector->at(foundSourceLabelIndex->second) = std::pair<float,float>(count,count);
+      std::pair< boost::unordered_map<const Factor*, std::vector< std::pair<float,float> >* >::iterator, bool > insertedJointCount =
+        m_labelPairProbabilities.insert( std::pair<const Factor*, std::vector< std::pair<float,float> >* >(targetLabelFactor,sourceVector) );
+      UTIL_THROW_IF2(!insertedJointCount.second, GetScoreProducerDescription()
+                     << ": Loading target/source label joint counts from file " << m_targetSourceLHSJointCountFile << " failed.");
+    }
+  }
+
+  // normalization
+  for (boost::unordered_map<const Factor*, std::vector< std::pair<float,float> >* >::iterator iter=m_labelPairProbabilities.begin();
+       iter!=m_labelPairProbabilities.end(); ++iter) {
+    float targetLHSCount = 0;
+    boost::unordered_map<const Factor*,float >::const_iterator targetLHSCountIt = targetLHSCounts.find( iter->first );
+    if ( targetLHSCountIt != targetLHSCounts.end() ) {
+      targetLHSCount = targetLHSCountIt->second;
+    }
+    std::vector< std::pair<float,float> > &probabilities = *(iter->second);
+    for (size_t index=0; index<probabilities.size(); ++index) {
+
+      if ( probabilities[index].first != 0 ) {
+        assert(targetLHSCount != 0);
+        probabilities[index].first  /= targetLHSCount;
+      }
+      if ( probabilities[index].second != 0 ) {
+        assert(sourceLHSCounts[index] != 0);
+        probabilities[index].second /= sourceLHSCounts[index];
+      }
+    }
+  }
+
+  inFile.Close();
+  FEATUREVERBOSE2(2, " Done." << std::endl);
+}
+
+
+void SoftSourceSyntacticConstraintsFeature::EvaluateWithSourceContext(const InputType &input
+    , const InputPath &inputPath
+    , const TargetPhrase &targetPhrase
+    , const StackVec *stackVec
+    , ScoreComponentCollection &scoreBreakdown
+    , ScoreComponentCollection *estimatedScores) const
+{
+  assert(stackVec);
+
+  IFFEATUREVERBOSE(3) {
+    FEATUREVERBOSE(3, targetPhrase << std::endl);
+    FEATUREVERBOSE(3, inputPath << std::endl);
+    for (size_t i = 0; i < stackVec->size(); ++i) {
+      const ChartCellLabel &cell = *stackVec->at(i);
+      const Range &ntRange = cell.GetCoverage();
+      FEATUREVERBOSE(3, "stackVec[ " << i << " ] : " << ntRange.GetStartPos() << " - " << ntRange.GetEndPos() << std::endl);
+    }
+
+    for (AlignmentInfo::const_iterator it=targetPhrase.GetAlignNonTerm().begin();
+         it!=targetPhrase.GetAlignNonTerm().end(); ++it) {
+      FEATUREVERBOSE(3, "alignNonTerm " << it->first << " " << it->second << std::endl);
+    }
+  }
+
+  // dense scores
+  std::vector<float> newScores(m_numScoreComponents,0);
+
+  const TreeInput& treeInput = static_cast<const TreeInput&>(input);
+  // const StaticData& staticData = StaticData::Instance();
+  // const Word& outputDefaultNonTerminal = staticData.GetOutputDefaultNonTerminal();
+
+  size_t nNTs = 1;
+  bool treeInputMismatchLHSBinary = true;
+  size_t treeInputMismatchRHSCount = 0;
+  bool hasCompleteTreeInputMatch = false;
+  float ruleLabelledProbability = 0.0;
+  float treeInputMatchProbRHS = 0.0;
+  float treeInputMatchProbLHS = 0.0;
+
+  // read SourceLabels property
+  const Factor* targetLHS = targetPhrase.GetTargetLHS()[0];
+  bool isGlueGrammarRule = false;
+  bool isUnkRule = false;
+
+  if (const PhraseProperty *property = targetPhrase.GetProperty("SourceLabels")) {
+
+    const SourceLabelsPhraseProperty *sourceLabelsPhraseProperty = static_cast<const SourceLabelsPhraseProperty*>(property);
+
+    nNTs = sourceLabelsPhraseProperty->GetNumberOfNonTerminals();
+    float totalCount = sourceLabelsPhraseProperty->GetTotalCount();
+
+    // prepare for input tree label matching
+    std::vector< boost::unordered_set<size_t> > treeInputLabelsRHS(nNTs-1);
+    boost::unordered_set<size_t> treeInputLabelsLHS;
+
+    // get index map for underlying hypotheses
+    const Range& range = inputPath.GetWordsRange();
+    size_t startPos = range.GetStartPos();
+    size_t endPos = range.GetEndPos();
+    const Phrase *sourcePhrase = targetPhrase.GetRuleSource();
+
+    if (nNTs > 1) { // rule has right-hand side non-terminals, i.e. it's a hierarchical rule
+      size_t nonTerminalNumber = 0;
+      size_t sourceSentPos = startPos;
+
+      for (size_t sourcePhrasePos=0; sourcePhrasePos<sourcePhrase->GetSize(); ++sourcePhrasePos) {
+        // consult rule for either word or non-terminal
+        const Word &word = sourcePhrase->GetWord(sourcePhrasePos);
+        size_t symbolStartPos = sourceSentPos;
+        size_t symbolEndPos = sourceSentPos;
+        if ( word.IsNonTerminal() ) {
+          // retrieve information that is required for input tree label matching (RHS)
+          const ChartCellLabel &cell = *stackVec->at(nonTerminalNumber);
+          const Range& prevWordsRange = cell.GetCoverage();
+          symbolStartPos = prevWordsRange.GetStartPos();
+          symbolEndPos = prevWordsRange.GetEndPos();
+        }
+
+        const NonTerminalSet& treeInputLabels = treeInput.GetLabelSet(symbolStartPos,symbolEndPos);
+
+        for (NonTerminalSet::const_iterator treeInputLabelsIt = treeInputLabels.begin();
+             treeInputLabelsIt != treeInputLabels.end(); ++treeInputLabelsIt) {
+          if (*treeInputLabelsIt != m_options->syntax.output_default_non_terminal) {
+            boost::unordered_map<const Factor*,size_t>::const_iterator foundTreeInputLabel
+            = m_sourceLabelIndexesByFactor.find((*treeInputLabelsIt)[0]);
+            if (foundTreeInputLabel != m_sourceLabelIndexesByFactor.end()) {
+              size_t treeInputLabelIndex = foundTreeInputLabel->second;
+              treeInputLabelsRHS[sourcePhrasePos].insert(treeInputLabelIndex);
+            }
+          }
+        }
+
+        if ( word.IsNonTerminal() ) {
+          ++nonTerminalNumber;
+        }
+        sourceSentPos = symbolEndPos + 1;
+      }
+    }
+
+    // retrieve information that is required for input tree label matching (LHS)
+    const NonTerminalSet& treeInputLabels = treeInput.GetLabelSet(startPos,endPos);
+
+    for (NonTerminalSet::const_iterator treeInputLabelsIt = treeInputLabels.begin();
+         treeInputLabelsIt != treeInputLabels.end(); ++treeInputLabelsIt) {
+      if (*treeInputLabelsIt != m_options->syntax.output_default_non_terminal) {
+        boost::unordered_map<const Factor*,size_t>::const_iterator foundTreeInputLabel
+        = m_sourceLabelIndexesByFactor.find((*treeInputLabelsIt)[0]);
+        if (foundTreeInputLabel != m_sourceLabelIndexesByFactor.end()) {
+          size_t treeInputLabelIndex = foundTreeInputLabel->second;
+          treeInputLabelsLHS.insert(treeInputLabelIndex);
+        }
+      }
+    }
+
+
+    // inspect source-labelled rule items
+
+    std::vector< boost::unordered_set<size_t> > sparseScoredTreeInputLabelsRHS(nNTs-1);
+    boost::unordered_set<size_t> sparseScoredTreeInputLabelsLHS;
+
+    std::vector<bool> sourceLabelSeenAsLHS(m_sourceLabels.size(),false);
+    std::vector<bool> treeInputMatchRHSCountByNonTerminal(nNTs-1,false);
+    std::vector<float> treeInputMatchProbRHSByNonTerminal(nNTs-1,0.0);
+
+    const std::list<SourceLabelsPhrasePropertyItem> &sourceLabelItems = sourceLabelsPhraseProperty->GetSourceLabelItems();
+
+    for (std::list<SourceLabelsPhrasePropertyItem>::const_iterator sourceLabelItem = sourceLabelItems.begin();
+         sourceLabelItem != sourceLabelItems.end() && !hasCompleteTreeInputMatch; ++sourceLabelItem) {
+
+      const std::list<size_t> &sourceLabelsRHS = sourceLabelItem->GetSourceLabelsRHS();
+      const std::list< std::pair<size_t,float> > &sourceLabelsLHSList = sourceLabelItem->GetSourceLabelsLHSList();
+      float sourceLabelsRHSCount = sourceLabelItem->GetSourceLabelsRHSCount();
+
+      assert(sourceLabelsRHS.size() == nNTs-1);
+
+      bool currentSourceLabelItemIsCompleteTreeInputMatch = true;
+
+      size_t nonTerminalNumber=0;
+      for (std::list<size_t>::const_iterator sourceLabelsRHSIt = sourceLabelsRHS.begin();
+           sourceLabelsRHSIt != sourceLabelsRHS.end(); ++sourceLabelsRHSIt, ++nonTerminalNumber) {
+
+        if (treeInputLabelsRHS[nonTerminalNumber].find(*sourceLabelsRHSIt) != treeInputLabelsRHS[nonTerminalNumber].end()) {
+
+          treeInputMatchRHSCountByNonTerminal[nonTerminalNumber] = true;
+          treeInputMatchProbRHSByNonTerminal[nonTerminalNumber] += sourceLabelsRHSCount; // to be normalized later on
+
+          if ( m_useSparse &&
+               (!m_useCoreSourceLabels || m_coreSourceLabels.find(*sourceLabelsRHSIt) != m_coreSourceLabels.end()) ) {
+            // score sparse features: RHS match
+            if (sparseScoredTreeInputLabelsRHS[nonTerminalNumber].find(*sourceLabelsRHSIt) == sparseScoredTreeInputLabelsRHS[nonTerminalNumber].end()) {
+              // (only if no match has been scored for this tree input label and rule non-terminal with a previous sourceLabelItem)
+              float score_RHS_1 = (float)1/treeInputLabelsRHS[nonTerminalNumber].size();
+              scoreBreakdown.PlusEquals(this,
+                                        m_sourceLabelsByIndex_RHS_1[*sourceLabelsRHSIt],
+                                        score_RHS_1);
+              sparseScoredTreeInputLabelsRHS[nonTerminalNumber].insert(*sourceLabelsRHSIt);
+            }
+          }
+
+        } else {
+
+          currentSourceLabelItemIsCompleteTreeInputMatch = false;
+
+        }
+      }
+
+      for (std::list< std::pair<size_t,float> >::const_iterator sourceLabelsLHSIt = sourceLabelsLHSList.begin();
+           sourceLabelsLHSIt != sourceLabelsLHSList.end(); ++sourceLabelsLHSIt) {
+
+        if ( sourceLabelsLHSIt->first == m_GlueTopLabel ) {
+          isGlueGrammarRule = true;
+        }
+
+        if (treeInputLabelsLHS.find(sourceLabelsLHSIt->first) != treeInputLabelsLHS.end()) {
+
+          treeInputMismatchLHSBinary = false;
+          treeInputMatchProbLHS += sourceLabelsLHSIt->second; // to be normalized later on
+
+          if ( m_useSparse &&
+               (!m_useCoreSourceLabels || m_coreSourceLabels.find(sourceLabelsLHSIt->first) != m_coreSourceLabels.end()) ) {
+            // score sparse features: LHS match
+            if (sparseScoredTreeInputLabelsLHS.find(sourceLabelsLHSIt->first) == sparseScoredTreeInputLabelsLHS.end()) {
+              // (only if no match has been scored for this tree input label and rule non-terminal with a previous sourceLabelItem)
+              float score_LHS_1 = (float)1/treeInputLabelsLHS.size();
+              scoreBreakdown.PlusEquals(this,
+                                        m_sourceLabelsByIndex_LHS_1[sourceLabelsLHSIt->first],
+                                        score_LHS_1);
+              sparseScoredTreeInputLabelsLHS.insert(sourceLabelsLHSIt->first);
+            }
+          }
+
+          if ( currentSourceLabelItemIsCompleteTreeInputMatch ) {
+            ruleLabelledProbability += sourceLabelsLHSIt->second; // to be normalized later on
+            hasCompleteTreeInputMatch = true;
+          }
+
+        }
+      }
+    }
+
+    // normalization
+    for (std::vector<float>::iterator treeInputMatchProbRHSByNonTerminalIt = treeInputMatchProbRHSByNonTerminal.begin();
+         treeInputMatchProbRHSByNonTerminalIt != treeInputMatchProbRHSByNonTerminal.end(); ++treeInputMatchProbRHSByNonTerminalIt) {
+      *treeInputMatchProbRHSByNonTerminalIt /= totalCount;
+      if ( *treeInputMatchProbRHSByNonTerminalIt != 0 ) {
+        treeInputMatchProbRHS += ( m_useLogprobs ? TransformScore(*treeInputMatchProbRHSByNonTerminalIt) : *treeInputMatchProbRHSByNonTerminalIt );
+      }
+    }
+    treeInputMatchProbLHS /= totalCount;
+    ruleLabelledProbability /= totalCount;
+
+    // input tree matching (RHS)
+    if ( !hasCompleteTreeInputMatch ) {
+      treeInputMismatchRHSCount = nNTs-1;
+      for (std::vector<bool>::const_iterator treeInputMatchRHSCountByNonTerminalIt = treeInputMatchRHSCountByNonTerminal.begin();
+           treeInputMatchRHSCountByNonTerminalIt != treeInputMatchRHSCountByNonTerminal.end(); ++treeInputMatchRHSCountByNonTerminalIt) {
+        if (*treeInputMatchRHSCountByNonTerminalIt) {
+          --treeInputMismatchRHSCount;
+        }
+      }
+    }
+
+    // score sparse features: mismatches
+    if ( m_useSparse ) {
+
+      // RHS
+
+      for (size_t nonTerminalNumber = 0; nonTerminalNumber < nNTs-1; ++nonTerminalNumber) {
+        // nNTs-1 because nNTs also counts the left-hand side non-terminal
+
+        float score_RHS_0 = (float)1/treeInputLabelsRHS[nonTerminalNumber].size();
+        for (boost::unordered_set<size_t>::const_iterator treeInputLabelsRHSIt = treeInputLabelsRHS[nonTerminalNumber].begin();
+             treeInputLabelsRHSIt != treeInputLabelsRHS[nonTerminalNumber].end(); ++treeInputLabelsRHSIt) {
+
+          if ( !m_useCoreSourceLabels || m_coreSourceLabels.find(*treeInputLabelsRHSIt) != m_coreSourceLabels.end() ) {
+
+            if (sparseScoredTreeInputLabelsRHS[nonTerminalNumber].find(*treeInputLabelsRHSIt) == sparseScoredTreeInputLabelsRHS[nonTerminalNumber].end()) {
+              // score sparse features: RHS mismatch
+              scoreBreakdown.PlusEquals(this,
+                                        m_sourceLabelsByIndex_RHS_0[*treeInputLabelsRHSIt],
+                                        score_RHS_0);
+            }
+          }
+        }
+      }
+
+      // LHS
+
+      float score_LHS_0 = (float)1/treeInputLabelsLHS.size();
+      for (boost::unordered_set<size_t>::const_iterator treeInputLabelsLHSIt = treeInputLabelsLHS.begin();
+           treeInputLabelsLHSIt != treeInputLabelsLHS.end(); ++treeInputLabelsLHSIt) {
+
+        if ( !m_useCoreSourceLabels || m_coreSourceLabels.find(*treeInputLabelsLHSIt) != m_coreSourceLabels.end() ) {
+
+          if (sparseScoredTreeInputLabelsLHS.find(*treeInputLabelsLHSIt) == sparseScoredTreeInputLabelsLHS.end()) {
+            // score sparse features: RHS mismatch
+            scoreBreakdown.PlusEquals(this,
+                                      m_sourceLabelsByIndex_LHS_0[*treeInputLabelsLHSIt],
+                                      score_LHS_0);
+          }
+        }
+      }
+
+    }
+
+    if ( m_useSparseLabelPairs && !isGlueGrammarRule ) {
+
+      // left-hand side label pairs (target NT, source NT)
+      float t2sLabelsScore = 0.0;
+      float s2tLabelsScore = 0.0;
+      for (boost::unordered_set<size_t>::const_iterator treeInputLabelsLHSIt = treeInputLabelsLHS.begin();
+           treeInputLabelsLHSIt != treeInputLabelsLHS.end(); ++treeInputLabelsLHSIt) {
+
+        scoreBreakdown.PlusEquals(this,
+                                  "LHSPAIR_" + targetLHS->GetString().as_string() + "_" + m_sourceLabelsByIndex[*treeInputLabelsLHSIt],
+                                  (float)1/treeInputLabelsLHS.size());
+
+        if (!m_targetSourceLHSJointCountFile.empty()) {
+          std::pair<float,float> probPair = GetLabelPairProbabilities( targetLHS, *treeInputLabelsLHSIt);
+          t2sLabelsScore += probPair.first;
+          s2tLabelsScore += probPair.second;
+        }
+      }
+      if ( treeInputLabelsLHS.size() == 0 ) {
+        scoreBreakdown.PlusEquals(this,
+                                  "LHSPAIR_" + targetLHS->GetString().as_string() + "_"
+                                  + m_options->syntax.output_default_non_terminal[0]
+                                  ->GetString().as_string(),
+                                  1);
+        if (!m_targetSourceLHSJointCountFile.empty()) {
+          t2sLabelsScore = TransformScore(m_floor);
+          s2tLabelsScore = TransformScore(m_floor);
+        }
+      } else {
+        if (!m_targetSourceLHSJointCountFile.empty()) {
+          float norm = TransformScore(treeInputLabelsLHS.size());
+          t2sLabelsScore = TransformScore(t2sLabelsScore) - norm;
+          s2tLabelsScore = TransformScore(s2tLabelsScore) - norm;
+        }
+      }
+      if (!m_targetSourceLHSJointCountFile.empty()) {
+        scoreBreakdown.PlusEquals(this, "LHST2S", t2sLabelsScore);
+        scoreBreakdown.PlusEquals(this, "LHSS2T", s2tLabelsScore);
+      }
+    }
+
+  } else {
+
+    // abort with error message if the phrase does not translate an unknown word
+    UTIL_THROW_IF2(!targetPhrase.GetWord(0).IsOOV(), GetScoreProducerDescription()
+                   << ": Missing SourceLabels property. "
+                   << "Please check phrase table and glue rules.");
+
+    // unknown word
+    isUnkRule = true;
+//    ruleLabelledProbability = 1;
+
+  }
+
+  // add scores
+
+  // input tree matching
+  newScores[0] = !hasCompleteTreeInputMatch;
+  if ( m_noMismatches ) {
+    newScores[0] = ( (hasCompleteTreeInputMatch || isGlueGrammarRule || isUnkRule) ? 0 : -std::numeric_limits<float>::infinity() );
+  }
+  newScores[1] = treeInputMismatchLHSBinary;
+  newScores[2] = treeInputMismatchRHSCount;
+
+  if ( m_useLogprobs ) {
+    if ( ruleLabelledProbability != 0 ) {
+      ruleLabelledProbability = TransformScore(ruleLabelledProbability);
+    }
+    if ( treeInputMatchProbLHS != 0 ) {
+      treeInputMatchProbLHS = TransformScore(treeInputMatchProbLHS);
+    }
+  }
+
+  newScores[3] = ruleLabelledProbability;
+  newScores[4] = treeInputMatchProbLHS;
+  newScores[5] = treeInputMatchProbRHS;
+
+  scoreBreakdown.PlusEquals(this, newScores);
+}
+
+
+std::pair<float,float> SoftSourceSyntacticConstraintsFeature::GetLabelPairProbabilities(
+  const Factor* target,
+  const size_t source) const
+{
+  boost::unordered_map<const Factor*, std::vector< std::pair<float,float> >* >::const_iterator found =
+    m_labelPairProbabilities.find(target);
+  if ( found == m_labelPairProbabilities.end() ) {
+    return std::pair<float,float>(m_floor,m_floor); // floor values
+  }
+  std::pair<float,float> ret = found->second->at(source);
+  if ( ret == std::pair<float,float>(0,0) ) {
+    return std::pair<float,float>(m_floor,m_floor); // floor values
+  }
+  return ret;
+}
+
+
+}
+

mosesdecoder/moses/FF/SourceGHKMTreeInputMatchFeature.cpp

@@ -0,0 +1,75 @@
+#include <map>
+#include <vector>
+#include <cassert>
+#include "SourceGHKMTreeInputMatchFeature.h"
+#include "moses/StaticData.h"
+#include "moses/InputFileStream.h"
+#include "moses/ScoreComponentCollection.h"
+#include "moses/Hypothesis.h"
+#include "moses/ChartHypothesis.h"
+#include "moses/Factor.h"
+#include "moses/FactorCollection.h"
+#include "moses/InputPath.h"
+#include "moses/TreeInput.h"
+
+
+using namespace std;
+
+namespace Moses
+{
+
+SourceGHKMTreeInputMatchFeature::SourceGHKMTreeInputMatchFeature(const std::string &line)
+  : StatelessFeatureFunction(2, line)
+{
+  std::cerr << GetScoreProducerDescription() << "Initializing feature...";
+  ReadParameters();
+  std::cerr << " Done." << std::endl;
+}
+
+void SourceGHKMTreeInputMatchFeature::SetParameter(const std::string& key, const std::string& value)
+{
+  UTIL_THROW(util::Exception, GetScoreProducerDescription() << ": Unknown parameter " << key << "=" << value);
+}
+
+// assumes that source-side syntax labels are stored in the target non-terminal field of the rules
+void SourceGHKMTreeInputMatchFeature::EvaluateWithSourceContext(const InputType &input
+    , const InputPath &inputPath
+    , const TargetPhrase &targetPhrase
+    , const StackVec *stackVec
+    , ScoreComponentCollection &scoreBreakdown
+    , ScoreComponentCollection *estimatedScores) const
+{
+  const Range& range = inputPath.GetWordsRange();
+  size_t startPos = range.GetStartPos();
+  size_t endPos = range.GetEndPos();
+  const TreeInput& treeInput = static_cast<const TreeInput&>(input);
+  const NonTerminalSet& treeInputLabels = treeInput.GetLabelSet(startPos,endPos);
+  const Word& lhsLabel = targetPhrase.GetTargetLHS();
+
+  const StaticData& staticData = StaticData::Instance();
+
+  std::vector<float> newScores(m_numScoreComponents,0.0);
+  // m_numScoreComponents == 2 // first fires for matches, second for mismatches
+
+  if ( (treeInputLabels.find(lhsLabel) != treeInputLabels.end())
+       && (lhsLabel != m_options->syntax.output_default_non_terminal) ) {
+    // match
+    newScores[0] = 1.0;
+  } else {
+    // mismatch
+    newScores[1] = 1.0;
+  }
+
+  scoreBreakdown.PlusEquals(this, newScores);
+}
+
+void
+SourceGHKMTreeInputMatchFeature::
+Load(AllOptions::ptr const& opts)
+{
+  m_options = opts;
+  // m_output_default_nonterminal = opts->syntax.output_default_non_terminal;
+}
+
+}
+

mosesdecoder/moses/FF/SourceGHKMTreeInputMatchFeature.h

@@ -0,0 +1,49 @@
+#pragma once
+
+#include "StatelessFeatureFunction.h"
+#include "moses/parameters/AllOptions.h"
+
+namespace Moses
+{
+
+// assumes that source-side syntax labels are stored in the target non-terminal field of the rules
+class SourceGHKMTreeInputMatchFeature : public StatelessFeatureFunction
+{
+public:
+  SourceGHKMTreeInputMatchFeature(const std::string &line);
+
+  bool IsUseable(const FactorMask &mask) const {
+    return true;
+  }
+
+  void SetParameter(const std::string& key, const std::string& value);
+
+  void EvaluateInIsolation(const Phrase &source
+                           , const TargetPhrase &targetPhrase
+                           , ScoreComponentCollection &scoreBreakdown
+                           , ScoreComponentCollection &estimatedScores) const {};
+
+  void EvaluateWithSourceContext(const InputType &input
+                                 , const InputPath &inputPath
+                                 , const TargetPhrase &targetPhrase
+                                 , const StackVec *stackVec
+                                 , ScoreComponentCollection &scoreBreakdown
+                                 , ScoreComponentCollection *estimatedScores = NULL) const;
+
+  void EvaluateTranslationOptionListWithSourceContext(const InputType &input
+      , const TranslationOptionList &translationOptionList) const {
+  }
+
+
+  void EvaluateWhenApplied(const Hypothesis& hypo,
+                           ScoreComponentCollection* accumulator) const {};
+
+  void EvaluateWhenApplied(const ChartHypothesis &hypo,
+                           ScoreComponentCollection* accumulator) const {};
+
+  void Load(AllOptions::ptr const& opts);
+};
+
+
+}
+

mosesdecoder/moses/FF/SourceWordDeletionFeature.h

@@ -0,0 +1,63 @@
+#pragma once
+
+#include <string>
+#include <boost/unordered_set.hpp>
+
+#include "StatelessFeatureFunction.h"
+#include "moses/FactorCollection.h"
+#include "moses/AlignmentInfo.h"
+
+namespace Moses
+{
+
+/** Sets the features for source word deletion
+ */
+class SourceWordDeletionFeature : public StatelessFeatureFunction
+{
+private:
+  boost::unordered_set<std::string> m_vocab;
+  FactorType m_factorType;
+  bool m_unrestricted;
+  std::string m_filename;
+
+public:
+  SourceWordDeletionFeature(const std::string &line);
+
+  void Load(AllOptions::ptr const& opts);
+
+  bool IsUseable(const FactorMask &mask) const;
+
+  void EvaluateInIsolation(const Phrase &source
+                           , const TargetPhrase &targetPhrase
+                           , ScoreComponentCollection &scoreBreakdown
+                           , ScoreComponentCollection &estimatedScores) const;
+  void EvaluateWithSourceContext(const InputType &input
+                                 , const InputPath &inputPath
+                                 , const TargetPhrase &targetPhrase
+                                 , const StackVec *stackVec
+                                 , ScoreComponentCollection &scoreBreakdown
+                                 , ScoreComponentCollection *estimatedScores = NULL) const {
+  }
+
+  void EvaluateTranslationOptionListWithSourceContext(const InputType &input
+      , const TranslationOptionList &translationOptionList) const {
+  }
+
+
+  void EvaluateWhenApplied(const Hypothesis& hypo,
+                           ScoreComponentCollection* accumulator) const {
+  }
+  void EvaluateWhenApplied(const ChartHypothesis &hypo,
+                           ScoreComponentCollection* accumulator) const {
+  }
+
+  void ComputeFeatures(const Phrase &source,
+                       const TargetPhrase& targetPhrase,
+                       ScoreComponentCollection* accumulator,
+                       const AlignmentInfo &alignmentInfo) const;
+  void SetParameter(const std::string& key, const std::string& value);
+
+};
+
+}
+

mosesdecoder/moses/FF/SpanLength.cpp

@@ -0,0 +1,88 @@
+#include <boost/shared_ptr.hpp>
+#include "SpanLength.h"
+#include "moses/StaticData.h"
+#include "moses/Word.h"
+#include "moses/ChartCellLabel.h"
+#include "moses/Range.h"
+#include "moses/StackVec.h"
+#include "moses/TargetPhrase.h"
+#include "moses/PP/PhraseProperty.h"
+#include "moses/PP/SpanLengthPhraseProperty.h"
+
+using namespace std;
+
+namespace Moses
+{
+SpanLength::SpanLength(const std::string &line)
+  :StatelessFeatureFunction(1, line)
+  ,m_smoothingMethod(None)
+  ,m_const(0)
+{
+  ReadParameters();
+}
+
+void SpanLength::EvaluateInIsolation(const Phrase &source
+                                     , const TargetPhrase &targetPhrase
+                                     , ScoreComponentCollection &scoreBreakdown
+                                     , ScoreComponentCollection &estimatedScores) const
+{
+  targetPhrase.SetRuleSource(source);
+}
+
+void SpanLength::EvaluateWithSourceContext(const InputType &input
+    , const InputPath &inputPath
+    , const TargetPhrase &targetPhrase
+    , const StackVec *stackVec
+    , ScoreComponentCollection &scoreBreakdown
+    , ScoreComponentCollection *estimatedScores) const
+{
+  assert(stackVec);
+
+  const PhraseProperty *property = targetPhrase.GetProperty("SpanLength");
+  if (property == NULL) {
+    return;
+  }
+
+  const SpanLengthPhraseProperty *slProp = static_cast<const SpanLengthPhraseProperty*>(property);
+
+  assert(targetPhrase.GetRuleSource());
+
+  float score = 0;
+  for (size_t i = 0; i < stackVec->size(); ++i) {
+    const ChartCellLabel &cell = *stackVec->at(i);
+    const Range &ntRange = cell.GetCoverage();
+    size_t sourceWidth = ntRange.GetNumWordsCovered();
+    float prob = slProp->GetProb(i, sourceWidth, m_const);
+    score += TransformScore(prob);
+  }
+
+  if (score < -100.0f) {
+    float weight = StaticData::Instance().GetWeight(this);
+    if (weight < 0) {
+      score = -100;
+    }
+  }
+
+  scoreBreakdown.PlusEquals(this, score);
+
+}
+
+void SpanLength::SetParameter(const std::string& key, const std::string& value)
+{
+  if (key == "smoothing") {
+    if (value == "plus-constant") {
+      m_smoothingMethod = PlusConst;
+    } else if (value == "none") {
+      m_smoothingMethod = None;
+    } else {
+      UTIL_THROW(util::Exception, "Unknown smoothing type " << value);
+    }
+  } else if (key == "constant") {
+    m_const = Scan<float>(value);
+  } else {
+    StatelessFeatureFunction::SetParameter(key, value);
+  }
+}
+
+}
+

mosesdecoder/moses/FF/SpanLength.h

@@ -0,0 +1,56 @@
+#pragma once
+#include <string>
+#include "StatelessFeatureFunction.h"
+
+namespace Moses
+{
+
+// Rule Scope - not quite completely implemented yet
+class SpanLength : public StatelessFeatureFunction
+{
+public:
+  SpanLength(const std::string &line);
+
+  virtual bool IsUseable(const FactorMask &mask) const {
+    return true;
+  }
+
+  virtual void EvaluateInIsolation(const Phrase &source
+                                   , const TargetPhrase &targetPhrase
+                                   , ScoreComponentCollection &scoreBreakdown
+                                   , ScoreComponentCollection &estimatedScores) const;
+
+  virtual void EvaluateWithSourceContext(const InputType &input
+                                         , const InputPath &inputPath
+                                         , const TargetPhrase &targetPhrase
+                                         , const StackVec *stackVec
+                                         , ScoreComponentCollection &scoreBreakdown
+                                         , ScoreComponentCollection *estimatedScores = NULL) const;
+
+  void EvaluateTranslationOptionListWithSourceContext(const InputType &input
+      , const TranslationOptionList &translationOptionList) const {
+  }
+
+
+  virtual void EvaluateWhenApplied(const Hypothesis& hypo,
+                                   ScoreComponentCollection* accumulator) const {
+  }
+
+  virtual void EvaluateWhenApplied(const ChartHypothesis &hypo,
+                                   ScoreComponentCollection* accumulator) const {
+  }
+
+  void SetParameter(const std::string& key, const std::string& value);
+
+protected:
+  enum SmoothingMethod {
+    None,
+    PlusConst,
+  };
+  SmoothingMethod m_smoothingMethod;
+
+  float m_const;
+};
+
+}
+

mosesdecoder/moses/FF/SparseHieroReorderingFeature.cpp

@@ -0,0 +1,224 @@
+#include <iostream>
+
+#include "moses/ChartHypothesis.h"
+#include "moses/ChartManager.h"
+#include "moses/FactorCollection.h"
+#include "moses/Sentence.h"
+
+#include "util/exception.hh"
+#include "util/string_stream.hh"
+
+#include "SparseHieroReorderingFeature.h"
+
+using namespace std;
+
+namespace Moses
+{
+
+SparseHieroReorderingFeature::SparseHieroReorderingFeature(const std::string &line)
+  :StatelessFeatureFunction(0, line),
+   m_type(SourceCombined),
+   m_sourceFactor(0),
+   m_targetFactor(0),
+   m_sourceVocabFile(""),
+   m_targetVocabFile("")
+{
+
+  /*
+    Configuration of features.
+      factor - Which factor should it apply to
+      type - what type of sparse reordering feature. e.g. block (modelled on Matthias
+        Huck's EAMT 2012 features)
+      word - which words to include, e.g. src_bdry, src_all, tgt_bdry , ...
+      vocab - vocab file to limit it to
+      orientation - e.g. lr, etc.
+  */
+  cerr << "Constructing a Sparse Reordering feature" << endl;
+  ReadParameters();
+  m_otherFactor = FactorCollection::Instance().AddFactor("##OTHER##");
+  LoadVocabulary(m_sourceVocabFile, m_sourceVocab);
+  LoadVocabulary(m_targetVocabFile, m_targetVocab);
+}
+
+void SparseHieroReorderingFeature::SetParameter(const std::string& key, const std::string& value)
+{
+  if (key == "input-factor") {
+    m_sourceFactor = Scan<FactorType>(value);
+  } else if (key == "output-factor") {
+    m_targetFactor = Scan<FactorType>(value);
+  } else if (key == "input-vocab-file") {
+    m_sourceVocabFile = value;
+  } else if (key == "output-vocab-file") {
+    m_targetVocabFile = value;
+  } else if (key == "type") {
+    if (value == "SourceCombined") {
+      m_type = SourceCombined;
+    } else if (value == "SourceLeft") {
+      m_type = SourceLeft;
+    } else if (value == "SourceRight") {
+      m_type = SourceRight;
+    } else {
+      UTIL_THROW(util::Exception, "Unknown sparse reordering type " << value);
+    }
+  } else {
+    FeatureFunction::SetParameter(key, value);
+  }
+}
+
+void SparseHieroReorderingFeature::LoadVocabulary(const std::string& filename, Vocab& vocab)
+{
+  if (filename.empty()) return;
+  ifstream in(filename.c_str());
+  UTIL_THROW_IF(!in, util::Exception, "Unable to open vocab file: " << filename);
+  string line;
+  while(getline(in,line)) {
+    vocab.insert(FactorCollection::Instance().AddFactor(line));
+  }
+  in.close();
+}
+
+const Factor* SparseHieroReorderingFeature::GetFactor(const Word& word, const Vocab& vocab, FactorType factorType) const
+{
+  const Factor* factor = word.GetFactor(factorType);
+  if (vocab.size() && vocab.find(factor) == vocab.end()) return m_otherFactor;
+  return factor;
+}
+
+void SparseHieroReorderingFeature::EvaluateWhenApplied(
+  const ChartHypothesis&  cur_hypo ,
+  ScoreComponentCollection* accumulator) const
+{
+  // get index map for underlying hypotheses
+  //const AlignmentInfo::NonTermIndexMap &nonTermIndexMap =
+  //  cur_hypo.GetCurrTargetPhrase().GetAlignNonTerm().GetNonTermIndexMap();
+
+  //The Huck features. For a rule with source side:
+  //   abXcdXef
+  //We first have to split into blocks:
+  // ab X cd X ef
+  //Then we extract features based in the boundary words of the neighbouring blocks
+  //For the block pair, we use the right word of the left block, and the left
+  //word of the right block.
+
+  //Need to get blocks, and their alignment. Each block has a word range (on the
+  // on the source), a non-terminal flag, and  a set of alignment points in the target phrase
+
+  //We need to be able to map source word position to target word position, as
+  //much as possible (don't need interior of non-terminals). The alignment info
+  //objects just give us the mappings between *rule* positions. So if we can
+  //map source word position to source rule position, and target rule position
+  //to target word position, then we can map right through.
+
+  size_t sourceStart = cur_hypo.GetCurrSourceRange().GetStartPos();
+  size_t sourceSize = cur_hypo.GetCurrSourceRange().GetNumWordsCovered();
+
+  vector<Range> sourceNTSpans;
+  for (size_t prevHypoId = 0; prevHypoId < cur_hypo.GetPrevHypos().size(); ++prevHypoId) {
+    sourceNTSpans.push_back(cur_hypo.GetPrevHypo(prevHypoId)->GetCurrSourceRange());
+  }
+  //put in source order. Is this necessary?
+  sort(sourceNTSpans.begin(), sourceNTSpans.end());
+  //cerr << "Source NTs: ";
+  //for (size_t i = 0; i < sourceNTSpans.size(); ++i) cerr << sourceNTSpans[i] << " ";
+  //cerr << endl;
+
+  typedef pair<Range,bool> Block;//flag indicates NT
+  vector<Block> sourceBlocks;
+  sourceBlocks.push_back(Block(cur_hypo.GetCurrSourceRange(),false));
+  for (vector<Range>::const_iterator i = sourceNTSpans.begin();
+       i != sourceNTSpans.end(); ++i) {
+    const Range& prevHypoRange = *i;
+    Block lastBlock = sourceBlocks.back();
+    sourceBlocks.pop_back();
+    //split this range into before NT, NT and after NT
+    if (prevHypoRange.GetStartPos() > lastBlock.first.GetStartPos()) {
+      sourceBlocks.push_back(Block(Range(lastBlock.first.GetStartPos(),prevHypoRange.GetStartPos()-1),false));
+    }
+    sourceBlocks.push_back(Block(prevHypoRange,true));
+    if (prevHypoRange.GetEndPos() < lastBlock.first.GetEndPos()) {
+      sourceBlocks.push_back(Block(Range(prevHypoRange.GetEndPos()+1,lastBlock.first.GetEndPos()), false));
+    }
+  }
+  /*
+  cerr << "Source Blocks: ";
+  for (size_t i = 0; i < sourceBlocks.size(); ++i) cerr << sourceBlocks[i].first << " "
+      << (sourceBlocks[i].second ? "NT" : "T") << " ";
+  cerr << endl;
+  */
+
+  //Mapping from source word to target rule position
+  vector<size_t> sourceWordToTargetRulePos(sourceSize);
+  map<size_t,size_t> alignMap;
+  alignMap.insert(
+    cur_hypo.GetCurrTargetPhrase().GetAlignTerm().begin(),
+    cur_hypo.GetCurrTargetPhrase().GetAlignTerm().end());
+  alignMap.insert(
+    cur_hypo.GetCurrTargetPhrase().GetAlignNonTerm().begin(),
+    cur_hypo.GetCurrTargetPhrase().GetAlignNonTerm().end());
+  //vector<size_t> alignMapTerm = cur_hypo.GetCurrTargetPhrase().GetAlignNonTerm()
+  size_t sourceRulePos = 0;
+  //cerr << "SW->RP ";
+  for (vector<Block>::const_iterator sourceBlockIt = sourceBlocks.begin();
+       sourceBlockIt != sourceBlocks.end(); ++sourceBlockIt) {
+    for (size_t sourceWordPos = sourceBlockIt->first.GetStartPos();
+         sourceWordPos <= sourceBlockIt->first.GetEndPos(); ++sourceWordPos) {
+      sourceWordToTargetRulePos[sourceWordPos - sourceStart] = alignMap[sourceRulePos];
+      //   cerr << sourceWordPos - sourceStart << "-" << alignMap[sourceRulePos] << " ";
+      if (! sourceBlockIt->second) {
+        //T
+        ++sourceRulePos;
+      }
+    }
+    if ( sourceBlockIt->second) {
+      //NT
+      ++sourceRulePos;
+    }
+  }
+  //cerr << endl;
+
+  //Iterate through block pairs
+  const Sentence& sentence =
+    static_cast<const Sentence&>(cur_hypo.GetManager().GetSource());
+  //const TargetPhrase& targetPhrase = cur_hypo.GetCurrTargetPhrase();
+  for (size_t i = 0; i < sourceBlocks.size()-1; ++i) {
+    Block& leftSourceBlock = sourceBlocks[i];
+    Block& rightSourceBlock = sourceBlocks[i+1];
+    size_t sourceLeftBoundaryPos = leftSourceBlock.first.GetEndPos();
+    size_t sourceRightBoundaryPos = rightSourceBlock.first.GetStartPos();
+    const Word& sourceLeftBoundaryWord = sentence.GetWord(sourceLeftBoundaryPos);
+    const Word& sourceRightBoundaryWord = sentence.GetWord(sourceRightBoundaryPos);
+    sourceLeftBoundaryPos -= sourceStart;
+    sourceRightBoundaryPos -= sourceStart;
+
+    // Need to figure out where these map to on the target.
+    size_t targetLeftRulePos =
+      sourceWordToTargetRulePos[sourceLeftBoundaryPos];
+    size_t targetRightRulePos =
+      sourceWordToTargetRulePos[sourceRightBoundaryPos];
+
+    bool isMonotone = true;
+    if ((sourceLeftBoundaryPos < sourceRightBoundaryPos  &&
+         targetLeftRulePos > targetRightRulePos) ||
+        ((sourceLeftBoundaryPos > sourceRightBoundaryPos  &&
+          targetLeftRulePos < targetRightRulePos))) {
+      isMonotone = false;
+    }
+    util::StringStream buf;
+    buf << "h_"; //sparse reordering, Huck
+    if (m_type == SourceLeft || m_type == SourceCombined) {
+      buf << GetFactor(sourceLeftBoundaryWord,m_sourceVocab,m_sourceFactor)->GetString();
+      buf << "_";
+    }
+    if (m_type == SourceRight || m_type == SourceCombined) {
+      buf << GetFactor(sourceRightBoundaryWord,m_sourceVocab,m_sourceFactor)->GetString();
+      buf << "_";
+    }
+    buf << (isMonotone ? "M" : "S");
+    accumulator->PlusEquals(this,buf.str(), 1);
+  }
+//  cerr << endl;
+}
+
+
+}
+

mosesdecoder/moses/FF/SparseHieroReorderingFeatureTest.cpp

@@ -0,0 +1,36 @@
+/***********************************************************************
+Moses - factored phrase-based language decoder
+Copyright (C) 2013- University of Edinburgh
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Lesser General Public
+License as published by the Free Software Foundation; either
+version 2.1 of the License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public
+License along with this library; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+***********************************************************************/
+#include <iostream>
+
+#include <boost/test/unit_test.hpp>
+
+#include "SparseHieroReorderingFeature.h"
+
+using namespace Moses;
+using namespace std;
+
+BOOST_AUTO_TEST_SUITE(shrf)
+
+BOOST_AUTO_TEST_CASE(lexical_rule)
+{
+  SparseHieroReorderingFeature feature("name=shrf");
+
+}
+
+BOOST_AUTO_TEST_SUITE_END()