Upload 440 files

.gitattributes

@@ -98,3 +98,6 @@ mosesdecoder/misc/bin/gcc-9/release/link-static/threading-multi/queryPhraseTable
 mosesdecoder/moses/bin/gcc-9/release/link-static/threading-multi/libmoses.a filter=lfs diff=lfs merge=lfs -text
 mosesdecoder/moses/bin/gcc-9/release/link-static/threading-multi/moses_test filter=lfs diff=lfs merge=lfs -text
 mosesdecoder/moses/LM/bin/BackwardTest.test/gcc-9/release/link-static/threading-multi/BackwardTest filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/moses2/bin/gcc-9/release/link-static/threading-multi/libmoses2_lib.a filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/moses2/bin/gcc-9/release/link-static/threading-multi/libmoses2decoder.a filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/moses2/bin/gcc-9/release/link-static/threading-multi/moses2 filter=lfs diff=lfs merge=lfs -text

mosesdecoder/moses2/AlignmentInfo.cpp

@@ -0,0 +1,176 @@
+/***********************************************************************
+ Moses - statistical machine translation system
+ Copyright (C) 2006-2011 University of Edinburgh
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+***********************************************************************/
+#include <algorithm>
+#include <set>
+#include <sstream>
+#include "AlignmentInfo.h"
+#include "legacy/Util2.h"
+#include "util/exception.hh"
+
+namespace Moses2
+{
+
+AlignmentInfo::AlignmentInfo(const std::set<std::pair<size_t,size_t> > &pairs)
+  : m_collection(pairs)
+{
+  BuildNonTermIndexMaps();
+}
+
+AlignmentInfo::AlignmentInfo(const std::vector<unsigned char> &aln)
+{
+  assert(aln.size()%2==0);
+  for (size_t i = 0; i < aln.size(); i+= 2)
+    m_collection.insert(std::make_pair(size_t(aln[i]),size_t(aln[i+1])));
+  BuildNonTermIndexMaps();
+}
+
+AlignmentInfo::AlignmentInfo(const std::string &str)
+{
+  std::vector<std::string> points = Tokenize(str, " ");
+  std::vector<std::string>::const_iterator iter;
+  for (iter = points.begin(); iter != points.end(); iter++) {
+    std::vector<size_t> point = Tokenize<size_t>(*iter, "-");
+    UTIL_THROW_IF2(point.size() != 2, "Bad format of word alignment point: " << *iter);
+    Add(point[0], point[1]);
+  }
+}
+
+void AlignmentInfo::BuildNonTermIndexMaps()
+{
+  if (m_collection.empty()) {
+    return;
+  }
+  const_iterator p = begin();
+  size_t maxIndex = p->second;
+  for (++p;  p != end(); ++p) {
+    if (p->second > maxIndex) {
+      maxIndex = p->second;
+    }
+  }
+  m_nonTermIndexMap.resize(maxIndex+1, NOT_FOUND);
+  m_nonTermIndexMap2.resize(maxIndex+1, NOT_FOUND);
+  size_t i = 0;
+  for (p = begin(); p != end(); ++p) {
+    if (m_nonTermIndexMap[p->second] != NOT_FOUND) {
+      // 1-to-many. Definitely a set of terminals. Don't bother storing 1-to-1 index map
+      m_nonTermIndexMap.clear();
+      m_nonTermIndexMap2.clear();
+      return;
+    }
+    m_nonTermIndexMap[p->second] = i++;
+    m_nonTermIndexMap2[p->second] = p->first;
+  }
+}
+
+std::set<size_t> AlignmentInfo::GetAlignmentsForSource(size_t sourcePos) const
+{
+  std::set<size_t> ret;
+  CollType::const_iterator iter;
+  for (iter = begin(); iter != end(); ++iter) {
+    // const std::pair<size_t,size_t> &align = *iter;
+    if (iter->first == sourcePos) {
+      ret.insert(iter->second);
+    }
+  }
+  return ret;
+}
+
+std::set<size_t> AlignmentInfo::GetAlignmentsForTarget(size_t targetPos) const
+{
+  std::set<size_t> ret;
+  CollType::const_iterator iter;
+  for (iter = begin(); iter != end(); ++iter) {
+    // const std::pair<size_t,size_t> &align = *iter;
+    if (iter->second == targetPos) {
+      ret.insert(iter->first);
+    }
+  }
+  return ret;
+}
+
+
+bool
+compare_target(std::pair<size_t,size_t> const* a,
+               std::pair<size_t,size_t> const* b)
+{
+  if(a->second < b->second)  return true;
+  if(a->second == b->second) return (a->first < b->first);
+  return false;
+}
+
+
+std::vector< const std::pair<size_t,size_t>* >
+AlignmentInfo::
+GetSortedAlignments(WordAlignmentSort SortOrder) const
+{
+  std::vector< const std::pair<size_t,size_t>* > ret;
+
+  CollType::const_iterator iter;
+  for (iter = m_collection.begin(); iter != m_collection.end(); ++iter) {
+    const std::pair<size_t,size_t> &alignPair = *iter;
+    ret.push_back(&alignPair);
+  }
+
+  switch (SortOrder) {
+  case NoSort:
+    break;
+
+  case TargetOrder:
+    std::sort(ret.begin(), ret.end(), compare_target);
+    break;
+
+  default:
+    UTIL_THROW(util::Exception, "Unknown word alignment sort option: "
+               << SortOrder);
+  }
+
+  return ret;
+
+}
+
+std::vector<size_t> AlignmentInfo::GetSourceIndex2PosMap() const
+{
+  std::set<size_t> sourcePoses;
+
+  CollType::const_iterator iter;
+  for (iter = m_collection.begin(); iter != m_collection.end(); ++iter) {
+    size_t sourcePos = iter->first;
+    sourcePoses.insert(sourcePos);
+  }
+  std::vector<size_t> ret(sourcePoses.begin(), sourcePoses.end());
+  return ret;
+}
+
+std::string AlignmentInfo::Debug(const System &system) const
+{
+  std::stringstream out;
+  out << *this;
+  return out.str();
+}
+
+std::ostream& operator<<(std::ostream& out, const AlignmentInfo& obj)
+{
+  AlignmentInfo::const_iterator iter;
+  for (iter = obj.begin(); iter != obj.end(); ++iter) {
+    out << iter->first << "-" << iter->second << " ";
+  }
+  return out;
+}
+
+}

mosesdecoder/moses2/AlignmentInfo.h

@@ -0,0 +1,148 @@
+/***********************************************************************
+ Moses - statistical machine translation system
+ Copyright (C) 2006-2011 University of Edinburgh
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+***********************************************************************/
+
+#pragma once
+
+#include <iostream>
+#include <ostream>
+#include <set>
+#include <vector>
+#include <cstdlib>
+
+#include <boost/functional/hash.hpp>
+#include "TypeDef.h"
+
+namespace Moses2
+{
+
+class AlignmentInfoCollection;
+class System;
+
+/** Collection of non-terminal alignment pairs, ordered by source index.
+  * Usually held by a TargetPhrase to map non-terms in hierarchical/syntax models
+ */
+class AlignmentInfo
+{
+  friend struct AlignmentInfoOrderer;
+  friend struct AlignmentInfoHasher;
+  friend class AlignmentInfoCollection;
+  friend class VW;
+
+  friend std::ostream& operator<<(std::ostream& out, const AlignmentInfo& obj);
+
+public:
+  typedef std::set<std::pair<size_t,size_t> > CollType;
+  typedef std::vector<size_t> NonTermIndexMap;
+  typedef CollType::const_iterator const_iterator;
+
+  const_iterator begin() const {
+    return m_collection.begin();
+  }
+  const_iterator end() const {
+    return m_collection.end();
+  }
+
+  void Add(size_t sourcePos, size_t targetPos) {
+    m_collection.insert(std::pair<size_t, size_t>(sourcePos, targetPos));
+  }
+  /** Provides a map from target-side to source-side non-terminal indices.
+    * The target-side index should be the rule symbol index (COUNTING terminals).
+    * The index returned is the rule non-terminal index (IGNORING terminals).
+   */
+  const NonTermIndexMap &GetNonTermIndexMap() const {
+    return m_nonTermIndexMap;
+  }
+
+  /** Like GetNonTermIndexMap but the return value is the symbol index (i.e.
+    * the index counting both terminals and non-terminals) */
+  const NonTermIndexMap &GetNonTermIndexMap2() const {
+    return m_nonTermIndexMap2;
+  }
+
+  const CollType &GetAlignments() const {
+    return m_collection;
+  }
+
+  std::set<size_t> GetAlignmentsForSource(size_t sourcePos) const;
+  std::set<size_t> GetAlignmentsForTarget(size_t targetPos) const;
+
+  size_t GetSize() const {
+    return m_collection.size();
+  }
+
+  std::vector< const std::pair<size_t,size_t>* >
+  GetSortedAlignments(Moses2::WordAlignmentSort SortOrder) const;
+
+  std::vector<size_t> GetSourceIndex2PosMap() const;
+
+  bool operator==(const AlignmentInfo& rhs) const {
+    return m_collection == rhs.m_collection &&
+           m_nonTermIndexMap == rhs.m_nonTermIndexMap;
+  }
+
+  std::string Debug(const System &system) const;
+
+private:
+  //! AlignmentInfo objects should only be created by an AlignmentInfoCollection
+  explicit AlignmentInfo(const std::set<std::pair<size_t,size_t> > &pairs);
+  explicit AlignmentInfo(const std::vector<unsigned char> &aln);
+
+  // used only by VW to load word alignment between sentences
+  explicit AlignmentInfo(const std::string &str);
+
+  void BuildNonTermIndexMaps();
+
+  CollType m_collection;
+  NonTermIndexMap m_nonTermIndexMap;
+  NonTermIndexMap m_nonTermIndexMap2;
+};
+
+/** Define an arbitrary strict weak ordering between AlignmentInfo objects
+ * for use by AlignmentInfoCollection.
+ */
+struct AlignmentInfoOrderer {
+  bool operator()(const AlignmentInfo &a, const AlignmentInfo &b) const {
+    if (a.m_collection == b.m_collection) {
+      return a.m_nonTermIndexMap < b.m_nonTermIndexMap;
+    } else {
+      return a.m_collection < b.m_collection;
+    }
+  }
+};
+
+/**
+ * Hashing functoid
+ **/
+struct AlignmentInfoHasher {
+  size_t operator()(const AlignmentInfo& a) const {
+    size_t seed = 0;
+    boost::hash_combine(seed,a.m_collection);
+    boost::hash_combine(seed,a.m_nonTermIndexMap);
+    return seed;
+  }
+
+};
+
+inline size_t hash_value(const AlignmentInfo& a)
+{
+  static AlignmentInfoHasher hasher;
+  return hasher(a);
+}
+
+}

mosesdecoder/moses2/AlignmentInfoCollection.cpp

@@ -0,0 +1,62 @@
+/***********************************************************************
+ Moses - statistical machine translation system
+ Copyright (C) 2006-2011 University of Edinburgh
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+***********************************************************************/
+
+#include "AlignmentInfoCollection.h"
+
+using namespace std;
+
+namespace Moses2
+{
+
+AlignmentInfoCollection AlignmentInfoCollection::s_instance;
+
+AlignmentInfoCollection::AlignmentInfoCollection()
+{
+  std::set<std::pair<size_t,size_t> > pairs;
+  m_emptyAlignmentInfo = Add(pairs);
+}
+
+AlignmentInfoCollection::~AlignmentInfoCollection()
+{}
+
+const AlignmentInfo &AlignmentInfoCollection::GetEmptyAlignmentInfo() const
+{
+  return *m_emptyAlignmentInfo;
+}
+
+AlignmentInfo const *
+AlignmentInfoCollection::
+Add(AlignmentInfo const& ainfo)
+{
+#ifdef WITH_THREADS
+  {
+    boost::shared_lock<boost::shared_mutex> read_lock(m_accessLock);
+    AlignmentInfoSet::const_iterator i = m_collection.find(ainfo);
+    if (i != m_collection.end())
+      return &*i;
+  }
+  boost::unique_lock<boost::shared_mutex> lock(m_accessLock);
+#endif
+  std::pair<AlignmentInfoSet::iterator, bool> ret = m_collection.insert(ainfo);
+  return &(*ret.first);
+}
+
+
+
+}

mosesdecoder/moses2/AlignmentInfoCollection.h

@@ -0,0 +1,81 @@
+/***********************************************************************
+ Moses - statistical machine translation system
+ Copyright (C) 2006-2011 University of Edinburgh
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+***********************************************************************/
+
+#pragma once
+
+#include "AlignmentInfo.h"
+
+#include <set>
+
+#ifdef WITH_THREADS
+#include <boost/thread/shared_mutex.hpp>
+#include <boost/thread/locks.hpp>
+#endif
+
+namespace Moses2
+{
+
+/** Singleton collection of all AlignmentInfo objects.
+ *  Used as a cache of all alignment info to save space.
+ */
+class AlignmentInfoCollection
+{
+public:
+  static AlignmentInfoCollection &Instance() {
+    return s_instance;
+  }
+
+  /** Returns a pointer to an AlignmentInfo object with the same source-target
+    * alignment pairs as given in the argument.  If the collection already
+    * contains such an object then returns a pointer to it; otherwise a new
+    * one is inserted.
+   */
+private:
+  const AlignmentInfo* Add(AlignmentInfo const& ainfo);
+
+public:
+  template<typename ALNREP>
+  AlignmentInfo const *
+  Add(ALNREP const & aln) {
+    return this->Add(AlignmentInfo(aln));
+  }
+
+  //! Returns a pointer to an empty AlignmentInfo object.
+  const AlignmentInfo &GetEmptyAlignmentInfo() const;
+
+private:
+  typedef std::set<AlignmentInfo, AlignmentInfoOrderer> AlignmentInfoSet;
+
+
+  //! Only a single static variable should be created.
+  AlignmentInfoCollection();
+  ~AlignmentInfoCollection();
+
+  static AlignmentInfoCollection s_instance;
+
+#ifdef WITH_THREADS
+  //reader-writer lock
+  mutable boost::shared_mutex m_accessLock;
+#endif
+
+  AlignmentInfoSet m_collection;
+  const AlignmentInfo *m_emptyAlignmentInfo;
+};
+
+}

mosesdecoder/moses2/ArcLists.cpp

@@ -0,0 +1,127 @@
+/*
+ * ArcList.cpp
+ *
+ *  Created on: 26 Oct 2015
+ *      Author: hieu
+ */
+#include <iostream>
+#include <sstream>
+#include <algorithm>
+#include <boost/foreach.hpp>
+#include "ArcLists.h"
+#include "HypothesisBase.h"
+#include "util/exception.hh"
+
+using namespace std;
+
+namespace Moses2
+{
+
+ArcLists::ArcLists()
+{
+  // TODO Auto-generated constructor stub
+
+}
+
+ArcLists::~ArcLists()
+{
+  BOOST_FOREACH(const Coll::value_type &collPair, m_coll) {
+    const ArcList *arcList = collPair.second;
+    delete arcList;
+  }
+}
+
+void ArcLists::AddArc(bool added, const HypothesisBase *currHypo,
+                      const HypothesisBase *otherHypo)
+{
+  //cerr << added << " " << currHypo << " " << otherHypo << endl;
+  ArcList *arcList;
+  if (added) {
+    // we're winners!
+    if (otherHypo) {
+      // there was a existing losing hypo
+      arcList = &GetAndDetachArcList(otherHypo);
+    } else {
+      // there was no existing hypo
+      arcList = new ArcList;
+    }
+    m_coll[currHypo] = arcList;
+  } else {
+    // we're losers!
+    // there should be a winner, we're not doing beam pruning
+    UTIL_THROW_IF2(otherHypo == NULL, "There must have been a winning hypo");
+    arcList = &GetArcList(otherHypo);
+  }
+
+  // in any case, add the curr hypo
+  arcList->push_back(currHypo);
+}
+
+ArcList &ArcLists::GetArcList(const HypothesisBase *hypo)
+{
+  Coll::iterator iter = m_coll.find(hypo);
+  UTIL_THROW_IF2(iter == m_coll.end(), "Can't find arc list");
+  ArcList &arcList = *iter->second;
+  return arcList;
+}
+
+const ArcList &ArcLists::GetArcList(const HypothesisBase *hypo) const
+{
+  Coll::const_iterator iter = m_coll.find(hypo);
+
+  if (iter == m_coll.end()) {
+    cerr << "looking for:" << hypo << " have " << m_coll.size() << " :";
+    BOOST_FOREACH(const Coll::value_type &collPair, m_coll) {
+      const HypothesisBase *hypo = collPair.first;
+      cerr << hypo << " ";
+    }
+  }
+
+  UTIL_THROW_IF2(iter == m_coll.end(), "Can't find arc list for " << hypo);
+  ArcList &arcList = *iter->second;
+  return arcList;
+}
+
+ArcList &ArcLists::GetAndDetachArcList(const HypothesisBase *hypo)
+{
+  Coll::iterator iter = m_coll.find(hypo);
+  UTIL_THROW_IF2(iter == m_coll.end(), "Can't find arc list");
+  ArcList &arcList = *iter->second;
+
+  m_coll.erase(iter);
+
+  return arcList;
+}
+
+void ArcLists::Sort()
+{
+  BOOST_FOREACH(Coll::value_type &collPair, m_coll) {
+    ArcList &list = *collPair.second;
+    std::sort(list.begin(), list.end(), HypothesisFutureScoreOrderer() );
+  }
+}
+
+void ArcLists::Delete(const HypothesisBase *hypo)
+{
+  //cerr << "hypo=" << hypo->Debug() << endl;
+  //cerr << "m_coll=" << m_coll.size() << endl;
+  Coll::iterator iter = m_coll.find(hypo);
+  UTIL_THROW_IF2(iter == m_coll.end(), "Can't find arc list");
+  ArcList *arcList = iter->second;
+
+  m_coll.erase(iter);
+  delete arcList;
+}
+
+std::string ArcLists::Debug(const System &system) const
+{
+  stringstream strm;
+  BOOST_FOREACH(const Coll::value_type &collPair, m_coll) {
+    const ArcList *arcList = collPair.second;
+    strm << arcList << "(" << arcList->size() << ") ";
+  }
+  return strm.str();
+}
+
+}
+

mosesdecoder/moses2/ArcLists.h

@@ -0,0 +1,43 @@
+/*
+ * ArcList.h
+ *
+ *  Created on: 26 Oct 2015
+ *      Author: hieu
+ */
+#pragma once
+#include <vector>
+#include <boost/unordered_map.hpp>
+
+namespace Moses2
+{
+class System;
+
+class HypothesisBase;
+
+typedef std::vector<const HypothesisBase*> ArcList;
+
+class ArcLists
+{
+public:
+  ArcLists();
+  virtual ~ArcLists();
+
+  void AddArc(bool added, const HypothesisBase *currHypo,
+              const HypothesisBase *otherHypo);
+  void Sort();
+  void Delete(const HypothesisBase *hypo);
+
+  const ArcList &GetArcList(const HypothesisBase *hypo) const;
+
+  std::string Debug(const System &system) const;
+protected:
+  typedef boost::unordered_map<const HypothesisBase*, ArcList*> Coll;
+  Coll m_coll;
+
+  ArcList &GetArcList(const HypothesisBase *hypo);
+  ArcList &GetAndDetachArcList(const HypothesisBase *hypo);
+
+};
+
+}
+

mosesdecoder/moses2/Array.h

@@ -0,0 +1,83 @@
+#pragma once
+#include <cassert>
+#include <boost/functional/hash.hpp>
+#include "MemPool.h"
+
+namespace Moses2
+{
+
+template<typename T>
+class Array
+{
+public:
+  typedef T* iterator;
+  typedef const T* const_iterator;
+  //! iterators
+  const_iterator begin() const {
+    return m_arr;
+  }
+  const_iterator end() const {
+    return m_arr + m_size;
+  }
+
+  iterator begin() {
+    return m_arr;
+  }
+  iterator end() {
+    return m_arr + m_size;
+  }
+
+  Array(MemPool &pool, size_t size = 0, const T &val = T()) {
+    m_size = size;
+    m_maxSize = size;
+    m_arr = pool.Allocate<T>(size);
+    for (size_t i = 0; i < size; ++i) {
+      m_arr[i] = val;
+    }
+  }
+
+  size_t size() const {
+    return m_size;
+  }
+
+  const T& operator[](size_t ind) const {
+    return m_arr[ind];
+  }
+
+  T& operator[](size_t ind) {
+    return m_arr[ind];
+  }
+
+  T *GetArray() {
+    return m_arr;
+  }
+
+  size_t hash() const {
+    size_t seed = 0;
+    for (size_t i = 0; i < m_size; ++i) {
+      boost::hash_combine(seed, m_arr[i]);
+    }
+    return seed;
+  }
+
+  int Compare(const Array &compare) const {
+
+    int cmp = memcmp(m_arr, compare.m_arr, sizeof(T) * m_size);
+    return cmp;
+  }
+
+  bool operator==(const Array &compare) const {
+    int cmp = Compare(compare);
+    return cmp == 0;
+  }
+
+  void resize(size_t newSize) {
+    assert(m_size <= m_maxSize);
+    m_size = newSize;
+  }
+protected:
+  size_t m_size, m_maxSize;
+  T *m_arr;
+};
+
+}

mosesdecoder/moses2/DLLEntryApi.cpp

@@ -0,0 +1,74 @@
+#include "Moses2Wrapper.h"
+#include <iostream>
+#include <string.h>
+
+
+// Generic helper definitions for shared library support
+#if defined _WIN32
+#define IMPORT __declspec(dllimport)
+#define EXPORT __declspec(dllexport)
+#else    // !(defined _WIN32 || defined __CYGWIN__) -- i.e., not Windows
+#define __stdcall
+#if __GNUC__ >= 4
+#define IMPORT __attribute__ ((visibility ("default")))
+#define EXPORT __attribute__ ((visibility ("default")))
+#else   // __GNUC__ < 4, which does not support the __attribute__ tag
+#define IMPORT
+#define EXPORT
+#endif  // __GNUC__ >= 4
+#endif 
+
+
+using namespace std;
+using namespace Moses2;
+
+extern "C" EXPORT MosesApiErrorCode __stdcall GetMosesSystem(const char* filePath, Moses2::Moses2Wrapper * *pObject) {
+
+	if (*pObject == NULL) {
+		*pObject = new Moses2::Moses2Wrapper(filePath);
+		return MS_API_OK;
+	}
+	else {
+		return MS_API_E_FAILURE;
+	}
+}
+
+extern "C" EXPORT MosesApiErrorCode __stdcall Translate(Moses2::Moses2Wrapper * pObject, long id, const char* input, char** output) {
+	if (pObject != NULL)
+	{
+		std::string tr = pObject->Translate(input, id);
+		*output = Moses2Wrapper::CopyString(tr.c_str());
+		return MS_API_OK;
+	}
+	else {
+		return MS_API_E_FAILURE;
+	}
+}
+
+extern "C" EXPORT MosesApiErrorCode __stdcall FreeMemory(char* output) {
+	if (output != nullptr) {
+		Moses2Wrapper::Free(output);
+		return MS_API_OK;
+	}
+	else {
+		return MS_API_E_FAILURE;
+	}
+}
+
+extern "C" EXPORT MosesApiErrorCode __stdcall ReleaseSystem(Moses2::Moses2Wrapper **pObject) {
+	if (*pObject != NULL)
+	{
+		delete* pObject;
+		*pObject = NULL;
+		return MS_API_OK;
+	}
+	else {
+		return MS_API_E_FAILURE;
+	}
+}
+
+extern "C" EXPORT MosesApiErrorCode __stdcall EngineVersion() {
+	//std::cout << "windows build on v1142/ msvc 14.27.29110"<< std::endl;
+	std::cout << "0.0.1" << std::endl;
+	return MS_API_OK;
+}

mosesdecoder/moses2/EstimatedScores.cpp

@@ -0,0 +1,117 @@
+// $Id$
+// vim:tabstop=2
+
+/***********************************************************************
+ Moses - factored phrase-based language decoder
+ Copyright (C) 2006 University of Edinburgh
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ ***********************************************************************/
+
+#include <string>
+#include <iostream>
+#include "EstimatedScores.h"
+
+using namespace std;
+
+namespace Moses2
+{
+/**
+ * Calculate future score estimate for a given coverage bitmap
+ *
+ * /param bitmap coverage bitmap
+ */
+
+float EstimatedScores::CalcEstimatedScore(Bitmap const &bitmap) const
+{
+  const size_t notInGap = numeric_limits<size_t>::max();
+  size_t startGap = notInGap;
+  float estimatedScore = 0.0f;
+  for (size_t currPos = 0; currPos < bitmap.GetSize(); currPos++) {
+    // start of a new gap?
+    if (bitmap.GetValue(currPos) == false && startGap == notInGap) {
+      startGap = currPos;
+    }
+    // end of a gap?
+    else if (bitmap.GetValue(currPos) == true && startGap != notInGap) {
+      estimatedScore += GetValue(startGap, currPos - 1);
+      startGap = notInGap;
+    }
+  }
+  // coverage ending with gap?
+  if (startGap != notInGap) {
+    estimatedScore += GetValue(startGap, bitmap.GetSize() - 1);
+  }
+
+  return estimatedScore;
+}
+
+/**
+ * Calculare future score estimate for a given coverage bitmap
+ * and an additional span that is also covered. This function is used
+ * to compute future score estimates for hypotheses that we may want
+ * build, but first want to check.
+ *
+ * Note: this function is implemented a bit more complex than
+ * the basic one (w/o additional phrase) for speed reasons,
+ * which is probably overkill.
+ *
+ * /param bitmap coverage bitmap
+ * /param startPos start of the span that is added to the coverage
+ * /param endPos end of the span that is added to the coverage
+ */
+
+float EstimatedScores::CalcEstimatedScore(Bitmap const &bitmap, size_t startPos,
+    size_t endPos) const
+{
+  const size_t notInGap = numeric_limits<size_t>::max();
+  float estimatedScore = 0.0f;
+  size_t startGap = bitmap.GetFirstGapPos();
+  if (startGap == NOT_FOUND) return estimatedScore; // everything filled
+
+  // start loop at first gap
+  size_t startLoop = startGap + 1;
+  if (startPos == startGap) { // unless covered by phrase
+    startGap = notInGap;
+    startLoop = endPos + 1; // -> postpone start
+  }
+
+  size_t lastCovered = bitmap.GetLastPos();
+  if (endPos > lastCovered || lastCovered == NOT_FOUND) lastCovered = endPos;
+
+  for (size_t currPos = startLoop; currPos <= lastCovered; currPos++) {
+    // start of a new gap?
+    if (startGap == notInGap && bitmap.GetValue(currPos) == false
+        && (currPos < startPos || currPos > endPos)) {
+      startGap = currPos;
+    }
+    // end of a gap?
+    else if (startGap != notInGap
+             && (bitmap.GetValue(currPos) == true
+                 || (startPos <= currPos && currPos <= endPos))) {
+      estimatedScore += GetValue(startGap, currPos - 1);
+      startGap = notInGap;
+    }
+  }
+  // coverage ending with gap?
+  if (lastCovered != bitmap.GetSize() - 1) {
+    estimatedScore += GetValue(lastCovered + 1, bitmap.GetSize() - 1);
+  }
+
+  return estimatedScore;
+}
+
+}
+

mosesdecoder/moses2/EstimatedScores.h

@@ -0,0 +1,59 @@
+// $Id$
+
+/***********************************************************************
+ Moses - factored phrase-based language decoder
+ Copyright (C) 2006 University of Edinburgh
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ ***********************************************************************/
+
+#pragma once
+
+#include <iostream>
+#include "legacy/Util2.h"
+#include "legacy/Bitmap.h"
+#include "legacy/Matrix.h"
+
+namespace Moses2
+{
+class MemPool;
+class System;
+
+//! A square array of floats to store future costs in the phrase-based decoder
+class EstimatedScores: public Matrix<float>
+{
+public:
+  EstimatedScores(MemPool &pool, size_t size) :
+    Matrix<float>(pool, size, size) {
+  }
+
+  ~EstimatedScores(); // not implemented
+
+  float CalcEstimatedScore(Bitmap const&) const;
+  float CalcEstimatedScore(Bitmap const&, size_t startPos, size_t endPos) const;
+
+  std::ostream &Debug(std::ostream &out, const System &system) const {
+    for (size_t endPos = 0; endPos < GetSize(); endPos++) {
+      for (size_t startPos = 0; startPos < GetSize(); startPos++)
+        out << GetValue(startPos, endPos) << " ";
+      out << std::endl;
+    }
+    return out;
+  }
+
+};
+
+}
+

mosesdecoder/moses2/FF/Distortion.cpp

@@ -0,0 +1,182 @@
+/*
+ * Distortion.cpp
+ *
+ *  Created on: 28 Oct 2015
+ *      Author: hieu
+ */
+#include <sstream>
+#include "Distortion.h"
+#include "../PhraseBased/Hypothesis.h"
+#include "../PhraseBased/Manager.h"
+#include "../legacy/Range.h"
+#include "../legacy/Bitmap.h"
+
+using namespace std;
+
+namespace Moses2
+{
+
+struct DistortionState_traditional: public FFState {
+  Range range;
+  int first_gap;
+
+  DistortionState_traditional() :
+    range() {
+    // uninitialised
+  }
+
+  void Set(const Range& wr, int fg) {
+    range = wr;
+    first_gap = fg;
+  }
+
+  size_t hash() const {
+    return range.GetEndPos();
+  }
+  virtual bool operator==(const FFState& other) const {
+    const DistortionState_traditional& o =
+      static_cast<const DistortionState_traditional&>(other);
+    return range.GetEndPos() == o.range.GetEndPos();
+  }
+
+  virtual std::string ToString() const {
+    stringstream sb;
+    sb << first_gap << " " << range;
+    return sb.str();
+  }
+
+};
+
+///////////////////////////////////////////////////////////////////////
+Distortion::Distortion(size_t startInd, const std::string &line) :
+  StatefulFeatureFunction(startInd, line)
+{
+  ReadParameters();
+}
+
+Distortion::~Distortion()
+{
+  // TODO Auto-generated destructor stub
+}
+
+FFState* Distortion::BlankState(MemPool &pool, const System &sys) const
+{
+  return new (pool.Allocate<DistortionState_traditional>()) DistortionState_traditional();
+}
+
+void Distortion::EmptyHypothesisState(FFState &state, const ManagerBase &mgr,
+                                      const InputType &input, const Hypothesis &hypo) const
+{
+  DistortionState_traditional &stateCast =
+    static_cast<DistortionState_traditional&>(state);
+
+  // fake previous translated phrase start and end
+  size_t start = NOT_FOUND;
+  size_t end = NOT_FOUND;
+  /*
+   if (input.m_frontSpanCoveredLength > 0) {
+   // can happen with --continue-partial-translation
+   start = 0;
+   end = input.m_frontSpanCoveredLength -1;
+   }
+   */
+
+  stateCast.range = Range(start, end);
+  stateCast.first_gap = NOT_FOUND;
+}
+
+void Distortion::EvaluateInIsolation(MemPool &pool, const System &system,
+                                     const Phrase<Moses2::Word> &source, const TargetPhraseImpl &targetPhrase, Scores &scores,
+                                     SCORE &estimatedScore) const
+{
+}
+
+void Distortion::EvaluateInIsolation(MemPool &pool, const System &system, const Phrase<SCFG::Word> &source,
+                                     const TargetPhrase<SCFG::Word> &targetPhrase, Scores &scores,
+                                     SCORE &estimatedScore) const
+{
+}
+
+void Distortion::EvaluateWhenApplied(const ManagerBase &mgr,
+                                     const Hypothesis &hypo, const FFState &prevState, Scores &scores,
+                                     FFState &state) const
+{
+  const DistortionState_traditional &prev =
+    static_cast<const DistortionState_traditional&>(prevState);
+  SCORE distortionScore = CalculateDistortionScore(prev.range,
+                          hypo.GetInputPath().range, prev.first_gap);
+  //cerr << "distortionScore=" << distortionScore << endl;
+
+  scores.PlusEquals(mgr.system, *this, distortionScore);
+
+  DistortionState_traditional &stateCast =
+    static_cast<DistortionState_traditional&>(state);
+  stateCast.Set(hypo.GetInputPath().range, hypo.GetBitmap().GetFirstGapPos());
+
+  //cerr << "hypo=" << hypo.Debug(mgr.system) << endl;
+}
+
+SCORE Distortion::CalculateDistortionScore(const Range &prev, const Range &curr,
+    const int FirstGap) const
+{
+  bool useEarlyDistortionCost = false;
+  if (!useEarlyDistortionCost) {
+    return -(SCORE) ComputeDistortionDistance(prev, curr);
+  } else {
+    /* Pay distortion score as soon as possible, from Moore and Quirk MT Summit 2007
+     Definitions:
+     S   : current source range
+     S'  : last translated source phrase range
+     S'' : longest fully-translated initial segment
+     */
+
+    int prefixEndPos = (int) FirstGap - 1;
+    if ((int) FirstGap == -1) prefixEndPos = -1;
+
+    // case1: S is adjacent to S'' => return 0
+    if ((int) curr.GetStartPos() == prefixEndPos + 1) {
+      //IFVERBOSE(4) std::cerr<< "MQ07disto:case1" << std::endl;
+      return 0;
+    }
+
+    // case2: S is to the left of S' => return 2(length(S))
+    if ((int) curr.GetEndPos() < (int) prev.GetEndPos()) {
+      //IFVERBOSE(4) std::cerr<< "MQ07disto:case2" << std::endl;
+      return (float) -2 * (int) curr.GetNumWordsCovered();
+    }
+
+    // case3: S' is a subsequence of S'' => return 2(nbWordBetween(S,S'')+length(S))
+    if ((int) prev.GetEndPos() <= prefixEndPos) {
+      //IFVERBOSE(4) std::cerr<< "MQ07disto:case3" << std::endl;
+      int z = (int) curr.GetStartPos() - prefixEndPos - 1;
+      return (float) -2 * (z + (int) curr.GetNumWordsCovered());
+    }
+
+    // case4: otherwise => return 2(nbWordBetween(S,S')+length(S))
+    //IFVERBOSE(4) std::cerr<< "MQ07disto:case4" << std::endl;
+    return (float) -2
+           * ((int) curr.GetNumWordsBetween(prev) + (int) curr.GetNumWordsCovered());
+
+  }
+}
+
+int Distortion::ComputeDistortionDistance(const Range& prev,
+    const Range& current) const
+{
+  int dist = 0;
+  if (prev.GetNumWordsCovered() == 0) {
+    dist = current.GetStartPos();
+  } else {
+    dist = (int) prev.GetEndPos() - (int) current.GetStartPos() + 1;
+  }
+  return abs(dist);
+}
+
+void Distortion::EvaluateWhenApplied(const SCFG::Manager &mgr,
+                                     const SCFG::Hypothesis &hypo, int featureID, Scores &scores,
+                                     FFState &state) const
+{
+  UTIL_THROW2("Not implemented");
+}
+
+}

mosesdecoder/moses2/FF/Distortion.h

@@ -0,0 +1,59 @@
+/*
+ * Distortion.h
+ *
+ *  Created on: 28 Oct 2015
+ *      Author: hieu
+ */
+
+#ifndef DISTORTION_H_
+#define DISTORTION_H_
+
+#include "StatefulFeatureFunction.h"
+#include "../legacy/Range.h"
+#include "../TypeDef.h"
+
+namespace Moses2
+{
+
+class Distortion: public StatefulFeatureFunction
+{
+public:
+  Distortion(size_t startInd, const std::string &line);
+  virtual ~Distortion();
+
+  virtual FFState* BlankState(MemPool &pool, const System &sys) const;
+  virtual void EmptyHypothesisState(FFState &state, const ManagerBase &mgr,
+                                    const InputType &input, const Hypothesis &hypo) const;
+
+  virtual void
+  EvaluateInIsolation(MemPool &pool, const System &system, const Phrase<Moses2::Word> &source,
+                      const TargetPhraseImpl &targetPhrase, Scores &scores,
+                      SCORE &estimatedScore) const;
+
+  virtual void
+  EvaluateInIsolation(MemPool &pool, const System &system, const Phrase<SCFG::Word> &source,
+                      const TargetPhrase<SCFG::Word> &targetPhrase, Scores &scores,
+                      SCORE &estimatedScore) const;
+
+  virtual void EvaluateWhenApplied(const std::deque<Hypothesis*> &hypos) const {
+  }
+
+  virtual void EvaluateWhenApplied(const ManagerBase &mgr,
+                                   const Hypothesis &hypo, const FFState &prevState, Scores &scores,
+                                   FFState &state) const;
+
+  virtual void EvaluateWhenApplied(const SCFG::Manager &mgr,
+                                   const SCFG::Hypothesis &hypo, int featureID, Scores &scores,
+                                   FFState &state) const;
+
+protected:
+  SCORE CalculateDistortionScore(const Range &prev, const Range &curr,
+                                 const int FirstGap) const;
+
+  int ComputeDistortionDistance(const Range& prev, const Range& current) const;
+
+};
+
+}
+
+#endif /* DISTORTION_H_ */

mosesdecoder/moses2/FF/ExampleStatefulFF.cpp

@@ -0,0 +1,96 @@
+/*
+ * ExampleStatefulFF.cpp
+ *
+ *  Created on: 27 Oct 2015
+ *      Author: hieu
+ */
+#include <sstream>
+#include "ExampleStatefulFF.h"
+#include "../PhraseBased/Manager.h"
+#include "../PhraseBased/Hypothesis.h"
+
+using namespace std;
+
+namespace Moses2
+{
+
+class ExampleState: public FFState
+{
+public:
+  int targetLen;
+
+  ExampleState() {
+    // uninitialised
+  }
+
+  virtual size_t hash() const {
+    return (size_t) targetLen;
+  }
+  virtual bool operator==(const FFState& o) const {
+    const ExampleState& other = static_cast<const ExampleState&>(o);
+    return targetLen == other.targetLen;
+  }
+
+  virtual std::string ToString() const {
+    stringstream sb;
+    sb << targetLen;
+    return sb.str();
+  }
+
+};
+
+////////////////////////////////////////////////////////////////////////////////////////
+ExampleStatefulFF::ExampleStatefulFF(size_t startInd, const std::string &line) :
+  StatefulFeatureFunction(startInd, line)
+{
+  ReadParameters();
+}
+
+ExampleStatefulFF::~ExampleStatefulFF()
+{
+  // TODO Auto-generated destructor stub
+}
+
+FFState* ExampleStatefulFF::BlankState(MemPool &pool, const System &sys) const
+{
+  return new (pool.Allocate<ExampleState>()) ExampleState();
+}
+
+void ExampleStatefulFF::EmptyHypothesisState(FFState &state,
+    const ManagerBase &mgr, const InputType &input,
+    const Hypothesis &hypo) const
+{
+  ExampleState &stateCast = static_cast<ExampleState&>(state);
+  stateCast.targetLen = 0;
+}
+
+void ExampleStatefulFF::EvaluateInIsolation(MemPool &pool,
+    const System &system, const Phrase<Moses2::Word> &source,
+    const TargetPhraseImpl &targetPhrase, Scores &scores,
+    SCORE &estimatedScore) const
+{
+}
+
+void ExampleStatefulFF::EvaluateInIsolation(MemPool &pool, const System &system, const Phrase<SCFG::Word> &source,
+    const TargetPhrase<SCFG::Word> &targetPhrase, Scores &scores,
+    SCORE &estimatedScore) const
+{
+}
+
+void ExampleStatefulFF::EvaluateWhenApplied(const ManagerBase &mgr,
+    const Hypothesis &hypo, const FFState &prevState, Scores &scores,
+    FFState &state) const
+{
+  ExampleState &stateCast = static_cast<ExampleState&>(state);
+  stateCast.targetLen = hypo.GetTargetPhrase().GetSize();
+}
+
+void ExampleStatefulFF::EvaluateWhenApplied(const SCFG::Manager &mgr,
+    const SCFG::Hypothesis &hypo, int featureID, Scores &scores,
+    FFState &state) const
+{
+  UTIL_THROW2("Not implemented");
+}
+
+}
+

mosesdecoder/moses2/FF/ExampleStatefulFF.h

@@ -0,0 +1,46 @@
+/*
+ * ExampleStatefulFF.h
+ *
+ *  Created on: 27 Oct 2015
+ *      Author: hieu
+ */
+
+#pragma once
+
+#include "StatefulFeatureFunction.h"
+
+namespace Moses2
+{
+
+class ExampleStatefulFF: public StatefulFeatureFunction
+{
+public:
+  ExampleStatefulFF(size_t startInd, const std::string &line);
+  virtual ~ExampleStatefulFF();
+
+  virtual FFState* BlankState(MemPool &pool, const System &sys) const;
+  virtual void EmptyHypothesisState(FFState &state, const ManagerBase &mgr,
+                                    const InputType &input, const Hypothesis &hypo) const;
+
+  virtual void
+  EvaluateInIsolation(MemPool &pool, const System &system, const Phrase<Moses2::Word> &source,
+                      const TargetPhraseImpl &targetPhrase, Scores &scores,
+                      SCORE &estimatedScore) const;
+
+  virtual void
+  EvaluateInIsolation(MemPool &pool, const System &system, const Phrase<SCFG::Word> &source,
+                      const TargetPhrase<SCFG::Word> &targetPhrase, Scores &scores,
+                      SCORE &estimatedScore) const;
+
+  virtual void EvaluateWhenApplied(const ManagerBase &mgr,
+                                   const Hypothesis &hypo, const FFState &prevState, Scores &scores,
+                                   FFState &state) const;
+
+  virtual void EvaluateWhenApplied(const SCFG::Manager &mgr,
+                                   const SCFG::Hypothesis &hypo, int featureID, Scores &scores,
+                                   FFState &state) const;
+
+};
+
+}
+

mosesdecoder/moses2/FF/ExampleStatelessFF.cpp

@@ -0,0 +1,40 @@
+/*
+ * SkeletonStatefulFF.cpp
+ *
+ *  Created on: 27 Oct 2015
+ *      Author: hieu
+ */
+#include "../Scores.h"
+
+#include "ExampleStatelessFF.h"
+
+namespace Moses2
+{
+
+ExampleStatelessFF::ExampleStatelessFF(size_t startInd,
+                                       const std::string &line) :
+  StatelessFeatureFunction(startInd, line)
+{
+  ReadParameters();
+}
+
+ExampleStatelessFF::~ExampleStatelessFF()
+{
+  // TODO Auto-generated destructor stub
+}
+
+void ExampleStatelessFF::EvaluateInIsolation(MemPool &pool,
+    const System &system, const Phrase<Moses2::Word> &source,
+    const TargetPhraseImpl &targetPhrase, Scores &scores,
+    SCORE &estimatedScore) const
+{
+}
+
+void ExampleStatelessFF::EvaluateInIsolation(MemPool &pool, const System &system, const Phrase<SCFG::Word> &source,
+    const TargetPhrase<SCFG::Word> &targetPhrase, Scores &scores,
+    SCORE &estimatedScore) const
+{
+}
+
+}
+

mosesdecoder/moses2/FF/ExampleStatelessFF.h

@@ -0,0 +1,34 @@
+/*
+ * SkeletonStatefulFF.h
+ *
+ *  Created on: 27 Oct 2015
+ *      Author: hieu
+ */
+
+#pragma once
+
+#include "StatelessFeatureFunction.h"
+
+namespace Moses2
+{
+
+class ExampleStatelessFF: public StatelessFeatureFunction
+{
+public:
+  ExampleStatelessFF(size_t startInd, const std::string &line);
+  virtual ~ExampleStatelessFF();
+
+  virtual void
+  EvaluateInIsolation(MemPool &pool, const System &system, const Phrase<Moses2::Word> &source,
+                      const TargetPhraseImpl &targetPhrase, Scores &scores,
+                      SCORE &estimatedScore) const;
+
+  virtual void
+  EvaluateInIsolation(MemPool &pool, const System &system, const Phrase<SCFG::Word> &source,
+                      const TargetPhrase<SCFG::Word> &targetPhrase, Scores &scores,
+                      SCORE &estimatedScore) const;
+
+};
+
+}
+

mosesdecoder/moses2/FF/FFState.cpp

@@ -0,0 +1 @@
+#include "FFState.h"

mosesdecoder/moses2/FF/FFState.h

@@ -0,0 +1,50 @@
+#pragma once
+
+#include <vector>
+#include <stddef.h>
+#include "util/exception.hh"
+
+namespace Moses2
+{
+
+class FFState
+{
+public:
+  virtual ~FFState() {
+  }
+  virtual size_t hash() const = 0;
+  virtual bool operator==(const FFState& other) const = 0;
+
+  virtual bool operator!=(const FFState& other) const {
+    return !(*this == other);
+  }
+
+  virtual std::string ToString() const = 0;
+};
+
+////////////////////////////////////////////////////////////////////////////////////////
+inline std::ostream& operator<<(std::ostream& out, const FFState& obj)
+{
+  out << obj.ToString();
+  return out;
+}
+
+////////////////////////////////////////////////////////////////////////////////////////
+class DummyState: public FFState
+{
+public:
+  DummyState() {
+  }
+
+  virtual size_t hash() const {
+    return 0;
+  }
+
+  virtual bool operator==(const FFState& other) const {
+    return true;
+  }
+
+};
+
+}
+

mosesdecoder/moses2/FF/FeatureFunction.cpp

@@ -0,0 +1,82 @@
+/*
+ * FeatureFunction.cpp
+ *
+ *  Created on: 23 Oct 2015
+ *      Author: hieu
+ */
+#include <string>
+#include <vector>
+#include "FeatureFunction.h"
+#include "../System.h"
+#include "../legacy/Util2.h"
+#include "util/exception.hh"
+
+using namespace std;
+
+namespace Moses2
+{
+
+FeatureFunction::FeatureFunction(size_t startInd, const std::string &line)
+  :m_startInd(startInd)
+  ,m_numScores(1)
+  ,m_PhraseTableInd(NOT_FOUND)
+  ,m_tuneable(true)
+{
+  ParseLine(line);
+  //cerr << GetName() << " " << m_startInd << "-" << (m_startInd + m_numScores - 1) << endl;
+}
+
+FeatureFunction::~FeatureFunction()
+{
+  // TODO Auto-generated destructor stub
+}
+
+void FeatureFunction::ParseLine(const std::string &line)
+{
+  vector<string> toks = Tokenize(line);
+  UTIL_THROW_IF2(toks.empty(), "Empty line");
+
+  string nameStub = toks[0];
+
+  set<string> keys;
+
+  for (size_t i = 1; i < toks.size(); ++i) {
+    vector<string> args = TokenizeFirstOnly(toks[i], "=");
+    UTIL_THROW_IF2(args.size() != 2,
+                   "Incorrect format for feature function arg: " << toks[i]);
+
+    pair<set<string>::iterator, bool> ret = keys.insert(args[0]);
+    UTIL_THROW_IF2(!ret.second, "Duplicate key in line " << line);
+
+    if (args[0] == "num-features") {
+      m_numScores = Scan<size_t>(args[1]);
+    } else if (args[0] == "name") {
+      m_name = args[1];
+    } else {
+      m_args.push_back(args);
+    }
+  }
+}
+
+void FeatureFunction::ReadParameters()
+{
+  while (!m_args.empty()) {
+    const vector<string> &args = m_args[0];
+    SetParameter(args[0], args[1]);
+
+    m_args.erase(m_args.begin());
+  }
+}
+
+void FeatureFunction::SetParameter(const std::string& key,
+                                   const std::string& value)
+{
+  if (key == "tuneable") {
+    m_tuneable = Scan<bool>(value);
+  } else {
+    UTIL_THROW2(GetName() << ": Unknown argument " << key << "=" << value);
+  }
+}
+
+}
+

mosesdecoder/moses2/FF/FeatureFunction.h

@@ -0,0 +1,118 @@
+/*
+ * FeatureFunction.h
+ *
+ *  Created on: 23 Oct 2015
+ *      Author: hieu
+ */
+
+#pragma once
+
+#include <cstddef>
+#include <string>
+#include <vector>
+#include "../TypeDef.h"
+#include "../Phrase.h"
+
+namespace Moses2
+{
+template<typename WORD>
+class TargetPhrase;
+
+class System;
+class PhraseImpl;
+class TargetPhrases;
+class TargetPhraseImpl;
+class Scores;
+class ManagerBase;
+class MemPool;
+class InputType;
+
+namespace SCFG
+{
+class TargetPhrase;
+class TargetPhrases;
+class Word;
+}
+
+class FeatureFunction
+{
+public:
+
+  FeatureFunction(size_t startInd, const std::string &line);
+  virtual ~FeatureFunction();
+  virtual void Load(System &system) {
+  }
+
+  size_t GetStartInd() const {
+    return m_startInd;
+  }
+  size_t GetNumScores() const {
+    return m_numScores;
+  }
+  const std::string &GetName() const {
+    return m_name;
+  }
+  void SetName(const std::string &val) {
+    m_name = val;
+  }
+
+  virtual size_t HasPhraseTableInd() const {
+    return false;
+  }
+  void SetPhraseTableInd(size_t ind) {
+    m_PhraseTableInd = ind;
+  }
+  size_t GetPhraseTableInd() const {
+    return m_PhraseTableInd;
+  }
+
+  //! if false, then this feature is not displayed in the n-best list.
+  // use with care
+  virtual bool IsTuneable() const {
+    return m_tuneable;
+  }
+
+  virtual void SetParameter(const std::string& key, const std::string& value);
+
+  // may have more factors than actually need, but not guaranteed.
+  virtual void
+  EvaluateInIsolation(MemPool &pool, const System &system, const Phrase<Moses2::Word> &source,
+                      const TargetPhraseImpl &targetPhrase, Scores &scores,
+                      SCORE &estimatedScore) const = 0;
+
+  // For SCFG decoding, the source can contain non-terminals, NOT the raw
+  // source from the input sentence
+  virtual void
+  EvaluateInIsolation(MemPool &pool, const System &system, const Phrase<SCFG::Word> &source,
+                      const TargetPhrase<SCFG::Word> &targetPhrase, Scores &scores,
+                      SCORE &estimatedScore) const = 0;
+
+  // used by lexicalised reordering model to add scores to tp data structures
+  virtual void EvaluateAfterTablePruning(MemPool &pool,
+                                         const TargetPhrases &tps, const Phrase<Moses2::Word> &sourcePhrase) const {
+  }
+
+  virtual void EvaluateAfterTablePruning(MemPool &pool,
+                                         const SCFG::TargetPhrases &tps, const Phrase<SCFG::Word> &sourcePhrase) const {
+  }
+
+  virtual void InitializeForInput(const ManagerBase &mgr, const InputType &input) { };
+
+  // clean up temporary memory, called after processing each sentence
+  virtual void CleanUpAfterSentenceProcessing(const System &system, const InputType &input) const {
+  }
+
+protected:
+  size_t m_startInd;
+  size_t m_numScores;
+  size_t m_PhraseTableInd;
+  std::string m_name;
+  std::vector<std::vector<std::string> > m_args;
+  bool m_tuneable;
+
+  virtual void ReadParameters();
+  void ParseLine(const std::string &line);
+};
+
+}
+

mosesdecoder/moses2/FF/FeatureFunctions.cpp

@@ -0,0 +1,291 @@
+/*
+ * FeatureFunctions.cpp
+ *
+ *  Created on: 27 Oct 2015
+ *      Author: hieu
+ */
+
+#include <boost/foreach.hpp>
+#include "FeatureRegistry.h"
+#include "FeatureFunctions.h"
+#include "StatefulFeatureFunction.h"
+#include "../System.h"
+#include "../Scores.h"
+#include "../MemPool.h"
+
+#include "../TranslationModel/PhraseTable.h"
+#include "../TranslationModel/UnknownWordPenalty.h"
+#include "../SCFG/TargetPhraseImpl.h"
+#include "../SCFG/Word.h"
+#include "../PhraseBased/TargetPhraseImpl.h"
+#include "util/exception.hh"
+
+using namespace std;
+
+namespace Moses2
+{
+FeatureFunctions::FeatureFunctions(System &system) :
+  m_system(system), m_ffStartInd(0)
+{
+}
+
+FeatureFunctions::~FeatureFunctions()
+{
+  RemoveAllInColl(m_featureFunctions);
+}
+
+void FeatureFunctions::Load()
+{
+  // load, everything but pts
+  BOOST_FOREACH(const FeatureFunction *ff, m_featureFunctions) {
+    FeatureFunction *nonConstFF = const_cast<FeatureFunction*>(ff);
+    PhraseTable *pt = dynamic_cast<PhraseTable*>(nonConstFF);
+
+    if (pt) {
+      // do nothing. load pt last
+    } else {
+      cerr << "Loading " << nonConstFF->GetName() << endl;
+      nonConstFF->Load(m_system);
+      cerr << "Finished loading " << nonConstFF->GetName() << endl;
+    }
+  }
+
+// load pt
+  BOOST_FOREACH(const PhraseTable *pt, phraseTables) {
+    PhraseTable *nonConstPT = const_cast<PhraseTable*>(pt);
+    cerr << "Loading " << nonConstPT->GetName() << endl;
+    nonConstPT->Load(m_system);
+    cerr << "Finished loading " << nonConstPT->GetName() << endl;
+  }
+}
+
+void FeatureFunctions::Create()
+{
+  const Parameter &params = m_system.params;
+
+  const PARAM_VEC *ffParams = params.GetParam("feature");
+  UTIL_THROW_IF2(ffParams == NULL, "Must have [feature] section");
+
+  BOOST_FOREACH(const std::string &line, *ffParams) {
+    FeatureFunction *ff = Create(line);
+
+    m_featureFunctions.push_back(ff);
+
+    StatefulFeatureFunction *sfff = dynamic_cast<StatefulFeatureFunction*>(ff);
+    if (sfff) {
+      sfff->SetStatefulInd(m_statefulFeatureFunctions.size());
+      m_statefulFeatureFunctions.push_back(sfff);
+    }
+
+    if (ff->HasPhraseTableInd()) {
+      ff->SetPhraseTableInd(m_withPhraseTableInd.size());
+      m_withPhraseTableInd.push_back(ff);
+    }
+
+    PhraseTable *pt = dynamic_cast<PhraseTable*>(ff);
+    if (pt) {
+      pt->SetPtInd(phraseTables.size());
+      phraseTables.push_back(pt);
+    }
+
+    UnknownWordPenalty *unkWP = dynamic_cast<UnknownWordPenalty *>(pt);
+    if (unkWP) {
+      m_unkWP = unkWP;
+
+      // legacy support
+      if (m_system.options.unk.drop) {
+        unkWP->SetParameter("drop", "true");
+      }
+      if (m_system.options.unk.mark) {
+        unkWP->SetParameter("prefix", m_system.options.unk.prefix);
+        unkWP->SetParameter("suffix", m_system.options.unk.suffix);
+      }
+    }
+  }
+
+  OverrideFeatures();
+}
+
+FeatureFunction *FeatureFunctions::Create(const std::string &line)
+{
+  vector<string> toks = Tokenize(line);
+
+  FeatureFunction *ff = FeatureRegistry::Instance().Construct(m_ffStartInd, toks[0], line);
+  UTIL_THROW_IF2(ff == NULL, "Feature function not created");
+
+  // name
+  if (ff->GetName() == "") {
+    ff->SetName(GetDefaultName(toks[0]));
+  }
+
+  m_ffStartInd += ff->GetNumScores();
+
+  return ff;
+}
+
+std::string FeatureFunctions::GetDefaultName(const std::string &stub)
+{
+  size_t ind;
+  boost::unordered_map<std::string, size_t>::iterator iter =
+    m_defaultNames.find(stub);
+  if (iter == m_defaultNames.end()) {
+    m_defaultNames[stub] = 0;
+    ind = 0;
+  } else {
+    ind = ++(iter->second);
+  }
+  return stub + SPrint(ind);
+}
+
+const FeatureFunction *FeatureFunctions::FindFeatureFunction(
+  const std::string &name) const
+{
+  BOOST_FOREACH(const FeatureFunction *ff, m_featureFunctions) {
+    if (ff->GetName() == name) {
+      return ff;
+    }
+  }
+  return NULL;
+}
+
+FeatureFunction *FeatureFunctions::FindFeatureFunction(
+  const std::string &name)
+{
+  BOOST_FOREACH(const FeatureFunction *ff, m_featureFunctions) {
+    if (ff->GetName() == name) {
+      return const_cast<FeatureFunction *>(ff);
+    }
+  }
+  return NULL;
+}
+
+const PhraseTable *FeatureFunctions::GetPhraseTableExcludeUnknownWordPenalty(size_t ptInd)
+{
+  // assume only 1 unk wp
+  std::vector<const PhraseTable*> tmpVec(phraseTables);
+  std::vector<const PhraseTable*>::iterator iter;
+  for (iter = tmpVec.begin(); iter != tmpVec.end(); ++iter) {
+    const PhraseTable *pt = *iter;
+    if (pt == m_unkWP) {
+      tmpVec.erase(iter);
+      break;
+    }
+  }
+
+  const PhraseTable *pt = tmpVec[ptInd];
+  return pt;
+}
+
+void FeatureFunctions::EvaluateInIsolation(MemPool &pool, const System &system,
+    const Phrase<Moses2::Word> &source, TargetPhraseImpl &targetPhrase) const
+{
+  SCORE estimatedScore = 0;
+
+  BOOST_FOREACH(const FeatureFunction *ff, m_featureFunctions) {
+    Scores& scores = targetPhrase.GetScores();
+    ff->EvaluateInIsolation(pool, system, source, targetPhrase, scores, estimatedScore);
+  }
+
+  targetPhrase.SetEstimatedScore(estimatedScore);
+}
+
+void FeatureFunctions::EvaluateInIsolation(
+  MemPool &pool,
+  const System &system,
+  const Phrase<SCFG::Word> &source,
+  SCFG::TargetPhraseImpl &targetPhrase) const
+{
+  SCORE estimatedScore = 0;
+
+  BOOST_FOREACH(const FeatureFunction *ff, m_featureFunctions) {
+    Scores& scores = targetPhrase.GetScores();
+    ff->EvaluateInIsolation(pool, system, source, targetPhrase, scores, estimatedScore);
+  }
+
+  targetPhrase.SetEstimatedScore(estimatedScore);
+}
+
+void FeatureFunctions::EvaluateAfterTablePruning(MemPool &pool,
+    const TargetPhrases &tps, const Phrase<Moses2::Word> &sourcePhrase) const
+{
+  BOOST_FOREACH(const FeatureFunction *ff, m_featureFunctions) {
+    ff->EvaluateAfterTablePruning(pool, tps, sourcePhrase);
+  }
+}
+
+void FeatureFunctions::EvaluateAfterTablePruning(MemPool &pool, const SCFG::TargetPhrases &tps,
+    const Phrase<SCFG::Word> &sourcePhrase) const
+{
+  BOOST_FOREACH(const FeatureFunction *ff, m_featureFunctions) {
+    ff->EvaluateAfterTablePruning(pool, tps, sourcePhrase);
+  }
+}
+
+void FeatureFunctions::EvaluateWhenAppliedBatch(const Batch &batch) const
+{
+  BOOST_FOREACH(const StatefulFeatureFunction *ff, m_statefulFeatureFunctions) {
+    ff->EvaluateWhenAppliedBatch(m_system, batch);
+  }
+}
+
+void FeatureFunctions::InitializeForInput(const ManagerBase &mgr, const InputType &input) 
+{
+  BOOST_FOREACH(FeatureFunction *ff, m_featureFunctions) {
+    ff->InitializeForInput(mgr, input);
+  }
+}
+
+void FeatureFunctions::CleanUpAfterSentenceProcessing(const InputType &input) const
+{
+  BOOST_FOREACH(const FeatureFunction *ff, m_featureFunctions) {
+    ff->CleanUpAfterSentenceProcessing(m_system, input);
+  }
+}
+
+void FeatureFunctions::ShowWeights(const Weights &allWeights)
+{
+  BOOST_FOREACH(const FeatureFunction *ff, m_featureFunctions) {
+    cout << ff->GetName();
+    if (ff->IsTuneable()) {
+      cout << "=";
+      vector<SCORE> weights = allWeights.GetWeights(*ff);
+      for (size_t i = 0; i < weights.size(); ++i) {
+        cout << " " << weights[i];
+      }
+      cout << endl;
+    } else {
+      cout << " UNTUNEABLE" << endl;
+    }
+  }
+}
+
+void FeatureFunctions::OverrideFeatures()
+{
+  const Parameter &parameter = m_system.params;
+
+  const PARAM_VEC *params = parameter.GetParam("feature-overwrite");
+  for (size_t i = 0; params && i < params->size(); ++i) {
+    const string &str = params->at(i);
+    vector<string> toks = Tokenize(str);
+    UTIL_THROW_IF2(toks.size() <= 1, "Incorrect format for feature override: " << str);
+
+    FeatureFunction *ff = FindFeatureFunction(toks[0]);
+    UTIL_THROW_IF2(ff == NULL, "Feature function not found: " << toks[0]);
+
+    for (size_t j = 1; j < toks.size(); ++j) {
+      const string &keyValStr = toks[j];
+      vector<string> keyVal = Tokenize(keyValStr, "=");
+      UTIL_THROW_IF2(keyVal.size() != 2, "Incorrect format for parameter override: " << keyValStr);
+
+      cerr << "Override " << ff->GetName() << " "
+           << keyVal[0] << "=" << keyVal[1] << endl;
+
+      ff->SetParameter(keyVal[0], keyVal[1]);
+
+    }
+  }
+
+}
+
+}
+

mosesdecoder/moses2/FF/FeatureFunctions.h

@@ -0,0 +1,113 @@
+/*
+ * FeatureFunctions.h
+ *
+ *  Created on: 27 Oct 2015
+ *      Author: hieu
+ */
+
+#pragma once
+
+#include <boost/unordered_map.hpp>
+#include <vector>
+#include <string>
+#include "../legacy/Parameter.h"
+#include "../Phrase.h"
+
+namespace Moses2
+{
+template<typename WORD>
+class TargetPhrase;
+
+class System;
+class FeatureFunction;
+class StatefulFeatureFunction;
+class PhraseTable;
+class Manager;
+class MemPool;
+class PhraseImpl;
+class TargetPhrases;
+class TargetPhraseImpl;
+class Scores;
+class Hypothesis;
+class UnknownWordPenalty;
+class Weights;
+class InputType;
+
+namespace SCFG
+{
+class TargetPhraseImpl;
+class TargetPhrases;
+class Word;
+}
+
+class FeatureFunctions
+{
+public:
+  std::vector<const PhraseTable*> phraseTables;
+
+  FeatureFunctions(System &system);
+  virtual ~FeatureFunctions();
+
+  const std::vector<FeatureFunction*> &GetFeatureFunctions() const {
+    return m_featureFunctions;
+  }
+
+  const std::vector<const StatefulFeatureFunction*> &GetStatefulFeatureFunctions() const {
+    return m_statefulFeatureFunctions;
+  }
+
+  const std::vector<const FeatureFunction*> &GetWithPhraseTableInd() const {
+    return m_withPhraseTableInd;
+  }
+
+  size_t GetNumScores() const {
+    return m_ffStartInd;
+  }
+
+  void Create();
+  void Load();
+
+  const FeatureFunction *FindFeatureFunction(const std::string &name) const;
+
+  const PhraseTable *GetPhraseTableExcludeUnknownWordPenalty(size_t ptInd);
+  const UnknownWordPenalty *GetUnknownWordPenalty() const {
+    return m_unkWP;
+  }
+
+  // the pool here must be the system pool if the rule was loaded during load, or the mgr pool if it was loaded on demand
+  void EvaluateInIsolation(MemPool &pool, const System &system,
+                           const Phrase<Moses2::Word> &source, TargetPhraseImpl &targetPhrase) const;
+  void EvaluateInIsolation(MemPool &pool, const System &system,
+                           const Phrase<SCFG::Word> &source, SCFG::TargetPhraseImpl &targetPhrase) const;
+
+  void EvaluateAfterTablePruning(MemPool &pool, const TargetPhrases &tps,
+                                 const Phrase<Moses2::Word> &sourcePhrase) const;
+  void EvaluateAfterTablePruning(MemPool &pool, const SCFG::TargetPhrases &tps,
+                                 const Phrase<SCFG::Word> &sourcePhrase) const;
+
+  void EvaluateWhenAppliedBatch(const Batch &batch) const;
+
+  void InitializeForInput(const ManagerBase &mgr, const InputType &input);
+  void CleanUpAfterSentenceProcessing(const InputType &input) const;
+
+  void ShowWeights(const Weights &allWeights);
+
+protected:
+  std::vector<FeatureFunction*> m_featureFunctions;
+  std::vector<const StatefulFeatureFunction*> m_statefulFeatureFunctions;
+  std::vector<const FeatureFunction*> m_withPhraseTableInd;
+  const UnknownWordPenalty *m_unkWP;
+
+  boost::unordered_map<std::string, size_t> m_defaultNames;
+  System &m_system;
+  size_t m_ffStartInd;
+
+  FeatureFunction *Create(const std::string &line);
+  std::string GetDefaultName(const std::string &stub);
+  void OverrideFeatures();
+  FeatureFunction *FindFeatureFunction(const std::string &name);
+
+};
+
+}
+

mosesdecoder/moses2/FF/FeatureRegistry.cpp

@@ -0,0 +1,128 @@
+#include "FeatureRegistry.h"
+
+#include "../TranslationModel/Memory/PhraseTableMemory.h"
+#include "../TranslationModel/ProbingPT.h"
+#include "../TranslationModel/UnknownWordPenalty.h"
+#include "../TranslationModel/Transliteration.h"
+#include "../TranslationModel/Dynamic/DynamicPhraseTable.h"
+
+#include "../LM/KENLM.h"
+#include "../LM/KENLMBatch.h"
+#include "../LM/LanguageModel.h"
+#include "../LM/GPULM.h"
+
+#include "Distortion.h"
+#include "LexicalReordering/LexicalReordering.h"
+#include "PhrasePenalty.h"
+#include "WordPenalty.h"
+#include "OSM/OpSequenceModel.h"
+
+#include "ExampleStatefulFF.h"
+#include "ExampleStatelessFF.h"
+
+using namespace std;
+
+
+namespace Moses2
+{
+FeatureRegistry FeatureRegistry::s_instance;
+
+template<class F>
+class DefaultFeatureFactory: public FeatureFactory
+{
+public:
+  FeatureFunction *Create(size_t startInd, const std::string &line) const {
+    return new F(startInd, line);
+  }
+};
+
+////////////////////////////////////////////////////////////////////
+class KenFactory: public FeatureFactory
+{
+public:
+  FeatureFunction *Create(size_t startInd, const std::string &line) const {
+    return ConstructKenLM(startInd, line);
+  }
+};
+
+////////////////////////////////////////////////////////////////////
+FeatureRegistry::FeatureRegistry()
+{
+  // Feature with same name as class
+#define MOSES_FNAME(name) Add(#name, new DefaultFeatureFactory< name >());
+  // Feature with different name than class.
+#define MOSES_FNAME2(name, type) Add(name, new DefaultFeatureFactory< type >());
+
+  MOSES_FNAME2("PhraseDictionaryMemory", PhraseTableMemory);
+  MOSES_FNAME(ProbingPT);
+  MOSES_FNAME2("PhraseDictionaryTransliteration", Transliteration);
+  MOSES_FNAME(UnknownWordPenalty);
+  MOSES_FNAME(DynamicPhraseTable);
+
+  Add("KENLM", new KenFactory());
+
+  MOSES_FNAME(KENLMBatch);
+  MOSES_FNAME(GPULM);
+
+  MOSES_FNAME(LanguageModel);
+
+  MOSES_FNAME(Distortion);
+  MOSES_FNAME(LexicalReordering);
+  MOSES_FNAME(PhrasePenalty);
+  MOSES_FNAME(WordPenalty);
+  MOSES_FNAME(OpSequenceModel);
+
+  MOSES_FNAME(ExampleStatefulFF);
+  MOSES_FNAME(ExampleStatelessFF);
+}
+
+FeatureRegistry::~FeatureRegistry()
+{
+
+}
+
+void FeatureRegistry::Add(const std::string &name, FeatureFactory *factory)
+{
+  std::pair<std::string, boost::shared_ptr<FeatureFactory> > to_ins(name,
+      boost::shared_ptr<FeatureFactory>(factory));
+  if (!registry_.insert(to_ins).second) {
+    cerr << "Duplicate feature name " << name << endl;
+    abort();
+  }
+}
+
+FeatureFunction *FeatureRegistry::Construct(size_t startInd,
+    const std::string &name, const std::string &line) const
+{
+  Map::const_iterator i = registry_.find(name);
+  if (i == registry_.end()) {
+    cerr << "Feature name " << name << " is not registered.";
+    abort();
+  }
+  FeatureFactory *fact = i->second.get();
+  FeatureFunction *ff = fact->Create(startInd, line);
+  return ff;
+}
+
+void FeatureRegistry::PrintFF() const
+{
+  std::vector<std::string> ffs;
+  std::cerr << "Available feature functions:" << std::endl;
+  Map::const_iterator iter;
+  for (iter = registry_.begin(); iter != registry_.end(); ++iter) {
+    const std::string &ffName = iter->first;
+    ffs.push_back(ffName);
+  }
+
+  std::vector<std::string>::const_iterator iterVec;
+  std::sort(ffs.begin(), ffs.end());
+  for (iterVec = ffs.begin(); iterVec != ffs.end(); ++iterVec) {
+    const std::string &ffName = *iterVec;
+    std::cerr << ffName << " ";
+  }
+
+  std::cerr << std::endl;
+}
+
+}
+

mosesdecoder/moses2/FF/FeatureRegistry.h

@@ -0,0 +1,52 @@
+#pragma once
+#include <boost/unordered_map.hpp>
+#include <boost/shared_ptr.hpp>
+
+namespace Moses2
+{
+class FeatureFunction;
+
+////////////////////////////////////////////////////////////////////
+class FeatureFactory
+{
+public:
+  virtual ~FeatureFactory() {
+  }
+
+  virtual FeatureFunction *Create(size_t startInd, const std::string &line) const = 0;
+
+protected:
+  FeatureFactory() {
+  }
+};
+
+////////////////////////////////////////////////////////////////////
+class FeatureRegistry
+{
+public:
+  static const FeatureRegistry &Instance() {
+    return s_instance;
+  }
+
+  ~FeatureRegistry();
+
+  FeatureFunction *Construct(size_t startInd, const std::string &name,
+                             const std::string &line) const;
+  void PrintFF() const;
+
+private:
+  static FeatureRegistry s_instance;
+
+  typedef boost::unordered_map<std::string, boost::shared_ptr<FeatureFactory> > Map;
+  Map registry_;
+
+  FeatureRegistry();
+
+  void Add(const std::string &name, FeatureFactory *factory);
+
+};
+
+////////////////////////////////////////////////////////////////////
+
+}
+

mosesdecoder/moses2/FF/LexicalReordering/BidirectionalReorderingState.cpp

@@ -0,0 +1,79 @@
+/*
+ * BidirectionalReorderingState.cpp
+ *
+ *  Created on: 22 Mar 2016
+ *      Author: hieu
+ */
+#include <boost/functional/hash_fwd.hpp>
+#include "BidirectionalReorderingState.h"
+#include "../../legacy/Util2.h"
+#include "../../PhraseBased/Manager.h"
+
+using namespace std;
+
+namespace Moses2
+{
+
+BidirectionalReorderingState::BidirectionalReorderingState(
+  const LRModel &config, LRState *bw, LRState *fw, size_t offset) :
+  LRState(config, LRModel::Bidirectional, offset), m_backward(bw), m_forward(
+    fw)
+{
+}
+
+BidirectionalReorderingState::~BidirectionalReorderingState()
+{
+  // TODO Auto-generated destructor stub
+}
+
+void BidirectionalReorderingState::Init(const LRState *prev,
+                                        const TargetPhrase<Moses2::Word> &topt, const InputPathBase &path, bool first,
+                                        const Bitmap *coverage)
+{
+  if (m_backward) {
+    m_backward->Init(prev, topt, path, first, coverage);
+  }
+  if (m_forward) {
+    m_forward->Init(prev, topt, path, first, coverage);
+  }
+}
+
+std::string BidirectionalReorderingState::ToString() const
+{
+  return "BidirectionalReorderingState " + SPrint(this) + " "
+         + SPrint(m_backward) + " " + SPrint(m_forward);
+}
+
+size_t BidirectionalReorderingState::hash() const
+{
+  size_t ret = m_backward->hash();
+  boost::hash_combine(ret, m_forward->hash());
+
+  return ret;
+}
+
+bool BidirectionalReorderingState::operator==(const FFState& o) const
+{
+  if (&o == this) return true;
+
+  BidirectionalReorderingState const &other =
+    static_cast<BidirectionalReorderingState const&>(o);
+
+  bool ret = (*m_backward == *other.m_backward)
+             && (*m_forward == *other.m_forward);
+  return ret;
+}
+
+void BidirectionalReorderingState::Expand(const ManagerBase &mgr,
+    const LexicalReordering &ff, const Hypothesis &hypo, size_t phraseTableInd,
+    Scores &scores, FFState &state) const
+{
+  BidirectionalReorderingState &stateCast =
+    static_cast<BidirectionalReorderingState&>(state);
+  m_backward->Expand(mgr, ff, hypo, phraseTableInd, scores,
+                     *stateCast.m_backward);
+  m_forward->Expand(mgr, ff, hypo, phraseTableInd, scores,
+                    *stateCast.m_forward);
+}
+
+} /* namespace Moses2 */

mosesdecoder/moses2/FF/LexicalReordering/BidirectionalReorderingState.h

@@ -0,0 +1,40 @@
+/*
+ * BidirectionalReorderingState.h
+ *
+ *  Created on: 22 Mar 2016
+ *      Author: hieu
+ */
+#pragma once
+#include "LRState.h"
+
+namespace Moses2
+{
+
+class BidirectionalReorderingState: public LRState
+{
+public:
+  BidirectionalReorderingState(const LRModel &config, LRState *bw, LRState *fw,
+                               size_t offset);
+
+  virtual ~BidirectionalReorderingState();
+
+  void Init(const LRState *prev, const TargetPhrase<Moses2::Word> &topt,
+            const InputPathBase &path, bool first, const Bitmap *coverage);
+
+  size_t hash() const;
+  virtual bool operator==(const FFState& other) const;
+
+  virtual std::string ToString() const;
+
+  void Expand(const ManagerBase &mgr, const LexicalReordering &ff,
+              const Hypothesis &hypo, size_t phraseTableInd, Scores &scores,
+              FFState &state) const;
+
+protected:
+  LRState *m_backward;
+  LRState *m_forward;
+
+};
+
+} /* namespace Moses2 */
+

mosesdecoder/moses2/FF/LexicalReordering/HReorderingBackwardState.cpp

@@ -0,0 +1,71 @@
+/*
+ * HReorderingBackwardState.cpp
+ *
+ *  Created on: 22 Mar 2016
+ *      Author: hieu
+ */
+
+#include "HReorderingBackwardState.h"
+#include "../../PhraseBased/Hypothesis.h"
+#include "../../PhraseBased/Manager.h"
+
+namespace Moses2
+{
+
+HReorderingBackwardState::HReorderingBackwardState(MemPool &pool,
+    const LRModel &config, size_t offset) :
+  LRState(config, LRModel::Backward, offset), reoStack(pool)
+{
+  // TODO Auto-generated constructor stub
+
+}
+
+HReorderingBackwardState::~HReorderingBackwardState()
+{
+  // TODO Auto-generated destructor stub
+}
+
+void HReorderingBackwardState::Init(const LRState *prev,
+                                    const TargetPhrase<Moses2::Word> &topt, const InputPathBase &path, bool first,
+                                    const Bitmap *coverage)
+{
+  prevTP = &topt;
+  reoStack.Init();
+}
+
+size_t HReorderingBackwardState::hash() const
+{
+  size_t ret = reoStack.hash();
+  return ret;
+}
+
+bool HReorderingBackwardState::operator==(const FFState& o) const
+{
+  const HReorderingBackwardState& other =
+    static_cast<const HReorderingBackwardState&>(o);
+  bool ret = reoStack == other.reoStack;
+  return ret;
+}
+
+std::string HReorderingBackwardState::ToString() const
+{
+  return "HReorderingBackwardState " + SPrint(m_offset);
+}
+
+void HReorderingBackwardState::Expand(const ManagerBase &mgr,
+                                      const LexicalReordering &ff, const Hypothesis &hypo, size_t phraseTableInd,
+                                      Scores &scores, FFState &state) const
+{
+  HReorderingBackwardState &nextState =
+    static_cast<HReorderingBackwardState&>(state);
+  nextState.Init(this, hypo.GetTargetPhrase(), hypo.GetInputPath(), false,
+                 NULL);
+  nextState.reoStack = reoStack;
+
+  const Range &swrange = hypo.GetInputPath().range;
+  int reoDistance = nextState.reoStack.ShiftReduce(swrange);
+  ReorderingType reoType = m_configuration.GetOrientation(reoDistance);
+  CopyScores(mgr.system, scores, hypo.GetTargetPhrase(), reoType);
+}
+
+} /* namespace Moses2 */

mosesdecoder/moses2/FF/LexicalReordering/HReorderingBackwardState.h

@@ -0,0 +1,37 @@
+/*
+ * HReorderingBackwardState.h
+ *
+ *  Created on: 22 Mar 2016
+ *      Author: hieu
+ */
+#pragma once
+#include "LRState.h"
+#include "ReorderingStack.h"
+
+namespace Moses2
+{
+
+class HReorderingBackwardState: public LRState
+{
+private:
+  ReorderingStack reoStack;
+
+public:
+  HReorderingBackwardState(MemPool &pool, const LRModel &config, size_t offset);
+
+  virtual void Init(const LRState *prev, const TargetPhrase<Moses2::Word> &topt,
+                    const InputPathBase &path, bool first, const Bitmap *coverage);
+
+  virtual ~HReorderingBackwardState();
+
+  size_t hash() const;
+  virtual bool operator==(const FFState& other) const;
+  virtual std::string ToString() const;
+  void Expand(const ManagerBase &mgr, const LexicalReordering &ff,
+              const Hypothesis &hypo, size_t phraseTableInd, Scores &scores,
+              FFState &state) const;
+
+};
+
+} /* namespace Moses2 */
+

mosesdecoder/moses2/FF/LexicalReordering/HReorderingForwardState.cpp

@@ -0,0 +1,87 @@
+/*
+ * HReorderingForwardState.cpp
+ *
+ *  Created on: 22 Mar 2016
+ *      Author: hieu
+ */
+
+#include "HReorderingForwardState.h"
+#include "../../InputPathBase.h"
+#include "../../PhraseBased/Manager.h"
+#include "../../PhraseBased/Hypothesis.h"
+
+namespace Moses2
+{
+
+HReorderingForwardState::HReorderingForwardState(const LRModel &config,
+    size_t offset) :
+  LRState(config, LRModel::Forward, offset), m_first(true)
+{
+  prevPath = NULL;
+  m_coverage = NULL;
+}
+
+HReorderingForwardState::~HReorderingForwardState()
+{
+  // TODO Auto-generated destructor stub
+}
+
+void HReorderingForwardState::Init(const LRState *prev,
+                                   const TargetPhrase<Moses2::Word> &topt, const InputPathBase &path, bool first,
+                                   const Bitmap *coverage)
+{
+  prevTP = &topt;
+  prevPath = &path;
+  m_first = first;
+  m_coverage = coverage;
+}
+
+size_t HReorderingForwardState::hash() const
+{
+  size_t ret;
+  ret = hash_value(prevPath->range);
+  return ret;
+}
+
+bool HReorderingForwardState::operator==(const FFState& o) const
+{
+  if (&o == this) return true;
+
+  HReorderingForwardState const& other =
+    static_cast<HReorderingForwardState const&>(o);
+
+  int compareScores = (
+                        (prevPath->range == other.prevPath->range) ?
+                        ComparePrevScores(other.prevTP) :
+                        (prevPath->range < other.prevPath->range) ? -1 : 1);
+  return compareScores == 0;
+}
+
+std::string HReorderingForwardState::ToString() const
+{
+  return "HReorderingForwardState " + SPrint(m_offset);
+}
+
+void HReorderingForwardState::Expand(const ManagerBase &mgr,
+                                     const LexicalReordering &ff, const Hypothesis &hypo, size_t phraseTableInd,
+                                     Scores &scores, FFState &state) const
+{
+  const Range &cur = hypo.GetInputPath().range;
+  // keep track of the current coverage ourselves so we don't need the hypothesis
+  Manager &mgrCast = const_cast<Manager&>(static_cast<const Manager&>(mgr));
+  Bitmaps &bms = mgrCast.GetBitmaps();
+  const Bitmap &cov = bms.GetBitmap(*m_coverage, cur);
+
+  if (!m_first) {
+    LRModel::ReorderingType reoType;
+    reoType = m_configuration.GetOrientation(prevPath->range, cur, cov);
+    CopyScores(mgr.system, scores, hypo.GetTargetPhrase(), reoType);
+  }
+
+  HReorderingForwardState &stateCast =
+    static_cast<HReorderingForwardState&>(state);
+  stateCast.Init(this, hypo.GetTargetPhrase(), hypo.GetInputPath(), false,
+                 &cov);
+}
+
+} /* namespace Moses2 */

mosesdecoder/moses2/FF/LexicalReordering/HReorderingForwardState.h

@@ -0,0 +1,41 @@
+/*
+ * HReorderingForwardState.h
+ *
+ *  Created on: 22 Mar 2016
+ *      Author: hieu
+ */
+#pragma once
+#include "LRState.h"
+
+namespace Moses2
+{
+class Range;
+class Bitmap;
+class InputPathBase;
+
+class HReorderingForwardState: public LRState
+{
+public:
+  HReorderingForwardState(const LRModel &config, size_t offset);
+  virtual ~HReorderingForwardState();
+
+  void Init(const LRState *prev, const TargetPhrase<Moses2::Word> &topt,
+            const InputPathBase &path, bool first, const Bitmap *coverage);
+
+  size_t hash() const;
+  virtual bool operator==(const FFState& other) const;
+  virtual std::string ToString() const;
+  void Expand(const ManagerBase &mgr, const LexicalReordering &ff,
+              const Hypothesis &hypo, size_t phraseTableInd, Scores &scores,
+              FFState &state) const;
+
+protected:
+  bool m_first;
+  //const Range &m_prevRange;
+  const InputPathBase *prevPath;
+  const Bitmap *m_coverage;
+
+};
+
+} /* namespace Moses2 */
+

mosesdecoder/moses2/FF/LexicalReordering/LRModel.cpp

@@ -0,0 +1,196 @@
+/*
+ * LRModel.cpp
+ *
+ *  Created on: 23 Mar 2016
+ *      Author: hieu
+ */
+
+#include "LRModel.h"
+#include "../../legacy/Util2.h"
+#include "../../legacy/Range.h"
+#include "../../legacy/Bitmap.h"
+#include "../../MemPool.h"
+#include "util/exception.hh"
+#include "PhraseBasedReorderingState.h"
+#include "BidirectionalReorderingState.h"
+#include "HReorderingBackwardState.h"
+#include "HReorderingForwardState.h"
+
+using namespace std;
+
+namespace Moses2
+{
+
+bool IsMonotonicStep(Range const& prev, // words range of last source phrase
+                     Range const& cur,  // words range of current source phrase
+                     Bitmap const& cov)  // coverage bitmap
+{
+  size_t e = prev.GetEndPos() + 1;
+  size_t s = cur.GetStartPos();
+  return (s == e || (s >= e && !cov.GetValue(e)));
+}
+
+bool IsSwap(Range const& prev, Range const& cur, Bitmap const& cov)
+{
+  size_t s = prev.GetStartPos();
+  size_t e = cur.GetEndPos();
+  return (e + 1 == s || (e < s && !cov.GetValue(s - 1)));
+}
+
+LRModel::LRModel(const std::string &modelType, LexicalReordering &ff) :
+  m_modelType(None), m_phraseBased(true), m_collapseScores(false), m_direction(
+    Backward), m_scoreProducer(&ff)
+{
+  std::vector<std::string> config = Tokenize(modelType, "-");
+
+  for (size_t i = 0; i < config.size(); ++i) {
+    if (config[i] == "hier") {
+      m_phraseBased = false;
+    } else if (config[i] == "phrase") {
+      m_phraseBased = true;
+    } else if (config[i] == "wbe") {
+      m_phraseBased = true;
+    }
+    // no word-based decoding available, fall-back to phrase-based
+    // This is the old lexical reordering model combination of moses
+
+    else if (config[i] == "msd") {
+      m_modelType = MSD;
+    } else if (config[i] == "mslr") {
+      m_modelType = MSLR;
+    } else if (config[i] == "monotonicity") {
+      m_modelType = Monotonic;
+    } else if (config[i] == "leftright") {
+      m_modelType = LeftRight;
+    }
+
+    // unidirectional is deprecated, use backward instead
+    else if (config[i] == "unidirectional") {
+      m_direction = Backward;
+    } else if (config[i] == "backward") {
+      m_direction = Backward;
+    } else if (config[i] == "forward") {
+      m_direction = Forward;
+    } else if (config[i] == "bidirectional") {
+      m_direction = Bidirectional;
+    }
+
+    else if (config[i] == "f") {
+      m_condition = F;
+    } else if (config[i] == "fe") {
+      m_condition = FE;
+    }
+
+    else if (config[i] == "collapseff") {
+      m_collapseScores = true;
+    } else if (config[i] == "allff") {
+      m_collapseScores = false;
+    } else {
+      std::cerr
+          << "Illegal part in the lexical reordering configuration string: "
+          << config[i] << std::endl;
+      exit(1);
+    }
+  }
+
+  if (m_modelType == None) {
+    std::cerr << "You need to specify the type of the reordering model "
+              << "(msd, monotonicity,...)" << std::endl;
+    exit(1);
+  }
+
+}
+
+LRModel::~LRModel()
+{
+  // TODO Auto-generated destructor stub
+}
+
+size_t LRModel::GetNumberOfTypes() const
+{
+  return ((m_modelType == MSD) ? 3 : (m_modelType == MSLR) ? 4 : 2);
+}
+
+/// return orientation for the first phrase
+LRModel::ReorderingType LRModel::GetOrientation(Range const& cur) const
+{
+  UTIL_THROW_IF2(m_modelType == None, "Reordering Model Type is None");
+  return ((m_modelType == LeftRight) ? R : (cur.GetStartPos() == 0) ? M :
+          (m_modelType == MSD) ? D : (m_modelType == MSLR) ? DR : NM);
+}
+
+LRModel::ReorderingType LRModel::GetOrientation(Range const& prev,
+    Range const& cur) const
+{
+  UTIL_THROW_IF2(m_modelType == None, "No reordering model type specified");
+  return (
+           (m_modelType == LeftRight) ? prev.GetEndPos() <= cur.GetStartPos() ? R : L
+         : (cur.GetStartPos() == prev.GetEndPos() + 1) ? M :
+           (m_modelType == Monotonic) ? NM :
+           (prev.GetStartPos() == cur.GetEndPos() + 1) ? S :
+           (m_modelType == MSD) ? D :
+           (cur.GetStartPos() > prev.GetEndPos()) ? DR : DL);
+}
+
+LRModel::ReorderingType LRModel::GetOrientation(int const reoDistance) const
+{
+  // this one is for HierarchicalReorderingBackwardState
+  return ((m_modelType == LeftRight) ? (reoDistance >= 1) ? R : L
+        : (reoDistance == 1) ? M : (m_modelType == Monotonic) ? NM :
+          (reoDistance == -1) ? S : (m_modelType == MSD) ? D :
+          (reoDistance > 1) ? DR : DL);
+}
+
+LRState *LRModel::CreateLRState(MemPool &pool) const
+{
+  LRState *bwd = NULL, *fwd = NULL;
+  size_t offset = 0;
+
+  switch (m_direction) {
+  case Backward:
+  case Bidirectional:
+    if (m_phraseBased) {
+      bwd =
+        new (pool.Allocate<PhraseBasedReorderingState>()) PhraseBasedReorderingState(
+        *this, Backward, offset);
+      //cerr << "bwd=" << bwd << bwd->ToString() << endl;
+    } else {
+      bwd =
+        new (pool.Allocate<HReorderingBackwardState>()) HReorderingBackwardState(
+        pool, *this, offset);
+    }
+    offset += m_collapseScores ? 1 : GetNumberOfTypes();
+    if (m_direction == Backward) return bwd; // else fall through
+  case Forward:
+    if (m_phraseBased) {
+      fwd =
+        new (pool.Allocate<PhraseBasedReorderingState>()) PhraseBasedReorderingState(
+        *this, Forward, offset);
+      //cerr << "fwd=" << fwd << fwd->ToString() << endl;
+    } else {
+      fwd =
+        new (pool.Allocate<HReorderingForwardState>()) HReorderingForwardState(
+        *this, offset);
+    }
+    offset += m_collapseScores ? 1 : GetNumberOfTypes();
+    if (m_direction == Forward) return fwd;
+  }
+
+  //cerr << "LRStates:" << *bwd << endl << *fwd << endl;
+  BidirectionalReorderingState *ret =
+    new (pool.Allocate<BidirectionalReorderingState>()) BidirectionalReorderingState(
+    *this, bwd, fwd, 0);
+  return ret;
+}
+
+LRModel::ReorderingType LRModel::GetOrientation(Range const& prev,
+    Range const& cur, Bitmap const& cov) const
+{
+  return (
+           (m_modelType == LeftRight) ? cur.GetStartPos() > prev.GetEndPos() ? R : L
+         : IsMonotonicStep(prev, cur, cov) ? M : (m_modelType == Monotonic) ? NM :
+           IsSwap(prev, cur, cov) ? S : (m_modelType == MSD) ? D :
+           cur.GetStartPos() > prev.GetEndPos() ? DR : DL);
+}
+
+} /* namespace Moses2 */

mosesdecoder/moses2/FF/LexicalReordering/LRModel.h

@@ -0,0 +1,99 @@
+/*
+ * LRModel.h
+ *
+ *  Created on: 23 Mar 2016
+ *      Author: hieu
+ */
+#pragma once
+#include <string>
+
+namespace Moses2
+{
+
+class MemPool;
+class Range;
+class Bitmap;
+class LRState;
+class LexicalReordering;
+
+class LRModel
+{
+public:
+  enum ModelType {
+    Monotonic, MSD, MSLR, LeftRight, None
+  };
+  enum Direction {
+    Forward, Backward, Bidirectional
+  };
+  enum Condition {
+    F, E, FE
+  };
+
+  enum ReorderingType {
+    M = 0, // monotonic
+    NM = 1, // non-monotonic
+    S = 1, // swap
+    D = 2, // discontinuous
+    DL = 2, // discontinuous, left
+    DR = 3, // discontinuous, right
+    R = 0, // right
+    L = 1, // left
+    MAX = 3, // largest possible
+    NONE = 4  // largest possible
+  };
+
+  LRModel(const std::string &modelType, LexicalReordering &ff);
+  virtual ~LRModel();
+
+  ModelType GetModelType() const {
+    return m_modelType;
+  }
+  Direction GetDirection() const {
+    return m_direction;
+  }
+  Condition GetCondition() const {
+    return m_condition;
+  }
+
+  bool IsPhraseBased() const {
+    return m_phraseBased;
+  }
+
+  bool CollapseScores() const {
+    return m_collapseScores;
+  }
+
+  size_t GetNumberOfTypes() const;
+
+  LexicalReordering*
+  GetScoreProducer() const {
+    return m_scoreProducer;
+  }
+
+  LRState *CreateLRState(MemPool &pool) const;
+
+  ReorderingType // for first phrase in phrase-based
+  GetOrientation(Range const& cur) const;
+
+  ReorderingType // for non-first phrases in phrase-based
+  GetOrientation(Range const& prev, Range const& cur) const;
+
+  ReorderingType // for HReorderingForwardState
+  GetOrientation(Range const& prev, Range const& cur, Bitmap const& cov) const;
+
+  ReorderingType // for HReorderingBackwarddState
+  GetOrientation(int const reoDistance) const;
+
+protected:
+
+  ModelType m_modelType;
+  bool m_phraseBased;
+  bool m_collapseScores;
+  Direction m_direction;
+  Condition m_condition;
+  LexicalReordering *m_scoreProducer;
+
+};
+
+} /* namespace Moses2 */
+

mosesdecoder/moses2/FF/LexicalReordering/LRState.cpp

@@ -0,0 +1,93 @@
+/*
+ * LRState.cpp
+ *
+ *  Created on: 22 Mar 2016
+ *      Author: hieu
+ */
+#include "LRState.h"
+#include "LexicalReordering.h"
+#include "../../Scores.h"
+#include "../../TargetPhrase.h"
+
+using namespace std;
+
+namespace Moses2
+{
+
+class InputType;
+
+LRState::LRState(const LRModel &config, LRModel::Direction dir, size_t offset) :
+  m_configuration(config), m_direction(dir), m_offset(offset)
+{
+}
+
+int LRState::ComparePrevScores(const TargetPhrase<Moses2::Word> *other) const
+{
+  LexicalReordering* producer = m_configuration.GetScoreProducer();
+  size_t phraseTableInd = producer->GetPhraseTableInd();
+  const SCORE *myScores = (const SCORE*) prevTP->ffData[phraseTableInd]; //producer->
+  const SCORE *yrScores = (const SCORE*) other->ffData[phraseTableInd]; //producer->
+
+  if (myScores == yrScores) return 0;
+
+  // The pointers are NULL if a phrase pair isn't found in the reordering table.
+  if (yrScores == NULL) return -1;
+  if (myScores == NULL) return 1;
+
+  size_t stop = m_offset + m_configuration.GetNumberOfTypes();
+  for (size_t i = m_offset; i < stop; i++) {
+    if ((myScores)[i] < (yrScores)[i]) return -1;
+    if ((myScores)[i] > (yrScores)[i]) return 1;
+  }
+  return 0;
+}
+
+void LRState::CopyScores(const System &system, Scores &accum,
+                         const TargetPhrase<Moses2::Word> &topt, ReorderingType reoType) const
+{
+  // don't call this on a bidirectional object
+  UTIL_THROW_IF2(
+    m_direction != LRModel::Backward && m_direction != LRModel::Forward,
+    "Unknown direction: " << m_direction);
+
+  TargetPhrase<Moses2::Word> const* relevantOpt = (
+        (m_direction == LRModel::Backward) ? &topt : prevTP);
+
+  LexicalReordering* producer = m_configuration.GetScoreProducer();
+  size_t phraseTableInd = producer->GetPhraseTableInd();
+  const SCORE *cached = (const SCORE*) relevantOpt->ffData[phraseTableInd]; //producer->
+
+  if (cached == NULL) {
+    return;
+  }
+
+  size_t off_remote = m_offset + reoType;
+  size_t off_local = m_configuration.CollapseScores() ? m_offset : off_remote;
+
+  UTIL_THROW_IF2(off_local >= producer->GetNumScores(),
+                 "offset out of vector bounds!");
+
+  // look up applicable score from vector of scores
+  //UTIL_THROW_IF2(off_remote >= cached->size(), "offset out of vector bounds!");
+  //Scores scores(producer->GetNumScoreComponents(),0);
+  SCORE score = cached[off_remote];
+  accum.PlusEquals(system, *producer, score, off_local);
+
+  // else: use default scores (if specified)
+  /*
+   else if (producer->GetHaveDefaultScores()) {
+   Scores scores(producer->GetNumScoreComponents(),0);
+   scores[off_local] = producer->GetDefaultScore(off_remote);
+   accum->PlusEquals(m_configuration.GetScoreProducer(), scores);
+   }
+   */
+  // note: if no default score, no cost
+  /*
+   const SparseReordering* sparse = m_configuration.GetSparseReordering();
+   if (sparse) sparse->CopyScores(*relevantOpt, m_prevOption, input, reoType,
+   m_direction, accum);
+   */
+}
+
+}
+

mosesdecoder/moses2/FF/LexicalReordering/LRState.h

@@ -0,0 +1,48 @@
+#pragma once
+#include "../FFState.h"
+#include "LRModel.h"
+
+namespace Moses2
+{
+template<typename WORD>
+class TargetPhrase;
+
+class LexicalReordering;
+class Hypothesis;
+class System;
+class Scores;
+class Bitmap;
+class ManagerBase;
+class InputType;
+class InputPathBase;
+class Word;
+
+class LRState: public FFState
+{
+public:
+  typedef LRModel::ReorderingType ReorderingType;
+  const TargetPhrase<Moses2::Word> *prevTP;
+
+  LRState(const LRModel &config, LRModel::Direction dir, size_t offset);
+
+  virtual void Init(const LRState *prev, const TargetPhrase<Moses2::Word> &topt,
+                    const InputPathBase &path, bool first, const Bitmap *coverage) = 0;
+
+  virtual void Expand(const ManagerBase &mgr, const LexicalReordering &ff,
+                      const Hypothesis &hypo, size_t phraseTableInd, Scores &scores,
+                      FFState &state) const = 0;
+
+  void CopyScores(const System &system, Scores &accum, const TargetPhrase<Moses2::Word> &topt,
+                  ReorderingType reoType) const;
+
+protected:
+  const LRModel& m_configuration;
+  LRModel::Direction m_direction;
+  size_t m_offset;
+
+  int
+  ComparePrevScores(const TargetPhrase<Moses2::Word> *other) const;
+
+};
+
+}

mosesdecoder/moses2/FF/LexicalReordering/LexicalReordering.cpp

@@ -0,0 +1,226 @@
+/*
+ * LexicalReordering.cpp
+ *
+ *  Created on: 15 Dec 2015
+ *      Author: hieu
+ */
+
+#include <boost/foreach.hpp>
+#include "util/exception.hh"
+#include "LexicalReordering.h"
+#include "LRModel.h"
+#include "PhraseBasedReorderingState.h"
+#include "BidirectionalReorderingState.h"
+#include "../../TranslationModel/PhraseTable.h"
+#include "../../System.h"
+#include "../../PhraseBased/PhraseImpl.h"
+#include "../../PhraseBased/Manager.h"
+#include "../../PhraseBased/Hypothesis.h"
+#include "../../PhraseBased/TargetPhrases.h"
+#include "../../PhraseBased/TargetPhraseImpl.h"
+#include "../../legacy/InputFileStream.h"
+#include "../../legacy/Util2.h"
+
+#ifdef HAVE_CMPH
+#include "../../TranslationModel/CompactPT/LexicalReorderingTableCompact.h"
+#endif
+
+
+using namespace std;
+
+namespace Moses2
+{
+
+///////////////////////////////////////////////////////////////////////
+
+LexicalReordering::LexicalReordering(size_t startInd, const std::string &line)
+  : StatefulFeatureFunction(startInd, line)
+  , m_blank(NULL)
+  , m_propertyInd(-1)
+  , m_coll(NULL)
+  , m_configuration(NULL)
+#ifdef HAVE_CMPH
+  , m_compactModel(NULL)
+#endif
+{
+  ReadParameters();
+  assert(m_configuration);
+  //assert(m_numScores == 6);
+}
+
+LexicalReordering::~LexicalReordering()
+{
+  delete m_coll;
+  delete m_configuration;
+#ifdef HAVE_CMPH
+  delete m_compactModel;
+#endif
+}
+
+void LexicalReordering::Load(System &system)
+{
+  MemPool &pool = system.GetSystemPool();
+
+  if (m_propertyInd >= 0) {
+    // Using integrate Lex RO. No loading needed
+#ifdef HAVE_CMPH
+  } else if (FileExists(m_path + ".minlexr")) {
+    m_compactModel = new LexicalReorderingTableCompact(m_path + ".minlexr",
+        m_FactorsF, m_FactorsE, m_FactorsC);
+    m_blank = new (pool.Allocate<PhraseImpl>()) PhraseImpl(pool, 0);
+#endif
+  } else {
+    m_coll = new Coll();
+    InputFileStream file(m_path);
+    string line;
+    size_t lineNum = 0;
+
+    while (getline(file, line)) {
+      if (++lineNum % 1000000 == 0) {
+        cerr << lineNum << " ";
+      }
+
+      std::vector<std::string> toks = TokenizeMultiCharSeparator(line, "|||");
+      assert(toks.size() == 3);
+      PhraseImpl *source = PhraseImpl::CreateFromString(pool, system.GetVocab(),
+                           system, toks[0]);
+      PhraseImpl *target = PhraseImpl::CreateFromString(pool, system.GetVocab(),
+                           system, toks[1]);
+      std::vector<SCORE> scores = Tokenize<SCORE>(toks[2]);
+      std::transform(scores.begin(), scores.end(), scores.begin(),
+                     TransformScore);
+      std::transform(scores.begin(), scores.end(), scores.begin(), FloorScore);
+
+      Key key(source, target);
+      (*m_coll)[key] = scores;
+    }
+  }
+}
+
+void LexicalReordering::SetParameter(const std::string& key,
+                                     const std::string& value)
+{
+  if (key == "path") {
+    m_path = value;
+  } else if (key == "type") {
+    m_configuration = new LRModel(value, *this);
+  } else if (key == "input-factor") {
+    m_FactorsF = Tokenize<FactorType>(value);
+  } else if (key == "output-factor") {
+    m_FactorsE = Tokenize<FactorType>(value);
+  } else if (key == "property-index") {
+    m_propertyInd = Scan<int>(value);
+  } else {
+    StatefulFeatureFunction::SetParameter(key, value);
+  }
+}
+
+FFState* LexicalReordering::BlankState(MemPool &pool, const System &sys) const
+{
+  FFState *ret = m_configuration->CreateLRState(pool);
+  return ret;
+}
+
+void LexicalReordering::EmptyHypothesisState(FFState &state,
+    const ManagerBase &mgr, const InputType &input,
+    const Hypothesis &hypo) const
+{
+  BidirectionalReorderingState &stateCast =
+    static_cast<BidirectionalReorderingState&>(state);
+  stateCast.Init(NULL, hypo.GetTargetPhrase(), hypo.GetInputPath(), true,
+                 &hypo.GetBitmap());
+}
+
+void LexicalReordering::EvaluateInIsolation(MemPool &pool, const System &system,
+    const Phrase<Moses2::Word> &source, const TargetPhraseImpl &targetPhrase, Scores &scores,
+    SCORE &estimatedScore) const
+{
+}
+
+void LexicalReordering::EvaluateInIsolation(MemPool &pool, const System &system, const Phrase<SCFG::Word> &source,
+    const TargetPhrase<SCFG::Word> &targetPhrase, Scores &scores,
+    SCORE &estimatedScore) const
+{
+  UTIL_THROW2("Don't use with SCFG models");
+}
+
+
+void LexicalReordering::EvaluateAfterTablePruning(MemPool &pool,
+    const TargetPhrases &tps, const Phrase<Moses2::Word> &sourcePhrase) const
+{
+  BOOST_FOREACH(const TargetPhraseImpl *tp, tps) {
+    EvaluateAfterTablePruning(pool, *tp, sourcePhrase);
+  }
+}
+
+void LexicalReordering::EvaluateAfterTablePruning(MemPool &pool,
+    const TargetPhraseImpl &targetPhrase, const Phrase<Moses2::Word> &sourcePhrase) const
+{
+  if (m_propertyInd >= 0) {
+    SCORE *scoreArr = targetPhrase.GetScoresProperty(m_propertyInd);
+    targetPhrase.ffData[m_PhraseTableInd] = scoreArr;
+#ifdef HAVE_CMPH
+  } else if (m_compactModel) {
+    // using external compact binary model
+    const Values values = m_compactModel->GetScore(sourcePhrase, targetPhrase,
+                          *m_blank);
+    if (values.size()) {
+      assert(values.size() == m_numScores);
+
+      SCORE *scoreArr = pool.Allocate<SCORE>(m_numScores);
+      for (size_t i = 0; i < m_numScores; ++i) {
+        scoreArr[i] = values[i];
+      }
+      targetPhrase.ffData[m_PhraseTableInd] = scoreArr;
+    } else {
+      targetPhrase.ffData[m_PhraseTableInd] = NULL;
+    }
+#endif
+  } else if (m_coll) {
+    // using external memory model
+
+    // cache data in target phrase
+    const Values *values = GetValues(sourcePhrase, targetPhrase);
+    assert(values->size() == m_numScores);
+
+    if (values) {
+      SCORE *scoreArr = pool.Allocate<SCORE>(m_numScores);
+      for (size_t i = 0; i < m_numScores; ++i) {
+        scoreArr[i] = (*values)[i];
+      }
+      targetPhrase.ffData[m_PhraseTableInd] = scoreArr;
+    } else {
+      targetPhrase.ffData[m_PhraseTableInd] = NULL;
+    }
+  }
+}
+
+void LexicalReordering::EvaluateWhenApplied(const ManagerBase &mgr,
+    const Hypothesis &hypo, const FFState &prevState, Scores &scores,
+    FFState &state) const
+{
+  const LRState &prevStateCast = static_cast<const LRState&>(prevState);
+  prevStateCast.Expand(mgr, *this, hypo, m_PhraseTableInd, scores, state);
+}
+
+const LexicalReordering::Values *LexicalReordering::GetValues(
+  const Phrase<Moses2::Word> &source, const Phrase<Moses2::Word> &target) const
+{
+  Key key(&source, &target);
+  Coll::const_iterator iter;
+  iter = m_coll->find(key);
+  if (iter == m_coll->end()) {
+    return NULL;
+  } else {
+    return &iter->second;
+  }
+}
+
+void LexicalReordering::EvaluateWhenApplied(const SCFG::Manager &mgr,
+    const SCFG::Hypothesis &hypo, int featureID, Scores &scores,
+    FFState &state) const
+{
+  UTIL_THROW2("Not implemented");
+}
+
+} /* namespace Moses2 */

mosesdecoder/moses2/FF/LexicalReordering/LexicalReordering.h

@@ -0,0 +1,115 @@
+/*
+ * LexicalReordering.h
+ *
+ *  Created on: 15 Dec 2015
+ *      Author: hieu
+ */
+
+#pragma once
+#include <vector>
+#include <boost/unordered_map.hpp>
+#include "../StatefulFeatureFunction.h"
+#include "../../TypeDef.h"
+#include "../../Phrase.h"
+#include "../../legacy/Range.h"
+
+namespace Moses2
+{
+
+class LexicalReorderingTableCompact;
+class LRModel;
+class TargetPhraseImpl;
+
+class LexicalReordering: public StatefulFeatureFunction
+{
+public:
+  LexicalReordering(size_t startInd, const std::string &line);
+  virtual ~LexicalReordering();
+
+  virtual void Load(System &system);
+
+  virtual void SetParameter(const std::string& key, const std::string& value);
+
+  virtual size_t HasPhraseTableInd() const {
+    return true;
+  }
+
+  virtual FFState* BlankState(MemPool &pool, const System &sys) const;
+  virtual void EmptyHypothesisState(FFState &state, const ManagerBase &mgr,
+                                    const InputType &input, const Hypothesis &hypo) const;
+
+  virtual void
+  EvaluateInIsolation(MemPool &pool, const System &system, const Phrase<Moses2::Word> &source,
+                      const TargetPhraseImpl &targetPhrase, Scores &scores,
+                      SCORE &estimatedScore) const;
+
+  virtual void
+  EvaluateInIsolation(MemPool &pool, const System &system, const Phrase<SCFG::Word> &source,
+                      const TargetPhrase<SCFG::Word> &targetPhrase, Scores &scores,
+                      SCORE &estimatedScore) const;
+
+  virtual void
+  EvaluateAfterTablePruning(MemPool &pool, const TargetPhrases &tps,
+                            const Phrase<Moses2::Word> &sourcePhrase) const;
+
+  virtual void EvaluateWhenApplied(const ManagerBase &mgr,
+                                   const Hypothesis &hypo, const FFState &prevState, Scores &scores,
+                                   FFState &state) const;
+
+  virtual void EvaluateWhenApplied(const SCFG::Manager &mgr,
+                                   const SCFG::Hypothesis &hypo, int featureID, Scores &scores,
+                                   FFState &state) const;
+
+protected:
+  std::string m_path;
+  FactorList m_FactorsF;
+  FactorList m_FactorsE;
+  FactorList m_FactorsC;
+
+  LRModel *m_configuration;
+
+  virtual void
+  EvaluateAfterTablePruning(MemPool &pool, const TargetPhraseImpl &targetPhrase,
+                            const Phrase<Moses2::Word> &sourcePhrase) const;
+
+  // PROPERTY IN PT
+  int m_propertyInd;
+
+  // COMPACT MODEL
+#ifdef HAVE_CMPH
+  LexicalReorderingTableCompact *m_compactModel;
+#endif
+
+  Phrase<Moses2::Word> *m_blank;
+
+  // MEMORY MODEL
+  typedef std::pair<const Phrase<Moses2::Word>*, const Phrase<Moses2::Word>* > Key;
+  typedef std::vector<SCORE> Values;
+
+  struct KeyComparer {
+    size_t operator()(const Key &obj) const {
+      size_t seed = obj.first->hash();
+      boost::hash_combine(seed, obj.second->hash());
+      return seed;
+    }
+
+    bool operator()(const Key& a, const Key& b) const {
+      if ((*a.first) != (*b.first)) {
+        return false;
+      }
+      if ((*a.second) != (*b.second)) {
+        return false;
+      }
+      return true;
+    }
+
+  };
+
+  typedef boost::unordered_map<Key, Values, KeyComparer, KeyComparer> Coll;
+  Coll *m_coll;
+
+  const Values *GetValues(const Phrase<Moses2::Word> &source, const Phrase<Moses2::Word> &target) const;
+};
+
+} /* namespace Moses2 */
+

mosesdecoder/moses2/FF/LexicalReordering/PhraseBasedReorderingState.cpp

@@ -0,0 +1,84 @@
+/*
+ * PhraseLR.cpp
+ *
+ *  Created on: 22 Mar 2016
+ *      Author: hieu
+ */
+
+#include "PhraseBasedReorderingState.h"
+#include "LexicalReordering.h"
+#include "../../PhraseBased/Hypothesis.h"
+#include "../../InputPathBase.h"
+#include "../../PhraseBased/Manager.h"
+
+using namespace std;
+
+namespace Moses2
+{
+
+PhraseBasedReorderingState::PhraseBasedReorderingState(const LRModel &config,
+    LRModel::Direction dir, size_t offset) :
+  LRState(config, dir, offset)
+{
+  // uninitialised
+  prevPath = NULL;
+  prevTP = NULL;
+}
+
+void PhraseBasedReorderingState::Init(const LRState *prev,
+                                      const TargetPhrase<Moses2::Word> &topt, const InputPathBase &path, bool first,
+                                      const Bitmap *coverage)
+{
+  prevTP = &topt;
+  prevPath = &path;
+  m_first = first;
+}
+
+size_t PhraseBasedReorderingState::hash() const
+{
+  size_t ret;
+  ret = (size_t) &prevPath->range;
+  boost::hash_combine(ret, m_direction);
+
+  return ret;
+}
+
+bool PhraseBasedReorderingState::operator==(const FFState& o) const
+{
+  if (&o == this) return true;
+
+  const PhraseBasedReorderingState &other =
+    static_cast<const PhraseBasedReorderingState&>(o);
+  if (&prevPath->range == &other.prevPath->range) {
+    if (m_direction == LRModel::Forward) {
+      int compareScore = ComparePrevScores(other.prevTP);
+      return compareScore == 0;
+    } else {
+      return true;
+    }
+  } else {
+    return false;
+  }
+}
+
+void PhraseBasedReorderingState::Expand(const ManagerBase &mgr,
+                                        const LexicalReordering &ff, const Hypothesis &hypo, size_t phraseTableInd,
+                                        Scores &scores, FFState &state) const
+{
+  if ((m_direction != LRModel::Forward) || !m_first) {
+    LRModel const& lrmodel = m_configuration;
+    Range const &cur = hypo.GetInputPath().range;
+    LRModel::ReorderingType reoType = (
+                                        m_first ?
+                                        lrmodel.GetOrientation(cur) :
+                                        lrmodel.GetOrientation(prevPath->range, cur));
+    CopyScores(mgr.system, scores, hypo.GetTargetPhrase(), reoType);
+  }
+
+  PhraseBasedReorderingState &stateCast =
+    static_cast<PhraseBasedReorderingState&>(state);
+  stateCast.Init(this, hypo.GetTargetPhrase(), hypo.GetInputPath(), false,
+                 NULL);
+}
+
+} /* namespace Moses2 */

mosesdecoder/moses2/FF/LexicalReordering/PhraseBasedReorderingState.h

@@ -0,0 +1,44 @@
+/*
+ * PhraseLR.h
+ *
+ *  Created on: 22 Mar 2016
+ *      Author: hieu
+ */
+
+#pragma once
+#include "LRState.h"
+
+namespace Moses2
+{
+
+class InputPathBase;
+
+class PhraseBasedReorderingState: public LRState
+{
+public:
+  const InputPathBase *prevPath;
+  bool m_first;
+
+  PhraseBasedReorderingState(const LRModel &config, LRModel::Direction dir,
+                             size_t offset);
+
+  void Init(const LRState *prev, const TargetPhrase<Moses2::Word> &topt,
+            const InputPathBase &path, bool first, const Bitmap *coverage);
+
+  size_t hash() const;
+  virtual bool operator==(const FFState& other) const;
+
+  virtual std::string ToString() const {
+    return "PhraseBasedReorderingState";
+  }
+
+  void Expand(const ManagerBase &mgr, const LexicalReordering &ff,
+              const Hypothesis &hypo, size_t phraseTableInd, Scores &scores,
+              FFState &state) const;
+
+protected:
+
+};
+
+} /* namespace Moses2 */
+

mosesdecoder/moses2/FF/LexicalReordering/ReorderingStack.cpp

@@ -0,0 +1,99 @@
+/*
+ * ReorderingStack.cpp
+ ** Author: Ankit K. Srivastava
+ ** Date: Jan 26, 2010
+ */
+
+#include <vector>
+#include "ReorderingStack.h"
+#include "../../MemPool.h"
+
+namespace Moses2
+{
+ReorderingStack::ReorderingStack(MemPool &pool) :
+  m_stack(pool)
+{
+
+}
+
+void ReorderingStack::Init()
+{
+  m_stack.clear();
+}
+
+size_t ReorderingStack::hash() const
+{
+  std::size_t ret = boost::hash_range(m_stack.begin(), m_stack.end());
+  return ret;
+}
+
+bool ReorderingStack::operator==(const ReorderingStack& o) const
+{
+  const ReorderingStack& other = static_cast<const ReorderingStack&>(o);
+  return m_stack == other.m_stack;
+}
+
+// Method to push (shift element into the stack and reduce if reqd)
+int ReorderingStack::ShiftReduce(const Range &input_span)
+{
+  int distance; // value to return: the initial distance between this and previous span
+
+  // stack is empty
+  if (m_stack.empty()) {
+    m_stack.push_back(input_span);
+    return input_span.GetStartPos() + 1; // - (-1)
+  }
+
+  // stack is non-empty
+  Range prev_span = m_stack.back(); //access last element added
+
+  //calculate the distance we are returning
+  if (input_span.GetStartPos() > prev_span.GetStartPos()) {
+    distance = input_span.GetStartPos() - prev_span.GetEndPos();
+  } else {
+    distance = input_span.GetEndPos() - prev_span.GetStartPos();
+  }
+
+  if (distance == 1) { //monotone
+    m_stack.pop_back();
+    Range new_span(prev_span.GetStartPos(), input_span.GetEndPos());
+    Reduce(new_span);
+  } else if (distance == -1) { //swap
+    m_stack.pop_back();
+    Range new_span(input_span.GetStartPos(), prev_span.GetEndPos());
+    Reduce(new_span);
+  } else {    // discontinuous
+    m_stack.push_back(input_span);
+  }
+
+  return distance;
+}
+
+// Method to reduce, if possible the spans
+void ReorderingStack::Reduce(Range current)
+{
+  bool cont_loop = true;
+
+  while (cont_loop && m_stack.size() > 0) {
+
+    Range previous = m_stack.back();
+
+    if (current.GetStartPos() - previous.GetEndPos() == 1) { //mono&merge
+      m_stack.pop_back();
+      Range t(previous.GetStartPos(), current.GetEndPos());
+      current = t;
+    } else if (previous.GetStartPos() - current.GetEndPos() == 1) { //swap&merge
+      m_stack.pop_back();
+      Range t(current.GetStartPos(), previous.GetEndPos());
+      current = t;
+    } else { // discontinuous, no more merging
+      cont_loop = false;
+    }
+  } // finished reducing, exit
+
+  // add to stack
+  m_stack.push_back(current);
+}
+
+}
+

mosesdecoder/moses2/FF/LexicalReordering/ReorderingStack.h

@@ -0,0 +1,41 @@
+/*
+ * ReorderingStack.h
+ ** Author: Ankit K. Srivastava
+ ** Date: Jan 26, 2010
+ */
+
+#pragma once
+
+//#include <string>
+#include <vector>
+//#include "Factor.h"
+//#include "Phrase.h"
+//#include "TypeDef.h"
+//#include "Util.h"
+#include "../../legacy/Range.h"
+#include "../../Vector.h"
+
+namespace Moses2
+{
+class MemPool;
+
+class ReorderingStack
+{
+private:
+
+  Vector<Range> m_stack;
+
+public:
+  ReorderingStack(MemPool &pool);
+
+  size_t hash() const;
+  bool operator==(const ReorderingStack& other) const;
+
+  void Init();
+  int ShiftReduce(const Range &input_span);
+
+private:
+  void Reduce(Range input_span);
+};
+
+}

mosesdecoder/moses2/FF/OSM/KenOSM.cpp

@@ -0,0 +1,33 @@
+#include "KenOSM.h"
+
+namespace Moses2
+{
+
+OSMLM* ConstructOSMLM(const char *file, util::LoadMethod load_method)
+{
+  lm::ngram::ModelType model_type;
+  lm::ngram::Config config;
+  config.load_method = load_method;
+  if (lm::ngram::RecognizeBinary(file, model_type)) {
+    switch(model_type) {
+    case lm::ngram::PROBING:
+      return new KenOSM<lm::ngram::ProbingModel>(file, config);
+    case lm::ngram::REST_PROBING:
+      return new KenOSM<lm::ngram::RestProbingModel>(file, config);
+    case lm::ngram::TRIE:
+      return new KenOSM<lm::ngram::TrieModel>(file, config);
+    case lm::ngram::QUANT_TRIE:
+      return new KenOSM<lm::ngram::QuantTrieModel>(file, config);
+    case lm::ngram::ARRAY_TRIE:
+      return new KenOSM<lm::ngram::ArrayTrieModel>(file, config);
+    case lm::ngram::QUANT_ARRAY_TRIE:
+      return new KenOSM<lm::ngram::QuantArrayTrieModel>(file, config);
+    default:
+      UTIL_THROW2("Unrecognized kenlm model type " << model_type);
+    }
+  } else {
+    return new KenOSM<lm::ngram::ProbingModel>(file, config);
+  }
+}
+
+} // namespace

mosesdecoder/moses2/FF/OSM/KenOSM.h

@@ -0,0 +1,53 @@
+#pragma once
+
+#include <string>
+#include "lm/model.hh"
+
+namespace Moses2
+{
+
+class KenOSMBase
+{
+public:
+  virtual ~KenOSMBase() {}
+
+  virtual float Score(const lm::ngram::State&, StringPiece,
+                      lm::ngram::State&) const = 0;
+
+  virtual const lm::ngram::State &BeginSentenceState() const = 0;
+
+  virtual const lm::ngram::State &NullContextState() const = 0;
+};
+
+template <class KenModel>
+class KenOSM : public KenOSMBase
+{
+public:
+  KenOSM(const char *file, const lm::ngram::Config &config)
+    : m_kenlm(file, config) {}
+
+  float Score(const lm::ngram::State &in_state,
+              StringPiece word,
+              lm::ngram::State &out_state) const {
+    return m_kenlm.Score(in_state, m_kenlm.GetVocabulary().Index(word),
+                         out_state);
+  }
+
+  const lm::ngram::State &BeginSentenceState() const {
+    return m_kenlm.BeginSentenceState();
+  }
+
+  const lm::ngram::State &NullContextState() const {
+    return m_kenlm.NullContextState();
+  }
+
+private:
+  KenModel m_kenlm;
+};
+
+typedef KenOSMBase OSMLM;
+
+OSMLM* ConstructOSMLM(const char *file, util::LoadMethod load_method);
+
+
+} // namespace

mosesdecoder/moses2/FF/OSM/OpSequenceModel.cpp

@@ -0,0 +1,248 @@
+#include <sstream>
+#include "OpSequenceModel.h"
+#include "osmHyp.h"
+#include "lm/state.hh"
+#include "../../PhraseBased/Manager.h"
+#include "../../PhraseBased/Hypothesis.h"
+#include "../../PhraseBased/TargetPhraseImpl.h"
+#include "../../PhraseBased/Sentence.h"
+#include "../../TranslationModel/UnknownWordPenalty.h"
+#include "../../System.h"
+
+using namespace std;
+
+namespace Moses2
+{
+
+////////////////////////////////////////////////////////////////////////////////////////
+
+OpSequenceModel::OpSequenceModel(size_t startInd, const std::string &line) :
+  StatefulFeatureFunction(startInd, line)
+{
+  sFactor = 0;
+  tFactor = 0;
+  numFeatures = 5;
+  load_method = util::READ;
+
+  ReadParameters();
+}
+
+OpSequenceModel::~OpSequenceModel()
+{
+  // TODO Auto-generated destructor stub
+}
+
+void OpSequenceModel::Load(System &system)
+{
+  readLanguageModel(m_lmPath.c_str());
+}
+
+FFState* OpSequenceModel::BlankState(MemPool &pool, const System &sys) const
+{
+  return new (pool.Allocate<osmState>()) osmState();
+}
+
+void OpSequenceModel::EmptyHypothesisState(FFState &state,
+    const ManagerBase &mgr, const InputType &input,
+    const Hypothesis &hypo) const
+{
+  lm::ngram::State startState = OSM->BeginSentenceState();
+
+  osmState &stateCast = static_cast<osmState&>(state);
+  stateCast.setState(startState);
+}
+
+void OpSequenceModel::EvaluateInIsolation(MemPool &pool,
+    const System &system, const Phrase<Moses2::Word> &source,
+    const TargetPhraseImpl &targetPhrase, Scores &scores,
+    SCORE &estimatedScore) const
+{
+  osmHypothesis obj;
+  obj.setState(OSM->NullContextState());
+
+  Bitmap myBitmap (pool, source.GetSize());
+  myBitmap.Init(std::vector<bool>());
+
+  vector <string> mySourcePhrase;
+  vector <string> myTargetPhrase;
+  vector<float> scoresVec;
+  vector <int> alignments;
+  int startIndex = 0;
+  int endIndex = source.GetSize();
+
+  const AlignmentInfo &align = targetPhrase.GetAlignTerm();
+  AlignmentInfo::const_iterator iter;
+
+  for (iter = align.begin(); iter != align.end(); ++iter) {
+    alignments.push_back(iter->first);
+    alignments.push_back(iter->second);
+  }
+
+  for (size_t i = 0; i < targetPhrase.GetSize(); i++) {
+    if (&targetPhrase.pt == system.featureFunctions.GetUnknownWordPenalty() && sFactor == 0 && tFactor == 0)
+      myTargetPhrase.push_back("_TRANS_SLF_");
+    else
+      myTargetPhrase.push_back(targetPhrase[i][tFactor]->GetString().as_string());
+  }
+
+  for (size_t i = 0; i < source.GetSize(); i++) {
+    mySourcePhrase.push_back(source[i][sFactor]->GetString().as_string());
+  }
+
+  obj.setPhrases(mySourcePhrase , myTargetPhrase);
+  obj.constructCepts(alignments,startIndex,endIndex-1,targetPhrase.GetSize());
+  obj.computeOSMFeature(startIndex,myBitmap);
+  obj.calculateOSMProb(*OSM);
+  obj.populateScores(scoresVec,numFeatures);
+
+  SCORE weightedScore = Scores::CalcWeightedScore(system, *this,
+                        scoresVec.data());
+  estimatedScore += weightedScore;
+
+}
+
+void OpSequenceModel::EvaluateInIsolation(MemPool &pool, const System &system, const Phrase<SCFG::Word> &source,
+    const TargetPhrase<SCFG::Word> &targetPhrase, Scores &scores,
+    SCORE &estimatedScore) const
+{
+  UTIL_THROW2("Not implemented");
+}
+
+void OpSequenceModel::EvaluateWhenApplied(const ManagerBase &mgr,
+    const Hypothesis &hypo, const FFState &prevState, Scores &scores,
+    FFState &state) const
+{
+  const TargetPhrase<Moses2::Word> &target = hypo.GetTargetPhrase();
+  const Bitmap &bitmap = hypo.GetBitmap();
+  Bitmap myBitmap(bitmap);
+  const ManagerBase &manager = hypo.GetManager();
+  const InputType &source = manager.GetInput();
+  const Sentence &sourceSentence = static_cast<const Sentence&>(source);
+
+  osmHypothesis obj;
+  vector <string> mySourcePhrase;
+  vector <string> myTargetPhrase;
+  vector<float> scoresVec;
+
+
+  //target.GetWord(0)
+
+  //cerr << target <<" --- "<<target.GetSourcePhrase()<< endl;  // English ...
+
+  //cerr << align << endl;   // Alignments ...
+  //cerr << cur_hypo.GetCurrSourceWordsRange() << endl;
+
+  //cerr << source <<endl;
+
+// int a = sourceRange.GetStartPos();
+// cerr << source.GetWord(a);
+  //cerr <<a<<endl;
+
+  //const Sentence &sentence = static_cast<const Sentence&>(curr_hypo.GetManager().GetSource());
+
+
+  const Range & sourceRange = hypo.GetInputPath().range;
+  int startIndex  = sourceRange.GetStartPos();
+  int endIndex = sourceRange.GetEndPos();
+  const AlignmentInfo &align = hypo.GetTargetPhrase().GetAlignTerm();
+  // osmState * statePtr;
+
+  vector <int> alignments;
+
+
+
+  AlignmentInfo::const_iterator iter;
+
+  for (iter = align.begin(); iter != align.end(); ++iter) {
+    //cerr << iter->first << "----" << iter->second << " ";
+    alignments.push_back(iter->first);
+    alignments.push_back(iter->second);
+  }
+
+
+  //cerr<<bitmap<<endl;
+  //cerr<<startIndex<<" "<<endIndex<<endl;
+
+
+  for (int i = startIndex; i <= endIndex; i++) {
+    myBitmap.SetValue(i,0); // resetting coverage of this phrase ...
+    mySourcePhrase.push_back(sourceSentence[i][sFactor]->GetString().as_string());
+    // cerr<<mySourcePhrase[i]<<endl;
+  }
+
+  for (size_t i = 0; i < target.GetSize(); i++) {
+    if (&target.pt == mgr.system.featureFunctions.GetUnknownWordPenalty() && sFactor == 0 && tFactor == 0)
+      myTargetPhrase.push_back("_TRANS_SLF_");
+    else
+      myTargetPhrase.push_back(target[i][tFactor]->GetString().as_string());
+
+  }
+
+
+  //cerr<<myBitmap<<endl;
+
+  obj.setState(&prevState);
+  obj.constructCepts(alignments,startIndex,endIndex,target.GetSize());
+  obj.setPhrases(mySourcePhrase , myTargetPhrase);
+  obj.computeOSMFeature(startIndex,myBitmap);
+  obj.calculateOSMProb(*OSM);
+  obj.populateScores(scoresVec,numFeatures);
+  //obj.print();
+
+  scores.PlusEquals(mgr.system, *this, scoresVec);
+
+  osmState &stateCast = static_cast<osmState&>(state);
+  obj.saveState(stateCast);
+}
+
+void OpSequenceModel::EvaluateWhenApplied(const SCFG::Manager &mgr,
+    const SCFG::Hypothesis &hypo, int featureID, Scores &scores,
+    FFState &state) const
+{
+  UTIL_THROW2("Not implemented");
+}
+
+void OpSequenceModel::SetParameter(const std::string& key, const std::string& value)
+{
+
+  if (key == "path") {
+    m_lmPath = value;
+  } else if (key == "support-features") {
+    if(value == "no")
+      numFeatures = 1;
+    else
+      numFeatures = 5;
+  } else if (key == "input-factor") {
+    sFactor = Scan<int>(value);
+  } else if (key == "output-factor") {
+    tFactor = Scan<int>(value);
+  } else if (key == "load") {
+    if (value == "lazy") {
+      load_method = util::LAZY;
+    } else if (value == "populate_or_lazy") {
+      load_method = util::POPULATE_OR_LAZY;
+    } else if (value == "populate_or_read" || value == "populate") {
+      load_method = util::POPULATE_OR_READ;
+    } else if (value == "read") {
+      load_method = util::READ;
+    } else if (value == "parallel_read") {
+      load_method = util::PARALLEL_READ;
+    } else {
+      UTIL_THROW2("Unknown KenLM load method " << value);
+    }
+  } else {
+    StatefulFeatureFunction::SetParameter(key, value);
+  }
+}
+
+void OpSequenceModel :: readLanguageModel(const char *lmFile)
+{
+  string unkOp = "_TRANS_SLF_";
+  OSM = ConstructOSMLM(m_lmPath.c_str(), load_method);
+
+  lm::ngram::State startState = OSM->NullContextState();
+  lm::ngram::State endState;
+  unkOpProb = OSM->Score(startState,unkOp,endState);
+}
+
+}

mosesdecoder/moses2/FF/OSM/OpSequenceModel.h

@@ -0,0 +1,57 @@
+#include "../StatefulFeatureFunction.h"
+#include "util/mmap.hh"
+#include "KenOSM.h"
+
+namespace Moses2
+{
+
+
+class OpSequenceModel : public StatefulFeatureFunction
+{
+public:
+  OSMLM* OSM;
+  float unkOpProb;
+  int numFeatures;   // Number of features used ...
+  int sFactor;  // Source Factor ...
+  int tFactor;  // Target Factor ...
+  util::LoadMethod load_method; // method to load model
+
+  OpSequenceModel(size_t startInd, const std::string &line);
+  virtual ~OpSequenceModel();
+
+  virtual void Load(System &system);
+
+  virtual FFState* BlankState(MemPool &pool, const System &sys) const;
+  virtual void EmptyHypothesisState(FFState &state, const ManagerBase &mgr,
+                                    const InputType &input, const Hypothesis &hypo) const;
+
+  virtual void
+  EvaluateInIsolation(MemPool &pool, const System &system, const Phrase<Moses2::Word> &source,
+                      const TargetPhraseImpl &targetPhrase, Scores &scores,
+                      SCORE &estimatedScore) const;
+
+  virtual void
+  EvaluateInIsolation(MemPool &pool, const System &system, const Phrase<SCFG::Word> &source,
+                      const TargetPhrase<SCFG::Word> &targetPhrase, Scores &scores,
+                      SCORE &estimatedScore) const;
+
+  virtual void EvaluateWhenApplied(const ManagerBase &mgr,
+                                   const Hypothesis &hypo, const FFState &prevState, Scores &scores,
+                                   FFState &state) const;
+
+  virtual void EvaluateWhenApplied(const SCFG::Manager &mgr,
+                                   const SCFG::Hypothesis &hypo, int featureID, Scores &scores,
+                                   FFState &state) const;
+
+  void SetParameter(const std::string& key, const std::string& value);
+
+protected:
+  std::string m_lmPath;
+
+  void readLanguageModel(const char *);
+
+};
+
+}
+
+

mosesdecoder/moses2/FF/OSM/osmHyp.cpp

@@ -0,0 +1,601 @@
+#include "osmHyp.h"
+#include <sstream>
+
+using namespace std;
+using namespace lm::ngram;
+
+namespace Moses2
+{
+void osmState::setState(const lm::ngram::State & val)
+{
+  j = 0;
+  E = 0;
+  lmState = val;
+}
+
+void osmState::saveState(int jVal, int eVal, map <int , string> & gapVal)
+{
+  gap.clear();
+  gap = gapVal;
+  j = jVal;
+  E = eVal;
+}
+
+size_t osmState::hash() const
+{
+  size_t ret = j;
+
+  boost::hash_combine(ret, E);
+  boost::hash_combine(ret, gap);
+  boost::hash_combine(ret, lmState.length);
+
+  return ret;
+}
+
+bool osmState::operator==(const FFState& otherBase) const
+{
+  const osmState &other = static_cast<const osmState&>(otherBase);
+  if (j != other.j)
+    return false;
+  if (E != other.E)
+    return false;
+  if (gap != other.gap)
+    return false;
+  if (lmState.length != other.lmState.length)
+    return false;
+
+  return true;
+}
+
+std::string osmState :: getName() const
+{
+
+  return "done";
+}
+
+//////////////////////////////////////////////////
+
+osmHypothesis :: osmHypothesis()
+{
+  opProb = 0;
+  gapWidth = 0;
+  gapCount = 0;
+  openGapCount = 0;
+  deletionCount = 0;
+  gapCount = 0;
+  j = 0;
+  E = 0;
+  gap.clear();
+}
+
+void osmHypothesis :: setState(const FFState* prev_state)
+{
+
+  if(prev_state != NULL) {
+
+    j = static_cast <const osmState *> (prev_state)->getJ();
+    E =  static_cast <const osmState *> (prev_state)->getE();
+    gap = static_cast <const osmState *> (prev_state)->getGap();
+    lmState = static_cast <const osmState *> (prev_state)->getLMState();
+  }
+}
+
+void osmHypothesis :: saveState(osmState &state)
+{
+  state.setState(lmState);
+  state.saveState(j,E,gap);
+}
+
+int osmHypothesis :: isTranslationOperation(int x)
+{
+  if (operations[x].find("_JMP_BCK_") != -1)
+    return 0;
+
+  if (operations[x].find("_JMP_FWD_") != -1)
+    return 0;
+
+  if (operations[x].find("_CONT_CEPT_") != -1)
+    return 0;
+
+  if (operations[x].find("_INS_GAP_") != -1)
+    return 0;
+
+  return 1;
+
+}
+
+void osmHypothesis :: removeReorderingOperations()
+{
+  gapCount = 0;
+  deletionCount = 0;
+  openGapCount = 0;
+  gapWidth = 0;
+
+  std::vector <std::string> tupleSequence;
+
+  for (int x = 0; x < operations.size(); x++) {
+    // cout<<operations[x]<<endl;
+
+    if(isTranslationOperation(x) == 1) {
+      tupleSequence.push_back(operations[x]);
+    }
+
+  }
+
+  operations.clear();
+  operations = tupleSequence;
+}
+
+void osmHypothesis :: calculateOSMProb(OSMLM& ptrOp)
+{
+
+  opProb = 0;
+  State currState = lmState;
+  State temp;
+
+  for (size_t i = 0; i<operations.size(); i++) {
+    temp = currState;
+    opProb += ptrOp.Score(temp,operations[i],currState);
+  }
+
+  lmState = currState;
+
+  //print();
+}
+
+
+int osmHypothesis :: firstOpenGap(vector <int> & coverageVector)
+{
+
+  int firstOG =-1;
+
+  for(int nd = 0; nd < coverageVector.size(); nd++) {
+    if(coverageVector[nd]==0) {
+      firstOG = nd;
+      return firstOG;
+    }
+  }
+
+  return firstOG;
+
+}
+
+string osmHypothesis :: intToString(int num)
+{
+  return SPrint(num);
+
+}
+
+void osmHypothesis :: generateOperations(int & startIndex , int j1 , int contFlag , Bitmap & coverageVector , string english , string german , set <int> & targetNullWords , vector <string> & currF)
+{
+
+  int gFlag = 0;
+  int gp = 0;
+  int ans;
+
+
+  if ( j < j1) { // j1 is the index of the source word we are about to generate ...
+    //if(coverageVector[j]==0) // if source word at j is not generated yet ...
+    if(coverageVector.GetValue(j)==0) { // if source word at j is not generated yet ...
+      operations.push_back("_INS_GAP_");
+      gFlag++;
+      gap[j]="Unfilled";
+    }
+    if (j == E) {
+      j = j1;
+    } else {
+      operations.push_back("_JMP_FWD_");
+      j=E;
+    }
+  }
+
+  if (j1 < j) {
+    // if(j < E && coverageVector[j]==0)
+    if(j < E && coverageVector.GetValue(j)==0) {
+      operations.push_back("_INS_GAP_");
+      gFlag++;
+      gap[j]="Unfilled";
+    }
+
+    j=closestGap(gap,j1,gp);
+    operations.push_back("_JMP_BCK_"+ intToString(gp));
+
+    //cout<<"I am j "<<j<<endl;
+    //cout<<"I am j1 "<<j1<<endl;
+
+    if(j==j1)
+      gap[j]="Filled";
+  }
+
+  if (j < j1) {
+    operations.push_back("_INS_GAP_");
+    gap[j] = "Unfilled";
+    gFlag++;
+    j=j1;
+  }
+
+  if(contFlag == 0) { // First words of the multi-word cept ...
+
+    if(english == "_TRANS_SLF_") { // Unknown word ...
+      operations.push_back("_TRANS_SLF_");
+    } else {
+      operations.push_back("_TRANS_" + english + "_TO_" + german);
+    }
+
+    //ans = firstOpenGap(coverageVector);
+    ans = coverageVector.GetFirstGapPos();
+
+    if (ans != -1)
+      gapWidth += j - ans;
+
+  } else if (contFlag == 2) {
+
+    operations.push_back("_INS_" + german);
+    ans = coverageVector.GetFirstGapPos();
+
+    if (ans != -1)
+      gapWidth += j - ans;
+    deletionCount++;
+  } else {
+    operations.push_back("_CONT_CEPT_");
+  }
+
+  //coverageVector[j]=1;
+  coverageVector.SetValue(j,1);
+  j+=1;
+
+  if(E<j)
+    E=j;
+
+  if (gFlag > 0)
+    gapCount++;
+
+  openGapCount += getOpenGaps();
+
+  //if (coverageVector[j] == 0 && targetNullWords.find(j) != targetNullWords.end())
+  if (j < coverageVector.GetSize()) {
+    if (coverageVector.GetValue(j) == 0 && targetNullWords.find(j) != targetNullWords.end()) {
+      j1 = j;
+      german = currF[j1-startIndex];
+      english = "_INS_";
+      generateOperations(startIndex, j1, 2 , coverageVector , english , german , targetNullWords , currF);
+    }
+  }
+
+}
+
+void osmHypothesis :: print()
+{
+  for (int i = 0; i< operations.size(); i++) {
+    cerr<<operations[i]<<" ";
+
+  }
+
+  cerr<<endl<<endl;
+
+  cerr<<"Operation Probability "<<opProb<<endl;
+  cerr<<"Gap Count "<<gapCount<<endl;
+  cerr<<"Open Gap Count "<<openGapCount<<endl;
+  cerr<<"Gap Width "<<gapWidth<<endl;
+  cerr<<"Deletion Count "<<deletionCount<<endl;
+
+  cerr<<"_______________"<<endl;
+}
+
+int osmHypothesis :: closestGap(map <int,string> gap, int j1, int & gp)
+{
+
+  int dist=1172;
+  int value=-1;
+  int temp=0;
+  gp=0;
+  int opGap=0;
+
+  map <int,string> :: iterator iter;
+
+  iter=gap.end();
+
+  do {
+    iter--;
+    //cout<<"Trapped "<<iter->first<<endl;
+
+    if(iter->first==j1 && iter->second== "Unfilled") {
+      opGap++;
+      gp = opGap;
+      return j1;
+
+    }
+
+    if(iter->second =="Unfilled") {
+      opGap++;
+      temp = iter->first - j1;
+
+      if(temp<0)
+        temp=temp * -1;
+
+      if(dist>temp && iter->first < j1) {
+        dist=temp;
+        value=iter->first;
+        gp=opGap;
+      }
+    }
+
+
+  } while(iter!=gap.begin());
+
+  return value;
+}
+
+
+
+int osmHypothesis :: getOpenGaps()
+{
+  map <int,string> :: iterator iter;
+
+  int nd = 0;
+  for (iter = gap.begin(); iter!=gap.end(); iter++) {
+    if(iter->second == "Unfilled")
+      nd++;
+  }
+
+  return nd;
+
+}
+
+void osmHypothesis :: generateDeleteOperations(std::string english, int currTargetIndex, std::set <int> doneTargetIndexes)
+{
+
+  operations.push_back("_DEL_" + english);
+  currTargetIndex++;
+
+  while(doneTargetIndexes.find(currTargetIndex) != doneTargetIndexes.end()) {
+    currTargetIndex++;
+  }
+
+  if (sourceNullWords.find(currTargetIndex) != sourceNullWords.end()) {
+    english = currE[currTargetIndex];
+    generateDeleteOperations(english,currTargetIndex,doneTargetIndexes);
+  }
+
+}
+
+void osmHypothesis :: computeOSMFeature(int startIndex , Bitmap & coverageVector)
+{
+
+  set <int> doneTargetIndexes;
+  set <int> eSide;
+  set <int> fSide;
+  set <int> :: iterator iter;
+  string english;
+  string source;
+  int j1;
+  int targetIndex = 0;
+  doneTargetIndexes.clear();
+
+
+  if (targetNullWords.size() != 0) { // Source words to be deleted in the start of this phrase ...
+    iter = targetNullWords.begin();
+
+    if (*iter == startIndex) {
+
+      j1 = startIndex;
+      source = currF[j1-startIndex];
+      english = "_INS_";
+      generateOperations(startIndex, j1, 2 , coverageVector , english , source , targetNullWords , currF);
+    }
+  }
+
+  if (sourceNullWords.find(targetIndex) != sourceNullWords.end()) { // first word has to be deleted ...
+    english = currE[targetIndex];
+    generateDeleteOperations(english,targetIndex, doneTargetIndexes);
+  }
+
+
+  for (size_t i = 0; i < ceptsInPhrase.size(); i++) {
+    source = "";
+    english = "";
+
+    fSide = ceptsInPhrase[i].first;
+    eSide = ceptsInPhrase[i].second;
+
+    iter = eSide.begin();
+    targetIndex = *iter;
+    english += currE[*iter];
+    iter++;
+
+    for (; iter != eSide.end(); iter++) {
+      if(*iter == targetIndex+1)
+        targetIndex++;
+      else
+        doneTargetIndexes.insert(*iter);
+
+      english += "^_^";
+      english += currE[*iter];
+    }
+
+    iter = fSide.begin();
+    source += currF[*iter];
+    iter++;
+
+    for (; iter != fSide.end(); iter++) {
+      source += "^_^";
+      source += currF[*iter];
+    }
+
+    iter = fSide.begin();
+    j1 = *iter + startIndex;
+    iter++;
+
+    generateOperations(startIndex, j1, 0 , coverageVector , english , source , targetNullWords , currF);
+
+
+    for (; iter != fSide.end(); iter++) {
+      j1 = *iter + startIndex;
+      generateOperations(startIndex, j1, 1 , coverageVector , english , source , targetNullWords , currF);
+    }
+
+    targetIndex++; // Check whether the next target word is unaligned ...
+
+    while(doneTargetIndexes.find(targetIndex) != doneTargetIndexes.end()) {
+      targetIndex++;
+    }
+
+    if(sourceNullWords.find(targetIndex) != sourceNullWords.end()) {
+      english = currE[targetIndex];
+      generateDeleteOperations(english,targetIndex, doneTargetIndexes);
+    }
+  }
+
+  //removeReorderingOperations();
+
+  //print();
+
+}
+
+void osmHypothesis :: getMeCepts ( set <int> & eSide , set <int> & fSide , map <int , vector <int> > & tS , map <int , vector <int> > & sT)
+{
+  set <int> :: iterator iter;
+
+  int sz = eSide.size();
+  vector <int> t;
+
+  for (iter = eSide.begin(); iter != eSide.end(); iter++) {
+    t = tS[*iter];
+
+    for (size_t i = 0; i < t.size(); i++) {
+      fSide.insert(t[i]);
+    }
+
+  }
+
+  for (iter = fSide.begin(); iter != fSide.end(); iter++) {
+
+    t = sT[*iter];
+
+    for (size_t i = 0 ; i<t.size(); i++) {
+      eSide.insert(t[i]);
+    }
+
+  }
+
+  if (eSide.size () > sz) {
+    getMeCepts(eSide,fSide,tS,sT);
+  }
+
+}
+
+void osmHypothesis :: constructCepts(vector <int> & align , int startIndex , int endIndex, int targetPhraseLength)
+{
+
+  std::map <int , vector <int> > sT;
+  std::map <int , vector <int> > tS;
+  std::set <int> eSide;
+  std::set <int> fSide;
+  std::set <int> :: iterator iter;
+  std :: map <int , vector <int> > :: iterator iter2;
+  std :: pair < set <int> , set <int> > cept;
+  int src;
+  int tgt;
+
+
+  for (size_t i = 0;  i < align.size(); i+=2) {
+    src = align[i];
+    tgt = align[i+1];
+    tS[tgt].push_back(src);
+    sT[src].push_back(tgt);
+  }
+
+  for (int i = startIndex; i<= endIndex; i++) { // What are unaligned source words in this phrase ...
+    if (sT.find(i-startIndex) == sT.end()) {
+      targetNullWords.insert(i);
+    }
+  }
+
+  for (int i = 0; i < targetPhraseLength; i++) { // What are unaligned target words in this phrase ...
+    if (tS.find(i) == tS.end()) {
+      sourceNullWords.insert(i);
+    }
+  }
+
+
+  while (tS.size() != 0 && sT.size() != 0) {
+
+    iter2 = tS.begin();
+
+    eSide.clear();
+    fSide.clear();
+    eSide.insert (iter2->first);
+
+    getMeCepts(eSide, fSide, tS , sT);
+
+    for (iter = eSide.begin(); iter != eSide.end(); iter++) {
+      iter2 = tS.find(*iter);
+      tS.erase(iter2);
+    }
+
+    for (iter = fSide.begin(); iter != fSide.end(); iter++) {
+      iter2 = sT.find(*iter);
+      sT.erase(iter2);
+    }
+
+    cept = make_pair (fSide , eSide);
+    ceptsInPhrase.push_back(cept);
+  }
+
+
+
+  /*
+
+      cerr<<"Extracted Cepts "<<endl;
+      for (int i = 0; i < ceptsInPhrase.size(); i++)
+        {
+
+          fSide = ceptsInPhrase[i].first;
+          eSide = ceptsInPhrase[i].second;
+
+          for (iter = eSide.begin(); iter != eSide.end(); iter++)
+          {
+              cerr<<*iter<<" ";
+          }
+              cerr<<"<---> ";
+
+          for (iter = fSide.begin(); iter != fSide.end(); iter++)
+          {
+            cerr<<*iter<<" ";
+          }
+
+          cerr<<endl;
+        }
+        cerr<<endl;
+
+      cerr<<"Unaligned Target Words"<<endl;
+
+      for (iter = sourceNullWords.begin(); iter != sourceNullWords.end(); iter++)
+        cerr<<*iter<<"<--->"<<endl;
+
+      cerr<<"Unaligned Source Words"<<endl;
+
+      for (iter = targetNullWords.begin(); iter != targetNullWords.end(); iter++)
+        cerr<<*iter<<"<--->"<<endl;
+
+  */
+
+}
+
+void osmHypothesis :: populateScores(vector <float> & scores , const int numFeatures)
+{
+  scores.clear();
+  scores.push_back(opProb);
+
+  if (numFeatures == 1)
+    return;
+
+  scores.push_back(gapWidth);
+  scores.push_back(gapCount);
+  scores.push_back(openGapCount);
+  scores.push_back(deletionCount);
+}
+
+
+} // namespace
+

mosesdecoder/moses2/FF/OSM/osmHyp.h

@@ -0,0 +1,112 @@
+#pragma once
+
+# include <set>
+# include <map>
+# include <string>
+# include <vector>
+#include "KenOSM.h"
+# include "../FFState.h"
+# include "../../legacy/Bitmap.h"
+
+namespace Moses2
+{
+
+class osmState : public FFState
+{
+public:
+  osmState()
+  {}
+
+  void setState(const lm::ngram::State & val);
+
+  virtual size_t hash() const;
+  virtual bool operator==(const FFState& other) const;
+
+  virtual std::string ToString() const {
+    return "osmState";
+  }
+
+  void saveState(int jVal, int eVal, std::map <int , std::string> & gapVal);
+  int getJ()const {
+    return j;
+  }
+  int getE()const {
+    return E;
+  }
+  std::map <int , std::string> getGap() const {
+    return gap;
+  }
+
+  lm::ngram::State getLMState() const {
+    return lmState;
+  }
+
+  void print() const;
+  std::string getName() const;
+
+protected:
+  int j, E;
+  std::map <int,std::string> gap;
+  lm::ngram::State lmState;
+};
+
+class osmHypothesis
+{
+
+private:
+
+
+  std::vector <std::string> operations; // List of operations required to generated this hyp ...
+  std::map <int,std::string> gap; // Maintains gap history ...
+  int j;  // Position after the last source word generated ...
+  int E; // Position after the right most source word so far generated ...
+  lm::ngram::State lmState; // KenLM's Model State ...
+
+  int gapCount; // Number of gaps inserted ...
+  int deletionCount;
+  int openGapCount;
+  int gapWidth;
+  double opProb;
+
+  std::vector <std::string> currE;
+  std::vector <std::string> currF;
+  std::vector < std::pair < std::set <int> , std::set <int> > > ceptsInPhrase;
+  std::set <int> targetNullWords;
+  std::set <int> sourceNullWords;
+
+  int closestGap(std::map <int,std::string> gap,int j1, int & gp);
+  int firstOpenGap(std::vector <int> & coverageVector);
+  std::string intToString(int);
+  int  getOpenGaps();
+  int isTranslationOperation(int j);
+  void removeReorderingOperations();
+
+  void getMeCepts ( std::set <int> & eSide , std::set <int> & fSide , std::map <int , std::vector <int> > & tS , std::map <int , std::vector <int> > & sT);
+
+public:
+
+  osmHypothesis();
+  ~osmHypothesis() {};
+  void generateOperations(int & startIndex, int j1 , int contFlag , Bitmap & coverageVector , std::string english , std::string german , std::set <int> & targetNullWords , std::vector <std::string> & currF);
+  void generateDeleteOperations(std::string english, int currTargetIndex, std::set <int> doneTargetIndexes);
+  void calculateOSMProb(OSMLM& ptrOp);
+  void computeOSMFeature(int startIndex , Bitmap & coverageVector);
+  void constructCepts(std::vector <int> & align , int startIndex , int endIndex, int targetPhraseLength);
+  void setPhrases(std::vector <std::string> & val1 , std::vector <std::string> & val2) {
+    currF = val1;
+    currE = val2;
+  }
+  void setState(const FFState* prev_state);
+  void saveState(osmState &state);
+  void print();
+  void populateScores(std::vector <float> & scores , const int numFeatures);
+  void setState(const lm::ngram::State & val) {
+    lmState = val;
+  }
+
+};
+
+} // namespace
+
+
+

mosesdecoder/moses2/FF/PhrasePenalty.cpp

@@ -0,0 +1,40 @@
+/*
+ * SkeletonStatefulFF.cpp
+ *
+ *  Created on: 27 Oct 2015
+ *      Author: hieu
+ */
+
+#include "PhrasePenalty.h"
+#include "../Scores.h"
+
+namespace Moses2
+{
+
+PhrasePenalty::PhrasePenalty(size_t startInd, const std::string &line) :
+  StatelessFeatureFunction(startInd, line)
+{
+  ReadParameters();
+}
+
+PhrasePenalty::~PhrasePenalty()
+{
+  // TODO Auto-generated destructor stub
+}
+
+void PhrasePenalty::EvaluateInIsolation(MemPool &pool, const System &system,
+                                        const Phrase<Moses2::Word> &source, const TargetPhraseImpl &targetPhrase, Scores &scores,
+                                        SCORE &estimatedScore) const
+{
+  scores.PlusEquals(system, *this, 1);
+}
+
+void PhrasePenalty::EvaluateInIsolation(MemPool &pool, const System &system, const Phrase<SCFG::Word> &source,
+                                        const TargetPhrase<SCFG::Word> &targetPhrase, Scores &scores,
+                                        SCORE &estimatedScore) const
+{
+  scores.PlusEquals(system, *this, 1);
+}
+
+}
+

mosesdecoder/moses2/FF/PhrasePenalty.h

@@ -0,0 +1,34 @@
+/*
+ * SkeletonStatefulFF.h
+ *
+ *  Created on: 27 Oct 2015
+ *      Author: hieu
+ */
+
+#pragma once
+
+#include "StatelessFeatureFunction.h"
+
+namespace Moses2
+{
+
+class PhrasePenalty: public StatelessFeatureFunction
+{
+public:
+  PhrasePenalty(size_t startInd, const std::string &line);
+  virtual ~PhrasePenalty();
+
+  virtual void
+  EvaluateInIsolation(MemPool &pool, const System &system, const Phrase<Moses2::Word> &source,
+                      const TargetPhraseImpl &targetPhrase, Scores &scores,
+                      SCORE &estimatedScore) const;
+
+  virtual void
+  EvaluateInIsolation(MemPool &pool, const System &system, const Phrase<SCFG::Word> &source,
+                      const TargetPhrase<SCFG::Word> &targetPhrase, Scores &scores,
+                      SCORE &estimatedScore) const;
+
+};
+
+}
+

mosesdecoder/moses2/FF/PointerState.cpp

@@ -0,0 +1,6 @@
+#include "PointerState.h"
+
+namespace Moses2
+{
+
+}