Upload 575 files

.gitattributes

@@ -95,3 +95,6 @@ mosesdecoder/misc/bin/gcc-9/release/link-static/threading-multi/processPhraseTab
 mosesdecoder/misc/bin/gcc-9/release/link-static/threading-multi/prunePhraseTable filter=lfs diff=lfs merge=lfs -text
 mosesdecoder/misc/bin/gcc-9/release/link-static/threading-multi/queryLexicalTable filter=lfs diff=lfs merge=lfs -text
 mosesdecoder/misc/bin/gcc-9/release/link-static/threading-multi/queryPhraseTableMin filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/moses/bin/gcc-9/release/link-static/threading-multi/libmoses.a filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/moses/bin/gcc-9/release/link-static/threading-multi/moses_test filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/moses/LM/bin/BackwardTest.test/gcc-9/release/link-static/threading-multi/BackwardTest filter=lfs diff=lfs merge=lfs -text

mosesdecoder/moses/AlignmentInfo.cpp

@@ -0,0 +1,170 @@
+/***********************************************************************
+ Moses - statistical machine translation system
+ Copyright (C) 2006-2011 University of Edinburgh
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+***********************************************************************/
+#include <algorithm>
+#include <set>
+#include "AlignmentInfo.h"
+#include "TypeDef.h"
+#include "StaticData.h"
+#include "Util.h"
+#include "util/exception.hh"
+
+namespace Moses
+{
+
+AlignmentInfo::AlignmentInfo(const std::set<std::pair<size_t,size_t> > &pairs)
+  : m_collection(pairs)
+{
+  BuildNonTermIndexMaps();
+}
+
+AlignmentInfo::AlignmentInfo(const std::vector<unsigned char> &aln)
+{
+  assert(aln.size()%2==0);
+  for (size_t i = 0; i < aln.size(); i+= 2)
+    m_collection.insert(std::make_pair(size_t(aln[i]),size_t(aln[i+1])));
+  BuildNonTermIndexMaps();
+}
+
+AlignmentInfo::AlignmentInfo(const std::string &str)
+{
+  std::vector<std::string> points = Tokenize(str, " ");
+  std::vector<std::string>::const_iterator iter;
+  for (iter = points.begin(); iter != points.end(); iter++) {
+    std::vector<size_t> point = Tokenize<size_t>(*iter, "-");
+    UTIL_THROW_IF2(point.size() != 2, "Bad format of word alignment point: " << *iter);
+    Add(point[0], point[1]);
+  }
+}
+
+void AlignmentInfo::BuildNonTermIndexMaps()
+{
+  if (m_collection.empty()) {
+    return;
+  }
+  const_iterator p = begin();
+  size_t maxIndex = p->second;
+  for (++p;  p != end(); ++p) {
+    if (p->second > maxIndex) {
+      maxIndex = p->second;
+    }
+  }
+  m_nonTermIndexMap.resize(maxIndex+1, NOT_FOUND);
+  m_nonTermIndexMap2.resize(maxIndex+1, NOT_FOUND);
+  size_t i = 0;
+  for (p = begin(); p != end(); ++p) {
+    if (m_nonTermIndexMap[p->second] != NOT_FOUND) {
+      // 1-to-many. Definitely a set of terminals. Don't bother storing 1-to-1 index map
+      m_nonTermIndexMap.clear();
+      m_nonTermIndexMap2.clear();
+      return;
+    }
+    m_nonTermIndexMap[p->second] = i++;
+    m_nonTermIndexMap2[p->second] = p->first;
+  }
+}
+
+std::set<size_t> AlignmentInfo::GetAlignmentsForSource(size_t sourcePos) const
+{
+  std::set<size_t> ret;
+  CollType::const_iterator iter;
+  for (iter = begin(); iter != end(); ++iter) {
+    // const std::pair<size_t,size_t> &align = *iter;
+    if (iter->first == sourcePos) {
+      ret.insert(iter->second);
+    }
+  }
+  return ret;
+}
+
+std::set<size_t> AlignmentInfo::GetAlignmentsForTarget(size_t targetPos) const
+{
+  std::set<size_t> ret;
+  CollType::const_iterator iter;
+  for (iter = begin(); iter != end(); ++iter) {
+    // const std::pair<size_t,size_t> &align = *iter;
+    if (iter->second == targetPos) {
+      ret.insert(iter->first);
+    }
+  }
+  return ret;
+}
+
+
+bool
+compare_target(std::pair<size_t,size_t> const* a,
+               std::pair<size_t,size_t> const* b)
+{
+  if(a->second < b->second)  return true;
+  if(a->second == b->second) return (a->first < b->first);
+  return false;
+}
+
+
+std::vector< const std::pair<size_t,size_t>* >
+AlignmentInfo::
+GetSortedAlignments(WordAlignmentSort SortOrder) const
+{
+  std::vector< const std::pair<size_t,size_t>* > ret;
+
+  CollType::const_iterator iter;
+  for (iter = m_collection.begin(); iter != m_collection.end(); ++iter) {
+    const std::pair<size_t,size_t> &alignPair = *iter;
+    ret.push_back(&alignPair);
+  }
+
+  switch (SortOrder) {
+  case NoSort:
+    break;
+
+  case TargetOrder:
+    std::sort(ret.begin(), ret.end(), compare_target);
+    break;
+
+  default:
+    UTIL_THROW(util::Exception, "Unknown word alignment sort option: "
+               << SortOrder);
+  }
+
+  return ret;
+
+}
+
+std::vector<size_t> AlignmentInfo::GetSourceIndex2PosMap() const
+{
+  std::set<size_t> sourcePoses;
+
+  CollType::const_iterator iter;
+  for (iter = m_collection.begin(); iter != m_collection.end(); ++iter) {
+    size_t sourcePos = iter->first;
+    sourcePoses.insert(sourcePos);
+  }
+  std::vector<size_t> ret(sourcePoses.begin(), sourcePoses.end());
+  return ret;
+}
+
+std::ostream& operator<<(std::ostream &out, const AlignmentInfo &alignmentInfo)
+{
+  AlignmentInfo::const_iterator iter;
+  for (iter = alignmentInfo.begin(); iter != alignmentInfo.end(); ++iter) {
+    out << iter->first << "-" << iter->second << " ";
+  }
+  return out;
+}
+
+}

mosesdecoder/moses/AlignmentInfo.h

@@ -0,0 +1,143 @@
+/***********************************************************************
+ Moses - statistical machine translation system
+ Copyright (C) 2006-2011 University of Edinburgh
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+***********************************************************************/
+
+#pragma once
+
+#include <iostream>
+#include <ostream>
+#include <set>
+#include <vector>
+#include <cstdlib>
+
+#include <boost/functional/hash.hpp>
+#include "TypeDef.h"
+namespace Moses
+{
+
+class AlignmentInfoCollection;
+
+/** Collection of non-terminal alignment pairs, ordered by source index.
+  * Usually held by a TargetPhrase to map non-terms in hierarchical/syntax models
+ */
+class AlignmentInfo
+{
+  friend std::ostream& operator<<(std::ostream &, const AlignmentInfo &);
+  friend struct AlignmentInfoOrderer;
+  friend struct AlignmentInfoHasher;
+  friend class AlignmentInfoCollection;
+  friend class VW;
+
+public:
+  typedef std::set<std::pair<size_t,size_t> > CollType;
+  typedef std::vector<size_t> NonTermIndexMap;
+  typedef CollType::const_iterator const_iterator;
+
+  const_iterator begin() const {
+    return m_collection.begin();
+  }
+  const_iterator end() const {
+    return m_collection.end();
+  }
+
+  void Add(size_t sourcePos, size_t targetPos) {
+    m_collection.insert(std::pair<size_t, size_t>(sourcePos, targetPos));
+  }
+  /** Provides a map from target-side to source-side non-terminal indices.
+    * The target-side index should be the rule symbol index (COUNTING terminals).
+    * The index returned is the rule non-terminal index (IGNORING terminals).
+   */
+  const NonTermIndexMap &GetNonTermIndexMap() const {
+    return m_nonTermIndexMap;
+  }
+
+  /** Like GetNonTermIndexMap but the return value is the symbol index (i.e.
+    * the index counting both terminals and non-terminals) */
+  const NonTermIndexMap &GetNonTermIndexMap2() const {
+    return m_nonTermIndexMap2;
+  }
+
+  const CollType &GetAlignments() const {
+    return m_collection;
+  }
+
+  std::set<size_t> GetAlignmentsForSource(size_t sourcePos) const;
+  std::set<size_t> GetAlignmentsForTarget(size_t targetPos) const;
+
+  size_t GetSize() const {
+    return m_collection.size();
+  }
+
+  std::vector< const std::pair<size_t,size_t>* >
+  GetSortedAlignments(WordAlignmentSort SortOrder) const;
+
+  std::vector<size_t> GetSourceIndex2PosMap() const;
+
+  bool operator==(const AlignmentInfo& rhs) const {
+    return m_collection == rhs.m_collection &&
+           m_nonTermIndexMap == rhs.m_nonTermIndexMap;
+  }
+
+private:
+  //! AlignmentInfo objects should only be created by an AlignmentInfoCollection
+  explicit AlignmentInfo(const std::set<std::pair<size_t,size_t> > &pairs);
+  explicit AlignmentInfo(const std::vector<unsigned char> &aln);
+
+  // used only by VW to load word alignment between sentences
+  explicit AlignmentInfo(const std::string &str);
+
+  void BuildNonTermIndexMaps();
+
+  CollType m_collection;
+  NonTermIndexMap m_nonTermIndexMap;
+  NonTermIndexMap m_nonTermIndexMap2;
+};
+
+/** Define an arbitrary strict weak ordering between AlignmentInfo objects
+ * for use by AlignmentInfoCollection.
+ */
+struct AlignmentInfoOrderer {
+  bool operator()(const AlignmentInfo &a, const AlignmentInfo &b) const {
+    if (a.m_collection == b.m_collection) {
+      return a.m_nonTermIndexMap < b.m_nonTermIndexMap;
+    } else {
+      return a.m_collection < b.m_collection;
+    }
+  }
+};
+
+/**
+ * Hashing functoid
+ **/
+struct AlignmentInfoHasher {
+  size_t operator()(const AlignmentInfo& a) const {
+    size_t seed = 0;
+    boost::hash_combine(seed,a.m_collection);
+    boost::hash_combine(seed,a.m_nonTermIndexMap);
+    return seed;
+  }
+
+};
+
+inline size_t hash_value(const AlignmentInfo& a)
+{
+  static AlignmentInfoHasher hasher;
+  return hasher(a);
+}
+
+}

mosesdecoder/moses/AlignmentInfoCollection.cpp

@@ -0,0 +1,60 @@
+/***********************************************************************
+ Moses - statistical machine translation system
+ Copyright (C) 2006-2011 University of Edinburgh
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+***********************************************************************/
+
+#include "AlignmentInfoCollection.h"
+
+namespace Moses
+{
+
+AlignmentInfoCollection AlignmentInfoCollection::s_instance;
+
+AlignmentInfoCollection::AlignmentInfoCollection()
+{
+  std::set<std::pair<size_t,size_t> > pairs;
+  m_emptyAlignmentInfo = Add(pairs);
+}
+
+AlignmentInfoCollection::~AlignmentInfoCollection()
+{}
+
+const AlignmentInfo &AlignmentInfoCollection::GetEmptyAlignmentInfo() const
+{
+  return *m_emptyAlignmentInfo;
+}
+
+AlignmentInfo const *
+AlignmentInfoCollection::
+Add(AlignmentInfo const& ainfo)
+{
+#ifdef WITH_THREADS
+  {
+    boost::shared_lock<boost::shared_mutex> read_lock(m_accessLock);
+    AlignmentInfoSet::const_iterator i = m_collection.find(ainfo);
+    if (i != m_collection.end())
+      return &*i;
+  }
+  boost::unique_lock<boost::shared_mutex> lock(m_accessLock);
+#endif
+  std::pair<AlignmentInfoSet::iterator, bool> ret = m_collection.insert(ainfo);
+  return &(*ret.first);
+}
+
+
+
+}

mosesdecoder/moses/AlignmentInfoCollection.h

@@ -0,0 +1,81 @@
+/***********************************************************************
+ Moses - statistical machine translation system
+ Copyright (C) 2006-2011 University of Edinburgh
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+***********************************************************************/
+
+#pragma once
+
+#include "AlignmentInfo.h"
+
+#include <set>
+
+#ifdef WITH_THREADS
+#include <boost/thread/shared_mutex.hpp>
+#include <boost/thread/locks.hpp>
+#endif
+
+namespace Moses
+{
+
+/** Singleton collection of all AlignmentInfo objects.
+ *  Used as a cache of all alignment info to save space.
+ */
+class AlignmentInfoCollection
+{
+public:
+  static AlignmentInfoCollection &Instance() {
+    return s_instance;
+  }
+
+  /** Returns a pointer to an AlignmentInfo object with the same source-target
+    * alignment pairs as given in the argument.  If the collection already
+    * contains such an object then returns a pointer to it; otherwise a new
+    * one is inserted.
+   */
+private:
+  const AlignmentInfo* Add(AlignmentInfo const& ainfo);
+
+public:
+  template<typename ALNREP>
+  AlignmentInfo const *
+  Add(ALNREP const & aln) {
+    return this->Add(AlignmentInfo(aln));
+  }
+
+  //! Returns a pointer to an empty AlignmentInfo object.
+  const AlignmentInfo &GetEmptyAlignmentInfo() const;
+
+private:
+  typedef std::set<AlignmentInfo, AlignmentInfoOrderer> AlignmentInfoSet;
+
+
+  //! Only a single static variable should be created.
+  AlignmentInfoCollection();
+  ~AlignmentInfoCollection();
+
+  static AlignmentInfoCollection s_instance;
+
+#ifdef WITH_THREADS
+  //reader-writer lock
+  mutable boost::shared_mutex m_accessLock;
+#endif
+
+  AlignmentInfoSet m_collection;
+  const AlignmentInfo *m_emptyAlignmentInfo;
+};
+
+}

mosesdecoder/moses/AlignmentInfoTest.cpp

@@ -0,0 +1,71 @@
+/***********************************************************************
+Moses - factored phrase-based language decoder
+Copyright (C) 2010- University of Edinburgh
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Lesser General Public
+License as published by the Free Software Foundation; either
+version 2.1 of the License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public
+License along with this library; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+***********************************************************************/
+
+#include <boost/test/unit_test.hpp>
+
+#include "AlignmentInfo.h"
+#include "AlignmentInfoCollection.h"
+
+using namespace Moses;
+using namespace std;
+
+BOOST_AUTO_TEST_SUITE(alignment_info)
+
+typedef pair<size_t,size_t> IndexPair;
+typedef set<pair<size_t,size_t> > IndexSet;
+
+struct AlignmentInfoFixture {
+  const AlignmentInfo* ai1;
+  const AlignmentInfo* ai2;
+  const AlignmentInfo* ai3;
+
+  AlignmentInfoFixture() {
+    AlignmentInfoCollection& collection = AlignmentInfoCollection::Instance();
+    IndexSet aligns1,aligns2,aligns3;
+    aligns1.insert(IndexPair(1,1));
+    aligns1.insert(IndexPair(2,1));
+    aligns2.insert(IndexPair(1,1));
+    aligns2.insert(IndexPair(2,1));
+    aligns3.insert(IndexPair(1,2));
+    aligns3.insert(IndexPair(2,1));
+    ai1 = collection.Add(aligns1);
+    ai2 = collection.Add(aligns2);
+    ai3 = collection.Add(aligns3);
+  }
+
+};
+
+BOOST_FIXTURE_TEST_CASE(comparator, AlignmentInfoFixture)
+{
+  BOOST_CHECK(*ai1 == *ai2);
+  BOOST_CHECK(*ai1 == *ai1);
+  BOOST_CHECK(*ai2 == *ai2);
+  BOOST_CHECK(*ai3 == *ai3);
+  BOOST_CHECK(!(*ai2 == *ai3));
+  BOOST_CHECK(!(*ai1 == *ai3));
+}
+
+BOOST_FIXTURE_TEST_CASE(hasher, AlignmentInfoFixture)
+{
+  //simple test that same objects give same hash
+  AlignmentInfoHasher hash;
+  BOOST_CHECK_EQUAL(hash(*ai1), hash(*ai2));
+}
+
+BOOST_AUTO_TEST_SUITE_END()

mosesdecoder/moses/BaseManager.cpp

@@ -0,0 +1,160 @@
+#include "BaseManager.h"
+#include "StaticData.h"
+#include "moses/FF/StatelessFeatureFunction.h"
+#include "moses/FF/StatefulFeatureFunction.h"
+#include "moses/TranslationTask.h"
+
+#include <vector>
+#include <boost/algorithm/string/predicate.hpp>
+#include <boost/iostreams/device/file.hpp>
+#include <boost/iostreams/filter/bzip2.hpp>
+#include <boost/iostreams/filter/gzip.hpp>
+#include <boost/iostreams/filtering_stream.hpp>
+#include <boost/filesystem.hpp>
+
+using namespace std;
+
+namespace Moses
+{
+
+BaseManager::BaseManager(ttasksptr const& ttask)
+  : m_ttask(ttask), m_source(*(ttask->GetSource().get()))
+{ }
+
+const InputType&
+BaseManager::GetSource() const
+{
+  return m_source;
+}
+
+const ttasksptr
+BaseManager::GetTtask() const
+{
+  return m_ttask.lock();
+}
+
+void
+BaseManager::
+OutputSearchGraphAsHypergraph(std::ostream& out) const
+{
+  // This virtual function that may not be implemented everywhere, but it should for
+  // derived classes that use it
+  UTIL_THROW2("Not implemented.");
+}
+
+void
+BaseManager::
+OutputSearchGraphAsHypergraph(std::string const& fname, size_t const precision) const
+{
+  std::string odir = boost::filesystem::path(fname).parent_path().string();
+  if (! boost::filesystem::exists(odir))
+    boost::filesystem::create_directory(odir);
+  UTIL_THROW_IF2(!boost::filesystem::is_directory(odir),
+                 "Cannot output hypergraphs to " << odir
+                 << " because that path exists but is not a directory.");
+
+  // not clear why we need to output the weights every time we dump a search
+  // graph into a file again, but that's what the old code did.
+
+  string weightsFile = odir + "/weights";
+  TRACE_ERR("The weights file is " << weightsFile << "\n");
+  ofstream weightsOut;
+  weightsOut.open(weightsFile.c_str());
+  weightsOut.setf(std::ios::fixed);
+  weightsOut.precision(6);
+  // just temporarily, till we've implemented weight scoring in the manager
+  // (or the translation task)
+  StaticData::Instance().GetAllWeights().Save(weightsOut);
+  weightsOut.close();
+
+  boost::iostreams::filtering_ostream file;
+  if (boost::ends_with(fname, ".gz"))
+    file.push(boost::iostreams::gzip_compressor());
+  else if (boost::ends_with(fname, ".bz2"))
+    file.push( boost::iostreams::bzip2_compressor() );
+  file.push( boost::iostreams::file_sink(fname, ios_base::out) );
+  if (file.is_complete() && file.good()) {
+    file.setf(std::ios::fixed);
+    file.precision(precision);
+    this->OutputSearchGraphAsHypergraph(file);
+    file.flush();
+  } else {
+    TRACE_ERR("Cannot output hypergraph for line "
+              << this->GetSource().GetTranslationId()
+              << " because the output file " << fname
+              << " is not open or not ready for writing"
+              << std::endl);
+  }
+  file.pop();
+}
+
+
+
+
+/***
+ * print surface factor only for the given phrase
+ */
+void
+BaseManager::
+OutputSurface(std::ostream &out, Phrase const& phrase) const
+{
+  std::vector<FactorType> const& factor_order = options()->output.factor_order;
+
+  bool markUnknown = options()->unk.mark;
+  std::string const& fd = options()->output.factor_delimiter;
+
+  size_t size = phrase.GetSize();
+  for (size_t pos = 0 ; pos < size ; pos++) {
+    const Factor *factor = phrase.GetFactor(pos, factor_order[0]);
+    UTIL_THROW_IF2(factor == NULL, "Empty factor 0 at position " << pos);
+
+    const Word &word = phrase.GetWord(pos);
+    if(markUnknown && word.IsOOV()) {
+      out << options()->unk.prefix;
+    }
+
+    out << *factor;
+
+    for (size_t i = 1 ; i < factor_order.size() ; i++) {
+      const Factor *factor = phrase.GetFactor(pos, factor_order[i]);
+      UTIL_THROW_IF2(!factor, "Empty factor " << i << " at position " << pos);
+      out << fd << *factor;
+    }
+
+    if(markUnknown && word.IsOOV()) {
+      out << options()->unk.suffix;
+    }
+
+    out << " ";
+  }
+}
+
+// Emulates the old operator<<(ostream &, const DottedRule &) function.  The
+// output format is a bit odd (reverse order and double spacing between symbols)
+// but there are scripts and tools that expect the output of -T to look like
+// that.
+void BaseManager::WriteApplicationContext(std::ostream &out,
+    const ApplicationContext &context) const
+{
+  assert(!context.empty());
+  ApplicationContext::const_reverse_iterator p = context.rbegin();
+  while (true) {
+    out << p->second << "=" << p->first << " ";
+    if (++p == context.rend()) {
+      break;
+    }
+    out << " ";
+  }
+}
+
+AllOptions::ptr const&
+BaseManager::
+options() const
+{
+  return GetTtask()->options();
+}
+
+
+} // namespace
+
+

mosesdecoder/moses/BaseManager.h

@@ -0,0 +1,78 @@
+// -*- c++ -*-
+#pragma once
+
+#include <iostream>
+#include <string>
+#include "ScoreComponentCollection.h"
+#include "InputType.h"
+#include "moses/parameters/AllOptions.h"
+namespace Moses
+{
+class ScoreComponentCollection;
+class FeatureFunction;
+class OutputCollector;
+
+class BaseManager
+{
+protected:
+  // const InputType &m_source; /**< source sentence to be translated */
+  ttaskwptr m_ttask;
+  InputType const& m_source;
+
+  BaseManager(ttasksptr const& ttask);
+
+  // output
+  typedef std::vector<std::pair<Moses::Word, Moses::Range> > ApplicationContext;
+  typedef std::set< std::pair<size_t, size_t>  > Alignments;
+
+  void OutputSurface(std::ostream &out, Phrase const& phrase) const;
+
+  void WriteApplicationContext(std::ostream &out,
+                               const ApplicationContext &context) const;
+
+  template <class T>
+  void ShiftOffsets(std::vector<T> &offsets, T shift) const {
+    T currPos = shift;
+    for (size_t i = 0; i < offsets.size(); ++i) {
+      if (offsets[i] == 0) {
+        offsets[i] = currPos;
+        ++currPos;
+      } else {
+        currPos += offsets[i];
+      }
+    }
+  }
+
+public:
+  virtual ~BaseManager() { }
+
+  //! the input sentence being decoded
+  const InputType& GetSource() const;
+  const ttasksptr  GetTtask() const;
+  AllOptions::ptr const& options() const;
+
+  virtual void Decode() = 0;
+  // outputs
+  virtual void OutputBest(OutputCollector *collector) const = 0;
+  virtual void OutputNBest(OutputCollector *collector) const = 0;
+  virtual void OutputLatticeSamples(OutputCollector *collector) const = 0;
+  virtual void OutputAlignment(OutputCollector *collector) const = 0;
+  virtual void OutputDetailedTranslationReport(OutputCollector *collector) const = 0;
+  virtual void OutputDetailedTreeFragmentsTranslationReport(OutputCollector *collector) const = 0;
+  virtual void OutputWordGraph(OutputCollector *collector) const = 0;
+  virtual void OutputSearchGraph(OutputCollector *collector) const = 0;
+  virtual void OutputUnknowns(OutputCollector *collector) const = 0;
+  virtual void OutputSearchGraphSLF() const = 0;
+  // virtual void OutputSearchGraphHypergraph() const = 0;
+
+  virtual void OutputSearchGraphAsHypergraph(std::ostream& out) const;
+  virtual void OutputSearchGraphAsHypergraph(std::string const& fname,
+      size_t const precision) const;
+  /***
+   * to be called after processing a sentence
+   */
+  virtual void CalcDecoderStatistics() const = 0;
+
+};
+
+}

mosesdecoder/moses/Bitmap.cpp

@@ -0,0 +1,96 @@
+// $Id$
+
+/***********************************************************************
+Moses - factored phrase-based language decoder
+Copyright (C) 2006 University of Edinburgh
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Lesser General Public
+License as published by the Free Software Foundation; either
+version 2.1 of the License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public
+License along with this library; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+***********************************************************************/
+
+#include <boost/functional/hash.hpp>
+#include "Bitmap.h"
+
+namespace Moses
+{
+
+TO_STRING_BODY(Bitmap);
+
+Bitmap::Bitmap(size_t size, const std::vector<bool>& initializer)
+  :m_bitmap(initializer.begin(), initializer.end())
+{
+
+  // The initializer may not be of the same length.  Change to the desired
+  // length.  If we need to add any elements, initialize them to false.
+  m_bitmap.resize(size, false);
+
+  m_numWordsCovered = std::count(m_bitmap.begin(), m_bitmap.end(), true);
+
+  // Find the first gap, and cache it.
+  std::vector<char>::const_iterator first_gap = std::find(
+        m_bitmap.begin(), m_bitmap.end(), false);
+  m_firstGap = (
+                 (first_gap == m_bitmap.end()) ?
+                 NOT_FOUND : first_gap - m_bitmap.begin());
+}
+
+//! Create Bitmap of length size and initialise.
+Bitmap::Bitmap(size_t size)
+  :m_bitmap(size, false)
+  ,m_firstGap(0)
+  ,m_numWordsCovered(0)
+
+{
+}
+
+//! Deep copy.
+Bitmap::Bitmap(const Bitmap &copy)
+  :m_bitmap(copy.m_bitmap)
+  ,m_firstGap(copy.m_firstGap)
+  ,m_numWordsCovered(copy.m_numWordsCovered)
+{
+}
+
+Bitmap::Bitmap(const Bitmap &copy, const Range &range)
+  :m_bitmap(copy.m_bitmap)
+  ,m_firstGap(copy.m_firstGap)
+  ,m_numWordsCovered(copy.m_numWordsCovered)
+{
+  SetValueNonOverlap(range);
+}
+
+// for unordered_set in stack
+size_t Bitmap::hash() const
+{
+  size_t ret = boost::hash_value(m_bitmap);
+  return ret;
+}
+
+bool Bitmap::operator==(const Bitmap& other) const
+{
+  return m_bitmap == other.m_bitmap;
+}
+
+// friend
+std::ostream& operator<<(std::ostream& out, const Bitmap& bitmap)
+{
+  for (size_t i = 0 ; i < bitmap.m_bitmap.size() ; i++) {
+    out << int(bitmap.GetValue(i));
+  }
+  return out;
+}
+
+} // namespace
+
+

mosesdecoder/moses/Bitmap.h

@@ -0,0 +1,249 @@
+// $Id$
+
+/***********************************************************************
+Moses - factored phrase-based language decoder
+Copyright (C) 2006 University of Edinburgh
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Lesser General Public
+License as published by the Free Software Foundation; either
+version 2.1 of the License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public
+License along with this library; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+***********************************************************************/
+
+#ifndef moses_WordsBitmap_h
+#define moses_WordsBitmap_h
+
+#include <algorithm>
+#include <limits>
+#include <vector>
+#include <iostream>
+#include <cstring>
+#include <cmath>
+#include <cstdlib>
+#include "TypeDef.h"
+#include "Range.h"
+
+namespace Moses
+{
+typedef unsigned long WordsBitmapID;
+
+/** Vector of boolean to represent whether a word has been translated or not.
+ *
+ * Implemented using a vector of char, which is usually the same representation
+ * for the elements that a C array of bool would use.  A vector of bool, or a
+ * Boost dynamic_bitset, could be much more efficient in theory.  Unfortunately
+ * algorithms like std::find() are not optimized for vector<bool> on gcc or
+ * clang, and dynamic_bitset lacks all the optimized search operations we want.
+ * Only benchmarking will tell what works best.  Perhaps dynamic_bitset could
+ * still be a dramatic improvement, if we flip the meaning of the bits around
+ * so we can use its find_first() and find_next() for the most common searches.
+ */
+class Bitmap
+{
+  friend std::ostream& operator<<(std::ostream& out, const Bitmap& bitmap);
+private:
+  std::vector<char> m_bitmap; //! Ticks of words in sentence that have been done.
+  size_t m_firstGap; //! Cached position of first gap, or NOT_FOUND.
+  size_t m_numWordsCovered;
+
+  Bitmap(); // not implemented
+  Bitmap& operator= (const Bitmap& other);
+
+  /** Update the first gap, when bits are flipped */
+  void UpdateFirstGap(size_t startPos, size_t endPos, bool value) {
+    if (value) {
+      //may remove gap
+      if (startPos <= m_firstGap && m_firstGap <= endPos) {
+        m_firstGap = NOT_FOUND;
+        for (size_t i = endPos + 1 ; i < m_bitmap.size(); ++i) {
+          if (!m_bitmap[i]) {
+            m_firstGap = i;
+            break;
+          }
+        }
+      }
+
+    } else {
+      //setting positions to false, may add new gap
+      if (startPos < m_firstGap) {
+        m_firstGap = startPos;
+      }
+    }
+  }
+
+  //! set value between 2 positions, inclusive
+  void
+  SetValueNonOverlap(Range const& range) {
+    size_t startPos = range.GetStartPos();
+    size_t endPos = range.GetEndPos();
+
+    for(size_t pos = startPos ; pos <= endPos ; pos++) {
+      m_bitmap[pos] = true;
+    }
+
+    m_numWordsCovered += range.GetNumWordsCovered();
+    UpdateFirstGap(startPos, endPos, true);
+  }
+
+public:
+  //! Create Bitmap of length size, and initialise with vector.
+  explicit Bitmap(size_t size, const std::vector<bool>& initializer);
+
+  //! Create Bitmap of length size and initialise.
+  explicit Bitmap(size_t size);
+
+  //! Deep copy.
+  explicit Bitmap(const Bitmap &copy);
+
+  explicit Bitmap(const Bitmap &copy, const Range &range);
+
+  //! Count of words translated.
+  size_t GetNumWordsCovered() const {
+    return m_numWordsCovered;
+  }
+
+  //! position of 1st word not yet translated, or NOT_FOUND if everything already translated
+  size_t GetFirstGapPos() const {
+    return m_firstGap;
+  }
+
+
+  //! position of last word not yet translated, or NOT_FOUND if everything already translated
+  size_t GetLastGapPos() const {
+    for (int pos = int(m_bitmap.size()) - 1 ; pos >= 0 ; pos--) {
+      if (!m_bitmap[pos]) {
+        return pos;
+      }
+    }
+    // no starting pos
+    return NOT_FOUND;
+  }
+
+
+  //! position of last translated word
+  size_t GetLastPos() const {
+    for (int pos = int(m_bitmap.size()) - 1 ; pos >= 0 ; pos--) {
+      if (m_bitmap[pos]) {
+        return pos;
+      }
+    }
+    // no starting pos
+    return NOT_FOUND;
+  }
+
+  //! whether a word has been translated at a particular position
+  bool GetValue(size_t pos) const {
+    return bool(m_bitmap[pos]);
+  }
+  //! set value at a particular position
+  void SetValue( size_t pos, bool value ) {
+    bool origValue = m_bitmap[pos];
+    if (origValue == value) {
+      // do nothing
+    } else {
+      m_bitmap[pos] = value;
+      UpdateFirstGap(pos, pos, value);
+      if (value) {
+        ++m_numWordsCovered;
+      } else {
+        --m_numWordsCovered;
+      }
+    }
+  }
+
+  //! whether every word has been translated
+  bool IsComplete() const {
+    return GetSize() == GetNumWordsCovered();
+  }
+  //! whether the wordrange overlaps with any translated word in this bitmap
+  bool Overlap(const Range &compare) const {
+    for (size_t pos = compare.GetStartPos() ; pos <= compare.GetEndPos() ; pos++) {
+      if (m_bitmap[pos])
+        return true;
+    }
+    return false;
+  }
+  //! number of elements
+  size_t GetSize() const {
+    return m_bitmap.size();
+  }
+
+  inline size_t GetEdgeToTheLeftOf(size_t l) const {
+    if (l == 0) return l;
+    while (l && !m_bitmap[l-1]) {
+      --l;
+    }
+    return l;
+  }
+
+  inline size_t GetEdgeToTheRightOf(size_t r) const {
+    if (r+1 == m_bitmap.size()) return r;
+    return (
+             std::find(m_bitmap.begin() + r + 1, m_bitmap.end(), true) -
+             m_bitmap.begin()
+           ) - 1;
+  }
+
+
+  //! converts bitmap into an integer ID: it consists of two parts: the first 16 bit are the pattern between the first gap and the last word-1, the second 16 bit are the number of filled positions. enforces a sentence length limit of 65535 and a max distortion of 16
+  WordsBitmapID GetID() const {
+    assert(m_bitmap.size() < (1<<16));
+
+    size_t start = GetFirstGapPos();
+    if (start == NOT_FOUND) start = m_bitmap.size(); // nothing left
+
+    size_t end = GetLastPos();
+    if (end == NOT_FOUND) end = 0; // nothing translated yet
+
+    assert(end < start || end-start <= 16);
+    WordsBitmapID id = 0;
+    for(size_t pos = end; pos > start; pos--) {
+      id = id*2 + (int) GetValue(pos);
+    }
+    return id + (1<<16) * start;
+  }
+
+  //! converts bitmap into an integer ID, with an additional span covered
+  WordsBitmapID GetIDPlus( size_t startPos, size_t endPos ) const {
+    assert(m_bitmap.size() < (1<<16));
+
+    size_t start = GetFirstGapPos();
+    if (start == NOT_FOUND) start = m_bitmap.size(); // nothing left
+
+    size_t end = GetLastPos();
+    if (end == NOT_FOUND) end = 0; // nothing translated yet
+
+    if (start == startPos) start = endPos+1;
+    if (end < endPos) end = endPos;
+
+    assert(end < start || end-start <= 16);
+    WordsBitmapID id = 0;
+    for(size_t pos = end; pos > start; pos--) {
+      id = id*2;
+      if (GetValue(pos) || (startPos<=pos && pos<=endPos))
+        id++;
+    }
+    return id + (1<<16) * start;
+  }
+
+  // for unordered_set in stack
+  size_t hash() const;
+  bool operator==(const Bitmap& other) const;
+  bool operator!=(const Bitmap& other) const {
+    return !(*this == other);
+  }
+
+  TO_STRING();
+};
+
+}
+#endif

mosesdecoder/moses/BitmapContainer.cpp

@@ -0,0 +1,498 @@
+// $Id$
+// vim:tabstop=2
+/***********************************************************************
+Moses - factored phrase-based language decoder
+Copyright (C) 2006 University of Edinburgh
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Lesser General Public
+License as published by the Free Software Foundation; either
+version 2.1 of the License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public
+License along with this library; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+***********************************************************************/
+
+#include <algorithm>
+#include <limits>
+#include <utility>
+
+#include "BitmapContainer.h"
+#include "HypothesisStackCubePruning.h"
+#include "moses/FF/DistortionScoreProducer.h"
+#include "TranslationOptionList.h"
+#include "Manager.h"
+
+namespace Moses
+{
+
+class HypothesisScoreOrdererNoDistortion
+{
+public:
+  bool operator()(const Hypothesis* hypoA, const Hypothesis* hypoB) const {
+    const float scoreA = hypoA->GetScore();
+    const float scoreB = hypoB->GetScore();
+
+    if (scoreA > scoreB) {
+      return true;
+    } else if (scoreA < scoreB) {
+      return false;
+    } else {
+      return hypoA < hypoB;
+    }
+  }
+};
+
+class HypothesisScoreOrdererWithDistortion
+{
+private:
+  bool m_deterministic;
+
+public:
+  HypothesisScoreOrdererWithDistortion(const Range* transOptRange,
+                                       const bool deterministic = false)
+    : m_deterministic(deterministic)
+    , m_transOptRange(transOptRange) {
+    m_totalWeightDistortion = 0;
+    const StaticData &staticData = StaticData::Instance();
+
+    const std::vector<const DistortionScoreProducer*> &ffs = DistortionScoreProducer::GetDistortionFeatureFunctions();
+    std::vector<const DistortionScoreProducer*>::const_iterator iter;
+    for (iter = ffs.begin(); iter != ffs.end(); ++iter) {
+      const DistortionScoreProducer *ff = *iter;
+
+      float weight =staticData.GetAllWeights().GetScoreForProducer(ff);
+      m_totalWeightDistortion += weight;
+    }
+  }
+
+  const Range* m_transOptRange;
+  float m_totalWeightDistortion;
+
+  bool operator()(const Hypothesis* hypoA, const Hypothesis* hypoB) const {
+    UTIL_THROW_IF2(m_transOptRange == NULL, "Words range not set");
+
+
+    const float distortionScoreA = DistortionScoreProducer::CalculateDistortionScore(
+                                     *hypoA,
+                                     hypoA->GetCurrSourceWordsRange(),
+                                     *m_transOptRange,
+                                     hypoA->GetWordsBitmap().GetFirstGapPos()
+                                   );
+    const float distortionScoreB = DistortionScoreProducer::CalculateDistortionScore(
+                                     *hypoB,
+                                     hypoB->GetCurrSourceWordsRange(),
+                                     *m_transOptRange,
+                                     hypoB->GetWordsBitmap().GetFirstGapPos()
+                                   );
+
+
+    const float scoreA = hypoA->GetScore() + distortionScoreA * m_totalWeightDistortion;
+    const float scoreB = hypoB->GetScore() + distortionScoreB * m_totalWeightDistortion;
+
+
+    if (scoreA > scoreB) {
+      return true;
+    } else if (scoreA < scoreB) {
+      return false;
+    } else {
+      if (m_deterministic) {
+        // Equal scores: break ties by comparing target phrases
+        return (hypoA->GetCurrTargetPhrase().Compare(hypoB->GetCurrTargetPhrase()) < 0);
+      }
+      // Fallback: non-deterministic sort
+      return hypoA < hypoB;
+    }
+  }
+
+};
+
+////////////////////////////////////////////////////////////////////////////////
+// BackwardsEdge Code
+////////////////////////////////////////////////////////////////////////////////
+
+BackwardsEdge::BackwardsEdge(const BitmapContainer &prevBitmapContainer
+                             , BitmapContainer &parent
+                             , const TranslationOptionList &translations
+                             , const SquareMatrix &estimatedScores,
+                             const InputType& itype,
+                             const bool deterministic)
+  : m_initialized(false)
+  , m_prevBitmapContainer(prevBitmapContainer)
+  , m_parent(parent)
+  , m_translations(translations)
+  , m_estimatedScores(estimatedScores)
+  , m_deterministic(deterministic)
+  , m_seenPosition()
+{
+
+  // If either dimension is empty, we haven't got anything to do.
+  if(m_prevBitmapContainer.GetHypotheses().size() == 0 || m_translations.size() == 0) {
+    VERBOSE(3, "Empty cube on BackwardsEdge" << std::endl);
+    return;
+  }
+
+  // Fetch the things we need for distortion cost computation.
+  // int maxDistortion = StaticData::Instance().GetMaxDistortion();
+  int maxDistortion  = itype.options()->reordering.max_distortion;
+
+  if (maxDistortion == -1) {
+    for (HypothesisSet::const_iterator iter = m_prevBitmapContainer.GetHypotheses().begin(); iter != m_prevBitmapContainer.GetHypotheses().end(); ++iter) {
+      m_hypotheses.push_back(*iter);
+    }
+    return;
+  }
+
+  const Range &transOptRange = translations.Get(0)->GetSourceWordsRange();
+
+  HypothesisSet::const_iterator iterHypo = m_prevBitmapContainer.GetHypotheses().begin();
+  HypothesisSet::const_iterator iterEnd = m_prevBitmapContainer.GetHypotheses().end();
+
+  while (iterHypo != iterEnd) {
+    const Hypothesis &hypo = **iterHypo;
+    // Special case: If this is the first hypothesis used to seed the search,
+    // it doesn't have a valid range, and we create the hypothesis, if the
+    // initial position is not further into the sentence than the distortion limit.
+    if (hypo.GetWordsBitmap().GetNumWordsCovered() == 0) {
+      if ((int)transOptRange.GetStartPos() <= maxDistortion)
+        m_hypotheses.push_back(&hypo);
+    } else {
+      int distortionDistance = itype.ComputeDistortionDistance(hypo.GetCurrSourceWordsRange()
+                               , transOptRange);
+
+      if (distortionDistance <= maxDistortion)
+        m_hypotheses.push_back(&hypo);
+    }
+
+    ++iterHypo;
+  }
+
+  if (m_translations.size() > 1) {
+    UTIL_THROW_IF2(m_translations.Get(0)->GetFutureScore() < m_translations.Get(1)->GetFutureScore(),
+                   "Non-monotonic future score: "
+                   << m_translations.Get(0)->GetFutureScore() << " vs. "
+                   << m_translations.Get(1)->GetFutureScore());
+  }
+
+  if (m_hypotheses.size() > 1) {
+    UTIL_THROW_IF2(m_hypotheses[0]->GetFutureScore() < m_hypotheses[1]->GetFutureScore(),
+                   "Non-monotonic total score"
+                   << m_hypotheses[0]->GetFutureScore() << " vs. "
+                   << m_hypotheses[1]->GetFutureScore());
+  }
+
+  HypothesisScoreOrdererWithDistortion orderer (&transOptRange, m_deterministic);
+  std::sort(m_hypotheses.begin(), m_hypotheses.end(), orderer);
+
+  // std::sort(m_hypotheses.begin(), m_hypotheses.end(), HypothesisScoreOrdererNoDistortion());
+}
+
+BackwardsEdge::~BackwardsEdge()
+{
+  m_seenPosition.clear();
+  m_hypotheses.clear();
+}
+
+
+void
+BackwardsEdge::Initialize()
+{
+  if(m_hypotheses.size() == 0 || m_translations.size() == 0) {
+    m_initialized = true;
+    return;
+  }
+
+  const Bitmap &bm = m_hypotheses[0]->GetWordsBitmap();
+  const Range &newRange = m_translations.Get(0)->GetSourceWordsRange();
+  m_estimatedScore = m_estimatedScores.CalcEstimatedScore(bm, newRange.GetStartPos(), newRange.GetEndPos());
+
+  Hypothesis *expanded = CreateHypothesis(*m_hypotheses[0], *m_translations.Get(0));
+  m_parent.Enqueue(0, 0, expanded, this);
+  SetSeenPosition(0, 0);
+  m_initialized = true;
+}
+
+Hypothesis *BackwardsEdge::CreateHypothesis(const Hypothesis &hypothesis, const TranslationOption &transOpt)
+{
+  // create hypothesis and calculate all its scores
+  IFVERBOSE(2) {
+    hypothesis.GetManager().GetSentenceStats().StartTimeBuildHyp();
+  }
+  const Bitmap &bitmap = m_parent.GetWordsBitmap();
+  Hypothesis *newHypo = new Hypothesis(hypothesis, transOpt, bitmap, hypothesis.GetManager().GetNextHypoId());
+  IFVERBOSE(2) {
+    hypothesis.GetManager().GetSentenceStats().StopTimeBuildHyp();
+  }
+  newHypo->EvaluateWhenApplied(m_estimatedScore);
+
+  return newHypo;
+}
+
+bool
+BackwardsEdge::SeenPosition(const size_t x, const size_t y)
+{
+  boost::unordered_set< int >::iterator iter = m_seenPosition.find((x<<16) + y);
+  return (iter != m_seenPosition.end());
+}
+
+void
+BackwardsEdge::SetSeenPosition(const size_t x, const size_t y)
+{
+  UTIL_THROW_IF2(x >= (1<<17), "Error");
+  UTIL_THROW_IF2(y >= (1<<17), "Error");
+
+  m_seenPosition.insert((x<<16) + y);
+}
+
+
+bool
+BackwardsEdge::GetInitialized()
+{
+  return m_initialized;
+}
+
+const BitmapContainer&
+BackwardsEdge::GetBitmapContainer() const
+{
+  return m_prevBitmapContainer;
+}
+
+void
+BackwardsEdge::PushSuccessors(const size_t x, const size_t y)
+{
+  Hypothesis *newHypo;
+
+  if(y + 1 < m_translations.size() && !SeenPosition(x, y + 1)) {
+    SetSeenPosition(x, y + 1);
+    newHypo = CreateHypothesis(*m_hypotheses[x], *m_translations.Get(y + 1));
+    if(newHypo != NULL) {
+      m_parent.Enqueue(x, y + 1, newHypo, (BackwardsEdge*)this);
+    }
+  }
+
+  if(x + 1 < m_hypotheses.size() && !SeenPosition(x + 1, y)) {
+    SetSeenPosition(x + 1, y);
+    newHypo = CreateHypothesis(*m_hypotheses[x + 1], *m_translations.Get(y));
+    if(newHypo != NULL) {
+      m_parent.Enqueue(x + 1, y, newHypo, (BackwardsEdge*)this);
+    }
+  }
+}
+
+
+////////////////////////////////////////////////////////////////////////////////
+// BitmapContainer Code
+////////////////////////////////////////////////////////////////////////////////
+
+BitmapContainer::BitmapContainer(const Bitmap &bitmap
+                                 , HypothesisStackCubePruning &stack
+                                 , bool deterministic)
+  : m_bitmap(bitmap)
+  , m_stack(stack)
+  , m_numStackInsertions(0)
+  , m_deterministic(deterministic)
+{
+  m_hypotheses = HypothesisSet();
+  m_edges = BackwardsEdgeSet();
+  m_queue = HypothesisQueue();
+}
+
+BitmapContainer::~BitmapContainer()
+{
+  // As we have created the square position objects we clean up now.
+
+  while (!m_queue.empty()) {
+    HypothesisQueueItem *item = m_queue.top();
+    m_queue.pop();
+
+    delete item->GetHypothesis();
+    delete item;
+  }
+
+  // Delete all edges.
+  RemoveAllInColl(m_edges);
+
+  m_hypotheses.clear();
+  m_edges.clear();
+}
+
+
+void
+BitmapContainer::Enqueue(int hypothesis_pos
+                         , int translation_pos
+                         , Hypothesis *hypothesis
+                         , BackwardsEdge *edge)
+{
+  // Only supply target phrase if running deterministic search mode
+  const TargetPhrase *target_phrase = m_deterministic ? &(hypothesis->GetCurrTargetPhrase()) : NULL;
+  HypothesisQueueItem *item = new HypothesisQueueItem(hypothesis_pos
+      , translation_pos
+      , hypothesis
+      , edge
+      , target_phrase);
+  IFVERBOSE(2) {
+    item->GetHypothesis()->GetManager().GetSentenceStats().StartTimeManageCubes();
+  }
+  m_queue.push(item);
+  IFVERBOSE(2) {
+    item->GetHypothesis()->GetManager().GetSentenceStats().StopTimeManageCubes();
+  }
+}
+
+HypothesisQueueItem*
+BitmapContainer::Dequeue(bool keepValue)
+{
+  if (!m_queue.empty()) {
+    HypothesisQueueItem *item = m_queue.top();
+
+    if (!keepValue) {
+      m_queue.pop();
+    }
+
+    return item;
+  }
+
+  return NULL;
+}
+
+HypothesisQueueItem*
+BitmapContainer::Top() const
+{
+  return m_queue.top();
+}
+
+size_t
+BitmapContainer::Size()
+{
+  return m_queue.size();
+}
+
+bool
+BitmapContainer::Empty() const
+{
+  return m_queue.empty();
+}
+
+const HypothesisSet&
+BitmapContainer::GetHypotheses() const
+{
+  return m_hypotheses;
+}
+
+size_t
+BitmapContainer::GetHypothesesSize() const
+{
+  return m_hypotheses.size();
+}
+
+const BackwardsEdgeSet&
+BitmapContainer::GetBackwardsEdges()
+{
+  return m_edges;
+}
+
+void
+BitmapContainer::AddHypothesis(Hypothesis *hypothesis)
+{
+  bool itemExists = false;
+  HypothesisSet::const_iterator iter = m_hypotheses.begin();
+  HypothesisSet::const_iterator iterEnd = m_hypotheses.end();
+
+  // cfedermann: do we actually need this check?
+  while (iter != iterEnd) {
+    if (*iter == hypothesis) {
+      itemExists = true;
+      break;
+    }
+
+    ++iter;
+  }
+  UTIL_THROW_IF2(itemExists, "Duplicate hypotheses");
+  m_hypotheses.push_back(hypothesis);
+}
+
+void
+BitmapContainer::AddBackwardsEdge(BackwardsEdge *edge)
+{
+  m_edges.insert(edge);
+}
+
+void
+BitmapContainer::InitializeEdges()
+{
+  BackwardsEdgeSet::iterator iter = m_edges.begin();
+  BackwardsEdgeSet::iterator iterEnd = m_edges.end();
+
+  while (iter != iterEnd) {
+    BackwardsEdge *edge = *iter;
+    edge->Initialize();
+
+    ++iter;
+  }
+}
+
+void
+BitmapContainer::EnsureMinStackHyps(const size_t minNumHyps)
+{
+  while ((!Empty()) && m_numStackInsertions < minNumHyps) {
+    ProcessBestHypothesis();
+  }
+}
+
+void
+BitmapContainer::ProcessBestHypothesis()
+{
+  if (m_queue.empty()) {
+    return;
+  }
+
+  // Get the currently best hypothesis from the queue.
+  HypothesisQueueItem *item = Dequeue();
+
+  // If the priority queue is exhausted, we are done and should have exited
+  UTIL_THROW_IF2(item == NULL, "Null object");
+
+  // check we are pulling things off of priority queue in right order
+  if (!Empty()) {
+    HypothesisQueueItem *check = Dequeue(true);
+    UTIL_THROW_IF2(item->GetHypothesis()->GetFutureScore() < check->GetHypothesis()->GetFutureScore(),
+                   "Non-monotonic total score: "
+                   << item->GetHypothesis()->GetFutureScore() << " vs. "
+                   << check->GetHypothesis()->GetFutureScore());
+  }
+
+  // Logging for the criminally insane
+  IFVERBOSE(3) {
+    item->GetHypothesis()->PrintHypothesis();
+  }
+
+  // Add best hypothesis to hypothesis stack.
+  const bool newstackentry = m_stack.AddPrune(item->GetHypothesis());
+  if (newstackentry)
+    m_numStackInsertions++;
+
+  IFVERBOSE(3) {
+    TRACE_ERR("new stack entry flag is " << newstackentry << std::endl);
+  }
+
+  // Create new hypotheses for the two successors of the hypothesis just added.
+  item->GetBackwardsEdge()->PushSuccessors(item->GetHypothesisPos(), item->GetTranslationPos());
+
+  // We are done with the queue item, we delete it.
+  delete item;
+}
+
+void
+BitmapContainer::SortHypotheses()
+{
+  std::sort(m_hypotheses.begin(), m_hypotheses.end(), HypothesisScoreOrderer(m_deterministic));
+}
+
+}
+

mosesdecoder/moses/BitmapContainer.h

@@ -0,0 +1,275 @@
+// $Id$
+// vim:tabstop=2
+/***********************************************************************
+Moses - factored phrase-based language decoder
+Copyright (C) 2006 University of Edinburgh
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Lesser General Public
+License as published by the Free Software Foundation; either
+version 2.1 of the License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public
+License along with this library; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+***********************************************************************/
+
+#ifndef moses_BitmapContainer_h
+#define moses_BitmapContainer_h
+
+#include <queue>
+#include <set>
+#include <vector>
+
+#include "Hypothesis.h"
+#include "HypothesisStackCubePruning.h"
+#include "SquareMatrix.h"
+#include "TranslationOption.h"
+#include "TypeDef.h"
+#include "Bitmap.h"
+
+#include <boost/unordered_set.hpp>
+
+namespace Moses
+{
+
+class BitmapContainer;
+class BackwardsEdge;
+class Hypothesis;
+class HypothesisStackCubePruning;
+class HypothesisQueueItem;
+class QueueItemOrderer;
+class TranslationOptionList;
+
+typedef std::vector< Hypothesis* > HypothesisSet;
+typedef std::set< BackwardsEdge* > BackwardsEdgeSet;
+typedef std::priority_queue< HypothesisQueueItem*, std::vector< HypothesisQueueItem* >, QueueItemOrderer> HypothesisQueue;
+
+////////////////////////////////////////////////////////////////////////////////
+// Hypothesis Priority Queue Code
+////////////////////////////////////////////////////////////////////////////////
+
+//! 1 item in the priority queue for stack decoding (phrase-based)
+class HypothesisQueueItem
+{
+private:
+  size_t m_hypothesis_pos, m_translation_pos;
+  Hypothesis *m_hypothesis;
+  BackwardsEdge *m_edge;
+  boost::shared_ptr<TargetPhrase> m_target_phrase;
+
+  HypothesisQueueItem();
+
+public:
+  HypothesisQueueItem(const size_t hypothesis_pos
+                      , const size_t translation_pos
+                      , Hypothesis *hypothesis
+                      , BackwardsEdge *edge
+                      , const TargetPhrase *target_phrase = NULL)
+    : m_hypothesis_pos(hypothesis_pos)
+    , m_translation_pos(translation_pos)
+    , m_hypothesis(hypothesis)
+    , m_edge(edge) {
+    if (target_phrase != NULL) {
+      m_target_phrase.reset(new TargetPhrase(*target_phrase));
+    }
+  }
+
+  ~HypothesisQueueItem() {
+  }
+
+  int GetHypothesisPos() {
+    return m_hypothesis_pos;
+  }
+
+  int GetTranslationPos() {
+    return m_translation_pos;
+  }
+
+  Hypothesis *GetHypothesis() {
+    return m_hypothesis;
+  }
+
+  BackwardsEdge *GetBackwardsEdge() {
+    return m_edge;
+  }
+
+  boost::shared_ptr<TargetPhrase> GetTargetPhrase() {
+    return m_target_phrase;
+  }
+};
+
+//! Allows comparison of two HypothesisQueueItem objects by the corresponding scores.
+class QueueItemOrderer
+{
+public:
+  bool operator()(HypothesisQueueItem* itemA, HypothesisQueueItem* itemB) const {
+    float scoreA = itemA->GetHypothesis()->GetFutureScore();
+    float scoreB = itemB->GetHypothesis()->GetFutureScore();
+
+    if (scoreA < scoreB) {
+      return true;
+    } else if (scoreA > scoreB) {
+      return false;
+    } else {
+      // Equal scores: break ties by comparing target phrases (if they exist)
+      // *Important*: these are pointers to copies of the target phrases from the
+      // hypotheses.  This class is used to keep priority queues ordered in the
+      // background, so comparisons made as those data structures are cleaned up
+      // may occur *after* the target phrases in hypotheses have been cleaned up,
+      // leading to segfaults if relying on hypotheses to provide target phrases.
+      boost::shared_ptr<TargetPhrase> phrA = itemA->GetTargetPhrase();
+      boost::shared_ptr<TargetPhrase> phrB = itemB->GetTargetPhrase();
+      if (!phrA || !phrB) {
+        // Fallback: scoreA < scoreB == false, non-deterministic sort
+        return false;
+      }
+      return (phrA->Compare(*phrB) > 0);
+    }
+  }
+};
+
+////////////////////////////////////////////////////////////////////////////////
+// Hypothesis Orderer Code
+////////////////////////////////////////////////////////////////////////////////
+// Allows to compare two Hypothesis objects by the corresponding scores.
+////////////////////////////////////////////////////////////////////////////////
+
+class HypothesisScoreOrderer
+{
+private:
+  bool m_deterministic;
+
+public:
+  HypothesisScoreOrderer(const bool deterministic = false)
+    : m_deterministic(deterministic) {}
+
+  bool operator()(const Hypothesis* hypoA, const Hypothesis* hypoB) const {
+
+    float scoreA = hypoA->GetFutureScore();
+    float scoreB = hypoB->GetFutureScore();
+
+    if (scoreA > scoreB) {
+      return true;
+    } else if (scoreA < scoreB) {
+      return false;
+    } else {
+      if (m_deterministic) {
+        // Equal scores: break ties by comparing target phrases
+        return (hypoA->GetCurrTargetPhrase().Compare(hypoB->GetCurrTargetPhrase()) < 0);
+      }
+      // Fallback: scoreA > scoreB == false, non-deterministic sort
+      return false;
+    }
+  }
+};
+
+////////////////////////////////////////////////////////////////////////////////
+// Backwards Edge Code
+////////////////////////////////////////////////////////////////////////////////
+// Encodes an edge pointing to a BitmapContainer.
+////////////////////////////////////////////////////////////////////////////////
+
+class BackwardsEdge
+{
+private:
+  friend class BitmapContainer;
+  bool m_initialized;
+
+  const BitmapContainer &m_prevBitmapContainer;
+  BitmapContainer &m_parent;
+  const TranslationOptionList &m_translations;
+  const SquareMatrix &m_estimatedScores;
+  float m_estimatedScore;
+
+  bool m_deterministic;
+
+  std::vector< const Hypothesis* > m_hypotheses;
+  boost::unordered_set< int > m_seenPosition;
+
+  // We don't want to instantiate "empty" objects.
+  BackwardsEdge();
+
+  Hypothesis *CreateHypothesis(const Hypothesis &hypothesis, const TranslationOption &transOpt);
+  bool SeenPosition(const size_t x, const size_t y);
+  void SetSeenPosition(const size_t x, const size_t y);
+
+protected:
+  void Initialize();
+
+public:
+  BackwardsEdge(const BitmapContainer &prevBitmapContainer
+                , BitmapContainer &parent
+                , const TranslationOptionList &translations
+                , const SquareMatrix &estimatedScores
+                , const InputType& source
+                , const bool deterministic = false);
+  ~BackwardsEdge();
+
+  bool GetInitialized();
+  const BitmapContainer &GetBitmapContainer() const;
+  int GetDistortionPenalty();
+  void PushSuccessors(const size_t x, const size_t y);
+};
+
+////////////////////////////////////////////////////////////////////////////////
+// Bitmap Container Code
+////////////////////////////////////////////////////////////////////////////////
+// A BitmapContainer encodes an ordered set of hypotheses and a set of edges
+// pointing to the "generating" BitmapContainers.  It also stores a priority
+// queue that contains expanded hypotheses from the connected edges.
+////////////////////////////////////////////////////////////////////////////////
+
+class BitmapContainer
+{
+private:
+  const Bitmap &m_bitmap;
+  HypothesisStackCubePruning &m_stack;
+  HypothesisSet m_hypotheses;
+  BackwardsEdgeSet m_edges;
+  HypothesisQueue m_queue;
+  size_t m_numStackInsertions;
+  bool m_deterministic;
+
+  // We always require a corresponding bitmap to be supplied.
+  BitmapContainer();
+  BitmapContainer(const BitmapContainer &);
+public:
+  BitmapContainer(const Bitmap &bitmap
+                  , HypothesisStackCubePruning &stack
+                  , bool deterministic = false);
+
+  // The destructor will also delete all the edges that are
+  // connected to this BitmapContainer.
+  ~BitmapContainer();
+
+  void Enqueue(int hypothesis_pos, int translation_pos, Hypothesis *hypothesis, BackwardsEdge *edge);
+  HypothesisQueueItem *Dequeue(bool keepValue=false);
+  HypothesisQueueItem *Top() const;
+  size_t Size();
+  bool Empty() const;
+
+  const Bitmap &GetWordsBitmap() const {
+    return m_bitmap;
+  }
+
+  const HypothesisSet &GetHypotheses() const;
+  size_t GetHypothesesSize() const;
+  const BackwardsEdgeSet &GetBackwardsEdges();
+
+  void InitializeEdges();
+  void ProcessBestHypothesis();
+  void EnsureMinStackHyps(const size_t minNumHyps);
+  void AddHypothesis(Hypothesis *hypothesis);
+  void AddBackwardsEdge(BackwardsEdge *edge);
+  void SortHypotheses();
+};
+
+}
+
+#endif

mosesdecoder/moses/Bitmaps.cpp

@@ -0,0 +1,58 @@
+#include <boost/foreach.hpp>
+#include "Bitmaps.h"
+#include "Util.h"
+
+using namespace std;
+
+namespace Moses
+{
+Bitmaps::Bitmaps(size_t inputSize, const std::vector<bool> &initSourceCompleted)
+{
+  m_initBitmap = new Bitmap(inputSize, initSourceCompleted);
+  m_coll[m_initBitmap];
+}
+
+Bitmaps::~Bitmaps()
+{
+  BOOST_FOREACH (const Coll::value_type& myPair, m_coll) {
+    const Bitmap *bm = myPair.first;
+    delete bm;
+  }
+}
+
+const Bitmap &Bitmaps::GetNextBitmap(const Bitmap &bm, const Range &range)
+{
+  Bitmap *newBM = new Bitmap(bm, range);
+
+  Coll::const_iterator iter = m_coll.find(newBM);
+  if (iter == m_coll.end()) {
+    m_coll[newBM] = NextBitmaps();
+    return *newBM;
+  } else {
+    delete newBM;
+    return *iter->first;
+  }
+}
+
+const Bitmap &Bitmaps::GetBitmap(const Bitmap &bm, const Range &range)
+{
+  Coll::iterator iter = m_coll.find(&bm);
+  assert(iter != m_coll.end());
+
+  const Bitmap *newBM;
+  NextBitmaps &next = iter->second;
+  NextBitmaps::const_iterator iterNext = next.find(range);
+  if (iterNext == next.end()) {
+    // not seen the link yet.
+    newBM = &GetNextBitmap(bm, range);
+    next[range] = newBM;
+  } else {
+    // link exist
+    //std::cerr << "link exists" << endl;
+    newBM = iterNext->second;
+  }
+  return *newBM;
+}
+
+}
+

mosesdecoder/moses/Bitmaps.h

@@ -0,0 +1,32 @@
+#pragma once
+
+#include <boost/unordered_set.hpp>
+#include <boost/unordered_map.hpp>
+#include <set>
+#include "Bitmap.h"
+#include "Util.h"
+
+namespace Moses
+{
+
+class Bitmaps
+{
+  typedef boost::unordered_map<Range, const Bitmap*> NextBitmaps;
+  typedef boost::unordered_map<const Bitmap*, NextBitmaps, UnorderedComparer<Bitmap>, UnorderedComparer<Bitmap> > Coll;
+  //typedef std::set<const Bitmap*, OrderedComparer<Bitmap> > Coll;
+  Coll m_coll;
+  const Bitmap *m_initBitmap;
+
+  const Bitmap &GetNextBitmap(const Bitmap &bm, const Range &range);
+public:
+  Bitmaps(size_t inputSize, const std::vector<bool> &initSourceCompleted);
+  virtual ~Bitmaps();
+
+  const Bitmap &GetInitialBitmap() const {
+    return *m_initBitmap;
+  }
+  const Bitmap &GetBitmap(const Bitmap &bm, const Range &range);
+
+};
+
+}

mosesdecoder/moses/CMakeLists.txt

@@ -0,0 +1,13 @@
+project(moses)
+
+FILE(GLOB source_moses *.cpp)
+FILE(GLOB source_moses_ff FF/*.cpp)
+FILE(GLOB source_moses_ff_lexicalReordering FF/LexicalReordering/*.cpp)
+FILE(GLOB source_moses_ff_osm FF/OSM-Feature/*.cpp)
+FILE(GLOB source_moses_lm LM/*.cpp)
+FILE(GLOB source_moses_tm TranslationModel/*.cpp)
+FILE(GLOB source_moses_tm_compactPT TranslationModel/CompactPT/*.cpp)
+FILE(GLOB source_moses_tm_cky TranslationModel/CKYPlusParser/*.cpp)
+
+
+add_library(biconcor ${biconcor_source})

mosesdecoder/moses/ChartCell.cpp

@@ -0,0 +1,233 @@
+// $Id$
+// vim:tabstop=2
+/***********************************************************************
+ Moses - factored phrase-based language decoder
+ Copyright (C) 2010 Hieu Hoang
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ ***********************************************************************/
+
+#include <algorithm>
+#include "ChartCell.h"
+#include "ChartCellCollection.h"
+#include "HypergraphOutput.h"
+#include "RuleCubeQueue.h"
+#include "RuleCube.h"
+#include "Range.h"
+#include "Util.h"
+#include "ChartTranslationOptions.h"
+#include "ChartTranslationOptionList.h"
+#include "ChartManager.h"
+#include "util/exception.hh"
+
+using namespace std;
+
+namespace Moses
+{
+
+ChartCellBase::ChartCellBase(size_t startPos, size_t endPos) :
+  m_coverage(startPos, endPos),
+  m_targetLabelSet(m_coverage) {}
+
+ChartCellBase::~ChartCellBase() {}
+
+/** Constructor
+ * \param startPos endPos range of this cell
+ * \param manager pointer back to the manager
+ */
+ChartCell::ChartCell(size_t startPos, size_t endPos, ChartManager &manager) :
+  ChartCellBase(startPos, endPos), m_manager(manager)
+{
+  m_nBestIsEnabled = manager.options()->nbest.enabled;
+}
+
+ChartCell::~ChartCell() {}
+
+/** Add the given hypothesis to the cell.
+ *  Returns true if added, false if not. Maybe it already exists in the collection or score falls below threshold etc.
+ *  This function just calls the corresponding AddHypothesis() in ChartHypothesisCollection
+ *  \param hypo Hypothesis to be added
+ */
+bool ChartCell::AddHypothesis(ChartHypothesis *hypo)
+{
+  const Word &targetLHS = hypo->GetTargetLHS();
+  MapType::iterator m = m_hypoColl.find(targetLHS);
+  if (m == m_hypoColl.end()) {
+    std::pair<Word, ChartHypothesisCollection>
+    e(targetLHS, ChartHypothesisCollection(*m_manager.options()));
+    m = m_hypoColl.insert(e).first;
+  }
+  return m->second.AddHypothesis(hypo, m_manager);
+}
+
+/** Prune each collection in this cell to a particular size */
+void ChartCell::PruneToSize()
+{
+  MapType::iterator iter;
+  for (iter = m_hypoColl.begin(); iter != m_hypoColl.end(); ++iter) {
+    ChartHypothesisCollection &coll = iter->second;
+    coll.PruneToSize(m_manager);
+  }
+}
+
+/** Decoding at span level: fill chart cell with hypotheses
+ *  (implementation of cube pruning)
+ * \param transOptList list of applicable rules to create hypotheses for the cell
+ * \param allChartCells entire chart - needed to look up underlying hypotheses
+ */
+void ChartCell::Decode(const ChartTranslationOptionList &transOptList
+                       , const ChartCellCollection &allChartCells)
+{
+  // priority queue for applicable rules with selected hypotheses
+  RuleCubeQueue queue(m_manager);
+
+  // add all trans opt into queue. using only 1st child node.
+  for (size_t i = 0; i < transOptList.GetSize(); ++i) {
+    const ChartTranslationOptions &transOpt = transOptList.Get(i);
+    RuleCube *ruleCube = new RuleCube(transOpt, allChartCells, m_manager);
+    queue.Add(ruleCube);
+  }
+
+  // pluck things out of queue and add to hypo collection
+  const size_t popLimit = m_manager.options()->cube.pop_limit;
+  for (size_t numPops = 0; numPops < popLimit && !queue.IsEmpty(); ++numPops) {
+    ChartHypothesis *hypo = queue.Pop();
+    AddHypothesis(hypo);
+  }
+}
+
+//! call SortHypotheses() in each hypo collection in this cell
+void ChartCell::SortHypotheses()
+{
+  UTIL_THROW_IF2(!m_targetLabelSet.Empty(), "Already sorted");
+
+  MapType::iterator iter;
+  for (iter = m_hypoColl.begin(); iter != m_hypoColl.end(); ++iter) {
+    ChartHypothesisCollection &coll = iter->second;
+
+    if (coll.GetSize()) {
+      coll.SortHypotheses();
+      m_targetLabelSet.AddConstituent(iter->first, &coll.GetSortedHypotheses());
+    }
+  }
+}
+
+/** Return the highest scoring hypothesis out of all the  hypo collection in this cell */
+const ChartHypothesis *ChartCell::GetBestHypothesis() const
+{
+  const ChartHypothesis *ret = NULL;
+  float bestScore = -std::numeric_limits<float>::infinity();
+
+  MapType::const_iterator iter;
+  for (iter = m_hypoColl.begin(); iter != m_hypoColl.end(); ++iter) {
+    const HypoList &sortedList = iter->second.GetSortedHypotheses();
+    if (sortedList.size() > 0) {
+      const ChartHypothesis *hypo = sortedList[0];
+      if (hypo->GetFutureScore() > bestScore) {
+        bestScore = hypo->GetFutureScore();
+        ret = hypo;
+      }
+    }
+  }
+
+  return ret;
+}
+
+//! call CleanupArcList() in each hypo collection in this cell
+void ChartCell::CleanupArcList()
+{
+  // only necessary if n-best calculations are enabled
+  if (!m_nBestIsEnabled) return;
+
+  MapType::iterator iter;
+  for (iter = m_hypoColl.begin(); iter != m_hypoColl.end(); ++iter) {
+    ChartHypothesisCollection &coll = iter->second;
+    coll.CleanupArcList();
+  }
+}
+
+//! debug info - size of each hypo collection in this cell
+void ChartCell::OutputSizes(std::ostream &out) const
+{
+  MapType::const_iterator iter;
+  for (iter = m_hypoColl.begin(); iter != m_hypoColl.end(); ++iter) {
+    const Word &targetLHS = iter->first;
+    const ChartHypothesisCollection &coll = iter->second;
+
+    out << targetLHS << "=" << coll.GetSize() << " ";
+  }
+}
+
+//! debug info - total number of hypos in all hypo collection in this cell
+size_t ChartCell::GetSize() const
+{
+  size_t ret = 0;
+  MapType::const_iterator iter;
+  for (iter = m_hypoColl.begin(); iter != m_hypoColl.end(); ++iter) {
+    const ChartHypothesisCollection &coll = iter->second;
+
+    ret += coll.GetSize();
+  }
+
+  return ret;
+}
+
+const HypoList *ChartCell::GetAllSortedHypotheses() const
+{
+  HypoList *ret = new HypoList();
+
+  MapType::const_iterator iter;
+  for (iter = m_hypoColl.begin(); iter != m_hypoColl.end(); ++iter) {
+    const ChartHypothesisCollection &coll = iter->second;
+    const HypoList &list = coll.GetSortedHypotheses();
+    std::copy(list.begin(), list.end(), std::inserter(*ret, ret->end()));
+  }
+  return ret;
+}
+
+//! call WriteSearchGraph() for each hypo collection
+void ChartCell::WriteSearchGraph(const ChartSearchGraphWriter& writer, const std::map<unsigned, bool> &reachable) const
+{
+  MapType::const_iterator iterOutside;
+  for (iterOutside = m_hypoColl.begin(); iterOutside != m_hypoColl.end(); ++iterOutside) {
+    const ChartHypothesisCollection &coll = iterOutside->second;
+    coll.WriteSearchGraph(writer, reachable);
+  }
+}
+
+std::ostream& operator<<(std::ostream &out, const ChartCell &cell)
+{
+  ChartCell::MapType::const_iterator iterOutside;
+  for (iterOutside = cell.m_hypoColl.begin(); iterOutside != cell.m_hypoColl.end(); ++iterOutside) {
+    const Word &targetLHS = iterOutside->first;
+    cerr << targetLHS << ":" << endl;
+
+    const ChartHypothesisCollection &coll = iterOutside->second;
+    cerr << coll;
+  }
+
+  /*
+  ChartCell::HCType::const_iterator iter;
+  for (iter = cell.m_hypos.begin(); iter != cell.m_hypos.end(); ++iter)
+  {
+  	const ChartHypothesis &hypo = **iter;
+  	out << hypo << endl;
+  }
+   */
+
+  return out;
+}
+
+} // namespace

mosesdecoder/moses/ChartCell.h

@@ -0,0 +1,128 @@
+// $Id$
+// vim:tabstop=2
+/***********************************************************************
+ Moses - factored phrase-based language decoder
+ Copyright (C) 2010 Hieu Hoang
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ ***********************************************************************/
+
+#pragma once
+
+#include <iostream>
+#include <queue>
+#include <map>
+#include <vector>
+#include "Word.h"
+#include "Range.h"
+#include "NonTerminal.h"
+#include "ChartHypothesis.h"
+#include "ChartHypothesisCollection.h"
+#include "RuleCube.h"
+#include "ChartCellLabelSet.h"
+
+#include <boost/scoped_ptr.hpp>
+#include <boost/functional/hash.hpp>
+#include <boost/unordered_map.hpp>
+#include <boost/version.hpp>
+
+namespace Moses
+{
+class ChartSearchGraphWriter;
+class ChartTranslationOptionList;
+class ChartCellCollection;
+class ChartManager;
+
+class ChartCellBase
+{
+public:
+  ChartCellBase(size_t startPos, size_t endPos);
+
+  virtual ~ChartCellBase();
+
+  const ChartCellLabelSet &GetTargetLabelSet() const {
+    return m_targetLabelSet;
+  }
+
+  ChartCellLabelSet &MutableTargetLabelSet() {
+    return m_targetLabelSet;
+  }
+
+  const Range &GetCoverage() const {
+    return m_coverage;
+  }
+
+protected:
+  const Range m_coverage;
+  ChartCellLabelSet m_targetLabelSet;
+};
+
+/** 1 cell in chart decoder.
+ *  Doesn't directly hold hypotheses. Each cell contain a map of ChartHypothesisCollection that have different constituent labels
+ */
+class ChartCell : public ChartCellBase
+{
+  friend std::ostream& operator<<(std::ostream&, const ChartCell&);
+public:
+#if defined(BOOST_VERSION) && (BOOST_VERSION >= 104200)
+  typedef boost::unordered_map<Word,
+          ChartHypothesisCollection,
+          NonTerminalHasher,
+          NonTerminalEqualityPred
+          > MapType;
+#else
+  typedef std::map<Word, ChartHypothesisCollection> MapType;
+#endif
+
+protected:
+  MapType m_hypoColl;
+
+  bool m_nBestIsEnabled; /**< flag to determine whether to keep track of old arcs */
+  ChartManager &m_manager;
+
+public:
+  ChartCell(size_t startPos, size_t endPos, ChartManager &manager);
+  ~ChartCell();
+
+  void Decode(const ChartTranslationOptionList &transOptList
+              ,const ChartCellCollection &allChartCells);
+
+  //! Get all hypotheses in the cell that have the specified constituent label
+  const HypoList *GetSortedHypotheses(const Word &constituentLabel) const {
+    MapType::const_iterator p = m_hypoColl.find(constituentLabel);
+    return (p == m_hypoColl.end()) ? NULL : &(p->second.GetSortedHypotheses());
+  }
+
+  //! for n-best list
+  const HypoList *GetAllSortedHypotheses() const;
+
+  bool AddHypothesis(ChartHypothesis *hypo);
+
+  void SortHypotheses();
+  void PruneToSize();
+
+  const ChartHypothesis *GetBestHypothesis() const;
+
+  void CleanupArcList();
+
+  void OutputSizes(std::ostream &out) const;
+  size_t GetSize() const;
+
+  void WriteSearchGraph(const ChartSearchGraphWriter& writer, const std::map<unsigned,bool> &reachable) const;
+
+};
+
+}
+

mosesdecoder/moses/ChartCellCollection.cpp

@@ -0,0 +1,58 @@
+// $Id$
+// vim:tabstop=2
+/***********************************************************************
+ Moses - factored phrase-based language decoder
+ Copyright (C) 2010 Hieu Hoang
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ ***********************************************************************/
+
+#include "ChartCellCollection.h"
+#include "InputType.h"
+#include "Range.h"
+#include "ChartManager.h"
+
+namespace Moses
+{
+
+ChartCellCollectionBase::~ChartCellCollectionBase()
+{
+  m_source.clear();
+  for (std::vector<std::vector<ChartCellBase*> >::iterator i = m_cells.begin(); i != m_cells.end(); ++i)
+    RemoveAllInColl(*i);
+}
+
+class CubeCellFactory
+{
+public:
+  explicit CubeCellFactory(ChartManager &manager) : m_manager(manager) {}
+
+  ChartCell *operator()(size_t start, size_t end) const {
+    return new ChartCell(start, end, m_manager);
+  }
+
+private:
+  ChartManager &m_manager;
+};
+
+/** Costructor
+ \param input the input sentence
+ \param manager reference back to the manager
+ */
+ChartCellCollection::ChartCellCollection(const InputType &input, ChartManager &manager)
+  :ChartCellCollectionBase(input, CubeCellFactory(manager), manager.GetParser()) {}
+
+} // namespace
+

mosesdecoder/moses/ChartCellCollection.h

@@ -0,0 +1,102 @@
+// $Id$
+// vim:tabstop=2
+/***********************************************************************
+ Moses - factored phrase-based language decoder
+ Copyright (C) 2010 Hieu Hoang
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ ***********************************************************************/
+#pragma once
+
+#include <boost/ptr_container/ptr_vector.hpp>
+#include "InputType.h"
+#include "ChartCell.h"
+#include "Range.h"
+#include "InputPath.h"
+
+namespace Moses
+{
+class InputType;
+class ChartManager;
+class ChartParser;
+
+class ChartCellCollectionBase
+{
+public:
+  template <class Factory> ChartCellCollectionBase(const InputType &input,
+      const Factory &factory,
+      const ChartParser &parser)
+    :m_cells(input.GetSize()) {
+
+    size_t size = input.GetSize();
+    for (size_t startPos = 0; startPos < size; ++startPos) {
+      std::vector<ChartCellBase*> &inner = m_cells[startPos];
+      inner.reserve(size - startPos);
+      for (size_t endPos = startPos; endPos < size; ++endPos) {
+        inner.push_back(factory(startPos, endPos));
+      }
+      /* Hack: ChartCellLabel shouldn't need to know its span, but the parser
+       * gets it from there :-(.  The span is actually stored as a reference,
+       * which needs to point somewhere, so I have it refer to the ChartCell.
+       */
+      const Range &range = inner[0]->GetCoverage();
+
+      m_source.push_back(new ChartCellLabel(range, input.GetWord(startPos)));
+    }
+  }
+
+  virtual ~ChartCellCollectionBase();
+
+
+  const ChartCellBase &GetBase(const Range &coverage) const {
+    return *m_cells[coverage.GetStartPos()][coverage.GetEndPos() - coverage.GetStartPos()];
+  }
+
+  ChartCellBase &MutableBase(const Range &coverage) {
+    return *m_cells[coverage.GetStartPos()][coverage.GetEndPos() - coverage.GetStartPos()];
+  }
+
+
+  const ChartCellLabel &GetSourceWordLabel(size_t at) const {
+    return m_source[at];
+  }
+
+private:
+  std::vector<std::vector<ChartCellBase*> > m_cells;
+
+  boost::ptr_vector<ChartCellLabel> m_source;
+
+};
+
+/** Hold all the chart cells for 1 input sentence. A variable of this type is held by the ChartManager
+ */
+class ChartCellCollection : public ChartCellCollectionBase
+{
+public:
+  ChartCellCollection(const InputType &input, ChartManager &manager);
+
+  //! get a chart cell for a particular range
+  ChartCell &Get(const Range &coverage) {
+    return static_cast<ChartCell&>(MutableBase(coverage));
+  }
+
+  //! get a chart cell for a particular range
+  const ChartCell &Get(const Range &coverage) const {
+    return static_cast<const ChartCell&>(GetBase(coverage));
+  }
+};
+
+}
+

mosesdecoder/moses/ChartCellLabel.h

@@ -0,0 +1,89 @@
+/***********************************************************************
+ Moses - statistical machine translation system
+ Copyright (C) 2006-2011 University of Edinburgh
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ ***********************************************************************/
+
+#pragma once
+
+#include "HypoList.h"
+#include "Word.h"
+#include "Range.h"
+#include "ChartParserCallback.h"
+
+namespace search
+{
+class Vertex;
+}
+
+namespace Moses
+{
+
+class Word;
+
+/** Contains a range, word (non-terms?) and a vector of hypotheses.
+ * @todo This is probably incompatible with lattice decoding when the word that spans
+ *   a position (or positions) can vary.
+ * @todo is this to hold sorted hypotheses that are in the queue for creating the next hypos?
+ */
+class ChartCellLabel
+{
+public:
+  union Stack {
+    const HypoList *cube; // cube pruning
+    search::Vertex *incr; // incremental search after filling.
+    void *incr_generator; // incremental search during filling.
+  };
+
+
+  ChartCellLabel(const Range &coverage, const Word &label,
+                 Stack stack=Stack())
+    : m_coverage(coverage)
+    , m_label(label)
+    , m_stack(stack)
+    , m_bestScore(0) {
+  }
+
+  const Range &GetCoverage() const {
+    return m_coverage;
+  }
+  const Word &GetLabel() const {
+    return m_label;
+  }
+  Stack GetStack() const {
+    return m_stack;
+  }
+  Stack &MutableStack() {
+    return m_stack;
+  }
+
+  //caching of best score on stack
+  float GetBestScore(const ChartParserCallback *outColl) const {
+    if (m_bestScore == 0) {
+      m_bestScore = outColl->GetBestScore(this);
+    }
+    return m_bestScore;
+  }
+
+private:
+  const Range &m_coverage;
+  const Word &m_label;
+  //const InputPath &m_inputPath;
+  Stack m_stack;
+  mutable float m_bestScore;
+};
+
+}

mosesdecoder/moses/ChartCellLabelSet.h

@@ -0,0 +1,147 @@
+/***********************************************************************
+ Moses - statistical machine translation system
+ Copyright (C) 2006-2011 University of Edinburgh
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ ***********************************************************************/
+
+#pragma once
+
+#include "ChartCellLabel.h"
+#include "NonTerminal.h"
+#include "moses/FactorCollection.h"
+
+#include <boost/functional/hash.hpp>
+#include <boost/unordered_map.hpp>
+#include <boost/version.hpp>
+
+namespace Moses
+{
+
+class ChartHypothesisCollection;
+
+/** @todo I have no idea what's in here
+ */
+class ChartCellLabelSet
+{
+private:
+
+  typedef std::vector<ChartCellLabel*> MapType;
+
+public:
+  typedef MapType::const_iterator const_iterator;
+  typedef MapType::iterator iterator;
+
+  ChartCellLabelSet(const Range &coverage)
+    : m_coverage(coverage)
+    , m_map(FactorCollection::Instance().GetNumNonTerminals(), NULL)
+    , m_size(0) { }
+
+  ~ChartCellLabelSet() {
+    RemoveAllInColl(m_map);
+  }
+
+  // TODO: skip empty elements when iterating, or deprecate this
+  const_iterator begin() const {
+    return m_map.begin();
+  }
+  const_iterator end() const {
+    return m_map.end();
+  }
+
+  iterator mutable_begin() {
+    return m_map.begin();
+  }
+  iterator mutable_end() {
+    return m_map.end();
+  }
+
+  void AddWord(const Word &w) {
+    size_t idx = w[0]->GetId();
+    if (! ChartCellExists(idx)) {
+      m_size++;
+
+
+      m_map[idx] = new ChartCellLabel(m_coverage, w);
+    }
+  }
+
+  // Stack is a HypoList or whatever the search algorithm uses.
+  void AddConstituent(const Word &w, const HypoList *stack) {
+    size_t idx = w[0]->GetId();
+    if (ChartCellExists(idx)) {
+      ChartCellLabel::Stack & s = m_map[idx]->MutableStack();
+      s.cube = stack;
+    } else {
+      ChartCellLabel::Stack s;
+      s.cube = stack;
+      m_size++;
+      m_map[idx] = new ChartCellLabel(m_coverage, w, s);
+    }
+  }
+
+  // grow vector if necessary
+  bool ChartCellExists(size_t idx) {
+    try {
+      if (m_map.at(idx) != NULL) {
+        return true;
+      }
+    } catch (const std::out_of_range& oor) {
+      m_map.resize(FactorCollection::Instance().GetNumNonTerminals(), NULL);
+    }
+    return false;
+  }
+
+  bool Empty() const {
+    return m_size == 0;
+  }
+
+  size_t GetSize() const {
+    return m_size;
+  }
+
+  const ChartCellLabel *Find(const Word &w) const {
+    size_t idx = w[0]->GetId();
+    try {
+      return m_map.at(idx);
+    } catch (const std::out_of_range& oor) {
+      return NULL;
+    }
+  }
+
+  const ChartCellLabel *Find(size_t idx) const {
+    try {
+      return m_map.at(idx);
+    } catch (const std::out_of_range& oor) {
+      return NULL;
+    }
+  }
+
+  ChartCellLabel::Stack &FindOrInsert(const Word &w) {
+    size_t idx = w[0]->GetId();
+    if (! ChartCellExists(idx)) {
+      m_size++;
+      m_map[idx] = new ChartCellLabel(m_coverage, w);
+    }
+    return m_map[idx]->MutableStack();
+  }
+
+private:
+  const Range &m_coverage;
+  MapType m_map;
+  size_t m_size;
+};
+
+}

mosesdecoder/moses/ChartHypothesis.cpp

@@ -0,0 +1,360 @@
+// vim:tabstop=2
+/***********************************************************************
+ Moses - factored phrase-based language decoder
+ Copyright (C) 2010 Hieu Hoang
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ ***********************************************************************/
+
+#include <algorithm>
+#include <vector>
+#include "ChartHypothesis.h"
+#include "RuleCubeItem.h"
+#include "ChartCell.h"
+#include "ChartManager.h"
+#include "TargetPhrase.h"
+#include "Phrase.h"
+#include "StaticData.h"
+#include "ChartTranslationOptions.h"
+#include "moses/FF/FFState.h"
+#include "moses/FF/StatefulFeatureFunction.h"
+#include "moses/FF/StatelessFeatureFunction.h"
+
+using namespace std;
+
+namespace Moses
+{
+
+/** Create a hypothesis from a rule
+ * \param transOpt wrapper around the rule
+ * \param item @todo dunno
+ * \param manager reference back to manager
+ */
+ChartHypothesis::ChartHypothesis(const ChartTranslationOptions &transOpt,
+                                 const RuleCubeItem &item,
+                                 ChartManager &manager)
+  :m_transOpt(item.GetTranslationDimension().GetTranslationOption())
+  ,m_currSourceWordsRange(transOpt.GetSourceWordsRange())
+  ,m_ffStates(StatefulFeatureFunction::GetStatefulFeatureFunctions().size())
+  ,m_arcList(NULL)
+  ,m_winningHypo(NULL)
+  ,m_manager(manager)
+  ,m_id(manager.GetNextHypoId())
+{
+  // underlying hypotheses for sub-spans
+  const std::vector<HypothesisDimension> &childEntries = item.GetHypothesisDimensions();
+  m_prevHypos.reserve(childEntries.size());
+  std::vector<HypothesisDimension>::const_iterator iter;
+  for (iter = childEntries.begin(); iter != childEntries.end(); ++iter) {
+    m_prevHypos.push_back(iter->GetHypothesis());
+  }
+}
+
+// Intended to be used by ChartKBestExtractor only.  This creates a mock
+// ChartHypothesis for use by the extractor's top-level target vertex.
+ChartHypothesis::ChartHypothesis(const ChartHypothesis &pred,
+                                 const ChartKBestExtractor & /*unused*/)
+  :m_currSourceWordsRange(pred.m_currSourceWordsRange)
+  ,m_totalScore(pred.m_totalScore)
+  ,m_arcList(NULL)
+  ,m_winningHypo(NULL)
+  ,m_manager(pred.m_manager)
+  ,m_id(pred.m_manager.GetNextHypoId())
+{
+  // One predecessor, which is an existing top-level ChartHypothesis.
+  m_prevHypos.push_back(&pred);
+}
+
+ChartHypothesis::~ChartHypothesis()
+{
+  // delete feature function states
+  for (unsigned i = 0; i < m_ffStates.size(); ++i) {
+    delete m_ffStates[i];
+  }
+
+  // delete hypotheses that are not in the chart (recombined away)
+  if (m_arcList) {
+    ChartArcList::iterator iter;
+    for (iter = m_arcList->begin() ; iter != m_arcList->end() ; ++iter) {
+      ChartHypothesis *hypo = *iter;
+      delete hypo;
+    }
+    m_arcList->clear();
+
+    delete m_arcList;
+  }
+}
+
+/** Create full output phrase that is contained in the hypothesis (and its children)
+ * \param outPhrase full output phrase as return argument
+ */
+void ChartHypothesis::GetOutputPhrase(Phrase &outPhrase) const
+{
+  FactorType placeholderFactor = StaticData::Instance().options()->input.placeholder_factor;
+
+  for (size_t pos = 0; pos < GetCurrTargetPhrase().GetSize(); ++pos) {
+    const Word &word = GetCurrTargetPhrase().GetWord(pos);
+    if (word.IsNonTerminal()) {
+      // non-term. fill out with prev hypo
+      size_t nonTermInd = GetCurrTargetPhrase().GetAlignNonTerm().GetNonTermIndexMap()[pos];
+      const ChartHypothesis *prevHypo = m_prevHypos[nonTermInd];
+      prevHypo->GetOutputPhrase(outPhrase);
+    } else {
+      outPhrase.AddWord(word);
+
+      if (placeholderFactor != NOT_FOUND) {
+        std::set<size_t> sourcePosSet = GetCurrTargetPhrase().GetAlignTerm().GetAlignmentsForTarget(pos);
+        if (sourcePosSet.size() == 1) {
+          const std::vector<const Word*> *ruleSourceFromInputPath = GetTranslationOption().GetSourceRuleFromInputPath();
+          UTIL_THROW_IF2(ruleSourceFromInputPath == NULL,
+                         "No source rule");
+
+          size_t sourcePos = *sourcePosSet.begin();
+          const Word *sourceWord = ruleSourceFromInputPath->at(sourcePos);
+          UTIL_THROW_IF2(sourceWord == NULL,
+                         "No source word");
+          const Factor *factor = sourceWord->GetFactor(placeholderFactor);
+          if (factor) {
+            outPhrase.Back()[0] = factor;
+          }
+        }
+      }
+
+    }
+  }
+}
+
+/** Return full output phrase */
+Phrase ChartHypothesis::GetOutputPhrase() const
+{
+  Phrase outPhrase(ARRAY_SIZE_INCR);
+  GetOutputPhrase(outPhrase);
+  return outPhrase;
+}
+
+/** TODO: this method isn't used anywhere. Remove? */
+void ChartHypothesis::GetOutputPhrase(size_t leftRightMost, size_t numWords, Phrase &outPhrase) const
+{
+  const TargetPhrase &tp = GetCurrTargetPhrase();
+
+  size_t targetSize = tp.GetSize();
+  for (size_t i = 0; i < targetSize; ++i) {
+    size_t pos;
+    if (leftRightMost == 1) {
+      pos = i;
+    } else if (leftRightMost == 2) {
+      pos = targetSize - i - 1;
+    } else {
+      abort();
+    }
+
+    const Word &word = tp.GetWord(pos);
+
+    if (word.IsNonTerminal()) {
+      // non-term. fill out with prev hypo
+      size_t nonTermInd = tp.GetAlignNonTerm().GetNonTermIndexMap()[pos];
+      const ChartHypothesis *prevHypo = m_prevHypos[nonTermInd];
+      prevHypo->GetOutputPhrase(outPhrase);
+    } else {
+      outPhrase.AddWord(word);
+    }
+
+    if (outPhrase.GetSize() >= numWords) {
+      return;
+    }
+  }
+}
+
+/** calculate total score */
+void ChartHypothesis::EvaluateWhenApplied()
+{
+  const StaticData &staticData = StaticData::Instance();
+
+  // compute values of stateless feature functions that were not
+  // cached in the translation option-- there is no principled distinction
+  const std::vector<const StatelessFeatureFunction*>& sfs =
+    StatelessFeatureFunction::GetStatelessFeatureFunctions();
+  for (unsigned i = 0; i < sfs.size(); ++i) {
+    if (! staticData.IsFeatureFunctionIgnored( *sfs[i] )) {
+      sfs[i]->EvaluateWhenApplied(*this,&m_currScoreBreakdown);
+    }
+  }
+
+  const std::vector<const StatefulFeatureFunction*>& ffs =
+    StatefulFeatureFunction::GetStatefulFeatureFunctions();
+  for (unsigned i = 0; i < ffs.size(); ++i) {
+    if (! staticData.IsFeatureFunctionIgnored( *ffs[i] )) {
+      m_ffStates[i] = ffs[i]->EvaluateWhenApplied(*this,i,&m_currScoreBreakdown);
+    }
+  }
+
+  // total score from current translation rule
+  m_totalScore = GetTranslationOption().GetScores().GetWeightedScore();
+  m_totalScore += m_currScoreBreakdown.GetWeightedScore();
+
+  // total scores from prev hypos
+  for (std::vector<const ChartHypothesis*>::const_iterator iter = m_prevHypos.begin(); iter != m_prevHypos.end(); ++iter) {
+    const ChartHypothesis &prevHypo = **iter;
+    m_totalScore += prevHypo.GetFutureScore();
+  }
+}
+
+void ChartHypothesis::AddArc(ChartHypothesis *loserHypo)
+{
+  if (!m_arcList) {
+    if (loserHypo->m_arcList) {
+      // we don't have an arcList, but loser does
+      this->m_arcList = loserHypo->m_arcList;  // take ownership, we'll delete
+      loserHypo->m_arcList = 0;                // prevent a double deletion
+    } else {
+      this->m_arcList = new ChartArcList();
+    }
+  } else {
+    if (loserHypo->m_arcList) {
+      // both have an arc list: merge. delete loser
+      size_t my_size = m_arcList->size();
+      size_t add_size = loserHypo->m_arcList->size();
+      this->m_arcList->resize(my_size + add_size, 0);
+      std::memcpy(&(*m_arcList)[0] + my_size, &(*loserHypo->m_arcList)[0], add_size * sizeof(ChartHypothesis *));
+      delete loserHypo->m_arcList;
+      loserHypo->m_arcList = 0;
+    } else {
+      // loserHypo doesn't have any arcs
+      // DO NOTHING
+    }
+  }
+  m_arcList->push_back(loserHypo);
+}
+
+// sorting helper
+struct CompareChartHypothesisTotalScore {
+  bool operator()(const ChartHypothesis* hypo1, const ChartHypothesis* hypo2) const {
+    return hypo1->GetFutureScore() > hypo2->GetFutureScore();
+  }
+};
+
+void ChartHypothesis::CleanupArcList()
+{
+  // point this hypo's main hypo to itself
+  m_winningHypo = this;
+
+  if (!m_arcList) return;
+
+  /* keep only number of arcs we need to create all n-best paths.
+   * However, may not be enough if only unique candidates are needed,
+   * so we'll keep all of arc list if nedd distinct n-best list
+   */
+  AllOptions const& opts = *StaticData::Instance().options();
+  size_t nBestSize = opts.nbest.nbest_size;
+  bool distinctNBest = (opts.nbest.only_distinct
+                        || opts.mbr.enabled
+                        || opts.output.NeedSearchGraph()
+                        || !opts.output.SearchGraphHG.empty());
+
+  if (!distinctNBest && m_arcList->size() > nBestSize) {
+    // prune arc list only if there too many arcs
+    NTH_ELEMENT4(m_arcList->begin()
+                 , m_arcList->begin() + nBestSize - 1
+                 , m_arcList->end()
+                 , CompareChartHypothesisTotalScore());
+
+    // delete bad ones
+    ChartArcList::iterator iter;
+    for (iter = m_arcList->begin() + nBestSize ; iter != m_arcList->end() ; ++iter) {
+      ChartHypothesis *arc = *iter;
+      delete arc;
+    }
+    m_arcList->erase(m_arcList->begin() + nBestSize
+                     , m_arcList->end());
+  }
+
+  // set all arc's main hypo variable to this hypo
+  ChartArcList::iterator iter = m_arcList->begin();
+  for (; iter != m_arcList->end() ; ++iter) {
+    ChartHypothesis *arc = *iter;
+    arc->SetWinningHypo(this);
+  }
+
+  //cerr << m_arcList->size() << " ";
+}
+
+void ChartHypothesis::SetWinningHypo(const ChartHypothesis *hypo)
+{
+  m_winningHypo = hypo;
+}
+
+size_t ChartHypothesis::hash() const
+{
+  size_t seed = 0;
+
+  // states
+  for (size_t i = 0; i < m_ffStates.size(); ++i) {
+    const FFState *state = m_ffStates[i];
+    size_t hash = state->hash();
+    boost::hash_combine(seed, hash);
+  }
+  return seed;
+
+}
+
+bool ChartHypothesis::operator==(const ChartHypothesis& other) const
+{
+  // states
+  for (size_t i = 0; i < m_ffStates.size(); ++i) {
+    const FFState &thisState = *m_ffStates[i];
+    const FFState &otherState = *other.m_ffStates[i];
+    if (thisState != otherState) {
+      return false;
+    }
+  }
+  return true;
+}
+
+TO_STRING_BODY(ChartHypothesis)
+
+// friend
+std::ostream& operator<<(std::ostream& out, const ChartHypothesis& hypo)
+{
+
+  out << hypo.GetId();
+
+  // recombination
+  if (hypo.GetWinningHypothesis() != NULL &&
+      hypo.GetWinningHypothesis() != &hypo) {
+    out << "->" << hypo.GetWinningHypothesis()->GetId();
+  }
+
+  if (hypo.GetManager().options()->output.include_lhs_in_search_graph) {
+    out << " " << hypo.GetTargetLHS() << "=>";
+  }
+  out << " " << hypo.GetCurrTargetPhrase()
+      //<< " " << outPhrase
+      << " " << hypo.GetCurrSourceRange();
+
+  HypoList::const_iterator iter;
+  for (iter = hypo.GetPrevHypos().begin(); iter != hypo.GetPrevHypos().end(); ++iter) {
+    const ChartHypothesis &prevHypo = **iter;
+    out << " " << prevHypo.GetId();
+  }
+
+  out << " [total=" << hypo.GetFutureScore() << "]";
+  out << " " << hypo.GetScoreBreakdown();
+
+  //out << endl;
+
+  return out;
+}
+
+}

mosesdecoder/moses/ChartHypothesis.h

@@ -0,0 +1,204 @@
+// vim:tabstop=2
+/***********************************************************************
+ Moses - factored phrase-based language decoder
+ Copyright (C) 2010 Hieu Hoang
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ ***********************************************************************/
+
+#pragma once
+
+#include <vector>
+#include <boost/scoped_ptr.hpp>
+#include "Util.h"
+#include "Range.h"
+#include "ScoreComponentCollection.h"
+#include "Phrase.h"
+#include "ChartTranslationOptions.h"
+#include "ObjectPool.h"
+
+namespace Moses
+{
+
+class ChartKBestExtractor;
+class ChartHypothesis;
+class ChartManager;
+class RuleCubeItem;
+class FFState;
+
+typedef std::vector<ChartHypothesis*> ChartArcList;
+
+/** a hypothesis in the hierarchical/syntax decoder.
+ * Contain a pointer to the current target phrase, a vector of previous hypos, and some scores
+ */
+class ChartHypothesis
+{
+  friend std::ostream& operator<<(std::ostream&, const ChartHypothesis&);
+//  friend class ChartKBestExtractor;
+
+protected:
+
+  boost::shared_ptr<ChartTranslationOption> m_transOpt;
+
+  Range m_currSourceWordsRange;
+  std::vector<const FFState*> m_ffStates; /*! stateful feature function states */
+  /*! sum of scores of this hypothesis, and previous hypotheses. Lazily initialised.  */
+  mutable boost::scoped_ptr<ScoreComponentCollection> m_scoreBreakdown;
+  mutable boost::scoped_ptr<ScoreComponentCollection> m_deltaScoreBreakdown;
+  ScoreComponentCollection m_currScoreBreakdown /*! scores for this hypothesis only */
+  ,m_lmNGram
+  ,m_lmPrefix;
+  float m_totalScore;
+
+  ChartArcList *m_arcList; /*! all arcs that end at the same trellis point as this hypothesis */
+  const ChartHypothesis *m_winningHypo;
+
+  std::vector<const ChartHypothesis*> m_prevHypos; // always sorted by source position?
+
+  ChartManager& m_manager;
+
+  unsigned m_id; /* pkoehn wants to log the order in which hypotheses were generated */
+
+  //! not implemented
+  ChartHypothesis();
+
+  //! not implemented
+  ChartHypothesis(const ChartHypothesis &copy);
+
+public:
+  ChartHypothesis(const ChartTranslationOptions &, const RuleCubeItem &item,
+                  ChartManager &manager);
+
+  //! only used by ChartKBestExtractor
+  ChartHypothesis(const ChartHypothesis &, const ChartKBestExtractor &);
+
+  ~ChartHypothesis();
+
+  unsigned GetId() const {
+    return m_id;
+  }
+
+  const ChartTranslationOption &GetTranslationOption() const {
+    return *m_transOpt;
+  }
+
+  //! Get the rule that created this hypothesis
+  const TargetPhrase &GetCurrTargetPhrase() const {
+    return m_transOpt->GetPhrase();
+  }
+
+  //! the source range that this hypothesis spans
+  const Range &GetCurrSourceRange() const {
+    return m_currSourceWordsRange;
+  }
+
+  //! the arc list when creating n-best lists
+  inline const ChartArcList* GetArcList() const {
+    return m_arcList;
+  }
+
+  //! the feature function states for a particular feature \param featureID
+  inline const FFState* GetFFState( size_t featureID ) const {
+    return m_ffStates[ featureID ];
+  }
+
+  //! reference back to the manager
+  inline const ChartManager& GetManager() const {
+    return m_manager;
+  }
+
+  void GetOutputPhrase(Phrase &outPhrase) const;
+  Phrase GetOutputPhrase() const;
+
+  // get leftmost/rightmost words only
+  // leftRightMost: 1=left, 2=right
+  void GetOutputPhrase(size_t leftRightMost, size_t numWords, Phrase &outPhrase) const;
+
+  void EvaluateWhenApplied();
+
+  void AddArc(ChartHypothesis *loserHypo);
+  void CleanupArcList();
+  void SetWinningHypo(const ChartHypothesis *hypo);
+
+  //! get the unweighted score for each feature function
+  const ScoreComponentCollection &GetScoreBreakdown() const {
+    // Note: never call this method before m_currScoreBreakdown is fully computed
+    if (!m_scoreBreakdown.get()) {
+      m_scoreBreakdown.reset(new ScoreComponentCollection());
+      // score breakdown from current translation rule
+      if (m_transOpt) {
+        m_scoreBreakdown->PlusEquals(GetTranslationOption().GetScores());
+      }
+      m_scoreBreakdown->PlusEquals(m_currScoreBreakdown);
+      // score breakdowns from prev hypos
+      for (std::vector<const ChartHypothesis*>::const_iterator iter = m_prevHypos.begin(); iter != m_prevHypos.end(); ++iter) {
+        const ChartHypothesis &prevHypo = **iter;
+        m_scoreBreakdown->PlusEquals(prevHypo.GetScoreBreakdown());
+      }
+    }
+    return *(m_scoreBreakdown.get());
+  }
+
+  //! get the unweighted score delta for each feature function
+  const ScoreComponentCollection &GetDeltaScoreBreakdown() const {
+    // Note: never call this method before m_currScoreBreakdown is fully computed
+    if (!m_deltaScoreBreakdown.get()) {
+      m_deltaScoreBreakdown.reset(new ScoreComponentCollection());
+      // score breakdown from current translation rule
+      if (m_transOpt) {
+        m_deltaScoreBreakdown->PlusEquals(GetTranslationOption().GetScores());
+      }
+      m_deltaScoreBreakdown->PlusEquals(m_currScoreBreakdown);
+      // delta: score breakdowns from prev hypos _not_ added
+    }
+    return *(m_deltaScoreBreakdown.get());
+  }
+
+  //! Get the weighted total score
+  float GetFutureScore() const {
+    // scores from current translation rule. eg. translation models & word penalty
+    return m_totalScore;
+  }
+
+  //! vector of previous hypotheses this hypo is built on
+  const std::vector<const ChartHypothesis*> &GetPrevHypos() const {
+    return m_prevHypos;
+  }
+
+  //! get a particular previous hypos
+  const ChartHypothesis* GetPrevHypo(size_t pos) const {
+    return m_prevHypos[pos];
+  }
+
+  //! get the constituency label that covers this hypo
+  const Word &GetTargetLHS() const {
+    return GetCurrTargetPhrase().GetTargetLHS();
+  }
+
+  //! get the best hypo in the arc list when doing n-best list creation. It's either this hypothesis, or the best hypo is this hypo is in the arc list
+  const ChartHypothesis* GetWinningHypothesis() const {
+    return m_winningHypo;
+  }
+
+  // for unordered_set in stack
+  size_t hash() const;
+  bool operator==(const ChartHypothesis& other) const;
+
+  TO_STRING();
+
+}; // class ChartHypothesis
+
+}
+

mosesdecoder/moses/ChartHypothesisCollection.cpp

@@ -0,0 +1,301 @@
+// $Id$
+// vim:tabstop=2
+/***********************************************************************
+ Moses - factored phrase-based language decoder
+ Copyright (C) 2010 Hieu Hoang
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ ***********************************************************************/
+
+#include <algorithm>
+#include "StaticData.h"
+#include "ChartHypothesisCollection.h"
+#include "ChartHypothesis.h"
+#include "ChartManager.h"
+#include "HypergraphOutput.h"
+#include "util/exception.hh"
+#include "parameters/AllOptions.h"
+
+using namespace std;
+using namespace Moses;
+
+namespace Moses
+{
+
+ChartHypothesisCollection::ChartHypothesisCollection(AllOptions const& opts)
+{
+  // const StaticData &staticData = StaticData::Instance();
+
+  m_beamWidth = opts.search.beam_width; // staticData.GetBeamWidth();
+  m_maxHypoStackSize = opts.search.stack_size; // staticData.options().search.stack_size;
+  m_nBestIsEnabled = opts.nbest.enabled; // staticData.options().nbest.enabled;
+  m_bestScore = -std::numeric_limits<float>::infinity();
+}
+
+ChartHypothesisCollection::~ChartHypothesisCollection()
+{
+  HCType::iterator iter;
+  for (iter = m_hypos.begin() ; iter != m_hypos.end() ; ++iter) {
+    ChartHypothesis *hypo = *iter;
+    delete hypo;
+  }
+  //RemoveAllInColl(m_hypos);
+}
+
+/** public function to add hypothesis to this collection.
+ * Returns false if equiv hypo exists in collection, otherwise returns true.
+ * Takes care of update arc list for n-best list creation.
+ * Will delete hypo if it exists - once this function is call don't delete hypothesis.
+ * \param hypo hypothesis to add
+ * \param manager pointer back to manager
+ */
+bool ChartHypothesisCollection::AddHypothesis(ChartHypothesis *hypo, ChartManager &manager)
+{
+  if (hypo->GetFutureScore() == - std::numeric_limits<float>::infinity()) {
+    manager.GetSentenceStats().AddDiscarded();
+    VERBOSE(3,"discarded, -inf score" << std::endl);
+    delete hypo;
+    return false;
+  }
+
+  if (hypo->GetFutureScore() < m_bestScore + m_beamWidth) {
+    // really bad score. don't bother adding hypo into collection
+    manager.GetSentenceStats().AddDiscarded();
+    VERBOSE(3,"discarded, too bad for stack" << std::endl);
+    delete hypo;
+    return false;
+  }
+
+  // over threshold, try to add to collection
+  std::pair<HCType::iterator, bool> addRet = Add(hypo, manager);
+
+  // does it have the same state as an existing hypothesis?
+  if (addRet.second) {
+    // nothing found. add to collection
+    return true;
+  }
+
+  // equiv hypo exists, recombine with other hypo
+  HCType::iterator &iterExisting = addRet.first;
+  ChartHypothesis *hypoExisting = *iterExisting;
+  UTIL_THROW_IF2(iterExisting == m_hypos.end(),
+                 "Adding a hypothesis should have returned a valid iterator");
+
+  //StaticData::Instance().GetSentenceStats().AddRecombination(*hypo, **iterExisting);
+
+  // found existing hypo with same target ending.
+  // keep the best 1
+  if (hypo->GetFutureScore() > hypoExisting->GetFutureScore()) {
+    // incoming hypo is better than the one we have
+    VERBOSE(3,"better than matching hyp " << hypoExisting->GetId() << ", recombining, ");
+    if (m_nBestIsEnabled) {
+      hypo->AddArc(hypoExisting);
+      Detach(iterExisting);
+    } else {
+      Remove(iterExisting);
+    }
+
+    bool added = Add(hypo, manager).second;
+    if (!added) {
+      iterExisting = m_hypos.find(hypo);
+      UTIL_THROW2("Offending hypo = " << **iterExisting);
+    }
+    return false;
+  } else {
+    // already storing the best hypo. discard current hypo
+    VERBOSE(3,"worse than matching hyp " << hypoExisting->GetId() << ", recombining" << std::endl)
+    if (m_nBestIsEnabled) {
+      hypoExisting->AddArc(hypo);
+    } else {
+      delete hypo;
+    }
+    return false;
+  }
+}
+
+/** add hypothesis to stack. Prune if necessary.
+ * Returns false if equiv hypo exists in collection, otherwise returns true, and the iterator that points to the place where the hypo was added
+ * \param hypo hypothesis to add
+ * \param manager pointer back to manager
+ */
+pair<ChartHypothesisCollection::HCType::iterator, bool> ChartHypothesisCollection::Add(ChartHypothesis *hypo, ChartManager &manager)
+{
+  std::pair<HCType::iterator, bool> ret = m_hypos.insert(hypo);
+  if (ret.second) {
+    // equiv hypo doesn't exists
+    VERBOSE(3,"added hyp to stack");
+
+    // Update best score, if this hypothesis is new best
+    if (hypo->GetFutureScore() > m_bestScore) {
+      VERBOSE(3,", best on stack");
+      m_bestScore = hypo->GetFutureScore();
+    }
+
+    // Prune only if stack is twice as big as needed (lazy pruning)
+    VERBOSE(3,", now size " << m_hypos.size());
+    if (m_hypos.size() > 2*m_maxHypoStackSize-1) {
+      PruneToSize(manager);
+    } else {
+      VERBOSE(3,std::endl);
+    }
+  }
+
+  return ret;
+}
+
+/** Remove hypothesis pointed to by iterator but DOES NOT delete the object.
+ * \param iter iterator to delete
+ */
+void ChartHypothesisCollection::Detach(const HCType::iterator &iter)
+{
+  m_hypos.erase(iter);
+}
+
+/** destroy iterator AND hypothesis pointed to by iterator. If in an object pool, takes care of that too
+ */
+void ChartHypothesisCollection::Remove(const HCType::iterator &iter)
+{
+  ChartHypothesis *h = *iter;
+  Detach(iter);
+  delete h;
+}
+
+/** prune number of hypo to a particular number of hypos, specified by m_maxHypoStackSize, according to score
+  * Don't prune of hypos have identical scores on the boundary, so occasionally number of hypo can remain above m_maxHypoStackSize.
+  * \param manager reference back to manager. Used for collecting stats
+ */
+void ChartHypothesisCollection::PruneToSize(ChartManager &manager)
+{
+  if (m_maxHypoStackSize == 0) return; // no limit
+
+  if (GetSize() > m_maxHypoStackSize) { // ok, if not over the limit
+    priority_queue<float> bestScores;
+
+    // push all scores to a heap
+    // (but never push scores below m_bestScore+m_beamWidth)
+    HCType::iterator iter = m_hypos.begin();
+    float score = 0;
+    while (iter != m_hypos.end()) {
+      ChartHypothesis *hypo = *iter;
+      score = hypo->GetFutureScore();
+      if (score > m_bestScore+m_beamWidth) {
+        bestScores.push(score);
+      }
+      ++iter;
+    }
+
+    // pop the top newSize scores (and ignore them, these are the scores of hyps that will remain)
+    //  ensure to never pop beyond heap size
+    size_t minNewSizeHeapSize = m_maxHypoStackSize > bestScores.size() ? bestScores.size() : m_maxHypoStackSize;
+    for (size_t i = 1 ; i < minNewSizeHeapSize ; i++)
+      bestScores.pop();
+
+    // and remember the threshold
+    float scoreThreshold = bestScores.top();
+
+    // delete all hypos under score threshold
+    iter = m_hypos.begin();
+    while (iter != m_hypos.end()) {
+      ChartHypothesis *hypo = *iter;
+      float score = hypo->GetFutureScore();
+      if (score < scoreThreshold) {
+        HCType::iterator iterRemove = iter++;
+        Remove(iterRemove);
+        manager.GetSentenceStats().AddPruning();
+      } else {
+        ++iter;
+      }
+    }
+    VERBOSE(3,", pruned to size " << m_hypos.size() << endl);
+
+    IFVERBOSE(3) {
+      TRACE_ERR("stack now contains: ");
+      for(iter = m_hypos.begin(); iter != m_hypos.end(); iter++) {
+        ChartHypothesis *hypo = *iter;
+        TRACE_ERR( hypo->GetId() << " (" << hypo->GetFutureScore() << ") ");
+      }
+      TRACE_ERR( endl);
+    }
+
+    // desperation pruning
+    if (m_hypos.size() > m_maxHypoStackSize * 2) {
+      std::vector<ChartHypothesis*> hyposOrdered;
+
+      // sort hypos
+      std::copy(m_hypos.begin(), m_hypos.end(), std::inserter(hyposOrdered, hyposOrdered.end()));
+      std::sort(hyposOrdered.begin(), hyposOrdered.end(), ChartHypothesisScoreOrderer());
+
+      //keep only |size|. delete the rest
+      std::vector<ChartHypothesis*>::iterator iter;
+      for (iter = hyposOrdered.begin() + (m_maxHypoStackSize * 2); iter != hyposOrdered.end(); ++iter) {
+        ChartHypothesis *hypo = *iter;
+        HCType::iterator iterFindHypo = m_hypos.find(hypo);
+        UTIL_THROW_IF2(iterFindHypo == m_hypos.end(),
+                       "Adding a hypothesis should have returned a valid iterator");
+
+        Remove(iterFindHypo);
+      }
+    }
+  }
+}
+
+//! sort hypothses  by descending score. Put these hypos into a vector m_hyposOrdered to be returned by function GetSortedHypotheses()
+void ChartHypothesisCollection::SortHypotheses()
+{
+  UTIL_THROW_IF2(!m_hyposOrdered.empty(), "Hypotheses already sorted");
+  if (!m_hypos.empty()) {
+    // done everything for this cell.
+    // sort
+    // put into vec
+    m_hyposOrdered.reserve(m_hypos.size());
+    std::copy(m_hypos.begin(), m_hypos.end(), back_inserter(m_hyposOrdered));
+    std::sort(m_hyposOrdered.begin(), m_hyposOrdered.end(), ChartHypothesisScoreOrderer());
+  }
+}
+
+//! Call CleanupArcList() for each main hypo in collection
+void ChartHypothesisCollection::CleanupArcList()
+{
+  HCType::iterator iter;
+  for (iter = m_hypos.begin() ; iter != m_hypos.end() ; ++iter) {
+    ChartHypothesis *mainHypo = *iter;
+    mainHypo->CleanupArcList();
+  }
+}
+
+/** Return all hypos, and all hypos in the arclist, in order to create the output searchgraph, ie. the hypergraph. The output is the debug hypo information.
+ * @todo this is a useful function. Make sure it outputs everything required, especially scores.
+ * \param translationId unique, contiguous id for the input sentence
+ * \param outputSearchGraphStream stream to output the info to
+ * \param reachable @todo don't know
+ */
+void ChartHypothesisCollection::WriteSearchGraph(const ChartSearchGraphWriter& writer, const std::map<unsigned, bool> &reachable) const
+{
+  writer.WriteHypos(*this,reachable);
+}
+
+std::ostream& operator<<(std::ostream &out, const ChartHypothesisCollection &coll)
+{
+  HypoList::const_iterator iterInside;
+  for (iterInside = coll.m_hyposOrdered.begin(); iterInside != coll.m_hyposOrdered.end(); ++iterInside) {
+    const ChartHypothesis &hypo = **iterInside;
+    out << hypo << endl;
+  }
+
+  return out;
+}
+
+
+} // namespace

mosesdecoder/moses/ChartHypothesisCollection.h

@@ -0,0 +1,108 @@
+// $Id$
+// vim:tabstop=2
+/***********************************************************************
+ Moses - factored phrase-based language decoder
+ Copyright (C) 2010 Hieu Hoang
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ ***********************************************************************/
+#pragma once
+
+#include <set>
+#include "ChartHypothesis.h"
+#include "RuleCube.h"
+
+
+namespace Moses
+{
+
+class ChartSearchGraphWriter;
+struct AllOptions;
+
+//! functor to compare (chart) hypotheses by (descending) score
+class ChartHypothesisScoreOrderer
+{
+public:
+  bool operator()(const ChartHypothesis* hypoA, const ChartHypothesis* hypoB) const {
+    return hypoA->GetFutureScore() > hypoB->GetFutureScore();
+  }
+};
+
+/** Contains a set of unique hypos that have the same HS non-term.
+  * ie. 1 of these for each target LHS in each cell
+  */
+class ChartHypothesisCollection
+{
+  friend std::ostream& operator<<(std::ostream&, const ChartHypothesisCollection&);
+
+protected:
+  //typedef std::set<ChartHypothesis*, ChartHypothesisRecombinationOrderer> HCType;
+  typedef boost::unordered_set< ChartHypothesis*, UnorderedComparer<ChartHypothesis>, UnorderedComparer<ChartHypothesis> > HCType;
+  HCType m_hypos;
+  HypoList m_hyposOrdered;
+
+  float m_bestScore; /**< score of the best hypothesis in collection */
+  float m_beamWidth; /**< minimum score due to threashold pruning */
+  size_t m_maxHypoStackSize; /**< maximum number of hypothesis allowed in this stack */
+  bool m_nBestIsEnabled; /**< flag to determine whether to keep track of old arcs */
+
+  std::pair<HCType::iterator, bool> Add(ChartHypothesis *hypo, ChartManager &manager);
+
+public:
+  typedef HCType::iterator iterator;
+  typedef HCType::const_iterator const_iterator;
+  //! iterators
+  const_iterator begin() const {
+    return m_hypos.begin();
+  }
+  const_iterator end() const {
+    return m_hypos.end();
+  }
+
+  ChartHypothesisCollection(AllOptions const& opts);
+  ~ChartHypothesisCollection();
+  bool AddHypothesis(ChartHypothesis *hypo, ChartManager &manager);
+
+  void Detach(const HCType::iterator &iter);
+  void Remove(const HCType::iterator &iter);
+
+  void PruneToSize(ChartManager &manager);
+
+  size_t GetSize() const {
+    return m_hypos.size();
+  }
+  size_t GetHypo() const {
+    return m_hypos.size();
+  }
+
+  void SortHypotheses();
+  void CleanupArcList();
+
+  //! return vector of hypothesis that has been sorted by score
+  const HypoList &GetSortedHypotheses() const {
+    return m_hyposOrdered;
+  }
+
+  //! return the best total score of all hypos in this collection
+  float GetBestScore() const {
+    return m_bestScore;
+  }
+
+  void WriteSearchGraph(const ChartSearchGraphWriter& writer, const std::map<unsigned,bool> &reachable) const;
+
+};
+
+} // namespace
+

mosesdecoder/moses/ChartKBestExtractor.cpp

@@ -0,0 +1,332 @@
+/***********************************************************************
+ Moses - statistical machine translation system
+ Copyright (C) 2006-2014 University of Edinburgh
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+***********************************************************************/
+
+#include "ChartKBestExtractor.h"
+
+#include "ChartHypothesis.h"
+#include "ScoreComponentCollection.h"
+#include "StaticData.h"
+
+#include <boost/scoped_ptr.hpp>
+
+#include <vector>
+
+using namespace std;
+
+namespace Moses
+{
+
+// Extract the k-best list from the search graph.
+void ChartKBestExtractor::Extract(
+  const std::vector<const ChartHypothesis*> &topLevelHypos, std::size_t k,
+  KBestVec &kBestList)
+{
+  kBestList.clear();
+  if (topLevelHypos.empty()) {
+    return;
+  }
+
+  // Create a new ChartHypothesis object, supremeHypo, that has the best
+  // top-level hypothesis as its predecessor and has the same score.
+  std::vector<const ChartHypothesis*>::const_iterator p = topLevelHypos.begin();
+  const ChartHypothesis &bestTopLevelHypo = **p;
+  boost::scoped_ptr<ChartHypothesis> supremeHypo(
+    new ChartHypothesis(bestTopLevelHypo, *this));
+
+  // Do the same for each alternative top-level hypothesis, but add the new
+  // ChartHypothesis objects as arcs from supremeHypo, as if they had been
+  // recombined.
+  for (++p; p != topLevelHypos.end(); ++p) {
+    // Check that the first item in topLevelHypos really was the best.
+    UTIL_THROW_IF2((*p)->GetFutureScore() > bestTopLevelHypo.GetFutureScore(),
+                   "top-level hypotheses are not correctly sorted");
+    // Note: there's no need for a smart pointer here: supremeHypo will take
+    // ownership of altHypo.
+    ChartHypothesis *altHypo = new ChartHypothesis(**p, *this);
+    supremeHypo->AddArc(altHypo);
+  }
+
+  // Create the target vertex then lazily fill its k-best list.
+  boost::shared_ptr<Vertex> targetVertex = FindOrCreateVertex(*supremeHypo);
+  LazyKthBest(*targetVertex, k, k);
+
+  // Copy the k-best list from the target vertex, but drop the top edge from
+  // each derivation.
+  kBestList.reserve(targetVertex->kBestList.size());
+  for (std::vector<boost::weak_ptr<Derivation> >::const_iterator
+       q = targetVertex->kBestList.begin();
+       q != targetVertex->kBestList.end(); ++q) {
+    const boost::shared_ptr<Derivation> d(*q);
+    assert(d);
+    assert(d->subderivations.size() == 1);
+    kBestList.push_back(d->subderivations[0]);
+  }
+}
+
+// Generate the target-side yield of the derivation d.
+Phrase ChartKBestExtractor::GetOutputPhrase(const Derivation &d)
+{
+  FactorType placeholderFactor = StaticData::Instance().options()->input.placeholder_factor;
+
+  Phrase ret(ARRAY_SIZE_INCR);
+
+  const ChartHypothesis &hypo = d.edge.head->hypothesis;
+  const TargetPhrase &phrase = hypo.GetCurrTargetPhrase();
+  const AlignmentInfo::NonTermIndexMap &nonTermIndexMap =
+    phrase.GetAlignNonTerm().GetNonTermIndexMap();
+  for (std::size_t pos = 0; pos < phrase.GetSize(); ++pos) {
+    const Word &word = phrase.GetWord(pos);
+    if (word.IsNonTerminal()) {
+      std::size_t nonTermInd = nonTermIndexMap[pos];
+      const Derivation &subderivation = *d.subderivations[nonTermInd];
+      Phrase subPhrase = GetOutputPhrase(subderivation);
+      ret.Append(subPhrase);
+    } else {
+      ret.AddWord(word);
+      if (placeholderFactor == NOT_FOUND) {
+        continue;
+      }
+      std::set<std::size_t> sourcePosSet =
+        phrase.GetAlignTerm().GetAlignmentsForTarget(pos);
+      if (sourcePosSet.size() == 1) {
+        const std::vector<const Word*> *ruleSourceFromInputPath =
+          hypo.GetTranslationOption().GetSourceRuleFromInputPath();
+        UTIL_THROW_IF2(ruleSourceFromInputPath == NULL,
+                       "Source Words in of the rules hasn't been filled out");
+        std::size_t sourcePos = *sourcePosSet.begin();
+        const Word *sourceWord = ruleSourceFromInputPath->at(sourcePos);
+        UTIL_THROW_IF2(sourceWord == NULL,
+                       "Null source word at position " << sourcePos);
+        const Factor *factor = sourceWord->GetFactor(placeholderFactor);
+        if (factor) {
+          ret.Back()[0] = factor;
+        }
+      }
+    }
+  }
+
+  return ret;
+}
+
+// Generate the score breakdown of the derivation d.
+boost::shared_ptr<ScoreComponentCollection>
+ChartKBestExtractor::GetOutputScoreBreakdown(const Derivation &d)
+{
+  const ChartHypothesis &hypo = d.edge.head->hypothesis;
+  boost::shared_ptr<ScoreComponentCollection> scoreBreakdown(new ScoreComponentCollection());
+  scoreBreakdown->PlusEquals(hypo.GetDeltaScoreBreakdown());
+  const TargetPhrase &phrase = hypo.GetCurrTargetPhrase();
+  const AlignmentInfo::NonTermIndexMap &nonTermIndexMap =
+    phrase.GetAlignNonTerm().GetNonTermIndexMap();
+  for (std::size_t pos = 0; pos < phrase.GetSize(); ++pos) {
+    const Word &word = phrase.GetWord(pos);
+    if (word.IsNonTerminal()) {
+      std::size_t nonTermInd = nonTermIndexMap[pos];
+      const Derivation &subderivation = *d.subderivations[nonTermInd];
+      scoreBreakdown->PlusEquals(*GetOutputScoreBreakdown(subderivation));
+    }
+  }
+
+  return scoreBreakdown;
+}
+
+// Generate the target tree of the derivation d.
+TreePointer ChartKBestExtractor::GetOutputTree(const Derivation &d)
+{
+  const ChartHypothesis &hypo = d.edge.head->hypothesis;
+  const TargetPhrase &phrase = hypo.GetCurrTargetPhrase();
+  if (const PhraseProperty *property = phrase.GetProperty("Tree")) {
+    const std::string *tree = property->GetValueString();
+    TreePointer mytree (boost::make_shared<InternalTree>(*tree));
+
+    //get subtrees (in target order)
+    std::vector<TreePointer> previous_trees;
+    for (size_t pos = 0; pos < phrase.GetSize(); ++pos) {
+      const Word &word = phrase.GetWord(pos);
+      if (word.IsNonTerminal()) {
+        size_t nonTermInd = phrase.GetAlignNonTerm().GetNonTermIndexMap()[pos];
+        const Derivation &subderivation = *d.subderivations[nonTermInd];
+        const TreePointer prev_tree = GetOutputTree(subderivation);
+        previous_trees.push_back(prev_tree);
+      }
+    }
+
+    mytree->Combine(previous_trees);
+    mytree->Unbinarize();
+    return mytree;
+  } else {
+    UTIL_THROW2("Error: k-best tree output active, but no internal tree structure found");
+  }
+}
+
+// Create an unweighted hyperarc corresponding to the given ChartHypothesis.
+ChartKBestExtractor::UnweightedHyperarc ChartKBestExtractor::CreateEdge(
+  const ChartHypothesis &h)
+{
+  UnweightedHyperarc edge;
+  edge.head = FindOrCreateVertex(h);
+  const std::vector<const ChartHypothesis*> &prevHypos = h.GetPrevHypos();
+  edge.tail.resize(prevHypos.size());
+  for (std::size_t i = 0; i < prevHypos.size(); ++i) {
+    const ChartHypothesis *prevHypo = prevHypos[i];
+    edge.tail[i] = FindOrCreateVertex(*prevHypo);
+  }
+  return edge;
+}
+
+// Look for the vertex corresponding to a given ChartHypothesis, creating
+// a new one if necessary.
+boost::shared_ptr<ChartKBestExtractor::Vertex>
+ChartKBestExtractor::FindOrCreateVertex(const ChartHypothesis &h)
+{
+  VertexMap::value_type element(&h, boost::shared_ptr<Vertex>());
+  std::pair<VertexMap::iterator, bool> p = m_vertexMap.insert(element);
+  boost::shared_ptr<Vertex> &sp = p.first->second;
+  if (!p.second) {
+    return sp;  // Vertex was already in m_vertexMap.
+  }
+  sp.reset(new Vertex(h));
+  // Create the 1-best derivation and add it to the vertex's kBestList.
+  UnweightedHyperarc bestEdge;
+  bestEdge.head = sp;
+  const std::vector<const ChartHypothesis*> &prevHypos = h.GetPrevHypos();
+  bestEdge.tail.resize(prevHypos.size());
+  for (std::size_t i = 0; i < prevHypos.size(); ++i) {
+    const ChartHypothesis *prevHypo = prevHypos[i];
+    bestEdge.tail[i] = FindOrCreateVertex(*prevHypo);
+  }
+  boost::shared_ptr<Derivation> bestDerivation(new Derivation(bestEdge));
+#ifndef NDEBUG
+  std::pair<DerivationSet::iterator, bool> q =
+#endif
+    m_derivations.insert(bestDerivation);
+  assert(q.second);
+  sp->kBestList.push_back(bestDerivation);
+  return sp;
+}
+
+// Create the 1-best derivation for each edge in BS(v) (except the best one)
+// and add it to v's candidate queue.
+void ChartKBestExtractor::GetCandidates(Vertex &v, std::size_t k)
+{
+  // Create derivations for all of v's incoming edges except the best.  This
+  // means everything in v.hypothesis.GetArcList() and not the edge defined
+  // by v.hypothesis itself.  The 1-best derivation for that edge will already
+  // have been created.
+  const ChartArcList *arcList = v.hypothesis.GetArcList();
+  if (arcList) {
+    for (std::size_t i = 0; i < arcList->size(); ++i) {
+      const ChartHypothesis &recombinedHypo = *(*arcList)[i];
+      boost::shared_ptr<Vertex> w = FindOrCreateVertex(recombinedHypo);
+      assert(w->kBestList.size() == 1);
+      v.candidates.push(w->kBestList[0]);
+    }
+  }
+}
+
+// Lazily fill v's k-best list.
+void ChartKBestExtractor::LazyKthBest(Vertex &v, std::size_t k,
+                                      std::size_t globalK)
+{
+  // If this is the first visit to vertex v then initialize the priority queue.
+  if (v.visited == false) {
+    // The 1-best derivation should already be in v's k-best list.
+    assert(v.kBestList.size() == 1);
+    // Initialize v's priority queue.
+    GetCandidates(v, globalK);
+    v.visited = true;
+  }
+  // Add derivations to the k-best list until it contains k or there are none
+  // left to add.
+  while (v.kBestList.size() < k) {
+    assert(!v.kBestList.empty());
+    // Update the priority queue by adding the successors of the last
+    // derivation (unless they've been seen before).
+    boost::shared_ptr<Derivation> d(v.kBestList.back());
+    LazyNext(v, *d, globalK);
+    // Check if there are any derivations left in the queue.
+    if (v.candidates.empty()) {
+      break;
+    }
+    // Get the next best derivation and delete it from the queue.
+    boost::weak_ptr<Derivation> next = v.candidates.top();
+    v.candidates.pop();
+    // Add it to the k-best list.
+    v.kBestList.push_back(next);
+  }
+}
+
+// Create the neighbours of Derivation d and add them to v's candidate queue.
+void ChartKBestExtractor::LazyNext(Vertex &v, const Derivation &d,
+                                   std::size_t globalK)
+{
+  for (std::size_t i = 0; i < d.edge.tail.size(); ++i) {
+    Vertex &pred = *d.edge.tail[i];
+    // Ensure that pred's k-best list contains enough derivations.
+    std::size_t k = d.backPointers[i] + 2;
+    LazyKthBest(pred, k, globalK);
+    if (pred.kBestList.size() < k) {
+      // pred's derivations have been exhausted.
+      continue;
+    }
+    // Create the neighbour.
+    boost::shared_ptr<Derivation> next(new Derivation(d, i));
+    // Check if it has been created before.
+    std::pair<DerivationSet::iterator, bool> p = m_derivations.insert(next);
+    if (p.second) {
+      v.candidates.push(next);  // Haven't previously seen it.
+    }
+  }
+}
+
+// Construct the 1-best Derivation that ends at edge e.
+ChartKBestExtractor::Derivation::Derivation(const UnweightedHyperarc &e)
+{
+  edge = e;
+  std::size_t arity = edge.tail.size();
+  backPointers.resize(arity, 0);
+  subderivations.reserve(arity);
+  for (std::size_t i = 0; i < arity; ++i) {
+    const Vertex &pred = *edge.tail[i];
+    assert(pred.kBestList.size() >= 1);
+    boost::shared_ptr<Derivation> sub(pred.kBestList[0]);
+    subderivations.push_back(sub);
+  }
+  score = edge.head->hypothesis.GetFutureScore();
+}
+
+// Construct a Derivation that neighbours an existing Derivation.
+ChartKBestExtractor::Derivation::Derivation(const Derivation &d, std::size_t i)
+{
+  edge.head = d.edge.head;
+  edge.tail = d.edge.tail;
+  backPointers = d.backPointers;
+  subderivations = d.subderivations;
+  std::size_t j = ++backPointers[i];
+  score = d.score;
+  // Deduct the score of the old subderivation.
+  score -= subderivations[i]->score;
+  // Update the subderivation pointer.
+  boost::shared_ptr<Derivation> newSub(edge.tail[i]->kBestList[j]);
+  subderivations[i] = newSub;
+  // Add the score of the new subderivation.
+  score += subderivations[i]->score;
+}
+
+}  // namespace Moses

mosesdecoder/moses/ChartKBestExtractor.h

@@ -0,0 +1,132 @@
+/***********************************************************************
+ Moses - statistical machine translation system
+ Copyright (C) 2006-2014 University of Edinburgh
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+***********************************************************************/
+
+#pragma once
+
+#include <cassert>
+#include "ChartHypothesis.h"
+#include "ScoreComponentCollection.h"
+#include "FF/InternalTree.h"
+
+#include <boost/unordered_set.hpp>
+#include <boost/weak_ptr.hpp>
+#include <boost/shared_ptr.hpp>
+
+#include <queue>
+#include <vector>
+
+namespace Moses
+{
+
+// k-best list extractor that implements algorithm 3 from this paper:
+//
+//  Liang Huang and David Chiang
+//  "Better k-best parsing"
+//  In Proceedings of IWPT 2005
+//
+class ChartKBestExtractor
+{
+public:
+  struct Vertex;
+
+  struct UnweightedHyperarc {
+    boost::shared_ptr<Vertex> head;
+    std::vector<boost::shared_ptr<Vertex> > tail;
+  };
+
+  struct Derivation {
+    Derivation(const UnweightedHyperarc &);
+    Derivation(const Derivation &, std::size_t);
+
+    UnweightedHyperarc edge;
+    std::vector<std::size_t> backPointers;
+    std::vector<boost::shared_ptr<Derivation> > subderivations;
+    float score;
+  };
+
+  struct DerivationOrderer {
+    bool operator()(const boost::weak_ptr<Derivation> &d1,
+                    const boost::weak_ptr<Derivation> &d2) const {
+      boost::shared_ptr<Derivation> s1(d1);
+      boost::shared_ptr<Derivation> s2(d2);
+      return s1->score < s2->score;
+    }
+  };
+
+  struct Vertex {
+    typedef std::priority_queue<boost::weak_ptr<Derivation>,
+            std::vector<boost::weak_ptr<Derivation> >,
+            DerivationOrderer> DerivationQueue;
+
+    Vertex(const ChartHypothesis &h) : hypothesis(h), visited(false) {}
+
+    const ChartHypothesis &hypothesis;
+    std::vector<boost::weak_ptr<Derivation> > kBestList;
+    DerivationQueue candidates;
+    bool visited;
+  };
+
+  typedef std::vector<boost::shared_ptr<Derivation> > KBestVec;
+
+  // Extract the k-best list from the search hypergraph given the full, sorted
+  // list of top-level vertices.
+  void Extract(const std::vector<const ChartHypothesis*> &topHypos,
+               std::size_t k, KBestVec &);
+
+  static Phrase GetOutputPhrase(const Derivation &);
+  static boost::shared_ptr<ScoreComponentCollection> GetOutputScoreBreakdown(const Derivation &);
+  static TreePointer GetOutputTree(const Derivation &);
+
+private:
+  typedef boost::unordered_map<const ChartHypothesis *,
+          boost::shared_ptr<Vertex> > VertexMap;
+
+  struct DerivationHasher {
+    std::size_t operator()(const boost::shared_ptr<Derivation> &d) const {
+      std::size_t seed = 0;
+      boost::hash_combine(seed, d->edge.head);
+      boost::hash_combine(seed, d->edge.tail);
+      boost::hash_combine(seed, d->backPointers);
+      return seed;
+    }
+  };
+
+  struct DerivationEqualityPred {
+    bool operator()(const boost::shared_ptr<Derivation> &d1,
+                    const boost::shared_ptr<Derivation> &d2) const {
+      return d1->edge.head == d2->edge.head &&
+             d1->edge.tail == d2->edge.tail &&
+             d1->backPointers == d2->backPointers;
+    }
+  };
+
+  typedef boost::unordered_set<boost::shared_ptr<Derivation>, DerivationHasher,
+          DerivationEqualityPred> DerivationSet;
+
+  UnweightedHyperarc CreateEdge(const ChartHypothesis &);
+  boost::shared_ptr<Vertex> FindOrCreateVertex(const ChartHypothesis &);
+  void GetCandidates(Vertex &, std::size_t);
+  void LazyKthBest(Vertex &, std::size_t, std::size_t);
+  void LazyNext(Vertex &, const Derivation &, std::size_t);
+
+  VertexMap m_vertexMap;
+  DerivationSet m_derivations;
+};
+
+}  // namespace Moses

mosesdecoder/moses/ChartManager.cpp

@@ -0,0 +1,867 @@
+// $Id$
+// vim:tabstop=2
+/***********************************************************************
+ Moses - factored phrase-based language decoder
+ Copyright (C) 2010 Hieu Hoang
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ ***********************************************************************/
+
+#include <cstdio>
+#include "ChartManager.h"
+#include "ChartCell.h"
+#include "ChartHypothesis.h"
+#include "ChartKBestExtractor.h"
+#include "ChartTranslationOptions.h"
+#include "HypergraphOutput.h"
+#include "StaticData.h"
+#include "DecodeStep.h"
+#include "TreeInput.h"
+#include "moses/FF/StatefulFeatureFunction.h"
+#include "moses/FF/WordPenaltyProducer.h"
+#include "moses/OutputCollector.h"
+#include "moses/ChartKBestExtractor.h"
+#include "moses/HypergraphOutput.h"
+#include "moses/TranslationTask.h"
+
+using namespace std;
+
+namespace Moses
+{
+
+/* constructor. Initialize everything prior to decoding a particular sentence.
+ * \param source the sentence to be decoded
+ * \param system which particular set of models to use.
+ */
+ChartManager::ChartManager(ttasksptr const& ttask)
+  : BaseManager(ttask)
+  , m_hypoStackColl(m_source, *this)
+  , m_start(clock())
+  , m_hypothesisId(0)
+  , m_parser(ttask, m_hypoStackColl)
+  , m_translationOptionList(ttask->options()->syntax.rule_limit, m_source)
+{ }
+
+ChartManager::~ChartManager()
+{
+  clock_t end = clock();
+  float et = (end - m_start);
+  et /= (float)CLOCKS_PER_SEC;
+  VERBOSE(1, "Translation took " << et << " seconds" << endl);
+
+}
+
+//! decode the sentence. This contains the main laps. Basically, the CKY++ algorithm
+void ChartManager::Decode()
+{
+
+  VERBOSE(1,"Translating: " << m_source << endl);
+
+  ResetSentenceStats(m_source);
+
+  VERBOSE(2,"Decoding: " << endl);
+  //ChartHypothesis::ResetHypoCount();
+
+  AddXmlChartOptions();
+
+  // MAIN LOOP
+  size_t size = m_source.GetSize();
+  for (int startPos = size-1; startPos >= 0; --startPos) {
+    for (size_t width = 1; width <= size-startPos; ++width) {
+      size_t endPos = startPos + width - 1;
+      Range range(startPos, endPos);
+
+      // create trans opt
+      m_translationOptionList.Clear();
+      m_parser.Create(range, m_translationOptionList);
+      m_translationOptionList.ApplyThreshold(options()->search.trans_opt_threshold);
+
+      const InputPath &inputPath = m_parser.GetInputPath(range);
+      m_translationOptionList.EvaluateWithSourceContext(m_source, inputPath);
+
+      // decode
+      ChartCell &cell = m_hypoStackColl.Get(range);
+      cell.Decode(m_translationOptionList, m_hypoStackColl);
+
+      m_translationOptionList.Clear();
+      cell.PruneToSize();
+      cell.CleanupArcList();
+      cell.SortHypotheses();
+    }
+  }
+
+  IFVERBOSE(1) {
+
+    for (size_t startPos = 0; startPos < size; ++startPos) {
+      cerr.width(3);
+      cerr << startPos << " ";
+    }
+    cerr << endl;
+    for (size_t width = 1; width <= size; width++) {
+      for( size_t space = 0; space < width-1; space++ ) {
+        cerr << "  ";
+      }
+      for (size_t startPos = 0; startPos <= size-width; ++startPos) {
+        Range range(startPos, startPos+width-1);
+        cerr.width(3);
+        cerr << m_hypoStackColl.Get(range).GetSize() << " ";
+      }
+      cerr << endl;
+    }
+  }
+}
+
+/** add specific translation options and hypotheses according to the XML override translation scheme.
+ *  Doesn't seem to do anything about walls and zones.
+ *  @todo check walls & zones. Check that the implementation doesn't leak, xml options sometimes does if you're not careful
+ */
+void ChartManager::AddXmlChartOptions()
+{
+  const std::vector <ChartTranslationOptions*> xmlChartOptionsList
+  = m_source.GetXmlChartTranslationOptions();
+  IFVERBOSE(2) {
+    cerr << "AddXmlChartOptions " << xmlChartOptionsList.size() << endl;
+  }
+  if (xmlChartOptionsList.size() == 0) return;
+
+  typedef std::vector<ChartTranslationOptions*>::const_iterator citer;
+  for(citer i = xmlChartOptionsList.begin(); i != xmlChartOptionsList.end(); ++i) {
+    ChartTranslationOptions* opt = *i;
+
+    const Range &range = opt->GetSourceWordsRange();
+
+    RuleCubeItem* item = new RuleCubeItem( *opt, m_hypoStackColl );
+    ChartHypothesis* hypo = new ChartHypothesis(*opt, *item, *this);
+    hypo->EvaluateWhenApplied();
+
+
+    ChartCell &cell = m_hypoStackColl.Get(range);
+    cell.AddHypothesis(hypo);
+  }
+}
+
+//! get best complete translation from the top chart cell.
+const ChartHypothesis *ChartManager::GetBestHypothesis() const
+{
+  size_t size = m_source.GetSize();
+
+  if (size == 0) // empty source
+    return NULL;
+  else {
+    Range range(0, size-1);
+    const ChartCell &lastCell = m_hypoStackColl.Get(range);
+    return lastCell.GetBestHypothesis();
+  }
+}
+
+/** Calculate the n-best paths through the output hypergraph.
+ * Return the list of paths with the variable ret
+ * \param n how may paths to return
+ * \param ret return argument
+ * \param onlyDistinct whether to check for distinct output sentence or not (default - don't check, just return top n-paths)
+ */
+void ChartManager::CalcNBest(
+  std::size_t n,
+  std::vector<boost::shared_ptr<ChartKBestExtractor::Derivation> > &nBestList,
+  bool onlyDistinct) const
+{
+  nBestList.clear();
+  if (n == 0 || m_source.GetSize() == 0) {
+    return;
+  }
+
+  // Get the list of top-level hypotheses, sorted by score.
+  Range range(0, m_source.GetSize()-1);
+  const ChartCell &lastCell = m_hypoStackColl.Get(range);
+  boost::scoped_ptr<const std::vector<const ChartHypothesis*> > topLevelHypos(
+    lastCell.GetAllSortedHypotheses());
+  if (!topLevelHypos) {
+    return;
+  }
+
+  ChartKBestExtractor extractor;
+
+  if (!onlyDistinct) {
+    // Return the n-best list as is, including duplicate translations.
+    extractor.Extract(*topLevelHypos, n, nBestList);
+    return;
+  }
+
+  // Determine how many derivations to extract.  If the n-best list is
+  // restricted to distinct translations then this limit should be bigger
+  // than n.  The n-best factor determines how much bigger the limit should be,
+  // with 0 being 'unlimited.'  This actually sets a large-ish limit in case
+  // too many translations are identical.
+  const std::size_t nBestFactor = options()->nbest.factor;
+  std::size_t numDerivations = (nBestFactor == 0) ? n*1000 : n*nBestFactor;
+
+  // Extract the derivations.
+  ChartKBestExtractor::KBestVec bigList;
+  bigList.reserve(numDerivations);
+  extractor.Extract(*topLevelHypos, numDerivations, bigList);
+
+  // Copy derivations into nBestList, skipping ones with repeated translations.
+  std::set<Phrase> distinct;
+  for (ChartKBestExtractor::KBestVec::const_iterator p = bigList.begin();
+       nBestList.size() < n && p != bigList.end(); ++p) {
+    boost::shared_ptr<ChartKBestExtractor::Derivation> derivation = *p;
+    Phrase translation = ChartKBestExtractor::GetOutputPhrase(*derivation);
+    if (distinct.insert(translation).second) {
+      nBestList.push_back(derivation);
+    }
+  }
+}
+
+void ChartManager::WriteSearchGraph(const ChartSearchGraphWriter& writer) const
+{
+
+  size_t size = m_source.GetSize();
+
+  // which hypotheses are reachable?
+  std::map<unsigned,bool> reachable;
+  Range fullRange(0, size-1);
+  const ChartCell &lastCell = m_hypoStackColl.Get(fullRange);
+  const ChartHypothesis *hypo = lastCell.GetBestHypothesis();
+
+  if (hypo == NULL) {
+    // no hypothesis
+    return;
+  }
+  size_t winners = 0;
+  size_t losers = 0;
+
+  FindReachableHypotheses( hypo, reachable, &winners, &losers);
+  writer.WriteHeader(winners, losers);
+
+  for (size_t width = 1; width <= size; ++width) {
+    for (size_t startPos = 0; startPos <= size-width; ++startPos) {
+      size_t endPos = startPos + width - 1;
+      Range range(startPos, endPos);
+      TRACE_ERR(" " << range << "=");
+
+      const ChartCell &cell = m_hypoStackColl.Get(range);
+      cell.WriteSearchGraph(writer, reachable);
+    }
+  }
+}
+
+void ChartManager::FindReachableHypotheses(
+  const ChartHypothesis *hypo, std::map<unsigned,bool> &reachable, size_t* winners, size_t* losers) const
+{
+  // do not recurse, if already visited
+  if (reachable.find(hypo->GetId()) != reachable.end()) {
+    return;
+  }
+
+  // recurse
+  reachable[ hypo->GetId() ] = true;
+  if (hypo->GetWinningHypothesis() == hypo) {
+    (*winners)++;
+  } else {
+    (*losers)++;
+  }
+  const std::vector<const ChartHypothesis*> &previous = hypo->GetPrevHypos();
+  for(std::vector<const ChartHypothesis*>::const_iterator i = previous.begin(); i != previous.end(); ++i) {
+    FindReachableHypotheses( *i, reachable, winners, losers );
+  }
+
+  // also loop over recombined hypotheses (arcs)
+  const ChartArcList *arcList = hypo->GetArcList();
+  if (arcList) {
+    ChartArcList::const_iterator iterArc;
+    for (iterArc = arcList->begin(); iterArc != arcList->end(); ++iterArc) {
+      const ChartHypothesis &arc = **iterArc;
+      FindReachableHypotheses( &arc, reachable, winners, losers );
+    }
+  }
+}
+
+void
+ChartManager::
+OutputSearchGraphAsHypergraph(std::ostream& out) const
+{
+  ChartSearchGraphWriterHypergraph writer(options(), &out);
+  WriteSearchGraph(writer);
+}
+
+void ChartManager::OutputSearchGraphMoses(std::ostream &outputSearchGraphStream) const
+{
+  ChartSearchGraphWriterMoses writer(options(), &outputSearchGraphStream,
+                                     m_source.GetTranslationId());
+  WriteSearchGraph(writer);
+}
+
+void ChartManager::OutputBest(OutputCollector *collector) const
+{
+  const ChartHypothesis *bestHypo = GetBestHypothesis();
+  if (collector && bestHypo) {
+    const size_t translationId = m_source.GetTranslationId();
+    const ChartHypothesis *bestHypo = GetBestHypothesis();
+    OutputBestHypo(collector, bestHypo, translationId);
+  }
+}
+
+void ChartManager::OutputNBest(OutputCollector *collector) const
+{
+  size_t nBestSize = options()->nbest.nbest_size;
+  if (nBestSize > 0) {
+    const size_t translationId = m_source.GetTranslationId();
+
+    VERBOSE(2,"WRITING " << nBestSize << " TRANSLATION ALTERNATIVES TO "
+            << options()->nbest.output_file_path << endl);
+    std::vector<boost::shared_ptr<ChartKBestExtractor::Derivation> > nBestList;
+    CalcNBest(nBestSize, nBestList, options()->nbest.only_distinct);
+    OutputNBestList(collector, nBestList, translationId);
+    IFVERBOSE(2) {
+      PrintUserTime("N-Best Hypotheses Generation Time:");
+    }
+  }
+
+}
+
+void ChartManager::OutputNBestList(OutputCollector *collector,
+                                   const ChartKBestExtractor::KBestVec &nBestList,
+                                   long translationId) const
+{
+  std::ostringstream out;
+
+  if (collector->OutputIsCout()) {
+    // Set precision only if we're writing the n-best list to cout.  This is to
+    // preserve existing behaviour, but should probably be done either way.
+    FixPrecision(out);
+  }
+
+  NBestOptions const& nbo = options()->nbest;
+  bool includeWordAlignment = nbo.include_alignment_info;
+  bool PrintNBestTrees = nbo.print_trees;
+
+  for (ChartKBestExtractor::KBestVec::const_iterator p = nBestList.begin();
+       p != nBestList.end(); ++p) {
+    const ChartKBestExtractor::Derivation &derivation = **p;
+
+    // get the derivation's target-side yield
+    Phrase outputPhrase = ChartKBestExtractor::GetOutputPhrase(derivation);
+
+    // delete <s> and </s>
+    UTIL_THROW_IF2(outputPhrase.GetSize() < 2,
+                   "Output phrase should have contained at least 2 words (beginning and end-of-sentence)");
+    outputPhrase.RemoveWord(0);
+    outputPhrase.RemoveWord(outputPhrase.GetSize() - 1);
+
+    // print the translation ID, surface factors, and scores
+    out << translationId << " ||| ";
+    OutputSurface(out, outputPhrase); // , outputFactorOrder, false);
+    out << " ||| ";
+    boost::shared_ptr<ScoreComponentCollection> scoreBreakdown = ChartKBestExtractor::GetOutputScoreBreakdown(derivation);
+    bool with_labels = options()->nbest.include_feature_labels;
+    scoreBreakdown->OutputAllFeatureScores(out, with_labels);
+    out << " ||| " << derivation.score;
+
+    // optionally, print word alignments
+    if (includeWordAlignment) {
+      out << " ||| ";
+      Alignments align;
+      OutputAlignmentNBest(align, derivation, 0);
+      for (Alignments::const_iterator q = align.begin(); q != align.end();
+           ++q) {
+        out << q->first << "-" << q->second << " ";
+      }
+    }
+
+    // optionally, print tree
+    if (PrintNBestTrees) {
+      TreePointer tree = ChartKBestExtractor::GetOutputTree(derivation);
+      out << " ||| " << tree->GetString();
+    }
+
+    out << std::endl;
+  }
+
+  assert(collector);
+  collector->Write(translationId, out.str());
+}
+
+size_t ChartManager::CalcSourceSize(const Moses::ChartHypothesis *hypo) const
+{
+  size_t ret = hypo->GetCurrSourceRange().GetNumWordsCovered();
+  const std::vector<const ChartHypothesis*> &prevHypos = hypo->GetPrevHypos();
+  for (size_t i = 0; i < prevHypos.size(); ++i) {
+    size_t childSize = prevHypos[i]->GetCurrSourceRange().GetNumWordsCovered();
+    ret -= (childSize - 1);
+  }
+  return ret;
+}
+
+size_t ChartManager::OutputAlignmentNBest(
+  Alignments &retAlign,
+  const Moses::ChartKBestExtractor::Derivation &derivation,
+  size_t startTarget) const
+{
+  const ChartHypothesis &hypo = derivation.edge.head->hypothesis;
+
+  size_t totalTargetSize = 0;
+  size_t startSource = hypo.GetCurrSourceRange().GetStartPos();
+
+  const TargetPhrase &tp = hypo.GetCurrTargetPhrase();
+
+  size_t thisSourceSize = CalcSourceSize(&hypo);
+
+  // position of each terminal word in translation rule, irrespective of alignment
+  // if non-term, number is undefined
+  vector<size_t> sourceOffsets(thisSourceSize, 0);
+  vector<size_t> targetOffsets(tp.GetSize(), 0);
+
+  const AlignmentInfo &aiNonTerm = hypo.GetCurrTargetPhrase().GetAlignNonTerm();
+  vector<size_t> sourceInd2pos = aiNonTerm.GetSourceIndex2PosMap();
+  const AlignmentInfo::NonTermIndexMap &targetPos2SourceInd = aiNonTerm.GetNonTermIndexMap();
+
+  UTIL_THROW_IF2(sourceInd2pos.size() != derivation.subderivations.size(),
+                 "Error");
+
+  size_t targetInd = 0;
+  for (size_t targetPos = 0; targetPos < tp.GetSize(); ++targetPos) {
+    if (tp.GetWord(targetPos).IsNonTerminal()) {
+      UTIL_THROW_IF2(targetPos >= targetPos2SourceInd.size(), "Error");
+      size_t sourceInd = targetPos2SourceInd[targetPos];
+      size_t sourcePos = sourceInd2pos[sourceInd];
+
+      const Moses::ChartKBestExtractor::Derivation &subderivation =
+        *derivation.subderivations[sourceInd];
+
+      // calc source size
+      size_t sourceSize = subderivation.edge.head->hypothesis.GetCurrSourceRange().GetNumWordsCovered();
+      sourceOffsets[sourcePos] = sourceSize;
+
+      // calc target size.
+      // Recursively look thru child hypos
+      size_t currStartTarget = startTarget + totalTargetSize;
+      size_t targetSize = OutputAlignmentNBest(retAlign, subderivation,
+                          currStartTarget);
+      targetOffsets[targetPos] = targetSize;
+
+      totalTargetSize += targetSize;
+      ++targetInd;
+    } else {
+      ++totalTargetSize;
+    }
+  }
+
+  // convert position within translation rule to absolute position within
+  // source sentence / output sentence
+  ShiftOffsets(sourceOffsets, startSource);
+  ShiftOffsets(targetOffsets, startTarget);
+
+  // get alignments from this hypo
+  const AlignmentInfo &aiTerm = hypo.GetCurrTargetPhrase().GetAlignTerm();
+
+  // add to output arg, offsetting by source & target
+  AlignmentInfo::const_iterator iter;
+  for (iter = aiTerm.begin(); iter != aiTerm.end(); ++iter) {
+    const std::pair<size_t,size_t> &align = *iter;
+    size_t relSource = align.first;
+    size_t relTarget = align.second;
+    size_t absSource = sourceOffsets[relSource];
+    size_t absTarget = targetOffsets[relTarget];
+
+    pair<size_t, size_t> alignPoint(absSource, absTarget);
+    pair<Alignments::iterator, bool> ret = retAlign.insert(alignPoint);
+    UTIL_THROW_IF2(!ret.second, "Error");
+  }
+
+  return totalTargetSize;
+}
+
+void ChartManager::OutputAlignment(OutputCollector *collector) const
+{
+  if (collector == NULL) {
+    return;
+  }
+
+  ostringstream out;
+
+  const ChartHypothesis *hypo = GetBestHypothesis();
+  if (hypo) {
+    Alignments retAlign;
+    OutputAlignment(retAlign, hypo, 0);
+
+    // output alignments
+    Alignments::const_iterator iter;
+    for (iter = retAlign.begin(); iter != retAlign.end(); ++iter) {
+      const pair<size_t, size_t> &alignPoint = *iter;
+      out << alignPoint.first << "-" << alignPoint.second << " ";
+    }
+  }
+  out << endl;
+
+  collector->Write(m_source.GetTranslationId(), out.str());
+
+}
+
+size_t ChartManager::OutputAlignment(Alignments &retAlign,
+                                     const Moses::ChartHypothesis *hypo,
+                                     size_t startTarget) const
+{
+  size_t totalTargetSize = 0;
+  size_t startSource = hypo->GetCurrSourceRange().GetStartPos();
+
+  const TargetPhrase &tp = hypo->GetCurrTargetPhrase();
+
+  size_t thisSourceSize = CalcSourceSize(hypo);
+
+  // position of each terminal word in translation rule, irrespective of alignment
+  // if non-term, number is undefined
+  vector<size_t> sourceOffsets(thisSourceSize, 0);
+  vector<size_t> targetOffsets(tp.GetSize(), 0);
+
+  const vector<const ChartHypothesis*> &prevHypos = hypo->GetPrevHypos();
+
+  const AlignmentInfo &aiNonTerm = hypo->GetCurrTargetPhrase().GetAlignNonTerm();
+  vector<size_t> sourceInd2pos = aiNonTerm.GetSourceIndex2PosMap();
+  const AlignmentInfo::NonTermIndexMap &targetPos2SourceInd = aiNonTerm.GetNonTermIndexMap();
+
+  UTIL_THROW_IF2(sourceInd2pos.size() != prevHypos.size(), "Error");
+
+  size_t targetInd = 0;
+  for (size_t targetPos = 0; targetPos < tp.GetSize(); ++targetPos) {
+    if (tp.GetWord(targetPos).IsNonTerminal()) {
+      UTIL_THROW_IF2(targetPos >= targetPos2SourceInd.size(), "Error");
+      size_t sourceInd = targetPos2SourceInd[targetPos];
+      size_t sourcePos = sourceInd2pos[sourceInd];
+
+      const ChartHypothesis *prevHypo = prevHypos[sourceInd];
+
+      // calc source size
+      size_t sourceSize = prevHypo->GetCurrSourceRange().GetNumWordsCovered();
+      sourceOffsets[sourcePos] = sourceSize;
+
+      // calc target size.
+      // Recursively look thru child hypos
+      size_t currStartTarget = startTarget + totalTargetSize;
+      size_t targetSize = OutputAlignment(retAlign, prevHypo, currStartTarget);
+      targetOffsets[targetPos] = targetSize;
+
+      totalTargetSize += targetSize;
+      ++targetInd;
+    } else {
+      ++totalTargetSize;
+    }
+  }
+
+  // convert position within translation rule to absolute position within
+  // source sentence / output sentence
+  ShiftOffsets(sourceOffsets, startSource);
+  ShiftOffsets(targetOffsets, startTarget);
+
+  // get alignments from this hypo
+  const AlignmentInfo &aiTerm = hypo->GetCurrTargetPhrase().GetAlignTerm();
+
+  // add to output arg, offsetting by source & target
+  AlignmentInfo::const_iterator iter;
+  for (iter = aiTerm.begin(); iter != aiTerm.end(); ++iter) {
+    const std::pair<size_t,size_t> &align = *iter;
+    size_t relSource = align.first;
+    size_t relTarget = align.second;
+    size_t absSource = sourceOffsets[relSource];
+    size_t absTarget = targetOffsets[relTarget];
+
+    pair<size_t, size_t> alignPoint(absSource, absTarget);
+    pair<Alignments::iterator, bool> ret = retAlign.insert(alignPoint);
+    UTIL_THROW_IF2(!ret.second, "Error");
+
+  }
+
+  return totalTargetSize;
+}
+
+void ChartManager::OutputDetailedTranslationReport(OutputCollector *collector) const
+{
+  if (collector) {
+    OutputDetailedTranslationReport(collector,
+                                    GetBestHypothesis(),
+                                    static_cast<const Sentence&>(m_source),
+                                    m_source.GetTranslationId());
+  }
+}
+
+void ChartManager::OutputDetailedTranslationReport(
+  OutputCollector *collector,
+  const ChartHypothesis *hypo,
+  const Sentence &sentence,
+  long translationId) const
+{
+  if (hypo == NULL) {
+    return;
+  }
+  std::ostringstream out;
+  ApplicationContext applicationContext;
+
+  OutputTranslationOptions(out, applicationContext, hypo, sentence, translationId);
+  collector->Write(translationId, out.str());
+
+  //DIMw
+  if (options()->output.detailed_all_transrep_filepath.size()) {
+    const Sentence &sentence = static_cast<const Sentence &>(m_source);
+    size_t nBestSize = options()->nbest.nbest_size;
+    std::vector<boost::shared_ptr<ChartKBestExtractor::Derivation> > nBestList;
+    CalcNBest(nBestSize, nBestList, options()->nbest.only_distinct);
+    OutputDetailedAllTranslationReport(collector, nBestList, sentence, translationId);
+  }
+
+}
+
+void ChartManager::OutputTranslationOptions(std::ostream &out,
+    ApplicationContext &applicationContext,
+    const ChartHypothesis *hypo,
+    const Sentence &sentence,
+    long translationId) const
+{
+  if (hypo != NULL) {
+    OutputTranslationOption(out, applicationContext, hypo, sentence, translationId);
+    out << std::endl;
+  }
+
+  // recursive
+  const std::vector<const ChartHypothesis*> &prevHypos = hypo->GetPrevHypos();
+  std::vector<const ChartHypothesis*>::const_iterator iter;
+  for (iter = prevHypos.begin(); iter != prevHypos.end(); ++iter) {
+    const ChartHypothesis *prevHypo = *iter;
+    OutputTranslationOptions(out, applicationContext, prevHypo, sentence, translationId);
+  }
+}
+
+void ChartManager::OutputTranslationOption(std::ostream &out,
+    ApplicationContext &applicationContext,
+    const ChartHypothesis *hypo,
+    const Sentence &sentence,
+    long translationId) const
+{
+  ReconstructApplicationContext(*hypo, sentence, applicationContext);
+  out << "Trans Opt " << translationId
+      << " " << hypo->GetCurrSourceRange()
+      << ": ";
+  WriteApplicationContext(out, applicationContext);
+  out << ": " << hypo->GetCurrTargetPhrase().GetTargetLHS()
+      << "->" << hypo->GetCurrTargetPhrase()
+      << " " << hypo->GetFutureScore() << hypo->GetScoreBreakdown();
+}
+
+// Given a hypothesis and sentence, reconstructs the 'application context' --
+// the source RHS symbols of the SCFG rule that was applied, plus their spans.
+void ChartManager::ReconstructApplicationContext(const ChartHypothesis &hypo,
+    const Sentence &sentence,
+    ApplicationContext &context) const
+{
+  context.clear();
+  const std::vector<const ChartHypothesis*> &prevHypos = hypo.GetPrevHypos();
+  std::vector<const ChartHypothesis*>::const_iterator p = prevHypos.begin();
+  std::vector<const ChartHypothesis*>::const_iterator end = prevHypos.end();
+  const Range &span = hypo.GetCurrSourceRange();
+  size_t i = span.GetStartPos();
+  while (i <= span.GetEndPos()) {
+    if (p == end || i < (*p)->GetCurrSourceRange().GetStartPos()) {
+      // Symbol is a terminal.
+      const Word &symbol = sentence.GetWord(i);
+      context.push_back(std::make_pair(symbol, Range(i, i)));
+      ++i;
+    } else {
+      // Symbol is a non-terminal.
+      const Word &symbol = (*p)->GetTargetLHS();
+      const Range &range = (*p)->GetCurrSourceRange();
+      context.push_back(std::make_pair(symbol, range));
+      i = range.GetEndPos()+1;
+      ++p;
+    }
+  }
+}
+
+void ChartManager::OutputUnknowns(OutputCollector *collector) const
+{
+  if (collector) {
+    long translationId = m_source.GetTranslationId();
+    const std::vector<Phrase*> &oovs = GetParser().GetUnknownSources();
+
+    std::ostringstream out;
+    for (std::vector<Phrase*>::const_iterator p = oovs.begin();
+         p != oovs.end(); ++p) {
+      out << **p;
+    }
+    out << std::endl;
+    collector->Write(translationId, out.str());
+  }
+
+}
+
+void ChartManager::OutputDetailedTreeFragmentsTranslationReport(OutputCollector *collector) const
+{
+  const ChartHypothesis *hypo = GetBestHypothesis();
+  if (collector == NULL || hypo == NULL) {
+    return;
+  }
+
+  std::ostringstream out;
+  ApplicationContext applicationContext;
+
+  const Sentence &sentence = static_cast<const Sentence &>(m_source);
+  const size_t translationId = m_source.GetTranslationId();
+
+  OutputTreeFragmentsTranslationOptions(out, applicationContext, hypo, sentence, translationId);
+
+  //Tree of full sentence
+  const StatefulFeatureFunction* treeStructure;
+  treeStructure = StaticData::Instance().GetTreeStructure();
+  if (treeStructure != NULL) {
+    const vector<const StatefulFeatureFunction*>& sff = StatefulFeatureFunction::GetStatefulFeatureFunctions();
+    for( size_t i=0; i<sff.size(); i++ ) {
+      if (sff[i] == treeStructure) {
+        const TreeState* tree = static_cast<const TreeState*>(hypo->GetFFState(i));
+        out << "Full Tree " << translationId << ": " << tree->GetTree()->GetString() << "\n";
+        break;
+      }
+    }
+  }
+
+  collector->Write(translationId, out.str());
+
+}
+
+void ChartManager::OutputTreeFragmentsTranslationOptions(std::ostream &out,
+    ApplicationContext &applicationContext,
+    const ChartHypothesis *hypo,
+    const Sentence &sentence,
+    long translationId) const
+{
+
+  if (hypo != NULL) {
+    OutputTranslationOption(out, applicationContext, hypo, sentence, translationId);
+
+    const TargetPhrase &currTarPhr = hypo->GetCurrTargetPhrase();
+
+    out << " ||| ";
+    if (const PhraseProperty *property = currTarPhr.GetProperty("Tree")) {
+      out << " " << *property->GetValueString();
+    } else {
+      out << " " << "noTreeInfo";
+    }
+    out << std::endl;
+  }
+
+  // recursive
+  const std::vector<const ChartHypothesis*> &prevHypos = hypo->GetPrevHypos();
+  std::vector<const ChartHypothesis*>::const_iterator iter;
+  for (iter = prevHypos.begin(); iter != prevHypos.end(); ++iter) {
+    const ChartHypothesis *prevHypo = *iter;
+    OutputTreeFragmentsTranslationOptions(out, applicationContext, prevHypo, sentence, translationId);
+  }
+}
+
+void ChartManager::OutputSearchGraph(OutputCollector *collector) const
+{
+  if (collector) {
+    long translationId = m_source.GetTranslationId();
+    std::ostringstream out;
+    OutputSearchGraphMoses( out);
+    collector->Write(translationId, out.str());
+  }
+}
+
+//DIMw
+void ChartManager::OutputDetailedAllTranslationReport(
+  OutputCollector *collector,
+  const std::vector<boost::shared_ptr<Moses::ChartKBestExtractor::Derivation> > &nBestList,
+  const Sentence &sentence,
+  long translationId) const
+{
+  std::ostringstream out;
+  ApplicationContext applicationContext;
+
+  const ChartCellCollection& cells = GetChartCellCollection();
+  size_t size = GetSource().GetSize();
+  for (size_t width = 1; width <= size; ++width) {
+    for (size_t startPos = 0; startPos <= size-width; ++startPos) {
+      size_t endPos = startPos + width - 1;
+      Range range(startPos, endPos);
+      const ChartCell& cell = cells.Get(range);
+      const HypoList* hyps = cell.GetAllSortedHypotheses();
+      out << "Chart Cell [" << startPos << ".." << endPos << "]" << endl;
+      HypoList::const_iterator iter;
+      size_t c = 1;
+      for (iter = hyps->begin(); iter != hyps->end(); ++iter) {
+        out << "----------------Item " << c++ << " ---------------------"
+            << endl;
+        OutputTranslationOptions(out, applicationContext, *iter,
+                                 sentence, translationId);
+      }
+    }
+  }
+  collector->Write(translationId, out.str());
+}
+
+void ChartManager::OutputBestHypo(OutputCollector *collector, const ChartHypothesis *hypo, long translationId) const
+{
+  if (!collector)
+    return;
+  std::ostringstream out;
+  FixPrecision(out);
+  if (hypo != NULL) {
+    VERBOSE(1,"BEST TRANSLATION: " << *hypo << endl);
+    VERBOSE(3,"Best path: ");
+    Backtrack(hypo);
+    VERBOSE(3,"0" << std::endl);
+
+    if (options()->output.ReportHypoScore) {
+      out << hypo->GetFutureScore() << " ";
+    }
+
+    if (options()->output.RecoverPath) {
+      out << "||| ";
+    }
+    Phrase outPhrase(ARRAY_SIZE_INCR);
+    hypo->GetOutputPhrase(outPhrase);
+
+    // delete 1st & last
+    UTIL_THROW_IF2(outPhrase.GetSize() < 2,
+                   "Output phrase should have contained at least 2 words (beginning and end-of-sentence)");
+
+    outPhrase.RemoveWord(0);
+    outPhrase.RemoveWord(outPhrase.GetSize() - 1);
+
+    string output = outPhrase.GetStringRep(options()->output.factor_order);
+    out << output << endl;
+  } else {
+    VERBOSE(1, "NO BEST TRANSLATION" << endl);
+
+    if (options()->output.ReportHypoScore) {
+      out << "0 ";
+    }
+
+    out << endl;
+  }
+  collector->Write(translationId, out.str());
+}
+
+void ChartManager::Backtrack(const ChartHypothesis *hypo) const
+{
+  const vector<const ChartHypothesis*> &prevHypos = hypo->GetPrevHypos();
+
+  vector<const ChartHypothesis*>::const_iterator iter;
+  for (iter = prevHypos.begin(); iter != prevHypos.end(); ++iter) {
+    const ChartHypothesis *prevHypo = *iter;
+
+    VERBOSE(3,prevHypo->GetId() << " <= ");
+    Backtrack(prevHypo);
+  }
+}
+
+} // namespace Moses

mosesdecoder/moses/ChartManager.h

@@ -0,0 +1,162 @@
+// $Id$
+// vim:tabstop=2
+/***********************************************************************
+ Moses - factored phrase-based language decoder
+ Copyright (C) 2010 Hieu Hoang
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ ***********************************************************************/
+
+#pragma once
+
+#include <vector>
+#include <boost/unordered_map.hpp>
+#include "ChartCell.h"
+#include "ChartCellCollection.h"
+#include "Range.h"
+#include "SentenceStats.h"
+#include "ChartTranslationOptionList.h"
+#include "ChartParser.h"
+#include "ChartKBestExtractor.h"
+#include "BaseManager.h"
+#include "moses/Syntax/KBestExtractor.h"
+
+namespace Moses
+{
+
+class ChartHypothesis;
+class ChartSearchGraphWriter;
+
+/** Holds everything you need to decode 1 sentence with the hierachical/syntax decoder
+ */
+class ChartManager : public BaseManager
+{
+private:
+  ChartCellCollection m_hypoStackColl;
+  std::auto_ptr<SentenceStats> m_sentenceStats;
+  clock_t m_start; /**< starting time, used for logging */
+  unsigned m_hypothesisId; /* For handing out hypothesis ids to ChartHypothesis */
+
+  ChartParser m_parser;
+
+  ChartTranslationOptionList m_translationOptionList; /**< pre-computed list of translation options for the phrases in this sentence */
+
+  /* auxilliary functions for SearchGraphs */
+  void FindReachableHypotheses(
+    const ChartHypothesis *hypo, std::map<unsigned,bool> &reachable , size_t* winners, size_t* losers) const;
+  void WriteSearchGraph(const ChartSearchGraphWriter& writer) const;
+
+  // output
+  void OutputNBestList(OutputCollector *collector,
+                       const ChartKBestExtractor::KBestVec &nBestList,
+                       long translationId) const;
+  size_t CalcSourceSize(const Moses::ChartHypothesis *hypo) const;
+  size_t OutputAlignmentNBest(Alignments &retAlign,
+                              const Moses::ChartKBestExtractor::Derivation &derivation,
+                              size_t startTarget) const;
+  size_t OutputAlignment(Alignments &retAlign,
+                         const Moses::ChartHypothesis *hypo,
+                         size_t startTarget) const;
+  void OutputDetailedTranslationReport(
+    OutputCollector *collector,
+    const ChartHypothesis *hypo,
+    const Sentence &sentence,
+    long translationId) const;
+  void OutputTranslationOptions(std::ostream &out,
+                                ApplicationContext &applicationContext,
+                                const ChartHypothesis *hypo,
+                                const Sentence &sentence,
+                                long translationId) const;
+  void OutputTranslationOption(std::ostream &out,
+                               ApplicationContext &applicationContext,
+                               const ChartHypothesis *hypo,
+                               const Sentence &sentence,
+                               long translationId) const;
+  void ReconstructApplicationContext(const ChartHypothesis &hypo,
+                                     const Sentence &sentence,
+                                     ApplicationContext &context) const;
+  void OutputTreeFragmentsTranslationOptions(std::ostream &out,
+      ApplicationContext &applicationContext,
+      const ChartHypothesis *hypo,
+      const Sentence &sentence,
+      long translationId) const;
+  void OutputDetailedAllTranslationReport(
+    OutputCollector *collector,
+    const std::vector<boost::shared_ptr<Moses::ChartKBestExtractor::Derivation> > &nBestList,
+    const Sentence &sentence,
+    long translationId) const;
+  void OutputBestHypo(OutputCollector *collector, const ChartHypothesis *hypo, long translationId) const;
+  void Backtrack(const ChartHypothesis *hypo) const;
+
+public:
+  ChartManager(ttasksptr const& ttask);
+  ~ChartManager();
+  void Decode();
+  void AddXmlChartOptions();
+  const ChartHypothesis *GetBestHypothesis() const;
+  void CalcNBest(size_t n, std::vector<boost::shared_ptr<ChartKBestExtractor::Derivation> > &nBestList, bool onlyDistinct=false) const;
+
+  /** "Moses" (osg)  type format */
+  void OutputSearchGraphMoses(std::ostream &outputSearchGraphStream) const;
+
+  /** Output in (modified) Kenneth hypergraph format */
+  void OutputSearchGraphAsHypergraph(std::ostream &outputSearchGraphStream) const;
+
+  //! debug data collected when decoding sentence
+  SentenceStats& GetSentenceStats() const {
+    return *m_sentenceStats;
+  }
+
+  //DIMw
+  const ChartCellCollection& GetChartCellCollection() const {
+    return m_hypoStackColl;
+  }
+
+  void CalcDecoderStatistics() const {
+  }
+
+  void ResetSentenceStats(const InputType& source) {
+    m_sentenceStats = std::auto_ptr<SentenceStats>(new SentenceStats(source));
+  }
+
+  //! contigious hypo id for each input sentence. For debugging purposes
+  unsigned GetNextHypoId() {
+    return m_hypothesisId++;
+  }
+
+  const ChartParser &GetParser() const {
+    return m_parser;
+  }
+
+  // outputs
+  void OutputBest(OutputCollector *collector) const;
+  void OutputNBest(OutputCollector *collector) const;
+  void OutputLatticeSamples(OutputCollector *collector) const {
+  }
+  void OutputAlignment(OutputCollector *collector) const;
+  void OutputDetailedTranslationReport(OutputCollector *collector) const;
+  void OutputUnknowns(OutputCollector *collector) const;
+  void OutputDetailedTreeFragmentsTranslationReport(OutputCollector *collector) const;
+  void OutputWordGraph(OutputCollector *collector) const {
+  }
+  void OutputSearchGraph(OutputCollector *collector) const;
+  void OutputSearchGraphSLF() const {
+  }
+  // void OutputSearchGraphHypergraph() const;
+
+};
+
+}
+

mosesdecoder/moses/ChartParser.cpp

@@ -0,0 +1,313 @@
+// $Id$
+// vim:tabstop=2
+/***********************************************************************
+ Moses - factored phrase-based language decoder
+ Copyright (C) 2010 Hieu Hoang
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ ***********************************************************************/
+
+#include "ChartParser.h"
+#include "ChartParserCallback.h"
+#include "ChartRuleLookupManager.h"
+#include "StaticData.h"
+#include "TreeInput.h"
+#include "Sentence.h"
+#include "DecodeGraph.h"
+#include "moses/FF/UnknownWordPenaltyProducer.h"
+#include "moses/TranslationModel/PhraseDictionary.h"
+#include "moses/TranslationTask.h"
+
+using namespace std;
+using namespace Moses;
+
+namespace Moses
+{
+
+ChartParserUnknown
+::ChartParserUnknown(ttasksptr const& ttask)
+  : m_ttask(ttask)
+{ }
+
+ChartParserUnknown::~ChartParserUnknown()
+{
+  RemoveAllInColl(m_unksrcs);
+}
+
+AllOptions::ptr const&
+ChartParserUnknown::
+options() const
+{
+  return m_ttask.lock()->options();
+}
+
+void
+ChartParserUnknown::
+Process(const Word &sourceWord, const Range &range, ChartParserCallback &to)
+{
+  // unknown word, add as trans opt
+  const StaticData &staticData = StaticData::Instance();
+  const UnknownWordPenaltyProducer &unknownWordPenaltyProducer
+  = UnknownWordPenaltyProducer::Instance();
+
+  size_t isDigit = 0;
+  if (options()->unk.drop) {
+    const Factor *f = sourceWord[0]; // TODO hack. shouldn't know which factor is surface
+    const StringPiece s = f->GetString();
+    isDigit = s.find_first_of("0123456789");
+    if (isDigit == string::npos)
+      isDigit = 0;
+    else
+      isDigit = 1;
+    // modify the starting bitmap
+  }
+
+  Phrase* unksrc = new Phrase(1);
+  unksrc->AddWord() = sourceWord;
+  Word &newWord = unksrc->GetWord(0);
+  newWord.SetIsOOV(true);
+
+  m_unksrcs.push_back(unksrc);
+
+  // hack. Once the OOV FF is a phrase table, get rid of this
+  PhraseDictionary *firstPt = NULL;
+  if (PhraseDictionary::GetColl().size() == 0) {
+    firstPt = PhraseDictionary::GetColl()[0];
+  }
+
+  //TranslationOption *transOpt;
+  if (! options()->unk.drop || isDigit) {
+    // loop
+    const UnknownLHSList &lhsList = options()->syntax.unknown_lhs; // staticData.GetUnknownLHS();
+    UnknownLHSList::const_iterator iterLHS;
+    for (iterLHS = lhsList.begin(); iterLHS != lhsList.end(); ++iterLHS) {
+      const string &targetLHSStr = iterLHS->first;
+      float prob = iterLHS->second;
+
+      // lhs
+      //const Word &sourceLHS = staticData.GetInputDefaultNonTerminal();
+      Word *targetLHS = new Word(true);
+
+      targetLHS->CreateFromString(Output, options()->output.factor_order,
+                                  targetLHSStr, true);
+      UTIL_THROW_IF2(targetLHS->GetFactor(0) == NULL, "Null factor for target LHS");
+
+      // add to dictionary
+      TargetPhrase *targetPhrase = new TargetPhrase(firstPt);
+      Word &targetWord = targetPhrase->AddWord();
+      targetWord.CreateUnknownWord(sourceWord);
+
+      // scores
+      float unknownScore = FloorScore(TransformScore(prob));
+
+      targetPhrase->GetScoreBreakdown().Assign(&unknownWordPenaltyProducer, unknownScore);
+      targetPhrase->SetTargetLHS(targetLHS);
+      targetPhrase->SetAlignmentInfo("0-0");
+      targetPhrase->EvaluateInIsolation(*unksrc);
+
+      if (!options()->output.detailed_tree_transrep_filepath.empty() ||
+          options()->nbest.print_trees || staticData.GetTreeStructure() != NULL) {
+        std::string prop = "[ ";
+        prop += (*targetLHS)[0]->GetString().as_string() + " ";
+        prop += sourceWord[0]->GetString().as_string() + " ]";
+        targetPhrase->SetProperty("Tree", prop);
+      }
+
+      // chart rule
+      to.AddPhraseOOV(*targetPhrase, m_cacheTargetPhraseCollection, range);
+    } // for (iterLHS
+  } else {
+    // drop source word. create blank trans opt
+    float unknownScore = FloorScore(-numeric_limits<float>::infinity());
+
+    TargetPhrase *targetPhrase = new TargetPhrase(firstPt);
+    // loop
+    const UnknownLHSList &lhsList = options()->syntax.unknown_lhs;//staticData.GetUnknownLHS();
+    UnknownLHSList::const_iterator iterLHS;
+    for (iterLHS = lhsList.begin(); iterLHS != lhsList.end(); ++iterLHS) {
+      const string &targetLHSStr = iterLHS->first;
+      //float prob = iterLHS->second;
+
+      Word *targetLHS = new Word(true);
+      targetLHS->CreateFromString(Output, staticData.options()->output.factor_order,
+                                  targetLHSStr, true);
+      UTIL_THROW_IF2(targetLHS->GetFactor(0) == NULL, "Null factor for target LHS");
+
+      targetPhrase->GetScoreBreakdown().Assign(&unknownWordPenaltyProducer, unknownScore);
+      targetPhrase->EvaluateInIsolation(*unksrc);
+
+      targetPhrase->SetTargetLHS(targetLHS);
+
+      // chart rule
+      to.AddPhraseOOV(*targetPhrase, m_cacheTargetPhraseCollection, range);
+    }
+  }
+}
+
+ChartParser
+::ChartParser(ttasksptr const& ttask, ChartCellCollectionBase &cells)
+  : m_ttask(ttask)
+  , m_unknown(ttask)
+  , m_decodeGraphList(StaticData::Instance().GetDecodeGraphs())
+  , m_source(*(ttask->GetSource().get()))
+{
+  const StaticData &staticData = StaticData::Instance();
+
+  staticData.InitializeForInput(ttask);
+  CreateInputPaths(m_source);
+
+  const std::vector<PhraseDictionary*> &dictionaries = PhraseDictionary::GetColl();
+  assert(dictionaries.size() == m_decodeGraphList.size());
+  m_ruleLookupManagers.reserve(dictionaries.size());
+  for (std::size_t i = 0; i < dictionaries.size(); ++i) {
+    const PhraseDictionary *dict = dictionaries[i];
+    PhraseDictionary *nonConstDict = const_cast<PhraseDictionary*>(dict);
+    std::size_t maxChartSpan = m_decodeGraphList[i]->GetMaxChartSpan();
+    ChartRuleLookupManager *lookupMgr = nonConstDict->CreateRuleLookupManager(*this, cells, maxChartSpan);
+    m_ruleLookupManagers.push_back(lookupMgr);
+  }
+
+}
+
+ChartParser::~ChartParser()
+{
+  RemoveAllInColl(m_ruleLookupManagers);
+  StaticData::Instance().CleanUpAfterSentenceProcessing(m_ttask.lock());
+
+  InputPathMatrix::const_iterator iterOuter;
+  for (iterOuter = m_inputPathMatrix.begin(); iterOuter != m_inputPathMatrix.end(); ++iterOuter) {
+    const std::vector<InputPath*> &outer = *iterOuter;
+
+    std::vector<InputPath*>::const_iterator iterInner;
+    for (iterInner = outer.begin(); iterInner != outer.end(); ++iterInner) {
+      InputPath *path = *iterInner;
+      delete path;
+    }
+  }
+}
+
+void ChartParser::Create(const Range &range, ChartParserCallback &to)
+{
+  assert(m_decodeGraphList.size() == m_ruleLookupManagers.size());
+
+  std::vector <DecodeGraph*>::const_iterator iterDecodeGraph;
+  std::vector <ChartRuleLookupManager*>::const_iterator iterRuleLookupManagers = m_ruleLookupManagers.begin();
+  for (iterDecodeGraph = m_decodeGraphList.begin(); iterDecodeGraph != m_decodeGraphList.end(); ++iterDecodeGraph, ++iterRuleLookupManagers) {
+    const DecodeGraph &decodeGraph = **iterDecodeGraph;
+    assert(decodeGraph.GetSize() == 1);
+    ChartRuleLookupManager &ruleLookupManager = **iterRuleLookupManagers;
+    size_t maxSpan = decodeGraph.GetMaxChartSpan();
+    size_t last = m_source.GetSize()-1;
+    if (maxSpan != 0) {
+      last = min(last, range.GetStartPos()+maxSpan);
+    }
+    if (maxSpan == 0 || range.GetNumWordsCovered() <= maxSpan) {
+      const InputPath &inputPath = GetInputPath(range);
+      ruleLookupManager.GetChartRuleCollection(inputPath, last, to);
+    }
+  }
+
+  if (range.GetNumWordsCovered() == 1
+      && range.GetStartPos() != 0
+      && range.GetStartPos() != m_source.GetSize()-1) {
+    bool always = options()->unk.always_create_direct_transopt;
+    if (to.Empty() || always) {
+      // create unknown words for 1 word coverage where we don't have any trans options
+      const Word &sourceWord = m_source.GetWord(range.GetStartPos());
+      m_unknown.Process(sourceWord, range, to);
+    }
+  }
+}
+
+void ChartParser::CreateInputPaths(const InputType &input)
+{
+  size_t size = input.GetSize();
+  m_inputPathMatrix.resize(size);
+
+  UTIL_THROW_IF2(input.GetType() != SentenceInput && input.GetType() != TreeInputType,
+                 "Input must be a sentence or a tree, " <<
+                 "not lattice or confusion networks");
+
+  TranslationTask const* ttask = m_ttask.lock().get();
+  for (size_t phaseSize = 1; phaseSize <= size; ++phaseSize) {
+    for (size_t startPos = 0; startPos < size - phaseSize + 1; ++startPos) {
+      size_t endPos = startPos + phaseSize -1;
+      vector<InputPath*> &vec = m_inputPathMatrix[startPos];
+
+      Range range(startPos, endPos);
+      Phrase subphrase(input.GetSubString(Range(startPos, endPos)));
+      const NonTerminalSet &labels = input.GetLabelSet(startPos, endPos);
+
+      InputPath *node;
+      if (range.GetNumWordsCovered() == 1) {
+        node = new InputPath(ttask, subphrase, labels, range, NULL, NULL);
+        vec.push_back(node);
+      } else {
+        const InputPath &prevNode = GetInputPath(startPos, endPos - 1);
+        node = new InputPath(ttask, subphrase, labels, range, &prevNode, NULL);
+        vec.push_back(node);
+      }
+
+      //m_inputPathQueue.push_back(node);
+    }
+  }
+}
+
+const InputPath &ChartParser::GetInputPath(const Range &range) const
+{
+  return GetInputPath(range.GetStartPos(), range.GetEndPos());
+}
+
+const InputPath &ChartParser::GetInputPath(size_t startPos, size_t endPos) const
+{
+  size_t offset = endPos - startPos;
+  UTIL_THROW_IF2(offset >= m_inputPathMatrix[startPos].size(),
+                 "Out of bound: " << offset);
+  return *m_inputPathMatrix[startPos][offset];
+}
+
+InputPath &ChartParser::GetInputPath(size_t startPos, size_t endPos)
+{
+  size_t offset = endPos - startPos;
+  UTIL_THROW_IF2(offset >= m_inputPathMatrix[startPos].size(),
+                 "Out of bound: " << offset);
+  return *m_inputPathMatrix[startPos][offset];
+}
+/*
+const Sentence &ChartParser::GetSentence() const {
+  const Sentence &sentence = static_cast<const Sentence&>(m_source);
+  return sentence;
+}
+*/
+size_t ChartParser::GetSize() const
+{
+  return m_source.GetSize();
+}
+
+long ChartParser::GetTranslationId() const
+{
+  return m_source.GetTranslationId();
+}
+
+
+AllOptions::ptr const&
+ChartParser::
+options() const
+{
+  return m_ttask.lock()->options();
+}
+
+
+} // namespace Moses

mosesdecoder/moses/ChartParser.h

@@ -0,0 +1,99 @@
+// -*- c++ -*-
+// $Id$
+// vim:tabstop=2
+/***********************************************************************
+ Moses - factored phrase-based language decoder
+ Copyright (C) 2010 Hieu Hoang
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ ***********************************************************************/
+
+#pragma once
+
+#include <list>
+#include <vector>
+#include "Range.h"
+#include "StackVec.h"
+#include "InputPath.h"
+#include "TargetPhraseCollection.h"
+namespace Moses
+{
+
+class ChartParserCallback;
+class ChartRuleLookupManager;
+class InputType;
+class Sentence;
+class ChartCellCollectionBase;
+class Word;
+class Phrase;
+// class TargetPhraseCollection;
+class DecodeGraph;
+
+class ChartParserUnknown
+{
+  ttaskwptr m_ttask;
+public:
+  ChartParserUnknown(ttasksptr const& ttask);
+  ~ChartParserUnknown();
+
+  void Process(const Word &sourceWord, const Range &range, ChartParserCallback &to);
+
+  const std::vector<Phrase*> &GetUnknownSources() const {
+    return m_unksrcs;
+  }
+
+private:
+  std::vector<Phrase*> m_unksrcs;
+  std::list<TargetPhraseCollection::shared_ptr> m_cacheTargetPhraseCollection;
+  AllOptions::ptr const& options() const;
+};
+
+class ChartParser
+{
+  ttaskwptr m_ttask;
+public:
+  ChartParser(ttasksptr const& ttask, ChartCellCollectionBase &cells);
+  ~ChartParser();
+
+  void Create(const Range &range, ChartParserCallback &to);
+
+  //! the sentence being decoded
+  //const Sentence &GetSentence() const;
+  long GetTranslationId() const;
+  size_t GetSize() const;
+  const InputPath &GetInputPath(size_t startPos, size_t endPos) const;
+  const InputPath &GetInputPath(const Range &range) const;
+  const std::vector<Phrase*> &GetUnknownSources() const {
+    return m_unknown.GetUnknownSources();
+  }
+
+  AllOptions::ptr const& options() const;
+
+private:
+  ChartParserUnknown m_unknown;
+  std::vector <DecodeGraph*> m_decodeGraphList;
+  std::vector<ChartRuleLookupManager*> m_ruleLookupManagers;
+  InputType const& m_source; /**< source sentence to be translated */
+
+  typedef std::vector< std::vector<InputPath*> > InputPathMatrix;
+  InputPathMatrix	m_inputPathMatrix;
+
+  void CreateInputPaths(const InputType &input);
+  InputPath &GetInputPath(size_t startPos, size_t endPos);
+
+};
+
+}
+

mosesdecoder/moses/ChartParserCallback.h

@@ -0,0 +1,35 @@
+#pragma once
+
+#include "StackVec.h"
+
+#include <list>
+#include "TargetPhraseCollection.h"
+
+namespace Moses
+{
+
+class TargetPhraseCollection;
+class Range;
+class TargetPhrase;
+class InputPath;
+class InputType;
+class ChartCellLabel;
+
+class ChartParserCallback
+{
+public:
+  virtual ~ChartParserCallback() {}
+
+  virtual void Add(const TargetPhraseCollection &, const StackVec &, const Range &) = 0;
+
+  virtual bool Empty() const = 0;
+
+  virtual void AddPhraseOOV(TargetPhrase &phrase, std::list<TargetPhraseCollection::shared_ptr > &waste_memory, const Range &range) = 0;
+
+  virtual void EvaluateWithSourceContext(const InputType &input, const InputPath &inputPath) = 0;
+
+  virtual float GetBestScore(const ChartCellLabel *chartCell) const = 0;
+
+};
+
+} // namespace Moses

mosesdecoder/moses/ChartRuleLookupManager.cpp

@@ -0,0 +1,9 @@
+#include "ChartRuleLookupManager.h"
+#include "ChartParser.h"
+
+namespace Moses
+{
+ChartRuleLookupManager::~ChartRuleLookupManager()
+{}
+}  // namespace Moses
+

mosesdecoder/moses/ChartRuleLookupManager.h

@@ -0,0 +1,84 @@
+/***********************************************************************
+  Moses - factored phrase-based language decoder
+  Copyright (C) 2011 University of Edinburgh
+
+  This library is free software; you can redistribute it and/or
+  modify it under the terms of the GNU Lesser General Public
+  License as published by the Free Software Foundation; either
+  version 2.1 of the License, or (at your option) any later version.
+
+  This library is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+  Lesser General Public License for more details.
+
+  You should have received a copy of the GNU Lesser General Public
+  License along with this library; if not, write to the Free Software
+  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ ***********************************************************************/
+
+#pragma once
+#ifndef moses_ChartRuleLookupManager_h
+#define moses_ChartRuleLookupManager_h
+
+#include "ChartCellCollection.h"
+#include "InputType.h"
+
+namespace Moses
+{
+class ChartParser;
+class ChartParserCallback;
+class Range;
+class Sentence;
+
+/** Defines an interface for looking up rules in a rule table.  Concrete
+ *  implementation classes should correspond to specific PhraseDictionary
+ *  subclasses (memory or on-disk).  Since a ChartRuleLookupManager object
+ *  maintains sentence-specific state, exactly one should be created for
+ *  each sentence that is to be decoded.
+ */
+class ChartRuleLookupManager
+{
+public:
+  ChartRuleLookupManager(const ChartParser &parser,
+                         const ChartCellCollectionBase &cellColl)
+    : m_parser(parser)
+    , m_cellCollection(cellColl) {}
+
+  virtual ~ChartRuleLookupManager();
+
+  const ChartCellLabelSet &GetTargetLabelSet(size_t begin, size_t end) const {
+    return m_cellCollection.GetBase(Range(begin, end)).GetTargetLabelSet();
+  }
+
+  const ChartParser &GetParser() const {
+    return m_parser;
+  }
+  //const Sentence &GetSentence() const;
+
+  const ChartCellLabel &GetSourceAt(size_t at) const {
+    return m_cellCollection.GetSourceWordLabel(at);
+  }
+
+  /** abstract function. Return a vector of translation options for given a range in the input sentence
+   *  \param range source range for which you want the translation options
+   *  \param outColl return argument
+   */
+  virtual void GetChartRuleCollection(
+    const InputPath &inputPath,
+    size_t lastPos,  // last position to consider if using lookahead
+    ChartParserCallback &outColl) = 0;
+
+private:
+  //! Non-copyable: copy constructor and assignment operator not implemented.
+  ChartRuleLookupManager(const ChartRuleLookupManager &);
+  //! Non-copyable: copy constructor and assignment operator not implemented.
+  ChartRuleLookupManager &operator=(const ChartRuleLookupManager &);
+
+  const ChartParser &m_parser;
+  const ChartCellCollectionBase &m_cellCollection;
+};
+
+}  // namespace Moses
+
+#endif

mosesdecoder/moses/ChartTranslationOption.cpp

@@ -0,0 +1,33 @@
+#include "ChartTranslationOptions.h"
+#include "InputType.h"
+#include "InputPath.h"
+
+namespace Moses
+{
+ChartTranslationOption::ChartTranslationOption(const TargetPhrase &targetPhrase)
+  :m_targetPhrase(targetPhrase)
+  ,m_scoreBreakdown(targetPhrase.GetScoreBreakdown())
+{
+}
+
+void ChartTranslationOption::EvaluateWithSourceContext(const InputType &input,
+    const InputPath &inputPath,
+    const StackVec &stackVec)
+{
+  const std::vector<FeatureFunction*> &ffs = FeatureFunction::GetFeatureFunctions();
+
+  for (size_t i = 0; i < ffs.size(); ++i) {
+    const FeatureFunction &ff = *ffs[i];
+    ff.EvaluateWithSourceContext(input, inputPath, m_targetPhrase, &stackVec, m_scoreBreakdown);
+  }
+}
+
+
+std::ostream& operator<<(std::ostream &out, const ChartTranslationOption &transOpt)
+{
+  out << transOpt.m_targetPhrase << " " << transOpt.m_scoreBreakdown;
+  return out;
+}
+
+}
+

mosesdecoder/moses/ChartTranslationOption.h

@@ -0,0 +1,54 @@
+#pragma once
+
+#include "ScoreComponentCollection.h"
+
+namespace Moses
+{
+class TargetPhrase;
+class InputPath;
+class InputType;
+class StackVec;
+
+class ChartTranslationOption
+{
+  friend std::ostream& operator<<(std::ostream&, const ChartTranslationOption&);
+
+protected:
+  const TargetPhrase &m_targetPhrase;
+  ScoreComponentCollection m_scoreBreakdown;
+  const InputPath *m_inputPath;
+  const std::vector<const Word*> *m_ruleSourceFromInputPath; // used by placeholders
+
+public:
+  ChartTranslationOption(const TargetPhrase &targetPhrase);
+
+  const TargetPhrase &GetPhrase() const {
+    return m_targetPhrase;
+  }
+
+  const InputPath *GetInputPath() const {
+    return m_inputPath;
+  }
+
+  void SetInputPath(const InputPath *inputPath) {
+    m_inputPath = inputPath;
+  }
+
+  const std::vector<const Word*> *GetSourceRuleFromInputPath() const {
+    return m_ruleSourceFromInputPath;
+  }
+  void SetSourceRuleFromInputPath(const std::vector<const Word*> *obj) {
+    m_ruleSourceFromInputPath = obj;
+  }
+
+  const ScoreComponentCollection &GetScores() const {
+    return m_scoreBreakdown;
+  }
+
+  void EvaluateWithSourceContext(const InputType &input,
+                                 const InputPath &inputPath,
+                                 const StackVec &stackVec);
+};
+
+}
+

mosesdecoder/moses/ChartTranslationOptionList.cpp

@@ -0,0 +1,219 @@
+/***********************************************************************
+ Moses - factored phrase-based language decoder
+ Copyright (C) 2010 Hieu Hoang
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ ***********************************************************************/
+
+#include <algorithm>
+#include <iostream>
+#include <vector>
+#include "StaticData.h"
+#include "ChartTranslationOptionList.h"
+#include "ChartTranslationOptions.h"
+#include "ChartCellCollection.h"
+#include "Range.h"
+#include "InputType.h"
+#include "InputPath.h"
+
+using namespace std;
+
+namespace Moses
+{
+
+ChartTranslationOptionList::
+ChartTranslationOptionList(size_t ruleLimit, const InputType &input)
+  : m_size(0)
+  , m_ruleLimit(ruleLimit)
+{
+  m_scoreThreshold = std::numeric_limits<float>::infinity();
+}
+
+ChartTranslationOptionList::~ChartTranslationOptionList()
+{
+  RemoveAllInColl(m_collection);
+}
+
+void ChartTranslationOptionList::Clear()
+{
+  m_size = 0;
+  m_scoreThreshold = std::numeric_limits<float>::infinity();
+}
+
+class ChartTranslationOptionOrderer
+{
+public:
+  bool operator()(const ChartTranslationOptions* itemA, const ChartTranslationOptions* itemB) const {
+    return itemA->GetEstimateOfBestScore() > itemB->GetEstimateOfBestScore();
+  }
+};
+
+void ChartTranslationOptionList::Add(const TargetPhraseCollection &tpc,
+                                     const StackVec &stackVec,
+                                     const Range &range)
+{
+  if (tpc.IsEmpty()) {
+    return;
+  }
+
+  for (size_t i = 0; i < stackVec.size(); ++i) {
+    const ChartCellLabel &chartCellLabel = *stackVec[i];
+    size_t numHypos = chartCellLabel.GetStack().cube->size();
+    if (numHypos == 0) {
+      return; // empty stack. These rules can't be used
+    }
+  }
+
+  const TargetPhrase &targetPhrase = **(tpc.begin());
+  float score = targetPhrase.GetFutureScore();
+  for (StackVec::const_iterator p = stackVec.begin(); p != stackVec.end(); ++p) {
+    score += (*p)->GetBestScore(this);
+  }
+
+  // If the rule limit has already been reached then don't add the option
+  // unless it is better than at least one existing option.
+  if (m_ruleLimit && m_size > m_ruleLimit && score < m_scoreThreshold) {
+    return;
+  }
+
+  // Add the option to the list.
+  if (m_size == m_collection.size()) {
+    // m_collection has reached capacity: create a new object.
+    m_collection.push_back(new ChartTranslationOptions(tpc, stackVec,
+                           range, score));
+  } else {
+    // Overwrite an unused object.
+    *(m_collection[m_size]) = ChartTranslationOptions(tpc, stackVec,
+                              range, score);
+  }
+  ++m_size;
+
+  // If the rule limit hasn't been exceeded then update the threshold.
+  if (!m_ruleLimit || m_size <= m_ruleLimit) {
+    m_scoreThreshold = (score < m_scoreThreshold) ? score : m_scoreThreshold;
+  }
+
+  // Prune if bursting
+  if (m_ruleLimit && m_size == m_ruleLimit * 2) {
+    NTH_ELEMENT4(m_collection.begin(),
+                 m_collection.begin() + m_ruleLimit - 1,
+                 m_collection.begin() + m_size,
+                 ChartTranslationOptionOrderer());
+    m_scoreThreshold = m_collection[m_ruleLimit-1]->GetEstimateOfBestScore();
+    m_size = m_ruleLimit;
+  }
+}
+
+void
+ChartTranslationOptionList::
+AddPhraseOOV(TargetPhrase &phrase,
+             std::list<TargetPhraseCollection::shared_ptr > &waste_memory,
+             const Range &range)
+{
+  TargetPhraseCollection::shared_ptr tpc(new TargetPhraseCollection);
+  tpc->Add(&phrase);
+  waste_memory.push_back(tpc);
+  StackVec empty;
+  Add(*tpc, empty, range);
+}
+
+void ChartTranslationOptionList::ApplyThreshold(float const threshold)
+{
+  if (m_ruleLimit && m_size > m_ruleLimit) {
+    // Something's gone wrong if the list has grown to m_ruleLimit * 2
+    // without being pruned.
+    assert(m_size < m_ruleLimit * 2);
+    // Reduce the list to the best m_ruleLimit options.  The remaining
+    // options can be overwritten on subsequent calls to Add().
+    NTH_ELEMENT4(m_collection.begin(),
+                 m_collection.begin()+m_ruleLimit,
+                 m_collection.begin()+m_size,
+                 ChartTranslationOptionOrderer());
+    m_size = m_ruleLimit;
+  }
+
+  // keep only those over best + threshold
+
+  float scoreThreshold = -std::numeric_limits<float>::infinity();
+
+  CollType::const_iterator iter;
+  for (iter = m_collection.begin(); iter != m_collection.begin()+m_size; ++iter) {
+    const ChartTranslationOptions *transOpt = *iter;
+    float score = transOpt->GetEstimateOfBestScore();
+    scoreThreshold = (score > scoreThreshold) ? score : scoreThreshold;
+  }
+
+  scoreThreshold += threshold; // StaticData::Instance().GetTranslationOptionThreshold();
+
+  CollType::iterator bound = std::partition(m_collection.begin(),
+                             m_collection.begin()+m_size,
+                             ScoreThresholdPred(scoreThreshold));
+
+  m_size = std::distance(m_collection.begin(), bound);
+}
+
+float ChartTranslationOptionList::GetBestScore(const ChartCellLabel *chartCell) const
+{
+  const HypoList *stack = chartCell->GetStack().cube;
+  assert(stack);
+  assert(!stack->empty());
+  const ChartHypothesis &bestHypo = **(stack->begin());
+  return bestHypo.GetFutureScore();
+}
+
+void ChartTranslationOptionList::EvaluateWithSourceContext(const InputType &input, const InputPath &inputPath)
+{
+  // NEVER iterate over ALL of the collection. Just over the first m_size
+  CollType::iterator iter;
+  for (iter = m_collection.begin(); iter != m_collection.begin() + m_size; ++iter) {
+    ChartTranslationOptions &transOpts = **iter;
+    transOpts.EvaluateWithSourceContext(input, inputPath);
+  }
+
+  // get rid of empty trans opts
+  size_t numDiscard = 0;
+  for (size_t i = 0; i < m_size; ++i) {
+    ChartTranslationOptions *transOpts = m_collection[i];
+    if (transOpts->GetSize() == 0) {
+      //delete transOpts;
+      ++numDiscard;
+    } else if (numDiscard) {
+      SwapTranslationOptions(i - numDiscard, i);
+      //m_collection[] = transOpts;
+    }
+  }
+
+  size_t newSize = m_size - numDiscard;
+  m_size = newSize;
+}
+
+void ChartTranslationOptionList::SwapTranslationOptions(size_t a, size_t b)
+{
+  ChartTranslationOptions *transOptsA = m_collection[a];
+  ChartTranslationOptions *transOptsB = m_collection[b];
+  m_collection[a] = transOptsB;
+  m_collection[b] = transOptsA;
+}
+
+std::ostream& operator<<(std::ostream &out, const ChartTranslationOptionList &obj)
+{
+  for (size_t i = 0; i < obj.m_collection.size(); ++i) {
+    const ChartTranslationOptions &transOpts = *obj.m_collection[i];
+    out << transOpts << endl;
+  }
+  return out;
+}
+
+}

mosesdecoder/moses/ChartTranslationOptionList.h

@@ -0,0 +1,90 @@
+/***********************************************************************
+Moses - factored phrase-based language decoder
+Copyright (C) 2006 University of Edinburgh
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Lesser General Public
+License as published by the Free Software Foundation; either
+version 2.1 of the License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public
+License along with this library; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+***********************************************************************/
+
+#pragma once
+
+#include "ChartTranslationOptions.h"
+#include "ChartParserCallback.h"
+#include "StackVec.h"
+
+#include <vector>
+
+namespace Moses
+{
+
+class TargetPhraseCollection;
+class Range;
+class InputType;
+class InputPath;
+class ChartCellLabel;
+
+//! a vector of translations options for a specific range, in a specific sentence
+class ChartTranslationOptionList : public ChartParserCallback
+{
+  friend std::ostream& operator<<(std::ostream&, const ChartTranslationOptionList&);
+
+public:
+  ChartTranslationOptionList(size_t ruleLimit, const InputType &input);
+  ~ChartTranslationOptionList();
+
+  const ChartTranslationOptions &Get(size_t i) const {
+    return *m_collection[i];
+  }
+
+  //! number of translation options
+  size_t GetSize() const {
+    return m_size;
+  }
+
+  void Add(const TargetPhraseCollection &, const StackVec &,
+           const Range &);
+
+  void AddPhraseOOV(TargetPhrase &phrase, std::list<TargetPhraseCollection::shared_ptr > &waste_memory, const Range &range);
+
+  bool Empty() const {
+    return m_size == 0;
+  }
+
+  float GetBestScore(const ChartCellLabel *chartCell) const;
+
+  void Clear();
+  void ApplyThreshold(float threshold);
+  void EvaluateWithSourceContext(const InputType &input, const InputPath &inputPath);
+
+private:
+  typedef std::vector<ChartTranslationOptions*> CollType;
+
+  struct ScoreThresholdPred {
+    ScoreThresholdPred(float threshold) : m_thresholdScore(threshold) {}
+    bool operator()(const ChartTranslationOptions *option)  {
+      return option->GetEstimateOfBestScore() >= m_thresholdScore;
+    }
+    float m_thresholdScore;
+  };
+
+  void SwapTranslationOptions(size_t a, size_t b);
+
+  CollType m_collection;
+  size_t m_size;
+  float m_scoreThreshold;
+  const size_t m_ruleLimit;
+
+};
+
+}

mosesdecoder/moses/ChartTranslationOptions.cpp

@@ -0,0 +1,168 @@
+/***********************************************************************
+ Moses - factored phrase-based language decoder
+ Copyright (C) 2010 Hieu Hoang
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ ***********************************************************************/
+
+#include "ChartTranslationOptions.h"
+#include "ChartHypothesis.h"
+#include "ChartCellLabel.h"
+#include "ChartTranslationOption.h"
+#include "InputPath.h"
+#include "StaticData.h"
+#include "TranslationTask.h"
+
+using namespace std;
+
+namespace Moses
+{
+
+ChartTranslationOptions::ChartTranslationOptions(const TargetPhraseCollection &targetPhraseColl,
+    const StackVec &stackVec,
+    const Range &range,
+    float score)
+  : m_stackVec(stackVec)
+  , m_wordsRange(&range)
+  , m_estimateOfBestScore(score)
+{
+  TargetPhraseCollection::const_iterator iter;
+  for (iter = targetPhraseColl.begin(); iter != targetPhraseColl.end(); ++iter) {
+    const TargetPhrase *origTP = *iter;
+
+    boost::shared_ptr<ChartTranslationOption> ptr(new ChartTranslationOption(*origTP));
+    m_collection.push_back(ptr);
+  }
+}
+
+ChartTranslationOptions::~ChartTranslationOptions()
+{
+
+}
+
+//! functor to compare (chart) hypotheses by (descending) score
+class ChartTranslationOptionScoreOrderer
+{
+public:
+  bool operator()(const boost::shared_ptr<ChartTranslationOption> &transOptA
+                  , const boost::shared_ptr<ChartTranslationOption> &transOptB) const {
+    const ScoreComponentCollection &scoresA = transOptA->GetScores();
+    const ScoreComponentCollection &scoresB = transOptB->GetScores();
+    return scoresA.GetWeightedScore() > scoresB.GetWeightedScore();
+  }
+};
+
+void ChartTranslationOptions::EvaluateWithSourceContext(const InputType &input, const InputPath &inputPath)
+{
+  SetInputPath(&inputPath);
+  // if (StaticData::Instance().GetPlaceholderFactor() != NOT_FOUND) {
+  if (inputPath.ttask->options()->input.placeholder_factor != NOT_FOUND) {
+    CreateSourceRuleFromInputPath();
+  }
+
+  CollType::iterator iter;
+  for (iter = m_collection.begin(); iter != m_collection.end(); ++iter) {
+    ChartTranslationOption &transOpt = **iter;
+    transOpt.SetInputPath(&inputPath);
+    transOpt.EvaluateWithSourceContext(input, inputPath, m_stackVec);
+  }
+
+  // get rid of -inf trans opts
+  size_t numDiscard = 0;
+  for (size_t i = 0; i < m_collection.size(); ++i) {
+    ChartTranslationOption *transOpt = m_collection[i].get();
+
+    if (transOpt->GetScores().GetWeightedScore() == - std::numeric_limits<float>::infinity()) {
+      ++numDiscard;
+    } else if (numDiscard) {
+      m_collection[i - numDiscard] = m_collection[i];
+    }
+  }
+
+  size_t newSize = m_collection.size() - numDiscard;
+  m_collection.resize(newSize);
+
+  // sort if necessary
+  const StaticData &staticData = StaticData::Instance();
+  if (staticData.RequireSortingAfterSourceContext()) {
+    std::sort(m_collection.begin()
+              , m_collection.begin() + newSize
+              , ChartTranslationOptionScoreOrderer());
+  }
+
+}
+
+void ChartTranslationOptions::SetInputPath(const InputPath *inputPath)
+{
+  CollType::iterator iter;
+  for (iter = m_collection.begin(); iter != m_collection.end(); ++iter) {
+    ChartTranslationOption &transOpt = **iter;
+    transOpt.SetInputPath(inputPath);
+  }
+}
+
+void ChartTranslationOptions::CreateSourceRuleFromInputPath()
+{
+  if (m_collection.size() == 0) {
+    return;
+  }
+
+  const InputPath *inputPath = m_collection.front()->GetInputPath();
+  assert(inputPath);
+  std::vector<const Word*> &ruleSourceFromInputPath = inputPath->AddRuleSourceFromInputPath();
+
+  size_t chartCellIndex = 0;
+  const ChartCellLabel *chartCellLabel = (chartCellIndex < m_stackVec.size()) ? m_stackVec[chartCellIndex] : NULL;
+
+  size_t ind = 0;
+  for (size_t sourcePos = m_wordsRange->GetStartPos(); sourcePos <= m_wordsRange->GetEndPos(); ++sourcePos, ++ind) {
+    if (chartCellLabel) {
+      if (sourcePos == chartCellLabel->GetCoverage().GetEndPos()) {
+        // end of child range. push an empty word to denote non-term
+        ruleSourceFromInputPath.push_back(NULL);
+        ++chartCellIndex;
+        chartCellLabel = (chartCellIndex < m_stackVec.size()) ? m_stackVec[chartCellIndex] : NULL;
+      } else if (sourcePos >= chartCellLabel->GetCoverage().GetStartPos()) {
+        // in the range of child hypo. do nothing
+      } else {
+        // not yet reached child range. add word
+        ruleSourceFromInputPath.push_back(&inputPath->GetPhrase().GetWord(ind));
+      }
+    } else {
+      // no child in sight. add word
+      ruleSourceFromInputPath.push_back(&inputPath->GetPhrase().GetWord(ind));
+    }
+  }
+
+  // save it to each trans opt
+  CollType::iterator iter;
+  for (iter = m_collection.begin(); iter != m_collection.end(); ++iter) {
+    ChartTranslationOption &transOpt = **iter;
+    transOpt.SetSourceRuleFromInputPath(&ruleSourceFromInputPath);
+  }
+
+}
+
+std::ostream& operator<<(std::ostream &out, const ChartTranslationOptions &obj)
+{
+  for (size_t i = 0; i < obj.m_collection.size(); ++i) {
+    const ChartTranslationOption &transOpt = *obj.m_collection[i];
+    out << transOpt << endl;
+  }
+
+  return out;
+}
+
+}

mosesdecoder/moses/ChartTranslationOptions.h

@@ -0,0 +1,104 @@
+/***********************************************************************
+ Moses - factored phrase-based language decoder
+ Copyright (C) 2010 Hieu Hoang
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ ***********************************************************************/
+
+#pragma once
+
+#include "StackVec.h"
+#include "TargetPhrase.h"
+#include "TargetPhraseCollection.h"
+#include "Range.h"
+
+#include <vector>
+#include <boost/shared_ptr.hpp>
+#include "ChartTranslationOption.h"
+
+namespace Moses
+{
+class ChartTranslationOption;
+class InputPath;
+class InputType;
+
+/** Similar to a DottedRule, but contains a direct reference to a list
+ * of translations and provdes an estimate of the best score. For a specific range in the input sentence
+ */
+class ChartTranslationOptions
+{
+  friend std::ostream& operator<<(std::ostream&, const ChartTranslationOptions&);
+
+public:
+  typedef std::vector<boost::shared_ptr<ChartTranslationOption> > CollType;
+
+  /** Constructor
+      \param targetPhraseColl @todo dunno
+      \param stackVec @todo dunno
+      \param range the range in the source sentence this translation option covers
+      \param score @todo dunno
+   */
+  ChartTranslationOptions(const TargetPhraseCollection &targetPhraseColl,
+                          const StackVec &stackVec,
+                          const Range &range,
+                          float score);
+  ~ChartTranslationOptions();
+
+  static float CalcEstimateOfBestScore(const TargetPhraseCollection &,
+                                       const StackVec &);
+
+  size_t GetSize() const {
+    return m_collection.size();
+  }
+
+  //! @todo dunno
+  const StackVec &GetStackVec() const {
+    return m_stackVec;
+  }
+
+  //! @todo isn't the translation suppose to just contain 1 target phrase, not a whole collection of them?
+  const CollType &GetTargetPhrases() const {
+    return m_collection;
+  }
+
+  //! the range in the source sentence this translation option covers
+  const Range &GetSourceWordsRange() const {
+    return *m_wordsRange;
+  }
+
+  /** return an estimate of the best score possible with this translation option.
+    * the estimate is the sum of the top target phrase's estimated score plus the
+    * scores of the best child hypotheses.
+    */
+  inline float GetEstimateOfBestScore() const {
+    return m_estimateOfBestScore;
+  }
+
+  void EvaluateWithSourceContext(const InputType &input, const InputPath &inputPath);
+
+  void SetInputPath(const InputPath *inputPath);
+
+  void CreateSourceRuleFromInputPath();
+
+private:
+
+  StackVec m_stackVec; //! vector of hypothesis list!
+  CollType m_collection;
+
+  const Range *m_wordsRange;
+  float m_estimateOfBestScore;
+};
+
+}

mosesdecoder/moses/ConfusionNet.cpp

@@ -0,0 +1,294 @@
+// -*- mode: c++; indent-tabs-mode: nil; tab-width: 2 -*-
+// $Id$
+
+#include "ConfusionNet.h"
+#include <sstream>
+
+#include "FactorCollection.h"
+#include "Util.h"
+#include "TranslationOptionCollectionConfusionNet.h"
+#include "StaticData.h"
+#include "Sentence.h"
+#include "moses/FF/InputFeature.h"
+#include "util/exception.hh"
+#include "moses/TranslationTask.h"
+namespace Moses
+{
+struct CNStats {
+  size_t created,destr,read,colls,words;
+
+  CNStats() : created(0),destr(0),read(0),colls(0),words(0) {}
+  ~CNStats() {
+    print(std::cerr);
+  }
+
+  void createOne() {
+    ++created;
+  }
+  void destroyOne() {
+    ++destr;
+  }
+
+  void collect(const ConfusionNet& cn) {
+    ++read;
+    colls+=cn.GetSize();
+    for(size_t i=0; i<cn.GetSize(); ++i)
+      words+=cn[i].size();
+  }
+  void print(std::ostream& out) const {
+    if(created>0) {
+      out<<"confusion net statistics:\n"
+         " created:\t"<<created<<"\n"
+         " destroyed:\t"<<destr<<"\n"
+         " succ. read:\t"<<read<<"\n"
+         " columns:\t"<<colls<<"\n"
+         " words:\t"<<words<<"\n"
+         " avg. word/column:\t"<<words/(1.0*colls)<<"\n"
+         " avg. cols/sent:\t"<<colls/(1.0*read)<<"\n"
+         "\n\n";
+    }
+  }
+};
+
+CNStats stats;
+
+size_t
+ConfusionNet::
+GetColumnIncrement(size_t i, size_t j) const
+{
+  (void) i;
+  (void) j;
+  return 1;
+}
+
+ConfusionNet::
+ConfusionNet(AllOptions::ptr const& opts) : InputType(opts)
+{
+  stats.createOne();
+
+  if (is_syntax(opts->search.algo)) {
+    m_defaultLabelSet.insert(opts->syntax.input_default_non_terminal);
+  }
+  UTIL_THROW_IF2(InputFeature::InstancePtr() == NULL, "Input feature must be specified");
+}
+
+ConfusionNet::
+~ConfusionNet()
+{
+  stats.destroyOne();
+}
+
+ConfusionNet::
+ConfusionNet(Sentence const& s) : InputType(s.options())
+{
+  data.resize(s.GetSize());
+  for(size_t i=0; i<s.GetSize(); ++i) {
+    ScorePair scorePair;
+    std::pair<Word, ScorePair > temp = std::make_pair(s.GetWord(i), scorePair);
+    data[i].push_back(temp);
+  }
+}
+
+bool
+ConfusionNet::
+ReadF(std::istream& in, int format)
+{
+  VERBOSE(2, "read confusion net with format "<<format<<"\n");
+  switch(format) {
+  case 0:
+    return ReadFormat0(in);
+  case 1:
+    return ReadFormat1(in);
+  default:
+    std::cerr << "ERROR: unknown format '"<<format
+              <<"' in ConfusionNet::Read";
+  }
+  return false;
+}
+
+int
+ConfusionNet::
+Read(std::istream& in)
+{
+  int rv=ReadF(in,0);
+  if(rv) stats.collect(*this);
+  return rv;
+}
+
+bool
+ConfusionNet::
+ReadFormat0(std::istream& in)
+{
+  Clear();
+  const std::vector<FactorType>& factorOrder = m_options->input.factor_order;
+
+  const InputFeature *inputFeature = InputFeature::InstancePtr();
+  size_t numInputScores   = inputFeature->GetNumInputScores();
+  size_t numRealWordCount = inputFeature->GetNumRealWordsInInput();
+
+  size_t totalCount = numInputScores + numRealWordCount;
+  bool addRealWordCount = (numRealWordCount > 0);
+
+  std::string line;
+  while(getline(in,line)) {
+    std::istringstream is(line);
+    std::string word;
+
+    Column col;
+    while(is>>word) {
+      Word w;
+      w.CreateFromString(Input,factorOrder,StringPiece(word),false,false);
+      std::vector<float> probs(totalCount, 0.0);
+      for(size_t i=0; i < numInputScores; i++) {
+        double prob;
+        if (!(is>>prob)) {
+          TRACE_ERR("ERROR: unable to parse CN input - bad link probability, "
+                    << "or wrong number of scores\n");
+          return false;
+        }
+        if(prob<0.0) {
+          VERBOSE(1, "WARN: negative prob: "<<prob<<" ->set to 0.0\n");
+          prob=0.0;
+        } else if (prob>1.0) {
+          VERBOSE(1, "WARN: prob > 1.0 : "<<prob<<" -> set to 1.0\n");
+          prob=1.0;
+        }
+        probs[i] = (std::max(static_cast<float>(log(prob)),LOWEST_SCORE));
+
+      }
+      // store 'real' word count in last feature if we have one more
+      // weight than we do arc scores and not epsilon
+      if (addRealWordCount && word!=EPSILON && word!="")
+        probs.back() = -1.0;
+
+      ScorePair scorePair(probs);
+
+      col.push_back(std::make_pair(w,scorePair));
+    }
+    if(col.size()) {
+      data.push_back(col);
+      ShrinkToFit(data.back());
+    } else break;
+  }
+  return !data.empty();
+}
+
+bool
+ConfusionNet::
+ReadFormat1(std::istream& in)
+{
+  Clear();
+  const std::vector<FactorType>& factorOrder = m_options->input.factor_order;
+  std::string line;
+  if(!getline(in,line)) return 0;
+  size_t s;
+  if(getline(in,line)) s=atoi(line.c_str());
+  else return 0;
+  data.resize(s);
+  for(size_t i=0; i<data.size(); ++i) {
+    if(!getline(in,line)) return 0;
+    std::istringstream is(line);
+    if(!(is>>s)) return 0;
+    std::string word;
+    double prob;
+    data[i].resize(s);
+    for(size_t j=0; j<s; ++j)
+      if(is>>word>>prob) {
+        //TODO: we are only reading one prob from this input format, should read many... but this function is unused anyway. -JS
+        data[i][j].second.denseScores = std::vector<float> (1);
+        data[i][j].second.denseScores.push_back((float) log(prob));
+        if(data[i][j].second.denseScores[0]<0) {
+          VERBOSE(1, "WARN: neg costs: "<<data[i][j].second.denseScores[0]<<" -> set to 0\n");
+          data[i][j].second.denseScores[0]=0.0;
+        }
+        // String2Word(word,data[i][j].first,factorOrder);
+        Word& w = data[i][j].first;
+        w.CreateFromString(Input,factorOrder,StringPiece(word),false,false);
+      } else return 0;
+  }
+  return !data.empty();
+}
+
+void ConfusionNet::Print(std::ostream& out) const
+{
+  out<<"conf net: "<<data.size()<<"\n";
+  for(size_t i=0; i<data.size(); ++i) {
+    out<<i<<" -- ";
+    for(size_t j=0; j<data[i].size(); ++j) {
+      out<<"("<<data[i][j].first.ToString()<<", ";
+
+      // dense
+      std::vector<float>::const_iterator iterDense;
+      for(iterDense = data[i][j].second.denseScores.begin();
+          iterDense < data[i][j].second.denseScores.end();
+          ++iterDense) {
+        out<<", "<<*iterDense;
+      }
+
+      // sparse
+      std::map<StringPiece, float>::const_iterator iterSparse;
+      for(iterSparse = data[i][j].second.sparseScores.begin();
+          iterSparse != data[i][j].second.sparseScores.end();
+          ++iterSparse) {
+        out << ", " << iterSparse->first << "=" << iterSparse->second;
+      }
+
+      out<<") ";
+    }
+    out<<"\n";
+  }
+  out<<"\n\n";
+}
+
+#ifdef _WIN32
+#pragma warning(disable:4716)
+#endif
+Phrase
+ConfusionNet::
+GetSubString(const Range&) const
+{
+  UTIL_THROW2("ERROR: call to ConfusionNet::GetSubString\n");
+  //return Phrase(Input);
+}
+
+std::string
+ConfusionNet::
+GetStringRep(const std::vector<FactorType> /* factorsToPrint */) const  //not well defined yet
+{
+  TRACE_ERR("ERROR: call to ConfusionNet::GeStringRep\n");
+  return "";
+}
+#ifdef _WIN32
+#pragma warning(disable:4716)
+#endif
+const Word& ConfusionNet::GetWord(size_t) const
+{
+  UTIL_THROW2("ERROR: call to ConfusionNet::GetFactorArray\n");
+}
+#ifdef _WIN32
+#pragma warning(default:4716)
+#endif
+std::ostream& operator<<(std::ostream& out,const ConfusionNet& cn)
+{
+  cn.Print(out);
+  return out;
+}
+
+TranslationOptionCollection*
+ConfusionNet::
+CreateTranslationOptionCollection(ttasksptr const& ttask) const
+{
+  // size_t maxNoTransOptPerCoverage
+  //   = ttask->options()->search.max_trans_opt_per_cov;
+  // float translationOptionThreshold
+  //   = ttask->options()->search.trans_opt_threshold;
+  TranslationOptionCollection *rv
+  = new TranslationOptionCollectionConfusionNet(ttask, *this);
+  //, maxNoTransOptPerCoverage, translationOptionThreshold);
+  assert(rv);
+  return rv;
+}
+
+}
+
+

mosesdecoder/moses/ConfusionNet.h

@@ -0,0 +1,92 @@
+// $Id$
+
+#ifndef moses_ConfusionNet_h
+#define moses_ConfusionNet_h
+
+#include <vector>
+#include <iostream>
+#include "Word.h"
+#include "InputType.h"
+#include "NonTerminal.h"
+#include "util/exception.hh"
+
+namespace Moses
+{
+
+class FactorCollection;
+class TranslationOptionCollection;
+class Sentence;
+class TranslationTask;
+
+/** An input to the decoder where each position can be 1 of a number of words,
+ *  each with an associated probability. Compared with a sentence, where each position is a word
+ */
+class ConfusionNet : public InputType
+{
+public:
+  typedef std::vector<std::pair<Word, ScorePair > > Column;
+
+protected:
+  std::vector<Column> data;
+  NonTerminalSet m_defaultLabelSet;
+
+  bool ReadFormat0(std::istream&);
+  bool ReadFormat1(std::istream&);
+  void String2Word(const std::string& s,Word& w,const std::vector<FactorType>& factorOrder);
+
+public:
+  ConfusionNet(AllOptions::ptr const& opts);
+  virtual ~ConfusionNet();
+
+  ConfusionNet(Sentence const& s);
+
+  InputTypeEnum GetType() const {
+    return ConfusionNetworkInput;
+  }
+
+  const Column& GetColumn(size_t i) const {
+    UTIL_THROW_IF2(i >= data.size(),
+                   "Out of bounds. Trying to access " << i
+                   << " when vector only contains " << data.size());
+    return data[i];
+  }
+  const Column& operator[](size_t i) const {
+    return GetColumn(i);
+  }
+  virtual size_t GetColumnIncrement(size_t i, size_t j) const; //! returns 1 for CNs
+
+  bool Empty() const {
+    return data.empty();
+  }
+  size_t GetSize() const {
+    return data.size();
+  }
+  void Clear() {
+    data.clear();
+  }
+
+  bool ReadF(std::istream&, int format=0);
+  virtual void Print(std::ostream&) const;
+
+  int Read(std::istream& in);
+
+  Phrase GetSubString(const Range&) const; //TODO not defined
+  std::string GetStringRep(const std::vector<FactorType> factorsToPrint) const; //TODO not defined
+  const Word& GetWord(size_t pos) const;
+
+  TranslationOptionCollection*
+  CreateTranslationOptionCollection(ttasksptr const& ttask) const;
+
+  const NonTerminalSet &GetLabelSet(size_t /*startPos*/, size_t /*endPos*/) const {
+    return m_defaultLabelSet;
+  }
+
+
+};
+
+std::ostream& operator<<(std::ostream& out,const ConfusionNet& cn);
+
+
+}
+
+#endif

mosesdecoder/moses/ContextScope.h

@@ -0,0 +1,124 @@
+// -*- mode: c++; indent-tabs-mode: nil; tab-width: 2 -*-
+// A class to store "local" information (such as task-specific caches).
+// The idea is for each translation task to have a scope, which stores
+// shared pointers to task-specific objects such as caches and priors.
+// Since these objects are referenced via shared pointers, sopes can
+// share information.
+#pragma once
+
+#ifdef WITH_THREADS
+#include <boost/thread/shared_mutex.hpp>
+#include <boost/thread/locks.hpp>
+#include <boost/foreach.hpp>
+#endif
+
+// for some reason, the xmlrpc_c headers must be included AFTER the
+// boost thread-related ones ...
+#include "xmlrpc-c.h"
+
+#include <map>
+#include <boost/shared_ptr.hpp>
+#include "TypeDef.h"
+#include "Util.h"
+
+namespace Moses
+{
+class ContextScope
+{
+protected:
+  typedef std::map<void const*, boost::shared_ptr<void> > scratchpad_t;
+  typedef scratchpad_t::iterator iter_t;
+  typedef scratchpad_t::value_type entry_t;
+  typedef scratchpad_t::const_iterator const_iter_t;
+  scratchpad_t m_scratchpad;
+#ifdef WITH_THREADS
+  mutable boost::shared_mutex m_lock;
+#endif
+  SPTR<std::map<std::string,float> const> m_context_weights;
+public:
+  typedef boost::shared_ptr<ContextScope> ptr;
+  template<typename T>
+  boost::shared_ptr<void> const&
+  set(void const* const key, boost::shared_ptr<T> const& val) {
+#ifdef WITH_THREADS
+    boost::unique_lock<boost::shared_mutex> lock(m_lock);
+#endif
+    return (m_scratchpad[key] = val);
+  }
+
+  template<typename T>
+  boost::shared_ptr<T> const
+  get(void const* key, bool CreateNewIfNecessary=false) {
+#ifdef WITH_THREADS
+    using boost::shared_mutex;
+    using boost::upgrade_lock;
+    // T const* key = reinterpret_cast<T const*>(xkey);
+    upgrade_lock<shared_mutex> lock(m_lock);
+#endif
+    iter_t m = m_scratchpad.find(key);
+    boost::shared_ptr< T > ret;
+    if (m != m_scratchpad.end()) {
+      if (m->second == NULL && CreateNewIfNecessary) {
+#ifdef WITH_THREADS
+        boost::upgrade_to_unique_lock<shared_mutex> xlock(lock);
+#endif
+        m->second.reset(new T);
+      }
+      ret = boost::static_pointer_cast< T >(m->second);
+      return ret;
+    }
+    if (!CreateNewIfNecessary) return ret;
+#ifdef WITH_THREADS
+    boost::upgrade_to_unique_lock<shared_mutex> xlock(lock);
+#endif
+    ret.reset(new T);
+    m_scratchpad[key] = ret;
+    return ret;
+  }
+
+  ContextScope() { }
+
+  ContextScope(ContextScope const& other) {
+#ifdef WITH_THREADS
+    boost::unique_lock<boost::shared_mutex> lock1(this->m_lock);
+    boost::unique_lock<boost::shared_mutex> lock2(other.m_lock);
+#endif
+    m_scratchpad = other.m_scratchpad;
+  }
+
+  SPTR<std::map<std::string,float> const>
+  GetContextWeights() {
+    return m_context_weights;
+  }
+
+  bool
+  SetContextWeights(std::string const& spec) {
+    if (m_context_weights) return false;
+    boost::unique_lock<boost::shared_mutex> lock(m_lock);
+    SPTR<std::map<std::string,float> > M(new std::map<std::string, float>);
+
+    // TO DO; This needs to be done with StringPiece.find, not Tokenize
+    // PRIORITY: low
+    std::vector<std::string> tokens = Tokenize(spec,":");
+    for (std::vector<std::string>::iterator it = tokens.begin();
+         it != tokens.end(); it++) {
+      std::vector<std::string> key_and_value = Tokenize(*it, ",");
+      (*M)[key_and_value[0]] = atof(key_and_value[1].c_str());
+    }
+    m_context_weights = M;
+    return true;
+  }
+
+  bool
+  SetContextWeights(SPTR<std::map<std::string,float> const> const& w) {
+    if (m_context_weights) return false;
+#ifdef WITH_THREADS
+    boost::unique_lock<boost::shared_mutex> lock(m_lock);
+#endif
+    m_context_weights = w;
+    return true;
+  }
+
+};
+
+};

mosesdecoder/moses/DecodeGraph.cpp

@@ -0,0 +1,43 @@
+// $Id$
+// vim:tabstop=2
+
+/***********************************************************************
+Moses - factored phrase-based language decoder
+Copyright (C) 2006 University of Edinburgh
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Lesser General Public
+License as published by the Free Software Foundation; either
+version 2.1 of the License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public
+License along with this library; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+***********************************************************************/
+
+#include "DecodeGraph.h"
+#include "DecodeStep.h"
+#include "TypeDef.h"
+#include "Util.h"
+
+namespace Moses
+{
+DecodeGraph::~DecodeGraph()
+{
+  RemoveAllInColl(m_steps);
+}
+
+//! Add another decode step to the graph
+void DecodeGraph::Add(DecodeStep *decodeStep)
+{
+  m_steps.push_back(decodeStep);
+  decodeStep->SetContainer(this);
+}
+
+}
+

mosesdecoder/moses/DecodeGraph.h

@@ -0,0 +1,101 @@
+// $Id$
+// vim:tabstop=2
+
+/***********************************************************************
+Moses - factored phrase-based language decoder
+Copyright (C) 2006 University of Edinburgh
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Lesser General Public
+License as published by the Free Software Foundation; either
+version 2.1 of the License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public
+License along with this library; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+***********************************************************************/
+
+#ifndef moses_DecodeGraph_h
+#define moses_DecodeGraph_h
+
+#include "util/exception.hh"
+#include <list>
+#include <iterator>
+#include "TypeDef.h"
+
+namespace Moses
+{
+
+class DecodeStep;
+
+//! list of DecodeSteps which factorizes the translation
+class DecodeGraph
+{
+protected:
+  std::list<const DecodeStep*> m_steps;
+  size_t m_id; // contiguous unique id, starting from 0
+  size_t m_maxChartSpan;
+  size_t m_backoff;
+
+public:
+  /**
+    * position: The position of this graph within the decode sequence.
+    **/
+  DecodeGraph(size_t id)
+    : m_id(id)
+    , m_maxChartSpan(NOT_FOUND)
+    , m_backoff(0) {
+  }
+
+  // for chart decoding
+  DecodeGraph(size_t id, size_t maxChartSpan)
+    : m_id(id)
+    , m_maxChartSpan(maxChartSpan) {
+  }
+
+  //! iterators
+  typedef std::list<const DecodeStep*>::iterator iterator;
+  typedef std::list<const DecodeStep*>::const_iterator const_iterator;
+  const_iterator begin() const {
+    return m_steps.begin();
+  }
+  const_iterator end() const {
+    return m_steps.end();
+  }
+
+  virtual ~DecodeGraph();
+
+  //! Add another decode step to the graph
+  void Add(DecodeStep *decodeStep);
+
+  size_t GetSize() const {
+    return m_steps.size();
+  }
+
+  size_t GetMaxChartSpan() const {
+    UTIL_THROW_IF2(m_maxChartSpan == NOT_FOUND, "Max chart span not specified");
+    return m_maxChartSpan;
+  }
+
+  size_t GetBackoff() const {
+    return m_backoff;
+  }
+
+  void SetBackoff(size_t backoff) {
+    m_backoff = backoff;
+  }
+
+  size_t GetId() const {
+    return m_id;
+  }
+
+};
+
+
+}
+#endif

mosesdecoder/moses/DecodeStep.cpp

@@ -0,0 +1,90 @@
+// $Id$
+
+/***********************************************************************
+Moses - factored phrase-based language decoder
+Copyright (C) 2006 University of Edinburgh
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Lesser General Public
+License as published by the Free Software Foundation; either
+version 2.1 of the License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public
+License along with this library; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+***********************************************************************/
+
+#include "DecodeStep.h"
+#include "GenerationDictionary.h"
+#include "StaticData.h"
+#include "moses/TranslationModel/PhraseDictionary.h"
+
+namespace Moses
+{
+DecodeStep::DecodeStep(DecodeFeature *decodeFeature,
+                       const DecodeStep* prev,
+                       const std::vector<FeatureFunction*> &features)
+  : m_decodeFeature(decodeFeature)
+{
+  FactorMask prevOutputFactors;
+  if (prev) prevOutputFactors = prev->m_outputFactors;
+  m_outputFactors = prevOutputFactors;
+  FactorMask conflictMask = (m_outputFactors & decodeFeature->GetOutputFactorMask());
+  m_outputFactors |= decodeFeature->GetOutputFactorMask();
+  FactorMask newOutputFactorMask = m_outputFactors ^ prevOutputFactors;  //xor
+  m_newOutputFactors.resize(newOutputFactorMask.count());
+  m_conflictFactors.resize(conflictMask.count());
+  size_t j=0, k=0;
+  for (size_t i = 0; i < MAX_NUM_FACTORS; i++) {
+    if (newOutputFactorMask[i]) m_newOutputFactors[j++] = i;
+    if (conflictMask[i]) m_conflictFactors[k++] = i;
+  }
+  VERBOSE(2,"DecodeStep():\n\toutputFactors=" << m_outputFactors
+          << "\n\tconflictFactors=" << conflictMask
+          << "\n\tnewOutputFactors=" << newOutputFactorMask << std::endl);
+
+  // find out which feature function can be applied in this decode step
+  for (size_t i = 0; i < features.size(); ++i) {
+    FeatureFunction *feature = features[i];
+    if (feature->IsUseable(m_outputFactors)) {
+      m_featuresToApply.push_back(feature);
+    } else {
+      m_featuresRemaining.push_back(feature);
+    }
+  }
+
+  decodeFeature->SetContainer(this);
+}
+
+DecodeStep::~DecodeStep() {}
+
+/** returns phrase feature (dictionary) for translation step */
+const PhraseDictionary* DecodeStep::GetPhraseDictionaryFeature() const
+{
+  return dynamic_cast<const PhraseDictionary*>(m_decodeFeature);
+}
+
+/** returns generation feature (dictionary) for generation step */
+const GenerationDictionary* DecodeStep::GetGenerationDictionaryFeature() const
+{
+  return dynamic_cast<const GenerationDictionary*>(m_decodeFeature);
+}
+
+void DecodeStep::RemoveFeature(const FeatureFunction *ff)
+{
+  for (size_t i = 0; i < m_featuresToApply.size(); ++i) {
+    if (ff == m_featuresToApply[i]) {
+      m_featuresToApply.erase(m_featuresToApply.begin() + i);
+      return;
+    }
+  }
+}
+
+}
+
+

mosesdecoder/moses/DecodeStep.h

@@ -0,0 +1,118 @@
+// $Id$
+
+/***********************************************************************
+Moses - factored phrase-based language decoder
+Copyright (C) 2006 University of Edinburgh
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Lesser General Public
+License as published by the Free Software Foundation; either
+version 2.1 of the License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public
+License along with this library; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+***********************************************************************/
+
+#ifndef moses_DecodeStep_h
+#define moses_DecodeStep_h
+
+#include "TypeDef.h"
+#include "FactorTypeSet.h"
+#include "Phrase.h"
+
+namespace Moses
+{
+
+class DecodeFeature;
+class PhraseDictionary;
+class GenerationDictionary;
+class TranslationOption;
+class TranslationOptionCollection;
+class PartialTranslOptColl;
+class FactorCollection;
+class InputType;
+class FeatureFunction;
+class DecodeGraph;
+
+/** Specification for a decoding step.
+ * The factored translation model consists of Translation and Generation
+ * steps, which consult a Dictionary of phrase translations or word
+ * generations. This class implements the specification for one of these
+ * steps, both the DecodeType and a pointer to the Translation or Generation Feature
+ **/
+class DecodeStep
+{
+protected:
+  FactorMask m_outputFactors; //! mask of what factors exist on the output side after this decode step
+  std::vector<FactorType> m_conflictFactors; //! list of the factors that may conflict during this step
+  std::vector<FactorType> m_newOutputFactors; //! list of the factors that are new in this step, may be empty
+  const DecodeFeature* m_decodeFeature;
+  const DecodeGraph *m_container;
+
+  std::vector<FeatureFunction*> m_featuresToApply, m_featuresRemaining;
+public:
+  DecodeStep(); //! not implemented
+  DecodeStep(DecodeFeature *featurePtr,
+             const DecodeStep* prevDecodeStep,
+             const std::vector<FeatureFunction*> &features);
+  virtual ~DecodeStep();
+
+  //! mask of factors that are present after this decode step
+  const FactorMask& GetOutputFactorMask() const {
+    return m_outputFactors;
+  }
+
+  //! returns true if this decode step must match some pre-existing factors
+  bool IsFilteringStep() const {
+    return !m_conflictFactors.empty();
+  }
+
+  //! returns true if this decode step produces one or more new factors
+  bool IsFactorProducingStep() const {
+    return !m_newOutputFactors.empty();
+  }
+
+  const std::vector<FeatureFunction*> &GetFeaturesRemaining() const {
+    return m_featuresRemaining;
+  }
+
+  /*! returns a list (possibly empty) of the (target side) factors that
+   * are produced in this decoding step.  For example, if a previous step
+   * generated factor 1, and this step generates 1,2, then only 2 will be
+   * in the returned vector. */
+  const std::vector<FactorType>& GetNewOutputFactors() const {
+    return m_newOutputFactors;
+  }
+
+  /*! returns a list (possibly empty) of the (target side) factors that
+   * are produced BUT ALREADY EXIST and therefore must be checked for
+   * conflict or compatibility */
+  const std::vector<FactorType>& GetConflictFactors() const {
+    return m_conflictFactors;
+  }
+
+  /*! returns phrase table feature for translation step */
+  const PhraseDictionary* GetPhraseDictionaryFeature() const;
+
+  /*! returns generation table feature for generation step */
+  const GenerationDictionary* GetGenerationDictionaryFeature() const;
+
+  void RemoveFeature(const FeatureFunction *ff);
+
+  void SetContainer(const DecodeGraph *container) {
+    m_container = container;
+  }
+  const DecodeGraph *GetContainer() const {
+    return m_container;
+  }
+
+};
+
+}
+#endif

mosesdecoder/moses/DecodeStepGeneration.cpp

@@ -0,0 +1,169 @@
+// $Id$
+
+/***********************************************************************
+Moses - factored phrase-based language decoder
+Copyright (C) 2006 University of Edinburgh
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Lesser General Public
+License as published by the Free Software Foundation; either
+version 2.1 of the License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public
+License along with this library; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+***********************************************************************/
+
+#include "DecodeStepGeneration.h"
+#include "GenerationDictionary.h"
+#include "TranslationOption.h"
+#include "TranslationOptionCollection.h"
+#include "PartialTranslOptColl.h"
+#include "FactorCollection.h"
+
+namespace Moses
+{
+using namespace std;
+
+DecodeStepGeneration::DecodeStepGeneration(GenerationDictionary* dict,
+    const DecodeStep* prev,
+    const std::vector<FeatureFunction*> &features)
+  : DecodeStep(dict, prev, features)
+{
+}
+
+// helpers
+typedef pair<Word, ScoreComponentCollection> WordPair;
+typedef list< WordPair > WordList;
+// 1st = word
+// 2nd = score
+typedef list< WordPair >::const_iterator WordListIterator;
+
+/** used in generation: increases iterators when looping through the exponential number of generation expansions */
+inline void IncrementIterators(vector< WordListIterator > &wordListIterVector
+                               , const vector< WordList > &wordListVector)
+{
+  for (size_t currPos = 0 ; currPos < wordListVector.size() ; currPos++) {
+    WordListIterator &iter = wordListIterVector[currPos];
+    iter++;
+    if (iter != wordListVector[currPos].end()) {
+      // eg. 4 -> 5
+      return;
+    } else {
+      //  eg 9 -> 10
+      iter = wordListVector[currPos].begin();
+    }
+  }
+}
+
+void DecodeStepGeneration::Process(const TranslationOption &inputPartialTranslOpt
+                                   , const DecodeStep &decodeStep
+                                   , PartialTranslOptColl &outputPartialTranslOptColl
+                                   , TranslationOptionCollection * /* toc */
+                                   , bool /*adhereTableLimit*/) const
+{
+  if (inputPartialTranslOpt.GetTargetPhrase().GetSize() == 0) {
+    // word deletion
+
+    TranslationOption *newTransOpt = new TranslationOption(inputPartialTranslOpt);
+    outputPartialTranslOptColl.Add(newTransOpt);
+
+    return;
+  }
+
+  // normal generation step
+  const GenerationDictionary* generationDictionary  = decodeStep.GetGenerationDictionaryFeature();
+
+  const Phrase &targetPhrase  = inputPartialTranslOpt.GetTargetPhrase();
+  const InputPath &inputPath = inputPartialTranslOpt.GetInputPath();
+  size_t targetLength         = targetPhrase.GetSize();
+
+  // generation list for each word in phrase
+  vector< WordList > wordListVector(targetLength);
+
+  // create generation list
+  int wordListVectorPos = 0;
+  for (size_t currPos = 0 ; currPos < targetLength ; currPos++) { // going thorugh all words
+    // generatable factors for this word to be put in wordList
+    WordList &wordList = wordListVector[wordListVectorPos];
+    const Word &word = targetPhrase.GetWord(currPos);
+
+    // consult dictionary for possible generations for this word
+    const OutputWordCollection *wordColl = generationDictionary->FindWord(word);
+
+    if (wordColl == NULL) {
+      // word not found in generation dictionary
+      //toc->ProcessUnknownWord(sourceWordsRange.GetStartPos(), factorCollection);
+      return; // can't be part of a phrase, special handling
+    } else {
+      // sort(*wordColl, CompareWordCollScore);
+      OutputWordCollection::const_iterator iterWordColl;
+      for (iterWordColl = wordColl->begin() ; iterWordColl != wordColl->end(); ++iterWordColl) {
+        const Word &outputWord = (*iterWordColl).first;
+        const ScoreComponentCollection& score = (*iterWordColl).second;
+        // enter into word list generated factor(s) and its(their) score(s)
+        wordList.push_back(WordPair(outputWord, score));
+      }
+
+      wordListVectorPos++; // done, next word
+    }
+  }
+
+  // use generation list (wordList)
+  // set up iterators (total number of expansions)
+  size_t numIteration = 1;
+  vector< WordListIterator >  wordListIterVector(targetLength);
+  vector< const Word* >       mergeWords(targetLength);
+  for (size_t currPos = 0 ; currPos < targetLength ; currPos++) {
+    wordListIterVector[currPos] = wordListVector[currPos].begin();
+    numIteration *= wordListVector[currPos].size();
+  }
+
+  // go thru each possible factor for each word & create hypothesis
+  for (size_t currIter = 0 ; currIter < numIteration ; currIter++) {
+    ScoreComponentCollection generationScore; // total score for this string of words
+
+    // create vector of words with new factors for last phrase
+    for (size_t currPos = 0 ; currPos < targetLength ; currPos++) {
+      const WordPair &wordPair = *wordListIterVector[currPos];
+      mergeWords[currPos] = &(wordPair.first);
+      generationScore.PlusEquals(wordPair.second);
+    }
+
+    // merge with existing trans opt
+    Phrase genPhrase( mergeWords);
+
+    if (IsFilteringStep()) {
+      if (!inputPartialTranslOpt.IsCompatible(genPhrase, m_conflictFactors))
+        continue;
+    }
+
+    const TargetPhrase &inPhrase = inputPartialTranslOpt.GetTargetPhrase();
+    TargetPhrase outPhrase(inPhrase);
+    outPhrase.GetScoreBreakdown().PlusEquals(generationScore);
+
+    outPhrase.MergeFactors(genPhrase, m_newOutputFactors);
+    outPhrase.EvaluateInIsolation(inputPath.GetPhrase(), m_featuresToApply);
+
+    const Range &sourceWordsRange = inputPartialTranslOpt.GetSourceWordsRange();
+
+    TranslationOption *newTransOpt = new TranslationOption(sourceWordsRange, outPhrase);
+    assert(newTransOpt);
+
+    newTransOpt->SetInputPath(inputPath);
+
+    outputPartialTranslOptColl.Add(newTransOpt);
+
+    // increment iterators
+    IncrementIterators(wordListIterVector, wordListVector);
+  }
+}
+
+}
+
+

mosesdecoder/moses/DecodeStepGeneration.h

@@ -0,0 +1,54 @@
+// $Id$
+
+/***********************************************************************
+Moses - factored phrase-based language decoder
+Copyright (C) 2006 University of Edinburgh
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Lesser General Public
+License as published by the Free Software Foundation; either
+version 2.1 of the License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public
+License along with this library; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+***********************************************************************/
+
+#ifndef moses_DecodeStepGeneration_h
+#define moses_DecodeStepGeneration_h
+
+#include "DecodeStep.h"
+
+namespace Moses
+{
+
+class GenerationDictionary;
+class Phrase;
+class ScoreComponentCollection;
+
+//! subclass of DecodeStep for generation step
+class DecodeStepGeneration : public DecodeStep
+{
+public:
+  DecodeStepGeneration(GenerationDictionary* dict,
+                       const DecodeStep* prev,
+                       const std::vector<FeatureFunction*> &features);
+
+
+  void Process(const TranslationOption &inputPartialTranslOpt
+               , const DecodeStep &decodeStep
+               , PartialTranslOptColl &outputPartialTranslOptColl
+               , TranslationOptionCollection *toc
+               , bool adhereTableLimit) const;
+
+private:
+};
+
+
+}
+#endif

mosesdecoder/moses/DecodeStepTranslation.cpp

@@ -0,0 +1,280 @@
+// $Id$
+
+/***********************************************************************
+Moses - factored phrase-based language decoder
+Copyright (C) 2006 University of Edinburgh
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Lesser General Public
+License as published by the Free Software Foundation; either
+version 2.1 of the License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public
+License along with this library; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+***********************************************************************/
+
+#include "DecodeStepTranslation.h"
+#include "TranslationOption.h"
+#include "TranslationOptionCollection.h"
+#include "PartialTranslOptColl.h"
+#include "FactorCollection.h"
+#include "util/exception.hh"
+
+using namespace std;
+
+namespace Moses
+{
+DecodeStepTranslation::DecodeStepTranslation(PhraseDictionary* pdf,
+    const DecodeStep* prev,
+    const std::vector<FeatureFunction*> &features)
+  : DecodeStep(pdf, prev, features)
+{
+  // don't apply feature functions that are from current phrase table.It should already have been
+  // dont by the phrase table.
+  const std::vector<FeatureFunction*> &pdfFeatures = pdf->GetFeaturesToApply();
+  for (size_t i = 0; i < pdfFeatures.size(); ++i) {
+    FeatureFunction *ff = pdfFeatures[i];
+    RemoveFeature(ff);
+  }
+}
+
+void DecodeStepTranslation::Process(const TranslationOption &inputPartialTranslOpt
+                                    , const DecodeStep &decodeStep
+                                    , PartialTranslOptColl &outputPartialTranslOptColl
+                                    , TranslationOptionCollection *toc
+                                    , bool adhereTableLimit
+                                    , TargetPhraseCollection::shared_ptr phraseColl) const
+{
+  if (inputPartialTranslOpt.GetTargetPhrase().GetSize() == 0) {
+    // word deletion
+    outputPartialTranslOptColl.Add(new TranslationOption(inputPartialTranslOpt));
+    return;
+  }
+
+  // normal trans step
+  const Range &sourceWordsRange        = inputPartialTranslOpt.GetSourceWordsRange();
+  const InputPath &inputPath = inputPartialTranslOpt.GetInputPath();
+  const PhraseDictionary* phraseDictionary  =
+    decodeStep.GetPhraseDictionaryFeature();
+  const TargetPhrase &inPhrase = inputPartialTranslOpt.GetTargetPhrase();
+  const size_t currSize = inPhrase.GetSize();
+  const size_t tableLimit = phraseDictionary->GetTableLimit();
+
+  if (phraseColl != NULL) {
+    TargetPhraseCollection::const_iterator iterTargetPhrase, iterEnd;
+    iterEnd = (!adhereTableLimit || tableLimit == 0 || phraseColl->GetSize() < tableLimit) ? phraseColl->end() : phraseColl->begin() + tableLimit;
+
+    for (iterTargetPhrase = phraseColl->begin(); iterTargetPhrase != iterEnd; ++iterTargetPhrase) {
+      const TargetPhrase& targetPhrase = **iterTargetPhrase;
+      // const ScoreComponentCollection &transScores = targetPhrase.GetScoreBreakdown();
+      // skip if the
+      if (targetPhrase.GetSize() != currSize) continue;
+
+      TargetPhrase outPhrase(inPhrase);
+
+      if (IsFilteringStep()) {
+        if (!inputPartialTranslOpt.IsCompatible(targetPhrase, m_conflictFactors))
+          continue;
+      }
+
+      outPhrase.Merge(targetPhrase, m_newOutputFactors);
+      outPhrase.EvaluateInIsolation(inputPath.GetPhrase(), m_featuresToApply); // need to do this as all non-transcores would be screwed up
+
+      TranslationOption *newTransOpt = new TranslationOption(sourceWordsRange, outPhrase);
+      assert(newTransOpt != NULL);
+
+      newTransOpt->SetInputPath(inputPath);
+
+      outputPartialTranslOptColl.Add(newTransOpt );
+
+    }
+  } else if (sourceWordsRange.GetNumWordsCovered() == 1) {
+    // unknown handler
+    //toc->ProcessUnknownWord(sourceWordsRange.GetStartPos(), factorCollection);
+  }
+}
+
+void
+DecodeStepTranslation::
+ProcessInitialTranslation(InputType const& source,
+                          PartialTranslOptColl &outputPartialTranslOptColl,
+                          size_t startPos, size_t endPos,
+                          bool adhereTableLimit,
+                          InputPath const& inputPath,
+                          TargetPhraseCollection::shared_ptr phraseColl) const
+{
+  const PhraseDictionary* phraseDictionary = GetPhraseDictionaryFeature();
+  const size_t tableLimit = phraseDictionary->GetTableLimit();
+
+  const Range range(startPos, endPos);
+
+  if (phraseColl != NULL) {
+    IFVERBOSE(3) {
+      if(source.GetType() == SentenceInput)
+        TRACE_ERR("[" << source.GetSubString(range) << "; "
+                  << startPos << "-" << endPos << "]\n");
+      else
+        TRACE_ERR("[" << startPos << "-" << endPos << "]" << std::endl);
+    }
+
+    TargetPhraseCollection::const_iterator iterTargetPhrase, iterEnd;
+    iterEnd = (!adhereTableLimit || tableLimit == 0 || phraseColl->GetSize() < tableLimit) ? phraseColl->end() : phraseColl->begin() + tableLimit;
+
+    for (iterTargetPhrase = phraseColl->begin() ; iterTargetPhrase != iterEnd ; ++iterTargetPhrase) {
+      const TargetPhrase	&targetPhrase = **iterTargetPhrase;
+      TranslationOption *transOpt = new TranslationOption(range, targetPhrase);
+
+      transOpt->SetInputPath(inputPath);
+
+      outputPartialTranslOptColl.Add (transOpt);
+
+      VERBOSE(3,"\t" << targetPhrase << "\n");
+    }
+    VERBOSE(3,std::endl);
+  }
+}
+
+void
+DecodeStepTranslation::
+ProcessInitialTransLEGACY(InputType const& source,
+                          PartialTranslOptColl &outputPartialTranslOptColl,
+                          size_t startPos, size_t endPos,
+                          bool adhereTableLimit,
+                          InputPathList const& inputPathList) const
+{
+  const PhraseDictionary* phraseDictionary = GetPhraseDictionaryFeature();
+  const size_t tableLimit = phraseDictionary->GetTableLimit();
+
+  const Range range(startPos, endPos);
+  TargetPhraseCollectionWithSourcePhrase::shared_ptr phraseColl
+  = phraseDictionary->GetTargetPhraseCollectionLEGACY(source,range);
+
+  if (phraseColl != NULL) {
+    IFVERBOSE(3) {
+      if(source.GetType() == SentenceInput)
+        TRACE_ERR("[" << source.GetSubString(range) << "; "
+                  << startPos << "-" << endPos << "]\n");
+      else
+        TRACE_ERR("[" << startPos << "-" << endPos << "]" << std::endl);
+    }
+
+    const std::vector<Phrase> &sourcePhrases = phraseColl->GetSourcePhrases();
+
+    TargetPhraseCollection::const_iterator iterTargetPhrase, iterEnd;
+    std::vector<Phrase>::const_iterator iterSourcePhrase;
+    iterEnd = (!adhereTableLimit || tableLimit == 0 || phraseColl->GetSize() < tableLimit) ? phraseColl->end() : phraseColl->begin() + tableLimit;
+
+    for (iterTargetPhrase = phraseColl->begin(), iterSourcePhrase =  sourcePhrases.begin()
+         ; iterTargetPhrase != iterEnd
+         ; ++iterTargetPhrase, ++iterSourcePhrase) {
+      assert(iterSourcePhrase != sourcePhrases.end());
+
+      const TargetPhrase	&targetPhrase = **iterTargetPhrase;
+      const Phrase			&sourcePhrase = *iterSourcePhrase;
+
+      const InputPath &inputPath = GetInputPathLEGACY(targetPhrase, sourcePhrase, inputPathList);
+
+      TranslationOption *transOpt = new TranslationOption(range, targetPhrase);
+      transOpt->SetInputPath(inputPath);
+
+      outputPartialTranslOptColl.Add (transOpt);
+
+      VERBOSE(3,"\t" << targetPhrase << "\n");
+    }
+    VERBOSE(3,std::endl);
+  }
+}
+
+const InputPath &DecodeStepTranslation::GetInputPathLEGACY(
+  const TargetPhrase targetPhrase,
+  const Phrase sourcePhrase,
+  const InputPathList &inputPathList) const
+{
+  const Word &wordFromPt =  sourcePhrase.GetWord(0);
+
+  InputPathList::const_iterator iter;
+  for (iter = inputPathList.begin(); iter != inputPathList.end(); ++iter) {
+    const InputPath &inputPath = **iter;
+    const Phrase &phraseFromIP = inputPath.GetPhrase();
+
+    const Word *wordIP = NULL;
+    for (size_t i = 0; i < phraseFromIP.GetSize(); ++i) {
+      const Word &tempWord =  phraseFromIP.GetWord(i);
+      if (!tempWord.IsEpsilon()) {
+        wordIP = &tempWord;
+        break;
+      }
+    }
+
+    // const Range &range = inputPath.GetWordsRange();
+
+    if (wordIP && *wordIP == wordFromPt) {
+      return inputPath;
+    }
+  }
+
+  UTIL_THROW(util::Exception, "Input path not found");
+}
+
+void
+DecodeStepTranslation::
+ProcessLEGACY(TranslationOption const& in,
+              DecodeStep const& decodeStep,
+              PartialTranslOptColl &out,
+              TranslationOptionCollection *toc,
+              bool adhereTableLimit) const
+{
+  if (in.GetTargetPhrase().GetSize() == 0) {
+    // word deletion
+    out.Add(new TranslationOption(in));
+    return;
+  }
+
+  // normal trans step
+  Range const& srcRange = in.GetSourceWordsRange();
+  InputPath const& inputPath = in.GetInputPath();
+  PhraseDictionary const* pdict  = decodeStep.GetPhraseDictionaryFeature();
+  TargetPhrase const& inPhrase = in.GetTargetPhrase();
+  size_t const currSize = inPhrase.GetSize();
+  size_t const tableLimit = pdict->GetTableLimit();
+
+  TargetPhraseCollectionWithSourcePhrase::shared_ptr phraseColl
+  = pdict->GetTargetPhraseCollectionLEGACY(toc->GetSource(),srcRange);
+
+  if (phraseColl != NULL) {
+    TargetPhraseCollection::const_iterator iterTargetPhrase, iterEnd;
+    iterEnd = ((adhereTableLimit && tableLimit && phraseColl->GetSize() >= tableLimit)
+               ? phraseColl->begin() + tableLimit : phraseColl->end());
+
+    for (iterTargetPhrase = phraseColl->begin();
+         iterTargetPhrase != iterEnd;
+         ++iterTargetPhrase) {
+      TargetPhrase const& targetPhrase = **iterTargetPhrase;
+      if (targetPhrase.GetSize() != currSize ||
+          (IsFilteringStep() && !in.IsCompatible(targetPhrase, m_conflictFactors)))
+        continue;
+
+      TargetPhrase outPhrase(inPhrase);
+      outPhrase.Merge(targetPhrase, m_newOutputFactors);
+      outPhrase.EvaluateInIsolation(inputPath.GetPhrase(), m_featuresToApply); // need to do this as all non-transcores would be screwed up
+
+      TranslationOption *newTransOpt = new TranslationOption(srcRange, outPhrase);
+      assert(newTransOpt != NULL);
+
+      newTransOpt->SetInputPath(inputPath);
+
+      out.Add(newTransOpt);
+
+    }
+  }
+}
+}
+
+
+