Add files using upload-large-folder tool

mosesdecoder/moses/AlignmentInfoCollection.cpp

@@ -0,0 +1,60 @@
+/***********************************************************************
+ Moses - statistical machine translation system
+ Copyright (C) 2006-2011 University of Edinburgh
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+***********************************************************************/
+
+#include "AlignmentInfoCollection.h"
+
+namespace Moses
+{
+
+AlignmentInfoCollection AlignmentInfoCollection::s_instance;
+
+AlignmentInfoCollection::AlignmentInfoCollection()
+{
+  std::set<std::pair<size_t,size_t> > pairs;
+  m_emptyAlignmentInfo = Add(pairs);
+}
+
+AlignmentInfoCollection::~AlignmentInfoCollection()
+{}
+
+const AlignmentInfo &AlignmentInfoCollection::GetEmptyAlignmentInfo() const
+{
+  return *m_emptyAlignmentInfo;
+}
+
+AlignmentInfo const *
+AlignmentInfoCollection::
+Add(AlignmentInfo const& ainfo)
+{
+#ifdef WITH_THREADS
+  {
+    boost::shared_lock<boost::shared_mutex> read_lock(m_accessLock);
+    AlignmentInfoSet::const_iterator i = m_collection.find(ainfo);
+    if (i != m_collection.end())
+      return &*i;
+  }
+  boost::unique_lock<boost::shared_mutex> lock(m_accessLock);
+#endif
+  std::pair<AlignmentInfoSet::iterator, bool> ret = m_collection.insert(ainfo);
+  return &(*ret.first);
+}
+
+
+
+}

mosesdecoder/moses/AlignmentInfoCollection.h

@@ -0,0 +1,81 @@
+/***********************************************************************
+ Moses - statistical machine translation system
+ Copyright (C) 2006-2011 University of Edinburgh
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+***********************************************************************/
+
+#pragma once
+
+#include "AlignmentInfo.h"
+
+#include <set>
+
+#ifdef WITH_THREADS
+#include <boost/thread/shared_mutex.hpp>
+#include <boost/thread/locks.hpp>
+#endif
+
+namespace Moses
+{
+
+/** Singleton collection of all AlignmentInfo objects.
+ *  Used as a cache of all alignment info to save space.
+ */
+class AlignmentInfoCollection
+{
+public:
+  static AlignmentInfoCollection &Instance() {
+    return s_instance;
+  }
+
+  /** Returns a pointer to an AlignmentInfo object with the same source-target
+    * alignment pairs as given in the argument.  If the collection already
+    * contains such an object then returns a pointer to it; otherwise a new
+    * one is inserted.
+   */
+private:
+  const AlignmentInfo* Add(AlignmentInfo const& ainfo);
+
+public:
+  template<typename ALNREP>
+  AlignmentInfo const *
+  Add(ALNREP const & aln) {
+    return this->Add(AlignmentInfo(aln));
+  }
+
+  //! Returns a pointer to an empty AlignmentInfo object.
+  const AlignmentInfo &GetEmptyAlignmentInfo() const;
+
+private:
+  typedef std::set<AlignmentInfo, AlignmentInfoOrderer> AlignmentInfoSet;
+
+
+  //! Only a single static variable should be created.
+  AlignmentInfoCollection();
+  ~AlignmentInfoCollection();
+
+  static AlignmentInfoCollection s_instance;
+
+#ifdef WITH_THREADS
+  //reader-writer lock
+  mutable boost::shared_mutex m_accessLock;
+#endif
+
+  AlignmentInfoSet m_collection;
+  const AlignmentInfo *m_emptyAlignmentInfo;
+};
+
+}

mosesdecoder/moses/AlignmentInfoTest.cpp

@@ -0,0 +1,71 @@
+/***********************************************************************
+Moses - factored phrase-based language decoder
+Copyright (C) 2010- University of Edinburgh
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Lesser General Public
+License as published by the Free Software Foundation; either
+version 2.1 of the License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public
+License along with this library; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+***********************************************************************/
+
+#include <boost/test/unit_test.hpp>
+
+#include "AlignmentInfo.h"
+#include "AlignmentInfoCollection.h"
+
+using namespace Moses;
+using namespace std;
+
+BOOST_AUTO_TEST_SUITE(alignment_info)
+
+typedef pair<size_t,size_t> IndexPair;
+typedef set<pair<size_t,size_t> > IndexSet;
+
+struct AlignmentInfoFixture {
+  const AlignmentInfo* ai1;
+  const AlignmentInfo* ai2;
+  const AlignmentInfo* ai3;
+
+  AlignmentInfoFixture() {
+    AlignmentInfoCollection& collection = AlignmentInfoCollection::Instance();
+    IndexSet aligns1,aligns2,aligns3;
+    aligns1.insert(IndexPair(1,1));
+    aligns1.insert(IndexPair(2,1));
+    aligns2.insert(IndexPair(1,1));
+    aligns2.insert(IndexPair(2,1));
+    aligns3.insert(IndexPair(1,2));
+    aligns3.insert(IndexPair(2,1));
+    ai1 = collection.Add(aligns1);
+    ai2 = collection.Add(aligns2);
+    ai3 = collection.Add(aligns3);
+  }
+
+};
+
+BOOST_FIXTURE_TEST_CASE(comparator, AlignmentInfoFixture)
+{
+  BOOST_CHECK(*ai1 == *ai2);
+  BOOST_CHECK(*ai1 == *ai1);
+  BOOST_CHECK(*ai2 == *ai2);
+  BOOST_CHECK(*ai3 == *ai3);
+  BOOST_CHECK(!(*ai2 == *ai3));
+  BOOST_CHECK(!(*ai1 == *ai3));
+}
+
+BOOST_FIXTURE_TEST_CASE(hasher, AlignmentInfoFixture)
+{
+  //simple test that same objects give same hash
+  AlignmentInfoHasher hash;
+  BOOST_CHECK_EQUAL(hash(*ai1), hash(*ai2));
+}
+
+BOOST_AUTO_TEST_SUITE_END()

mosesdecoder/moses/BaseManager.cpp

@@ -0,0 +1,160 @@
+#include "BaseManager.h"
+#include "StaticData.h"
+#include "moses/FF/StatelessFeatureFunction.h"
+#include "moses/FF/StatefulFeatureFunction.h"
+#include "moses/TranslationTask.h"
+
+#include <vector>
+#include <boost/algorithm/string/predicate.hpp>
+#include <boost/iostreams/device/file.hpp>
+#include <boost/iostreams/filter/bzip2.hpp>
+#include <boost/iostreams/filter/gzip.hpp>
+#include <boost/iostreams/filtering_stream.hpp>
+#include <boost/filesystem.hpp>
+
+using namespace std;
+
+namespace Moses
+{
+
+BaseManager::BaseManager(ttasksptr const& ttask)
+  : m_ttask(ttask), m_source(*(ttask->GetSource().get()))
+{ }
+
+const InputType&
+BaseManager::GetSource() const
+{
+  return m_source;
+}
+
+const ttasksptr
+BaseManager::GetTtask() const
+{
+  return m_ttask.lock();
+}
+
+void
+BaseManager::
+OutputSearchGraphAsHypergraph(std::ostream& out) const
+{
+  // This virtual function that may not be implemented everywhere, but it should for
+  // derived classes that use it
+  UTIL_THROW2("Not implemented.");
+}
+
+void
+BaseManager::
+OutputSearchGraphAsHypergraph(std::string const& fname, size_t const precision) const
+{
+  std::string odir = boost::filesystem::path(fname).parent_path().string();
+  if (! boost::filesystem::exists(odir))
+    boost::filesystem::create_directory(odir);
+  UTIL_THROW_IF2(!boost::filesystem::is_directory(odir),
+                 "Cannot output hypergraphs to " << odir
+                 << " because that path exists but is not a directory.");
+
+  // not clear why we need to output the weights every time we dump a search
+  // graph into a file again, but that's what the old code did.
+
+  string weightsFile = odir + "/weights";
+  TRACE_ERR("The weights file is " << weightsFile << "\n");
+  ofstream weightsOut;
+  weightsOut.open(weightsFile.c_str());
+  weightsOut.setf(std::ios::fixed);
+  weightsOut.precision(6);
+  // just temporarily, till we've implemented weight scoring in the manager
+  // (or the translation task)
+  StaticData::Instance().GetAllWeights().Save(weightsOut);
+  weightsOut.close();
+
+  boost::iostreams::filtering_ostream file;
+  if (boost::ends_with(fname, ".gz"))
+    file.push(boost::iostreams::gzip_compressor());
+  else if (boost::ends_with(fname, ".bz2"))
+    file.push( boost::iostreams::bzip2_compressor() );
+  file.push( boost::iostreams::file_sink(fname, ios_base::out) );
+  if (file.is_complete() && file.good()) {
+    file.setf(std::ios::fixed);
+    file.precision(precision);
+    this->OutputSearchGraphAsHypergraph(file);
+    file.flush();
+  } else {
+    TRACE_ERR("Cannot output hypergraph for line "
+              << this->GetSource().GetTranslationId()
+              << " because the output file " << fname
+              << " is not open or not ready for writing"
+              << std::endl);
+  }
+  file.pop();
+}
+
+
+
+
+/***
+ * print surface factor only for the given phrase
+ */
+void
+BaseManager::
+OutputSurface(std::ostream &out, Phrase const& phrase) const
+{
+  std::vector<FactorType> const& factor_order = options()->output.factor_order;
+
+  bool markUnknown = options()->unk.mark;
+  std::string const& fd = options()->output.factor_delimiter;
+
+  size_t size = phrase.GetSize();
+  for (size_t pos = 0 ; pos < size ; pos++) {
+    const Factor *factor = phrase.GetFactor(pos, factor_order[0]);
+    UTIL_THROW_IF2(factor == NULL, "Empty factor 0 at position " << pos);
+
+    const Word &word = phrase.GetWord(pos);
+    if(markUnknown && word.IsOOV()) {
+      out << options()->unk.prefix;
+    }
+
+    out << *factor;
+
+    for (size_t i = 1 ; i < factor_order.size() ; i++) {
+      const Factor *factor = phrase.GetFactor(pos, factor_order[i]);
+      UTIL_THROW_IF2(!factor, "Empty factor " << i << " at position " << pos);
+      out << fd << *factor;
+    }
+
+    if(markUnknown && word.IsOOV()) {
+      out << options()->unk.suffix;
+    }
+
+    out << " ";
+  }
+}
+
+// Emulates the old operator<<(ostream &, const DottedRule &) function.  The
+// output format is a bit odd (reverse order and double spacing between symbols)
+// but there are scripts and tools that expect the output of -T to look like
+// that.
+void BaseManager::WriteApplicationContext(std::ostream &out,
+    const ApplicationContext &context) const
+{
+  assert(!context.empty());
+  ApplicationContext::const_reverse_iterator p = context.rbegin();
+  while (true) {
+    out << p->second << "=" << p->first << " ";
+    if (++p == context.rend()) {
+      break;
+    }
+    out << " ";
+  }
+}
+
+AllOptions::ptr const&
+BaseManager::
+options() const
+{
+  return GetTtask()->options();
+}
+
+
+} // namespace
+
+

mosesdecoder/moses/Bitmaps.cpp

@@ -0,0 +1,58 @@
+#include <boost/foreach.hpp>
+#include "Bitmaps.h"
+#include "Util.h"
+
+using namespace std;
+
+namespace Moses
+{
+Bitmaps::Bitmaps(size_t inputSize, const std::vector<bool> &initSourceCompleted)
+{
+  m_initBitmap = new Bitmap(inputSize, initSourceCompleted);
+  m_coll[m_initBitmap];
+}
+
+Bitmaps::~Bitmaps()
+{
+  BOOST_FOREACH (const Coll::value_type& myPair, m_coll) {
+    const Bitmap *bm = myPair.first;
+    delete bm;
+  }
+}
+
+const Bitmap &Bitmaps::GetNextBitmap(const Bitmap &bm, const Range &range)
+{
+  Bitmap *newBM = new Bitmap(bm, range);
+
+  Coll::const_iterator iter = m_coll.find(newBM);
+  if (iter == m_coll.end()) {
+    m_coll[newBM] = NextBitmaps();
+    return *newBM;
+  } else {
+    delete newBM;
+    return *iter->first;
+  }
+}
+
+const Bitmap &Bitmaps::GetBitmap(const Bitmap &bm, const Range &range)
+{
+  Coll::iterator iter = m_coll.find(&bm);
+  assert(iter != m_coll.end());
+
+  const Bitmap *newBM;
+  NextBitmaps &next = iter->second;
+  NextBitmaps::const_iterator iterNext = next.find(range);
+  if (iterNext == next.end()) {
+    // not seen the link yet.
+    newBM = &GetNextBitmap(bm, range);
+    next[range] = newBM;
+  } else {
+    // link exist
+    //std::cerr << "link exists" << endl;
+    newBM = iterNext->second;
+  }
+  return *newBM;
+}
+
+}
+

mosesdecoder/moses/ChartCell.h

@@ -0,0 +1,128 @@
+// $Id$
+// vim:tabstop=2
+/***********************************************************************
+ Moses - factored phrase-based language decoder
+ Copyright (C) 2010 Hieu Hoang
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ ***********************************************************************/
+
+#pragma once
+
+#include <iostream>
+#include <queue>
+#include <map>
+#include <vector>
+#include "Word.h"
+#include "Range.h"
+#include "NonTerminal.h"
+#include "ChartHypothesis.h"
+#include "ChartHypothesisCollection.h"
+#include "RuleCube.h"
+#include "ChartCellLabelSet.h"
+
+#include <boost/scoped_ptr.hpp>
+#include <boost/functional/hash.hpp>
+#include <boost/unordered_map.hpp>
+#include <boost/version.hpp>
+
+namespace Moses
+{
+class ChartSearchGraphWriter;
+class ChartTranslationOptionList;
+class ChartCellCollection;
+class ChartManager;
+
+class ChartCellBase
+{
+public:
+  ChartCellBase(size_t startPos, size_t endPos);
+
+  virtual ~ChartCellBase();
+
+  const ChartCellLabelSet &GetTargetLabelSet() const {
+    return m_targetLabelSet;
+  }
+
+  ChartCellLabelSet &MutableTargetLabelSet() {
+    return m_targetLabelSet;
+  }
+
+  const Range &GetCoverage() const {
+    return m_coverage;
+  }
+
+protected:
+  const Range m_coverage;
+  ChartCellLabelSet m_targetLabelSet;
+};
+
+/** 1 cell in chart decoder.
+ *  Doesn't directly hold hypotheses. Each cell contain a map of ChartHypothesisCollection that have different constituent labels
+ */
+class ChartCell : public ChartCellBase
+{
+  friend std::ostream& operator<<(std::ostream&, const ChartCell&);
+public:
+#if defined(BOOST_VERSION) && (BOOST_VERSION >= 104200)
+  typedef boost::unordered_map<Word,
+          ChartHypothesisCollection,
+          NonTerminalHasher,
+          NonTerminalEqualityPred
+          > MapType;
+#else
+  typedef std::map<Word, ChartHypothesisCollection> MapType;
+#endif
+
+protected:
+  MapType m_hypoColl;
+
+  bool m_nBestIsEnabled; /**< flag to determine whether to keep track of old arcs */
+  ChartManager &m_manager;
+
+public:
+  ChartCell(size_t startPos, size_t endPos, ChartManager &manager);
+  ~ChartCell();
+
+  void Decode(const ChartTranslationOptionList &transOptList
+              ,const ChartCellCollection &allChartCells);
+
+  //! Get all hypotheses in the cell that have the specified constituent label
+  const HypoList *GetSortedHypotheses(const Word &constituentLabel) const {
+    MapType::const_iterator p = m_hypoColl.find(constituentLabel);
+    return (p == m_hypoColl.end()) ? NULL : &(p->second.GetSortedHypotheses());
+  }
+
+  //! for n-best list
+  const HypoList *GetAllSortedHypotheses() const;
+
+  bool AddHypothesis(ChartHypothesis *hypo);
+
+  void SortHypotheses();
+  void PruneToSize();
+
+  const ChartHypothesis *GetBestHypothesis() const;
+
+  void CleanupArcList();
+
+  void OutputSizes(std::ostream &out) const;
+  size_t GetSize() const;
+
+  void WriteSearchGraph(const ChartSearchGraphWriter& writer, const std::map<unsigned,bool> &reachable) const;
+
+};
+
+}
+

mosesdecoder/moses/ChartCellLabel.h

@@ -0,0 +1,89 @@
+/***********************************************************************
+ Moses - statistical machine translation system
+ Copyright (C) 2006-2011 University of Edinburgh
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ ***********************************************************************/
+
+#pragma once
+
+#include "HypoList.h"
+#include "Word.h"
+#include "Range.h"
+#include "ChartParserCallback.h"
+
+namespace search
+{
+class Vertex;
+}
+
+namespace Moses
+{
+
+class Word;
+
+/** Contains a range, word (non-terms?) and a vector of hypotheses.
+ * @todo This is probably incompatible with lattice decoding when the word that spans
+ *   a position (or positions) can vary.
+ * @todo is this to hold sorted hypotheses that are in the queue for creating the next hypos?
+ */
+class ChartCellLabel
+{
+public:
+  union Stack {
+    const HypoList *cube; // cube pruning
+    search::Vertex *incr; // incremental search after filling.
+    void *incr_generator; // incremental search during filling.
+  };
+
+
+  ChartCellLabel(const Range &coverage, const Word &label,
+                 Stack stack=Stack())
+    : m_coverage(coverage)
+    , m_label(label)
+    , m_stack(stack)
+    , m_bestScore(0) {
+  }
+
+  const Range &GetCoverage() const {
+    return m_coverage;
+  }
+  const Word &GetLabel() const {
+    return m_label;
+  }
+  Stack GetStack() const {
+    return m_stack;
+  }
+  Stack &MutableStack() {
+    return m_stack;
+  }
+
+  //caching of best score on stack
+  float GetBestScore(const ChartParserCallback *outColl) const {
+    if (m_bestScore == 0) {
+      m_bestScore = outColl->GetBestScore(this);
+    }
+    return m_bestScore;
+  }
+
+private:
+  const Range &m_coverage;
+  const Word &m_label;
+  //const InputPath &m_inputPath;
+  Stack m_stack;
+  mutable float m_bestScore;
+};
+
+}

mosesdecoder/moses/ChartHypothesis.h

@@ -0,0 +1,204 @@
+// vim:tabstop=2
+/***********************************************************************
+ Moses - factored phrase-based language decoder
+ Copyright (C) 2010 Hieu Hoang
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ ***********************************************************************/
+
+#pragma once
+
+#include <vector>
+#include <boost/scoped_ptr.hpp>
+#include "Util.h"
+#include "Range.h"
+#include "ScoreComponentCollection.h"
+#include "Phrase.h"
+#include "ChartTranslationOptions.h"
+#include "ObjectPool.h"
+
+namespace Moses
+{
+
+class ChartKBestExtractor;
+class ChartHypothesis;
+class ChartManager;
+class RuleCubeItem;
+class FFState;
+
+typedef std::vector<ChartHypothesis*> ChartArcList;
+
+/** a hypothesis in the hierarchical/syntax decoder.
+ * Contain a pointer to the current target phrase, a vector of previous hypos, and some scores
+ */
+class ChartHypothesis
+{
+  friend std::ostream& operator<<(std::ostream&, const ChartHypothesis&);
+//  friend class ChartKBestExtractor;
+
+protected:
+
+  boost::shared_ptr<ChartTranslationOption> m_transOpt;
+
+  Range m_currSourceWordsRange;
+  std::vector<const FFState*> m_ffStates; /*! stateful feature function states */
+  /*! sum of scores of this hypothesis, and previous hypotheses. Lazily initialised.  */
+  mutable boost::scoped_ptr<ScoreComponentCollection> m_scoreBreakdown;
+  mutable boost::scoped_ptr<ScoreComponentCollection> m_deltaScoreBreakdown;
+  ScoreComponentCollection m_currScoreBreakdown /*! scores for this hypothesis only */
+  ,m_lmNGram
+  ,m_lmPrefix;
+  float m_totalScore;
+
+  ChartArcList *m_arcList; /*! all arcs that end at the same trellis point as this hypothesis */
+  const ChartHypothesis *m_winningHypo;
+
+  std::vector<const ChartHypothesis*> m_prevHypos; // always sorted by source position?
+
+  ChartManager& m_manager;
+
+  unsigned m_id; /* pkoehn wants to log the order in which hypotheses were generated */
+
+  //! not implemented
+  ChartHypothesis();
+
+  //! not implemented
+  ChartHypothesis(const ChartHypothesis &copy);
+
+public:
+  ChartHypothesis(const ChartTranslationOptions &, const RuleCubeItem &item,
+                  ChartManager &manager);
+
+  //! only used by ChartKBestExtractor
+  ChartHypothesis(const ChartHypothesis &, const ChartKBestExtractor &);
+
+  ~ChartHypothesis();
+
+  unsigned GetId() const {
+    return m_id;
+  }
+
+  const ChartTranslationOption &GetTranslationOption() const {
+    return *m_transOpt;
+  }
+
+  //! Get the rule that created this hypothesis
+  const TargetPhrase &GetCurrTargetPhrase() const {
+    return m_transOpt->GetPhrase();
+  }
+
+  //! the source range that this hypothesis spans
+  const Range &GetCurrSourceRange() const {
+    return m_currSourceWordsRange;
+  }
+
+  //! the arc list when creating n-best lists
+  inline const ChartArcList* GetArcList() const {
+    return m_arcList;
+  }
+
+  //! the feature function states for a particular feature \param featureID
+  inline const FFState* GetFFState( size_t featureID ) const {
+    return m_ffStates[ featureID ];
+  }
+
+  //! reference back to the manager
+  inline const ChartManager& GetManager() const {
+    return m_manager;
+  }
+
+  void GetOutputPhrase(Phrase &outPhrase) const;
+  Phrase GetOutputPhrase() const;
+
+  // get leftmost/rightmost words only
+  // leftRightMost: 1=left, 2=right
+  void GetOutputPhrase(size_t leftRightMost, size_t numWords, Phrase &outPhrase) const;
+
+  void EvaluateWhenApplied();
+
+  void AddArc(ChartHypothesis *loserHypo);
+  void CleanupArcList();
+  void SetWinningHypo(const ChartHypothesis *hypo);
+
+  //! get the unweighted score for each feature function
+  const ScoreComponentCollection &GetScoreBreakdown() const {
+    // Note: never call this method before m_currScoreBreakdown is fully computed
+    if (!m_scoreBreakdown.get()) {
+      m_scoreBreakdown.reset(new ScoreComponentCollection());
+      // score breakdown from current translation rule
+      if (m_transOpt) {
+        m_scoreBreakdown->PlusEquals(GetTranslationOption().GetScores());
+      }
+      m_scoreBreakdown->PlusEquals(m_currScoreBreakdown);
+      // score breakdowns from prev hypos
+      for (std::vector<const ChartHypothesis*>::const_iterator iter = m_prevHypos.begin(); iter != m_prevHypos.end(); ++iter) {
+        const ChartHypothesis &prevHypo = **iter;
+        m_scoreBreakdown->PlusEquals(prevHypo.GetScoreBreakdown());
+      }
+    }
+    return *(m_scoreBreakdown.get());
+  }
+
+  //! get the unweighted score delta for each feature function
+  const ScoreComponentCollection &GetDeltaScoreBreakdown() const {
+    // Note: never call this method before m_currScoreBreakdown is fully computed
+    if (!m_deltaScoreBreakdown.get()) {
+      m_deltaScoreBreakdown.reset(new ScoreComponentCollection());
+      // score breakdown from current translation rule
+      if (m_transOpt) {
+        m_deltaScoreBreakdown->PlusEquals(GetTranslationOption().GetScores());
+      }
+      m_deltaScoreBreakdown->PlusEquals(m_currScoreBreakdown);
+      // delta: score breakdowns from prev hypos _not_ added
+    }
+    return *(m_deltaScoreBreakdown.get());
+  }
+
+  //! Get the weighted total score
+  float GetFutureScore() const {
+    // scores from current translation rule. eg. translation models & word penalty
+    return m_totalScore;
+  }
+
+  //! vector of previous hypotheses this hypo is built on
+  const std::vector<const ChartHypothesis*> &GetPrevHypos() const {
+    return m_prevHypos;
+  }
+
+  //! get a particular previous hypos
+  const ChartHypothesis* GetPrevHypo(size_t pos) const {
+    return m_prevHypos[pos];
+  }
+
+  //! get the constituency label that covers this hypo
+  const Word &GetTargetLHS() const {
+    return GetCurrTargetPhrase().GetTargetLHS();
+  }
+
+  //! get the best hypo in the arc list when doing n-best list creation. It's either this hypothesis, or the best hypo is this hypo is in the arc list
+  const ChartHypothesis* GetWinningHypothesis() const {
+    return m_winningHypo;
+  }
+
+  // for unordered_set in stack
+  size_t hash() const;
+  bool operator==(const ChartHypothesis& other) const;
+
+  TO_STRING();
+
+}; // class ChartHypothesis
+
+}
+

mosesdecoder/moses/ChartKBestExtractor.cpp

@@ -0,0 +1,332 @@
+/***********************************************************************
+ Moses - statistical machine translation system
+ Copyright (C) 2006-2014 University of Edinburgh
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+***********************************************************************/
+
+#include "ChartKBestExtractor.h"
+
+#include "ChartHypothesis.h"
+#include "ScoreComponentCollection.h"
+#include "StaticData.h"
+
+#include <boost/scoped_ptr.hpp>
+
+#include <vector>
+
+using namespace std;
+
+namespace Moses
+{
+
+// Extract the k-best list from the search graph.
+void ChartKBestExtractor::Extract(
+  const std::vector<const ChartHypothesis*> &topLevelHypos, std::size_t k,
+  KBestVec &kBestList)
+{
+  kBestList.clear();
+  if (topLevelHypos.empty()) {
+    return;
+  }
+
+  // Create a new ChartHypothesis object, supremeHypo, that has the best
+  // top-level hypothesis as its predecessor and has the same score.
+  std::vector<const ChartHypothesis*>::const_iterator p = topLevelHypos.begin();
+  const ChartHypothesis &bestTopLevelHypo = **p;
+  boost::scoped_ptr<ChartHypothesis> supremeHypo(
+    new ChartHypothesis(bestTopLevelHypo, *this));
+
+  // Do the same for each alternative top-level hypothesis, but add the new
+  // ChartHypothesis objects as arcs from supremeHypo, as if they had been
+  // recombined.
+  for (++p; p != topLevelHypos.end(); ++p) {
+    // Check that the first item in topLevelHypos really was the best.
+    UTIL_THROW_IF2((*p)->GetFutureScore() > bestTopLevelHypo.GetFutureScore(),
+                   "top-level hypotheses are not correctly sorted");
+    // Note: there's no need for a smart pointer here: supremeHypo will take
+    // ownership of altHypo.
+    ChartHypothesis *altHypo = new ChartHypothesis(**p, *this);
+    supremeHypo->AddArc(altHypo);
+  }
+
+  // Create the target vertex then lazily fill its k-best list.
+  boost::shared_ptr<Vertex> targetVertex = FindOrCreateVertex(*supremeHypo);
+  LazyKthBest(*targetVertex, k, k);
+
+  // Copy the k-best list from the target vertex, but drop the top edge from
+  // each derivation.
+  kBestList.reserve(targetVertex->kBestList.size());
+  for (std::vector<boost::weak_ptr<Derivation> >::const_iterator
+       q = targetVertex->kBestList.begin();
+       q != targetVertex->kBestList.end(); ++q) {
+    const boost::shared_ptr<Derivation> d(*q);
+    assert(d);
+    assert(d->subderivations.size() == 1);
+    kBestList.push_back(d->subderivations[0]);
+  }
+}
+
+// Generate the target-side yield of the derivation d.
+Phrase ChartKBestExtractor::GetOutputPhrase(const Derivation &d)
+{
+  FactorType placeholderFactor = StaticData::Instance().options()->input.placeholder_factor;
+
+  Phrase ret(ARRAY_SIZE_INCR);
+
+  const ChartHypothesis &hypo = d.edge.head->hypothesis;
+  const TargetPhrase &phrase = hypo.GetCurrTargetPhrase();
+  const AlignmentInfo::NonTermIndexMap &nonTermIndexMap =
+    phrase.GetAlignNonTerm().GetNonTermIndexMap();
+  for (std::size_t pos = 0; pos < phrase.GetSize(); ++pos) {
+    const Word &word = phrase.GetWord(pos);
+    if (word.IsNonTerminal()) {
+      std::size_t nonTermInd = nonTermIndexMap[pos];
+      const Derivation &subderivation = *d.subderivations[nonTermInd];
+      Phrase subPhrase = GetOutputPhrase(subderivation);
+      ret.Append(subPhrase);
+    } else {
+      ret.AddWord(word);
+      if (placeholderFactor == NOT_FOUND) {
+        continue;
+      }
+      std::set<std::size_t> sourcePosSet =
+        phrase.GetAlignTerm().GetAlignmentsForTarget(pos);
+      if (sourcePosSet.size() == 1) {
+        const std::vector<const Word*> *ruleSourceFromInputPath =
+          hypo.GetTranslationOption().GetSourceRuleFromInputPath();
+        UTIL_THROW_IF2(ruleSourceFromInputPath == NULL,
+                       "Source Words in of the rules hasn't been filled out");
+        std::size_t sourcePos = *sourcePosSet.begin();
+        const Word *sourceWord = ruleSourceFromInputPath->at(sourcePos);
+        UTIL_THROW_IF2(sourceWord == NULL,
+                       "Null source word at position " << sourcePos);
+        const Factor *factor = sourceWord->GetFactor(placeholderFactor);
+        if (factor) {
+          ret.Back()[0] = factor;
+        }
+      }
+    }
+  }
+
+  return ret;
+}
+
+// Generate the score breakdown of the derivation d.
+boost::shared_ptr<ScoreComponentCollection>
+ChartKBestExtractor::GetOutputScoreBreakdown(const Derivation &d)
+{
+  const ChartHypothesis &hypo = d.edge.head->hypothesis;
+  boost::shared_ptr<ScoreComponentCollection> scoreBreakdown(new ScoreComponentCollection());
+  scoreBreakdown->PlusEquals(hypo.GetDeltaScoreBreakdown());
+  const TargetPhrase &phrase = hypo.GetCurrTargetPhrase();
+  const AlignmentInfo::NonTermIndexMap &nonTermIndexMap =
+    phrase.GetAlignNonTerm().GetNonTermIndexMap();
+  for (std::size_t pos = 0; pos < phrase.GetSize(); ++pos) {
+    const Word &word = phrase.GetWord(pos);
+    if (word.IsNonTerminal()) {
+      std::size_t nonTermInd = nonTermIndexMap[pos];
+      const Derivation &subderivation = *d.subderivations[nonTermInd];
+      scoreBreakdown->PlusEquals(*GetOutputScoreBreakdown(subderivation));
+    }
+  }
+
+  return scoreBreakdown;
+}
+
+// Generate the target tree of the derivation d.
+TreePointer ChartKBestExtractor::GetOutputTree(const Derivation &d)
+{
+  const ChartHypothesis &hypo = d.edge.head->hypothesis;
+  const TargetPhrase &phrase = hypo.GetCurrTargetPhrase();
+  if (const PhraseProperty *property = phrase.GetProperty("Tree")) {
+    const std::string *tree = property->GetValueString();
+    TreePointer mytree (boost::make_shared<InternalTree>(*tree));
+
+    //get subtrees (in target order)
+    std::vector<TreePointer> previous_trees;
+    for (size_t pos = 0; pos < phrase.GetSize(); ++pos) {
+      const Word &word = phrase.GetWord(pos);
+      if (word.IsNonTerminal()) {
+        size_t nonTermInd = phrase.GetAlignNonTerm().GetNonTermIndexMap()[pos];
+        const Derivation &subderivation = *d.subderivations[nonTermInd];
+        const TreePointer prev_tree = GetOutputTree(subderivation);
+        previous_trees.push_back(prev_tree);
+      }
+    }
+
+    mytree->Combine(previous_trees);
+    mytree->Unbinarize();
+    return mytree;
+  } else {
+    UTIL_THROW2("Error: k-best tree output active, but no internal tree structure found");
+  }
+}
+
+// Create an unweighted hyperarc corresponding to the given ChartHypothesis.
+ChartKBestExtractor::UnweightedHyperarc ChartKBestExtractor::CreateEdge(
+  const ChartHypothesis &h)
+{
+  UnweightedHyperarc edge;
+  edge.head = FindOrCreateVertex(h);
+  const std::vector<const ChartHypothesis*> &prevHypos = h.GetPrevHypos();
+  edge.tail.resize(prevHypos.size());
+  for (std::size_t i = 0; i < prevHypos.size(); ++i) {
+    const ChartHypothesis *prevHypo = prevHypos[i];
+    edge.tail[i] = FindOrCreateVertex(*prevHypo);
+  }
+  return edge;
+}
+
+// Look for the vertex corresponding to a given ChartHypothesis, creating
+// a new one if necessary.
+boost::shared_ptr<ChartKBestExtractor::Vertex>
+ChartKBestExtractor::FindOrCreateVertex(const ChartHypothesis &h)
+{
+  VertexMap::value_type element(&h, boost::shared_ptr<Vertex>());
+  std::pair<VertexMap::iterator, bool> p = m_vertexMap.insert(element);
+  boost::shared_ptr<Vertex> &sp = p.first->second;
+  if (!p.second) {
+    return sp;  // Vertex was already in m_vertexMap.
+  }
+  sp.reset(new Vertex(h));
+  // Create the 1-best derivation and add it to the vertex's kBestList.
+  UnweightedHyperarc bestEdge;
+  bestEdge.head = sp;
+  const std::vector<const ChartHypothesis*> &prevHypos = h.GetPrevHypos();
+  bestEdge.tail.resize(prevHypos.size());
+  for (std::size_t i = 0; i < prevHypos.size(); ++i) {
+    const ChartHypothesis *prevHypo = prevHypos[i];
+    bestEdge.tail[i] = FindOrCreateVertex(*prevHypo);
+  }
+  boost::shared_ptr<Derivation> bestDerivation(new Derivation(bestEdge));
+#ifndef NDEBUG
+  std::pair<DerivationSet::iterator, bool> q =
+#endif
+    m_derivations.insert(bestDerivation);
+  assert(q.second);
+  sp->kBestList.push_back(bestDerivation);
+  return sp;
+}
+
+// Create the 1-best derivation for each edge in BS(v) (except the best one)
+// and add it to v's candidate queue.
+void ChartKBestExtractor::GetCandidates(Vertex &v, std::size_t k)
+{
+  // Create derivations for all of v's incoming edges except the best.  This
+  // means everything in v.hypothesis.GetArcList() and not the edge defined
+  // by v.hypothesis itself.  The 1-best derivation for that edge will already
+  // have been created.
+  const ChartArcList *arcList = v.hypothesis.GetArcList();
+  if (arcList) {
+    for (std::size_t i = 0; i < arcList->size(); ++i) {
+      const ChartHypothesis &recombinedHypo = *(*arcList)[i];
+      boost::shared_ptr<Vertex> w = FindOrCreateVertex(recombinedHypo);
+      assert(w->kBestList.size() == 1);
+      v.candidates.push(w->kBestList[0]);
+    }
+  }
+}
+
+// Lazily fill v's k-best list.
+void ChartKBestExtractor::LazyKthBest(Vertex &v, std::size_t k,
+                                      std::size_t globalK)
+{
+  // If this is the first visit to vertex v then initialize the priority queue.
+  if (v.visited == false) {
+    // The 1-best derivation should already be in v's k-best list.
+    assert(v.kBestList.size() == 1);
+    // Initialize v's priority queue.
+    GetCandidates(v, globalK);
+    v.visited = true;
+  }
+  // Add derivations to the k-best list until it contains k or there are none
+  // left to add.
+  while (v.kBestList.size() < k) {
+    assert(!v.kBestList.empty());
+    // Update the priority queue by adding the successors of the last
+    // derivation (unless they've been seen before).
+    boost::shared_ptr<Derivation> d(v.kBestList.back());
+    LazyNext(v, *d, globalK);
+    // Check if there are any derivations left in the queue.
+    if (v.candidates.empty()) {
+      break;
+    }
+    // Get the next best derivation and delete it from the queue.
+    boost::weak_ptr<Derivation> next = v.candidates.top();
+    v.candidates.pop();
+    // Add it to the k-best list.
+    v.kBestList.push_back(next);
+  }
+}
+
+// Create the neighbours of Derivation d and add them to v's candidate queue.
+void ChartKBestExtractor::LazyNext(Vertex &v, const Derivation &d,
+                                   std::size_t globalK)
+{
+  for (std::size_t i = 0; i < d.edge.tail.size(); ++i) {
+    Vertex &pred = *d.edge.tail[i];
+    // Ensure that pred's k-best list contains enough derivations.
+    std::size_t k = d.backPointers[i] + 2;
+    LazyKthBest(pred, k, globalK);
+    if (pred.kBestList.size() < k) {
+      // pred's derivations have been exhausted.
+      continue;
+    }
+    // Create the neighbour.
+    boost::shared_ptr<Derivation> next(new Derivation(d, i));
+    // Check if it has been created before.
+    std::pair<DerivationSet::iterator, bool> p = m_derivations.insert(next);
+    if (p.second) {
+      v.candidates.push(next);  // Haven't previously seen it.
+    }
+  }
+}
+
+// Construct the 1-best Derivation that ends at edge e.
+ChartKBestExtractor::Derivation::Derivation(const UnweightedHyperarc &e)
+{
+  edge = e;
+  std::size_t arity = edge.tail.size();
+  backPointers.resize(arity, 0);
+  subderivations.reserve(arity);
+  for (std::size_t i = 0; i < arity; ++i) {
+    const Vertex &pred = *edge.tail[i];
+    assert(pred.kBestList.size() >= 1);
+    boost::shared_ptr<Derivation> sub(pred.kBestList[0]);
+    subderivations.push_back(sub);
+  }
+  score = edge.head->hypothesis.GetFutureScore();
+}
+
+// Construct a Derivation that neighbours an existing Derivation.
+ChartKBestExtractor::Derivation::Derivation(const Derivation &d, std::size_t i)
+{
+  edge.head = d.edge.head;
+  edge.tail = d.edge.tail;
+  backPointers = d.backPointers;
+  subderivations = d.subderivations;
+  std::size_t j = ++backPointers[i];
+  score = d.score;
+  // Deduct the score of the old subderivation.
+  score -= subderivations[i]->score;
+  // Update the subderivation pointer.
+  boost::shared_ptr<Derivation> newSub(edge.tail[i]->kBestList[j]);
+  subderivations[i] = newSub;
+  // Add the score of the new subderivation.
+  score += subderivations[i]->score;
+}
+
+}  // namespace Moses

mosesdecoder/moses/ChartKBestExtractor.h

@@ -0,0 +1,132 @@
+/***********************************************************************
+ Moses - statistical machine translation system
+ Copyright (C) 2006-2014 University of Edinburgh
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+***********************************************************************/
+
+#pragma once
+
+#include <cassert>
+#include "ChartHypothesis.h"
+#include "ScoreComponentCollection.h"
+#include "FF/InternalTree.h"
+
+#include <boost/unordered_set.hpp>
+#include <boost/weak_ptr.hpp>
+#include <boost/shared_ptr.hpp>
+
+#include <queue>
+#include <vector>
+
+namespace Moses
+{
+
+// k-best list extractor that implements algorithm 3 from this paper:
+//
+//  Liang Huang and David Chiang
+//  "Better k-best parsing"
+//  In Proceedings of IWPT 2005
+//
+class ChartKBestExtractor
+{
+public:
+  struct Vertex;
+
+  struct UnweightedHyperarc {
+    boost::shared_ptr<Vertex> head;
+    std::vector<boost::shared_ptr<Vertex> > tail;
+  };
+
+  struct Derivation {
+    Derivation(const UnweightedHyperarc &);
+    Derivation(const Derivation &, std::size_t);
+
+    UnweightedHyperarc edge;
+    std::vector<std::size_t> backPointers;
+    std::vector<boost::shared_ptr<Derivation> > subderivations;
+    float score;
+  };
+
+  struct DerivationOrderer {
+    bool operator()(const boost::weak_ptr<Derivation> &d1,
+                    const boost::weak_ptr<Derivation> &d2) const {
+      boost::shared_ptr<Derivation> s1(d1);
+      boost::shared_ptr<Derivation> s2(d2);
+      return s1->score < s2->score;
+    }
+  };
+
+  struct Vertex {
+    typedef std::priority_queue<boost::weak_ptr<Derivation>,
+            std::vector<boost::weak_ptr<Derivation> >,
+            DerivationOrderer> DerivationQueue;
+
+    Vertex(const ChartHypothesis &h) : hypothesis(h), visited(false) {}
+
+    const ChartHypothesis &hypothesis;
+    std::vector<boost::weak_ptr<Derivation> > kBestList;
+    DerivationQueue candidates;
+    bool visited;
+  };
+
+  typedef std::vector<boost::shared_ptr<Derivation> > KBestVec;
+
+  // Extract the k-best list from the search hypergraph given the full, sorted
+  // list of top-level vertices.
+  void Extract(const std::vector<const ChartHypothesis*> &topHypos,
+               std::size_t k, KBestVec &);
+
+  static Phrase GetOutputPhrase(const Derivation &);
+  static boost::shared_ptr<ScoreComponentCollection> GetOutputScoreBreakdown(const Derivation &);
+  static TreePointer GetOutputTree(const Derivation &);
+
+private:
+  typedef boost::unordered_map<const ChartHypothesis *,
+          boost::shared_ptr<Vertex> > VertexMap;
+
+  struct DerivationHasher {
+    std::size_t operator()(const boost::shared_ptr<Derivation> &d) const {
+      std::size_t seed = 0;
+      boost::hash_combine(seed, d->edge.head);
+      boost::hash_combine(seed, d->edge.tail);
+      boost::hash_combine(seed, d->backPointers);
+      return seed;
+    }
+  };
+
+  struct DerivationEqualityPred {
+    bool operator()(const boost::shared_ptr<Derivation> &d1,
+                    const boost::shared_ptr<Derivation> &d2) const {
+      return d1->edge.head == d2->edge.head &&
+             d1->edge.tail == d2->edge.tail &&
+             d1->backPointers == d2->backPointers;
+    }
+  };
+
+  typedef boost::unordered_set<boost::shared_ptr<Derivation>, DerivationHasher,
+          DerivationEqualityPred> DerivationSet;
+
+  UnweightedHyperarc CreateEdge(const ChartHypothesis &);
+  boost::shared_ptr<Vertex> FindOrCreateVertex(const ChartHypothesis &);
+  void GetCandidates(Vertex &, std::size_t);
+  void LazyKthBest(Vertex &, std::size_t, std::size_t);
+  void LazyNext(Vertex &, const Derivation &, std::size_t);
+
+  VertexMap m_vertexMap;
+  DerivationSet m_derivations;
+};
+
+}  // namespace Moses

mosesdecoder/moses/ChartParser.cpp

@@ -0,0 +1,313 @@
+// $Id$
+// vim:tabstop=2
+/***********************************************************************
+ Moses - factored phrase-based language decoder
+ Copyright (C) 2010 Hieu Hoang
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ ***********************************************************************/
+
+#include "ChartParser.h"
+#include "ChartParserCallback.h"
+#include "ChartRuleLookupManager.h"
+#include "StaticData.h"
+#include "TreeInput.h"
+#include "Sentence.h"
+#include "DecodeGraph.h"
+#include "moses/FF/UnknownWordPenaltyProducer.h"
+#include "moses/TranslationModel/PhraseDictionary.h"
+#include "moses/TranslationTask.h"
+
+using namespace std;
+using namespace Moses;
+
+namespace Moses
+{
+
+ChartParserUnknown
+::ChartParserUnknown(ttasksptr const& ttask)
+  : m_ttask(ttask)
+{ }
+
+ChartParserUnknown::~ChartParserUnknown()
+{
+  RemoveAllInColl(m_unksrcs);
+}
+
+AllOptions::ptr const&
+ChartParserUnknown::
+options() const
+{
+  return m_ttask.lock()->options();
+}
+
+void
+ChartParserUnknown::
+Process(const Word &sourceWord, const Range &range, ChartParserCallback &to)
+{
+  // unknown word, add as trans opt
+  const StaticData &staticData = StaticData::Instance();
+  const UnknownWordPenaltyProducer &unknownWordPenaltyProducer
+  = UnknownWordPenaltyProducer::Instance();
+
+  size_t isDigit = 0;
+  if (options()->unk.drop) {
+    const Factor *f = sourceWord[0]; // TODO hack. shouldn't know which factor is surface
+    const StringPiece s = f->GetString();
+    isDigit = s.find_first_of("0123456789");
+    if (isDigit == string::npos)
+      isDigit = 0;
+    else
+      isDigit = 1;
+    // modify the starting bitmap
+  }
+
+  Phrase* unksrc = new Phrase(1);
+  unksrc->AddWord() = sourceWord;
+  Word &newWord = unksrc->GetWord(0);
+  newWord.SetIsOOV(true);
+
+  m_unksrcs.push_back(unksrc);
+
+  // hack. Once the OOV FF is a phrase table, get rid of this
+  PhraseDictionary *firstPt = NULL;
+  if (PhraseDictionary::GetColl().size() == 0) {
+    firstPt = PhraseDictionary::GetColl()[0];
+  }
+
+  //TranslationOption *transOpt;
+  if (! options()->unk.drop || isDigit) {
+    // loop
+    const UnknownLHSList &lhsList = options()->syntax.unknown_lhs; // staticData.GetUnknownLHS();
+    UnknownLHSList::const_iterator iterLHS;
+    for (iterLHS = lhsList.begin(); iterLHS != lhsList.end(); ++iterLHS) {
+      const string &targetLHSStr = iterLHS->first;
+      float prob = iterLHS->second;
+
+      // lhs
+      //const Word &sourceLHS = staticData.GetInputDefaultNonTerminal();
+      Word *targetLHS = new Word(true);
+
+      targetLHS->CreateFromString(Output, options()->output.factor_order,
+                                  targetLHSStr, true);
+      UTIL_THROW_IF2(targetLHS->GetFactor(0) == NULL, "Null factor for target LHS");
+
+      // add to dictionary
+      TargetPhrase *targetPhrase = new TargetPhrase(firstPt);
+      Word &targetWord = targetPhrase->AddWord();
+      targetWord.CreateUnknownWord(sourceWord);
+
+      // scores
+      float unknownScore = FloorScore(TransformScore(prob));
+
+      targetPhrase->GetScoreBreakdown().Assign(&unknownWordPenaltyProducer, unknownScore);
+      targetPhrase->SetTargetLHS(targetLHS);
+      targetPhrase->SetAlignmentInfo("0-0");
+      targetPhrase->EvaluateInIsolation(*unksrc);
+
+      if (!options()->output.detailed_tree_transrep_filepath.empty() ||
+          options()->nbest.print_trees || staticData.GetTreeStructure() != NULL) {
+        std::string prop = "[ ";
+        prop += (*targetLHS)[0]->GetString().as_string() + " ";
+        prop += sourceWord[0]->GetString().as_string() + " ]";
+        targetPhrase->SetProperty("Tree", prop);
+      }
+
+      // chart rule
+      to.AddPhraseOOV(*targetPhrase, m_cacheTargetPhraseCollection, range);
+    } // for (iterLHS
+  } else {
+    // drop source word. create blank trans opt
+    float unknownScore = FloorScore(-numeric_limits<float>::infinity());
+
+    TargetPhrase *targetPhrase = new TargetPhrase(firstPt);
+    // loop
+    const UnknownLHSList &lhsList = options()->syntax.unknown_lhs;//staticData.GetUnknownLHS();
+    UnknownLHSList::const_iterator iterLHS;
+    for (iterLHS = lhsList.begin(); iterLHS != lhsList.end(); ++iterLHS) {
+      const string &targetLHSStr = iterLHS->first;
+      //float prob = iterLHS->second;
+
+      Word *targetLHS = new Word(true);
+      targetLHS->CreateFromString(Output, staticData.options()->output.factor_order,
+                                  targetLHSStr, true);
+      UTIL_THROW_IF2(targetLHS->GetFactor(0) == NULL, "Null factor for target LHS");
+
+      targetPhrase->GetScoreBreakdown().Assign(&unknownWordPenaltyProducer, unknownScore);
+      targetPhrase->EvaluateInIsolation(*unksrc);
+
+      targetPhrase->SetTargetLHS(targetLHS);
+
+      // chart rule
+      to.AddPhraseOOV(*targetPhrase, m_cacheTargetPhraseCollection, range);
+    }
+  }
+}
+
+ChartParser
+::ChartParser(ttasksptr const& ttask, ChartCellCollectionBase &cells)
+  : m_ttask(ttask)
+  , m_unknown(ttask)
+  , m_decodeGraphList(StaticData::Instance().GetDecodeGraphs())
+  , m_source(*(ttask->GetSource().get()))
+{
+  const StaticData &staticData = StaticData::Instance();
+
+  staticData.InitializeForInput(ttask);
+  CreateInputPaths(m_source);
+
+  const std::vector<PhraseDictionary*> &dictionaries = PhraseDictionary::GetColl();
+  assert(dictionaries.size() == m_decodeGraphList.size());
+  m_ruleLookupManagers.reserve(dictionaries.size());
+  for (std::size_t i = 0; i < dictionaries.size(); ++i) {
+    const PhraseDictionary *dict = dictionaries[i];
+    PhraseDictionary *nonConstDict = const_cast<PhraseDictionary*>(dict);
+    std::size_t maxChartSpan = m_decodeGraphList[i]->GetMaxChartSpan();
+    ChartRuleLookupManager *lookupMgr = nonConstDict->CreateRuleLookupManager(*this, cells, maxChartSpan);
+    m_ruleLookupManagers.push_back(lookupMgr);
+  }
+
+}
+
+ChartParser::~ChartParser()
+{
+  RemoveAllInColl(m_ruleLookupManagers);
+  StaticData::Instance().CleanUpAfterSentenceProcessing(m_ttask.lock());
+
+  InputPathMatrix::const_iterator iterOuter;
+  for (iterOuter = m_inputPathMatrix.begin(); iterOuter != m_inputPathMatrix.end(); ++iterOuter) {
+    const std::vector<InputPath*> &outer = *iterOuter;
+
+    std::vector<InputPath*>::const_iterator iterInner;
+    for (iterInner = outer.begin(); iterInner != outer.end(); ++iterInner) {
+      InputPath *path = *iterInner;
+      delete path;
+    }
+  }
+}
+
+void ChartParser::Create(const Range &range, ChartParserCallback &to)
+{
+  assert(m_decodeGraphList.size() == m_ruleLookupManagers.size());
+
+  std::vector <DecodeGraph*>::const_iterator iterDecodeGraph;
+  std::vector <ChartRuleLookupManager*>::const_iterator iterRuleLookupManagers = m_ruleLookupManagers.begin();
+  for (iterDecodeGraph = m_decodeGraphList.begin(); iterDecodeGraph != m_decodeGraphList.end(); ++iterDecodeGraph, ++iterRuleLookupManagers) {
+    const DecodeGraph &decodeGraph = **iterDecodeGraph;
+    assert(decodeGraph.GetSize() == 1);
+    ChartRuleLookupManager &ruleLookupManager = **iterRuleLookupManagers;
+    size_t maxSpan = decodeGraph.GetMaxChartSpan();
+    size_t last = m_source.GetSize()-1;
+    if (maxSpan != 0) {
+      last = min(last, range.GetStartPos()+maxSpan);
+    }
+    if (maxSpan == 0 || range.GetNumWordsCovered() <= maxSpan) {
+      const InputPath &inputPath = GetInputPath(range);
+      ruleLookupManager.GetChartRuleCollection(inputPath, last, to);
+    }
+  }
+
+  if (range.GetNumWordsCovered() == 1
+      && range.GetStartPos() != 0
+      && range.GetStartPos() != m_source.GetSize()-1) {
+    bool always = options()->unk.always_create_direct_transopt;
+    if (to.Empty() || always) {
+      // create unknown words for 1 word coverage where we don't have any trans options
+      const Word &sourceWord = m_source.GetWord(range.GetStartPos());
+      m_unknown.Process(sourceWord, range, to);
+    }
+  }
+}
+
+void ChartParser::CreateInputPaths(const InputType &input)
+{
+  size_t size = input.GetSize();
+  m_inputPathMatrix.resize(size);
+
+  UTIL_THROW_IF2(input.GetType() != SentenceInput && input.GetType() != TreeInputType,
+                 "Input must be a sentence or a tree, " <<
+                 "not lattice or confusion networks");
+
+  TranslationTask const* ttask = m_ttask.lock().get();
+  for (size_t phaseSize = 1; phaseSize <= size; ++phaseSize) {
+    for (size_t startPos = 0; startPos < size - phaseSize + 1; ++startPos) {
+      size_t endPos = startPos + phaseSize -1;
+      vector<InputPath*> &vec = m_inputPathMatrix[startPos];
+
+      Range range(startPos, endPos);
+      Phrase subphrase(input.GetSubString(Range(startPos, endPos)));
+      const NonTerminalSet &labels = input.GetLabelSet(startPos, endPos);
+
+      InputPath *node;
+      if (range.GetNumWordsCovered() == 1) {
+        node = new InputPath(ttask, subphrase, labels, range, NULL, NULL);
+        vec.push_back(node);
+      } else {
+        const InputPath &prevNode = GetInputPath(startPos, endPos - 1);
+        node = new InputPath(ttask, subphrase, labels, range, &prevNode, NULL);
+        vec.push_back(node);
+      }
+
+      //m_inputPathQueue.push_back(node);
+    }
+  }
+}
+
+const InputPath &ChartParser::GetInputPath(const Range &range) const
+{
+  return GetInputPath(range.GetStartPos(), range.GetEndPos());
+}
+
+const InputPath &ChartParser::GetInputPath(size_t startPos, size_t endPos) const
+{
+  size_t offset = endPos - startPos;
+  UTIL_THROW_IF2(offset >= m_inputPathMatrix[startPos].size(),
+                 "Out of bound: " << offset);
+  return *m_inputPathMatrix[startPos][offset];
+}
+
+InputPath &ChartParser::GetInputPath(size_t startPos, size_t endPos)
+{
+  size_t offset = endPos - startPos;
+  UTIL_THROW_IF2(offset >= m_inputPathMatrix[startPos].size(),
+                 "Out of bound: " << offset);
+  return *m_inputPathMatrix[startPos][offset];
+}
+/*
+const Sentence &ChartParser::GetSentence() const {
+  const Sentence &sentence = static_cast<const Sentence&>(m_source);
+  return sentence;
+}
+*/
+size_t ChartParser::GetSize() const
+{
+  return m_source.GetSize();
+}
+
+long ChartParser::GetTranslationId() const
+{
+  return m_source.GetTranslationId();
+}
+
+
+AllOptions::ptr const&
+ChartParser::
+options() const
+{
+  return m_ttask.lock()->options();
+}
+
+
+} // namespace Moses

mosesdecoder/moses/ChartRuleLookupManager.cpp

@@ -0,0 +1,9 @@
+#include "ChartRuleLookupManager.h"
+#include "ChartParser.h"
+
+namespace Moses
+{
+ChartRuleLookupManager::~ChartRuleLookupManager()
+{}
+}  // namespace Moses
+

mosesdecoder/moses/ChartRuleLookupManager.h

@@ -0,0 +1,84 @@
+/***********************************************************************
+  Moses - factored phrase-based language decoder
+  Copyright (C) 2011 University of Edinburgh
+
+  This library is free software; you can redistribute it and/or
+  modify it under the terms of the GNU Lesser General Public
+  License as published by the Free Software Foundation; either
+  version 2.1 of the License, or (at your option) any later version.
+
+  This library is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+  Lesser General Public License for more details.
+
+  You should have received a copy of the GNU Lesser General Public
+  License along with this library; if not, write to the Free Software
+  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ ***********************************************************************/
+
+#pragma once
+#ifndef moses_ChartRuleLookupManager_h
+#define moses_ChartRuleLookupManager_h
+
+#include "ChartCellCollection.h"
+#include "InputType.h"
+
+namespace Moses
+{
+class ChartParser;
+class ChartParserCallback;
+class Range;
+class Sentence;
+
+/** Defines an interface for looking up rules in a rule table.  Concrete
+ *  implementation classes should correspond to specific PhraseDictionary
+ *  subclasses (memory or on-disk).  Since a ChartRuleLookupManager object
+ *  maintains sentence-specific state, exactly one should be created for
+ *  each sentence that is to be decoded.
+ */
+class ChartRuleLookupManager
+{
+public:
+  ChartRuleLookupManager(const ChartParser &parser,
+                         const ChartCellCollectionBase &cellColl)
+    : m_parser(parser)
+    , m_cellCollection(cellColl) {}
+
+  virtual ~ChartRuleLookupManager();
+
+  const ChartCellLabelSet &GetTargetLabelSet(size_t begin, size_t end) const {
+    return m_cellCollection.GetBase(Range(begin, end)).GetTargetLabelSet();
+  }
+
+  const ChartParser &GetParser() const {
+    return m_parser;
+  }
+  //const Sentence &GetSentence() const;
+
+  const ChartCellLabel &GetSourceAt(size_t at) const {
+    return m_cellCollection.GetSourceWordLabel(at);
+  }
+
+  /** abstract function. Return a vector of translation options for given a range in the input sentence
+   *  \param range source range for which you want the translation options
+   *  \param outColl return argument
+   */
+  virtual void GetChartRuleCollection(
+    const InputPath &inputPath,
+    size_t lastPos,  // last position to consider if using lookahead
+    ChartParserCallback &outColl) = 0;
+
+private:
+  //! Non-copyable: copy constructor and assignment operator not implemented.
+  ChartRuleLookupManager(const ChartRuleLookupManager &);
+  //! Non-copyable: copy constructor and assignment operator not implemented.
+  ChartRuleLookupManager &operator=(const ChartRuleLookupManager &);
+
+  const ChartParser &m_parser;
+  const ChartCellCollectionBase &m_cellCollection;
+};
+
+}  // namespace Moses
+
+#endif

mosesdecoder/moses/ChartTranslationOptionList.cpp

@@ -0,0 +1,219 @@
+/***********************************************************************
+ Moses - factored phrase-based language decoder
+ Copyright (C) 2010 Hieu Hoang
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ ***********************************************************************/
+
+#include <algorithm>
+#include <iostream>
+#include <vector>
+#include "StaticData.h"
+#include "ChartTranslationOptionList.h"
+#include "ChartTranslationOptions.h"
+#include "ChartCellCollection.h"
+#include "Range.h"
+#include "InputType.h"
+#include "InputPath.h"
+
+using namespace std;
+
+namespace Moses
+{
+
+ChartTranslationOptionList::
+ChartTranslationOptionList(size_t ruleLimit, const InputType &input)
+  : m_size(0)
+  , m_ruleLimit(ruleLimit)
+{
+  m_scoreThreshold = std::numeric_limits<float>::infinity();
+}
+
+ChartTranslationOptionList::~ChartTranslationOptionList()
+{
+  RemoveAllInColl(m_collection);
+}
+
+void ChartTranslationOptionList::Clear()
+{
+  m_size = 0;
+  m_scoreThreshold = std::numeric_limits<float>::infinity();
+}
+
+class ChartTranslationOptionOrderer
+{
+public:
+  bool operator()(const ChartTranslationOptions* itemA, const ChartTranslationOptions* itemB) const {
+    return itemA->GetEstimateOfBestScore() > itemB->GetEstimateOfBestScore();
+  }
+};
+
+void ChartTranslationOptionList::Add(const TargetPhraseCollection &tpc,
+                                     const StackVec &stackVec,
+                                     const Range &range)
+{
+  if (tpc.IsEmpty()) {
+    return;
+  }
+
+  for (size_t i = 0; i < stackVec.size(); ++i) {
+    const ChartCellLabel &chartCellLabel = *stackVec[i];
+    size_t numHypos = chartCellLabel.GetStack().cube->size();
+    if (numHypos == 0) {
+      return; // empty stack. These rules can't be used
+    }
+  }
+
+  const TargetPhrase &targetPhrase = **(tpc.begin());
+  float score = targetPhrase.GetFutureScore();
+  for (StackVec::const_iterator p = stackVec.begin(); p != stackVec.end(); ++p) {
+    score += (*p)->GetBestScore(this);
+  }
+
+  // If the rule limit has already been reached then don't add the option
+  // unless it is better than at least one existing option.
+  if (m_ruleLimit && m_size > m_ruleLimit && score < m_scoreThreshold) {
+    return;
+  }
+
+  // Add the option to the list.
+  if (m_size == m_collection.size()) {
+    // m_collection has reached capacity: create a new object.
+    m_collection.push_back(new ChartTranslationOptions(tpc, stackVec,
+                           range, score));
+  } else {
+    // Overwrite an unused object.
+    *(m_collection[m_size]) = ChartTranslationOptions(tpc, stackVec,
+                              range, score);
+  }
+  ++m_size;
+
+  // If the rule limit hasn't been exceeded then update the threshold.
+  if (!m_ruleLimit || m_size <= m_ruleLimit) {
+    m_scoreThreshold = (score < m_scoreThreshold) ? score : m_scoreThreshold;
+  }
+
+  // Prune if bursting
+  if (m_ruleLimit && m_size == m_ruleLimit * 2) {
+    NTH_ELEMENT4(m_collection.begin(),
+                 m_collection.begin() + m_ruleLimit - 1,
+                 m_collection.begin() + m_size,
+                 ChartTranslationOptionOrderer());
+    m_scoreThreshold = m_collection[m_ruleLimit-1]->GetEstimateOfBestScore();
+    m_size = m_ruleLimit;
+  }
+}
+
+void
+ChartTranslationOptionList::
+AddPhraseOOV(TargetPhrase &phrase,
+             std::list<TargetPhraseCollection::shared_ptr > &waste_memory,
+             const Range &range)
+{
+  TargetPhraseCollection::shared_ptr tpc(new TargetPhraseCollection);
+  tpc->Add(&phrase);
+  waste_memory.push_back(tpc);
+  StackVec empty;
+  Add(*tpc, empty, range);
+}
+
+void ChartTranslationOptionList::ApplyThreshold(float const threshold)
+{
+  if (m_ruleLimit && m_size > m_ruleLimit) {
+    // Something's gone wrong if the list has grown to m_ruleLimit * 2
+    // without being pruned.
+    assert(m_size < m_ruleLimit * 2);
+    // Reduce the list to the best m_ruleLimit options.  The remaining
+    // options can be overwritten on subsequent calls to Add().
+    NTH_ELEMENT4(m_collection.begin(),
+                 m_collection.begin()+m_ruleLimit,
+                 m_collection.begin()+m_size,
+                 ChartTranslationOptionOrderer());
+    m_size = m_ruleLimit;
+  }
+
+  // keep only those over best + threshold
+
+  float scoreThreshold = -std::numeric_limits<float>::infinity();
+
+  CollType::const_iterator iter;
+  for (iter = m_collection.begin(); iter != m_collection.begin()+m_size; ++iter) {
+    const ChartTranslationOptions *transOpt = *iter;
+    float score = transOpt->GetEstimateOfBestScore();
+    scoreThreshold = (score > scoreThreshold) ? score : scoreThreshold;
+  }
+
+  scoreThreshold += threshold; // StaticData::Instance().GetTranslationOptionThreshold();
+
+  CollType::iterator bound = std::partition(m_collection.begin(),
+                             m_collection.begin()+m_size,
+                             ScoreThresholdPred(scoreThreshold));
+
+  m_size = std::distance(m_collection.begin(), bound);
+}
+
+float ChartTranslationOptionList::GetBestScore(const ChartCellLabel *chartCell) const
+{
+  const HypoList *stack = chartCell->GetStack().cube;
+  assert(stack);
+  assert(!stack->empty());
+  const ChartHypothesis &bestHypo = **(stack->begin());
+  return bestHypo.GetFutureScore();
+}
+
+void ChartTranslationOptionList::EvaluateWithSourceContext(const InputType &input, const InputPath &inputPath)
+{
+  // NEVER iterate over ALL of the collection. Just over the first m_size
+  CollType::iterator iter;
+  for (iter = m_collection.begin(); iter != m_collection.begin() + m_size; ++iter) {
+    ChartTranslationOptions &transOpts = **iter;
+    transOpts.EvaluateWithSourceContext(input, inputPath);
+  }
+
+  // get rid of empty trans opts
+  size_t numDiscard = 0;
+  for (size_t i = 0; i < m_size; ++i) {
+    ChartTranslationOptions *transOpts = m_collection[i];
+    if (transOpts->GetSize() == 0) {
+      //delete transOpts;
+      ++numDiscard;
+    } else if (numDiscard) {
+      SwapTranslationOptions(i - numDiscard, i);
+      //m_collection[] = transOpts;
+    }
+  }
+
+  size_t newSize = m_size - numDiscard;
+  m_size = newSize;
+}
+
+void ChartTranslationOptionList::SwapTranslationOptions(size_t a, size_t b)
+{
+  ChartTranslationOptions *transOptsA = m_collection[a];
+  ChartTranslationOptions *transOptsB = m_collection[b];
+  m_collection[a] = transOptsB;
+  m_collection[b] = transOptsA;
+}
+
+std::ostream& operator<<(std::ostream &out, const ChartTranslationOptionList &obj)
+{
+  for (size_t i = 0; i < obj.m_collection.size(); ++i) {
+    const ChartTranslationOptions &transOpts = *obj.m_collection[i];
+    out << transOpts << endl;
+  }
+  return out;
+}
+
+}

mosesdecoder/moses/ChartTranslationOptions.h

@@ -0,0 +1,104 @@
+/***********************************************************************
+ Moses - factored phrase-based language decoder
+ Copyright (C) 2010 Hieu Hoang
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ ***********************************************************************/
+
+#pragma once
+
+#include "StackVec.h"
+#include "TargetPhrase.h"
+#include "TargetPhraseCollection.h"
+#include "Range.h"
+
+#include <vector>
+#include <boost/shared_ptr.hpp>
+#include "ChartTranslationOption.h"
+
+namespace Moses
+{
+class ChartTranslationOption;
+class InputPath;
+class InputType;
+
+/** Similar to a DottedRule, but contains a direct reference to a list
+ * of translations and provdes an estimate of the best score. For a specific range in the input sentence
+ */
+class ChartTranslationOptions
+{
+  friend std::ostream& operator<<(std::ostream&, const ChartTranslationOptions&);
+
+public:
+  typedef std::vector<boost::shared_ptr<ChartTranslationOption> > CollType;
+
+  /** Constructor
+      \param targetPhraseColl @todo dunno
+      \param stackVec @todo dunno
+      \param range the range in the source sentence this translation option covers
+      \param score @todo dunno
+   */
+  ChartTranslationOptions(const TargetPhraseCollection &targetPhraseColl,
+                          const StackVec &stackVec,
+                          const Range &range,
+                          float score);
+  ~ChartTranslationOptions();
+
+  static float CalcEstimateOfBestScore(const TargetPhraseCollection &,
+                                       const StackVec &);
+
+  size_t GetSize() const {
+    return m_collection.size();
+  }
+
+  //! @todo dunno
+  const StackVec &GetStackVec() const {
+    return m_stackVec;
+  }
+
+  //! @todo isn't the translation suppose to just contain 1 target phrase, not a whole collection of them?
+  const CollType &GetTargetPhrases() const {
+    return m_collection;
+  }
+
+  //! the range in the source sentence this translation option covers
+  const Range &GetSourceWordsRange() const {
+    return *m_wordsRange;
+  }
+
+  /** return an estimate of the best score possible with this translation option.
+    * the estimate is the sum of the top target phrase's estimated score plus the
+    * scores of the best child hypotheses.
+    */
+  inline float GetEstimateOfBestScore() const {
+    return m_estimateOfBestScore;
+  }
+
+  void EvaluateWithSourceContext(const InputType &input, const InputPath &inputPath);
+
+  void SetInputPath(const InputPath *inputPath);
+
+  void CreateSourceRuleFromInputPath();
+
+private:
+
+  StackVec m_stackVec; //! vector of hypothesis list!
+  CollType m_collection;
+
+  const Range *m_wordsRange;
+  float m_estimateOfBestScore;
+};
+
+}

mosesdecoder/moses/ConfusionNet.cpp

@@ -0,0 +1,294 @@
+// -*- mode: c++; indent-tabs-mode: nil; tab-width: 2 -*-
+// $Id$
+
+#include "ConfusionNet.h"
+#include <sstream>
+
+#include "FactorCollection.h"
+#include "Util.h"
+#include "TranslationOptionCollectionConfusionNet.h"
+#include "StaticData.h"
+#include "Sentence.h"
+#include "moses/FF/InputFeature.h"
+#include "util/exception.hh"
+#include "moses/TranslationTask.h"
+namespace Moses
+{
+struct CNStats {
+  size_t created,destr,read,colls,words;
+
+  CNStats() : created(0),destr(0),read(0),colls(0),words(0) {}
+  ~CNStats() {
+    print(std::cerr);
+  }
+
+  void createOne() {
+    ++created;
+  }
+  void destroyOne() {
+    ++destr;
+  }
+
+  void collect(const ConfusionNet& cn) {
+    ++read;
+    colls+=cn.GetSize();
+    for(size_t i=0; i<cn.GetSize(); ++i)
+      words+=cn[i].size();
+  }
+  void print(std::ostream& out) const {
+    if(created>0) {
+      out<<"confusion net statistics:\n"
+         " created:\t"<<created<<"\n"
+         " destroyed:\t"<<destr<<"\n"
+         " succ. read:\t"<<read<<"\n"
+         " columns:\t"<<colls<<"\n"
+         " words:\t"<<words<<"\n"
+         " avg. word/column:\t"<<words/(1.0*colls)<<"\n"
+         " avg. cols/sent:\t"<<colls/(1.0*read)<<"\n"
+         "\n\n";
+    }
+  }
+};
+
+CNStats stats;
+
+size_t
+ConfusionNet::
+GetColumnIncrement(size_t i, size_t j) const
+{
+  (void) i;
+  (void) j;
+  return 1;
+}
+
+ConfusionNet::
+ConfusionNet(AllOptions::ptr const& opts) : InputType(opts)
+{
+  stats.createOne();
+
+  if (is_syntax(opts->search.algo)) {
+    m_defaultLabelSet.insert(opts->syntax.input_default_non_terminal);
+  }
+  UTIL_THROW_IF2(InputFeature::InstancePtr() == NULL, "Input feature must be specified");
+}
+
+ConfusionNet::
+~ConfusionNet()
+{
+  stats.destroyOne();
+}
+
+ConfusionNet::
+ConfusionNet(Sentence const& s) : InputType(s.options())
+{
+  data.resize(s.GetSize());
+  for(size_t i=0; i<s.GetSize(); ++i) {
+    ScorePair scorePair;
+    std::pair<Word, ScorePair > temp = std::make_pair(s.GetWord(i), scorePair);
+    data[i].push_back(temp);
+  }
+}
+
+bool
+ConfusionNet::
+ReadF(std::istream& in, int format)
+{
+  VERBOSE(2, "read confusion net with format "<<format<<"\n");
+  switch(format) {
+  case 0:
+    return ReadFormat0(in);
+  case 1:
+    return ReadFormat1(in);
+  default:
+    std::cerr << "ERROR: unknown format '"<<format
+              <<"' in ConfusionNet::Read";
+  }
+  return false;
+}
+
+int
+ConfusionNet::
+Read(std::istream& in)
+{
+  int rv=ReadF(in,0);
+  if(rv) stats.collect(*this);
+  return rv;
+}
+
+bool
+ConfusionNet::
+ReadFormat0(std::istream& in)
+{
+  Clear();
+  const std::vector<FactorType>& factorOrder = m_options->input.factor_order;
+
+  const InputFeature *inputFeature = InputFeature::InstancePtr();
+  size_t numInputScores   = inputFeature->GetNumInputScores();
+  size_t numRealWordCount = inputFeature->GetNumRealWordsInInput();
+
+  size_t totalCount = numInputScores + numRealWordCount;
+  bool addRealWordCount = (numRealWordCount > 0);
+
+  std::string line;
+  while(getline(in,line)) {
+    std::istringstream is(line);
+    std::string word;
+
+    Column col;
+    while(is>>word) {
+      Word w;
+      w.CreateFromString(Input,factorOrder,StringPiece(word),false,false);
+      std::vector<float> probs(totalCount, 0.0);
+      for(size_t i=0; i < numInputScores; i++) {
+        double prob;
+        if (!(is>>prob)) {
+          TRACE_ERR("ERROR: unable to parse CN input - bad link probability, "
+                    << "or wrong number of scores\n");
+          return false;
+        }
+        if(prob<0.0) {
+          VERBOSE(1, "WARN: negative prob: "<<prob<<" ->set to 0.0\n");
+          prob=0.0;
+        } else if (prob>1.0) {
+          VERBOSE(1, "WARN: prob > 1.0 : "<<prob<<" -> set to 1.0\n");
+          prob=1.0;
+        }
+        probs[i] = (std::max(static_cast<float>(log(prob)),LOWEST_SCORE));
+
+      }
+      // store 'real' word count in last feature if we have one more
+      // weight than we do arc scores and not epsilon
+      if (addRealWordCount && word!=EPSILON && word!="")
+        probs.back() = -1.0;
+
+      ScorePair scorePair(probs);
+
+      col.push_back(std::make_pair(w,scorePair));
+    }
+    if(col.size()) {
+      data.push_back(col);
+      ShrinkToFit(data.back());
+    } else break;
+  }
+  return !data.empty();
+}
+
+bool
+ConfusionNet::
+ReadFormat1(std::istream& in)
+{
+  Clear();
+  const std::vector<FactorType>& factorOrder = m_options->input.factor_order;
+  std::string line;
+  if(!getline(in,line)) return 0;
+  size_t s;
+  if(getline(in,line)) s=atoi(line.c_str());
+  else return 0;
+  data.resize(s);
+  for(size_t i=0; i<data.size(); ++i) {
+    if(!getline(in,line)) return 0;
+    std::istringstream is(line);
+    if(!(is>>s)) return 0;
+    std::string word;
+    double prob;
+    data[i].resize(s);
+    for(size_t j=0; j<s; ++j)
+      if(is>>word>>prob) {
+        //TODO: we are only reading one prob from this input format, should read many... but this function is unused anyway. -JS
+        data[i][j].second.denseScores = std::vector<float> (1);
+        data[i][j].second.denseScores.push_back((float) log(prob));
+        if(data[i][j].second.denseScores[0]<0) {
+          VERBOSE(1, "WARN: neg costs: "<<data[i][j].second.denseScores[0]<<" -> set to 0\n");
+          data[i][j].second.denseScores[0]=0.0;
+        }
+        // String2Word(word,data[i][j].first,factorOrder);
+        Word& w = data[i][j].first;
+        w.CreateFromString(Input,factorOrder,StringPiece(word),false,false);
+      } else return 0;
+  }
+  return !data.empty();
+}
+
+void ConfusionNet::Print(std::ostream& out) const
+{
+  out<<"conf net: "<<data.size()<<"\n";
+  for(size_t i=0; i<data.size(); ++i) {
+    out<<i<<" -- ";
+    for(size_t j=0; j<data[i].size(); ++j) {
+      out<<"("<<data[i][j].first.ToString()<<", ";
+
+      // dense
+      std::vector<float>::const_iterator iterDense;
+      for(iterDense = data[i][j].second.denseScores.begin();
+          iterDense < data[i][j].second.denseScores.end();
+          ++iterDense) {
+        out<<", "<<*iterDense;
+      }
+
+      // sparse
+      std::map<StringPiece, float>::const_iterator iterSparse;
+      for(iterSparse = data[i][j].second.sparseScores.begin();
+          iterSparse != data[i][j].second.sparseScores.end();
+          ++iterSparse) {
+        out << ", " << iterSparse->first << "=" << iterSparse->second;
+      }
+
+      out<<") ";
+    }
+    out<<"\n";
+  }
+  out<<"\n\n";
+}
+
+#ifdef _WIN32
+#pragma warning(disable:4716)
+#endif
+Phrase
+ConfusionNet::
+GetSubString(const Range&) const
+{
+  UTIL_THROW2("ERROR: call to ConfusionNet::GetSubString\n");
+  //return Phrase(Input);
+}
+
+std::string
+ConfusionNet::
+GetStringRep(const std::vector<FactorType> /* factorsToPrint */) const  //not well defined yet
+{
+  TRACE_ERR("ERROR: call to ConfusionNet::GeStringRep\n");
+  return "";
+}
+#ifdef _WIN32
+#pragma warning(disable:4716)
+#endif
+const Word& ConfusionNet::GetWord(size_t) const
+{
+  UTIL_THROW2("ERROR: call to ConfusionNet::GetFactorArray\n");
+}
+#ifdef _WIN32
+#pragma warning(default:4716)
+#endif
+std::ostream& operator<<(std::ostream& out,const ConfusionNet& cn)
+{
+  cn.Print(out);
+  return out;
+}
+
+TranslationOptionCollection*
+ConfusionNet::
+CreateTranslationOptionCollection(ttasksptr const& ttask) const
+{
+  // size_t maxNoTransOptPerCoverage
+  //   = ttask->options()->search.max_trans_opt_per_cov;
+  // float translationOptionThreshold
+  //   = ttask->options()->search.trans_opt_threshold;
+  TranslationOptionCollection *rv
+  = new TranslationOptionCollectionConfusionNet(ttask, *this);
+  //, maxNoTransOptPerCoverage, translationOptionThreshold);
+  assert(rv);
+  return rv;
+}
+
+}
+
+

mosesdecoder/moses/ContextScope.h

@@ -0,0 +1,124 @@
+// -*- mode: c++; indent-tabs-mode: nil; tab-width: 2 -*-
+// A class to store "local" information (such as task-specific caches).
+// The idea is for each translation task to have a scope, which stores
+// shared pointers to task-specific objects such as caches and priors.
+// Since these objects are referenced via shared pointers, sopes can
+// share information.
+#pragma once
+
+#ifdef WITH_THREADS
+#include <boost/thread/shared_mutex.hpp>
+#include <boost/thread/locks.hpp>
+#include <boost/foreach.hpp>
+#endif
+
+// for some reason, the xmlrpc_c headers must be included AFTER the
+// boost thread-related ones ...
+#include "xmlrpc-c.h"
+
+#include <map>
+#include <boost/shared_ptr.hpp>
+#include "TypeDef.h"
+#include "Util.h"
+
+namespace Moses
+{
+class ContextScope
+{
+protected:
+  typedef std::map<void const*, boost::shared_ptr<void> > scratchpad_t;
+  typedef scratchpad_t::iterator iter_t;
+  typedef scratchpad_t::value_type entry_t;
+  typedef scratchpad_t::const_iterator const_iter_t;
+  scratchpad_t m_scratchpad;
+#ifdef WITH_THREADS
+  mutable boost::shared_mutex m_lock;
+#endif
+  SPTR<std::map<std::string,float> const> m_context_weights;
+public:
+  typedef boost::shared_ptr<ContextScope> ptr;
+  template<typename T>
+  boost::shared_ptr<void> const&
+  set(void const* const key, boost::shared_ptr<T> const& val) {
+#ifdef WITH_THREADS
+    boost::unique_lock<boost::shared_mutex> lock(m_lock);
+#endif
+    return (m_scratchpad[key] = val);
+  }
+
+  template<typename T>
+  boost::shared_ptr<T> const
+  get(void const* key, bool CreateNewIfNecessary=false) {
+#ifdef WITH_THREADS
+    using boost::shared_mutex;
+    using boost::upgrade_lock;
+    // T const* key = reinterpret_cast<T const*>(xkey);
+    upgrade_lock<shared_mutex> lock(m_lock);
+#endif
+    iter_t m = m_scratchpad.find(key);
+    boost::shared_ptr< T > ret;
+    if (m != m_scratchpad.end()) {
+      if (m->second == NULL && CreateNewIfNecessary) {
+#ifdef WITH_THREADS
+        boost::upgrade_to_unique_lock<shared_mutex> xlock(lock);
+#endif
+        m->second.reset(new T);
+      }
+      ret = boost::static_pointer_cast< T >(m->second);
+      return ret;
+    }
+    if (!CreateNewIfNecessary) return ret;
+#ifdef WITH_THREADS
+    boost::upgrade_to_unique_lock<shared_mutex> xlock(lock);
+#endif
+    ret.reset(new T);
+    m_scratchpad[key] = ret;
+    return ret;
+  }
+
+  ContextScope() { }
+
+  ContextScope(ContextScope const& other) {
+#ifdef WITH_THREADS
+    boost::unique_lock<boost::shared_mutex> lock1(this->m_lock);
+    boost::unique_lock<boost::shared_mutex> lock2(other.m_lock);
+#endif
+    m_scratchpad = other.m_scratchpad;
+  }
+
+  SPTR<std::map<std::string,float> const>
+  GetContextWeights() {
+    return m_context_weights;
+  }
+
+  bool
+  SetContextWeights(std::string const& spec) {
+    if (m_context_weights) return false;
+    boost::unique_lock<boost::shared_mutex> lock(m_lock);
+    SPTR<std::map<std::string,float> > M(new std::map<std::string, float>);
+
+    // TO DO; This needs to be done with StringPiece.find, not Tokenize
+    // PRIORITY: low
+    std::vector<std::string> tokens = Tokenize(spec,":");
+    for (std::vector<std::string>::iterator it = tokens.begin();
+         it != tokens.end(); it++) {
+      std::vector<std::string> key_and_value = Tokenize(*it, ",");
+      (*M)[key_and_value[0]] = atof(key_and_value[1].c_str());
+    }
+    m_context_weights = M;
+    return true;
+  }
+
+  bool
+  SetContextWeights(SPTR<std::map<std::string,float> const> const& w) {
+    if (m_context_weights) return false;
+#ifdef WITH_THREADS
+    boost::unique_lock<boost::shared_mutex> lock(m_lock);
+#endif
+    m_context_weights = w;
+    return true;
+  }
+
+};
+
+};

mosesdecoder/moses/DecodeGraph.cpp

@@ -0,0 +1,43 @@
+// $Id$
+// vim:tabstop=2
+
+/***********************************************************************
+Moses - factored phrase-based language decoder
+Copyright (C) 2006 University of Edinburgh
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Lesser General Public
+License as published by the Free Software Foundation; either
+version 2.1 of the License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public
+License along with this library; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+***********************************************************************/
+
+#include "DecodeGraph.h"
+#include "DecodeStep.h"
+#include "TypeDef.h"
+#include "Util.h"
+
+namespace Moses
+{
+DecodeGraph::~DecodeGraph()
+{
+  RemoveAllInColl(m_steps);
+}
+
+//! Add another decode step to the graph
+void DecodeGraph::Add(DecodeStep *decodeStep)
+{
+  m_steps.push_back(decodeStep);
+  decodeStep->SetContainer(this);
+}
+
+}
+

mosesdecoder/moses/DecodeStep.cpp

@@ -0,0 +1,90 @@
+// $Id$
+
+/***********************************************************************
+Moses - factored phrase-based language decoder
+Copyright (C) 2006 University of Edinburgh
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Lesser General Public
+License as published by the Free Software Foundation; either
+version 2.1 of the License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public
+License along with this library; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+***********************************************************************/
+
+#include "DecodeStep.h"
+#include "GenerationDictionary.h"
+#include "StaticData.h"
+#include "moses/TranslationModel/PhraseDictionary.h"
+
+namespace Moses
+{
+DecodeStep::DecodeStep(DecodeFeature *decodeFeature,
+                       const DecodeStep* prev,
+                       const std::vector<FeatureFunction*> &features)
+  : m_decodeFeature(decodeFeature)
+{
+  FactorMask prevOutputFactors;
+  if (prev) prevOutputFactors = prev->m_outputFactors;
+  m_outputFactors = prevOutputFactors;
+  FactorMask conflictMask = (m_outputFactors & decodeFeature->GetOutputFactorMask());
+  m_outputFactors |= decodeFeature->GetOutputFactorMask();
+  FactorMask newOutputFactorMask = m_outputFactors ^ prevOutputFactors;  //xor
+  m_newOutputFactors.resize(newOutputFactorMask.count());
+  m_conflictFactors.resize(conflictMask.count());
+  size_t j=0, k=0;
+  for (size_t i = 0; i < MAX_NUM_FACTORS; i++) {
+    if (newOutputFactorMask[i]) m_newOutputFactors[j++] = i;
+    if (conflictMask[i]) m_conflictFactors[k++] = i;
+  }
+  VERBOSE(2,"DecodeStep():\n\toutputFactors=" << m_outputFactors
+          << "\n\tconflictFactors=" << conflictMask
+          << "\n\tnewOutputFactors=" << newOutputFactorMask << std::endl);
+
+  // find out which feature function can be applied in this decode step
+  for (size_t i = 0; i < features.size(); ++i) {
+    FeatureFunction *feature = features[i];
+    if (feature->IsUseable(m_outputFactors)) {
+      m_featuresToApply.push_back(feature);
+    } else {
+      m_featuresRemaining.push_back(feature);
+    }
+  }
+
+  decodeFeature->SetContainer(this);
+}
+
+DecodeStep::~DecodeStep() {}
+
+/** returns phrase feature (dictionary) for translation step */
+const PhraseDictionary* DecodeStep::GetPhraseDictionaryFeature() const
+{
+  return dynamic_cast<const PhraseDictionary*>(m_decodeFeature);
+}
+
+/** returns generation feature (dictionary) for generation step */
+const GenerationDictionary* DecodeStep::GetGenerationDictionaryFeature() const
+{
+  return dynamic_cast<const GenerationDictionary*>(m_decodeFeature);
+}
+
+void DecodeStep::RemoveFeature(const FeatureFunction *ff)
+{
+  for (size_t i = 0; i < m_featuresToApply.size(); ++i) {
+    if (ff == m_featuresToApply[i]) {
+      m_featuresToApply.erase(m_featuresToApply.begin() + i);
+      return;
+    }
+  }
+}
+
+}
+
+

mosesdecoder/moses/DecodeStepGeneration.h

@@ -0,0 +1,54 @@
+// $Id$
+
+/***********************************************************************
+Moses - factored phrase-based language decoder
+Copyright (C) 2006 University of Edinburgh
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Lesser General Public
+License as published by the Free Software Foundation; either
+version 2.1 of the License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public
+License along with this library; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+***********************************************************************/
+
+#ifndef moses_DecodeStepGeneration_h
+#define moses_DecodeStepGeneration_h
+
+#include "DecodeStep.h"
+
+namespace Moses
+{
+
+class GenerationDictionary;
+class Phrase;
+class ScoreComponentCollection;
+
+//! subclass of DecodeStep for generation step
+class DecodeStepGeneration : public DecodeStep
+{
+public:
+  DecodeStepGeneration(GenerationDictionary* dict,
+                       const DecodeStep* prev,
+                       const std::vector<FeatureFunction*> &features);
+
+
+  void Process(const TranslationOption &inputPartialTranslOpt
+               , const DecodeStep &decodeStep
+               , PartialTranslOptColl &outputPartialTranslOptColl
+               , TranslationOptionCollection *toc
+               , bool adhereTableLimit) const;
+
+private:
+};
+
+
+}
+#endif

mosesdecoder/moses/DecodeStepTranslation.cpp

@@ -0,0 +1,280 @@
+// $Id$
+
+/***********************************************************************
+Moses - factored phrase-based language decoder
+Copyright (C) 2006 University of Edinburgh
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Lesser General Public
+License as published by the Free Software Foundation; either
+version 2.1 of the License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public
+License along with this library; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+***********************************************************************/
+
+#include "DecodeStepTranslation.h"
+#include "TranslationOption.h"
+#include "TranslationOptionCollection.h"
+#include "PartialTranslOptColl.h"
+#include "FactorCollection.h"
+#include "util/exception.hh"
+
+using namespace std;
+
+namespace Moses
+{
+DecodeStepTranslation::DecodeStepTranslation(PhraseDictionary* pdf,
+    const DecodeStep* prev,
+    const std::vector<FeatureFunction*> &features)
+  : DecodeStep(pdf, prev, features)
+{
+  // don't apply feature functions that are from current phrase table.It should already have been
+  // dont by the phrase table.
+  const std::vector<FeatureFunction*> &pdfFeatures = pdf->GetFeaturesToApply();
+  for (size_t i = 0; i < pdfFeatures.size(); ++i) {
+    FeatureFunction *ff = pdfFeatures[i];
+    RemoveFeature(ff);
+  }
+}
+
+void DecodeStepTranslation::Process(const TranslationOption &inputPartialTranslOpt
+                                    , const DecodeStep &decodeStep
+                                    , PartialTranslOptColl &outputPartialTranslOptColl
+                                    , TranslationOptionCollection *toc
+                                    , bool adhereTableLimit
+                                    , TargetPhraseCollection::shared_ptr phraseColl) const
+{
+  if (inputPartialTranslOpt.GetTargetPhrase().GetSize() == 0) {
+    // word deletion
+    outputPartialTranslOptColl.Add(new TranslationOption(inputPartialTranslOpt));
+    return;
+  }
+
+  // normal trans step
+  const Range &sourceWordsRange        = inputPartialTranslOpt.GetSourceWordsRange();
+  const InputPath &inputPath = inputPartialTranslOpt.GetInputPath();
+  const PhraseDictionary* phraseDictionary  =
+    decodeStep.GetPhraseDictionaryFeature();
+  const TargetPhrase &inPhrase = inputPartialTranslOpt.GetTargetPhrase();
+  const size_t currSize = inPhrase.GetSize();
+  const size_t tableLimit = phraseDictionary->GetTableLimit();
+
+  if (phraseColl != NULL) {
+    TargetPhraseCollection::const_iterator iterTargetPhrase, iterEnd;
+    iterEnd = (!adhereTableLimit || tableLimit == 0 || phraseColl->GetSize() < tableLimit) ? phraseColl->end() : phraseColl->begin() + tableLimit;
+
+    for (iterTargetPhrase = phraseColl->begin(); iterTargetPhrase != iterEnd; ++iterTargetPhrase) {
+      const TargetPhrase& targetPhrase = **iterTargetPhrase;
+      // const ScoreComponentCollection &transScores = targetPhrase.GetScoreBreakdown();
+      // skip if the
+      if (targetPhrase.GetSize() != currSize) continue;
+
+      TargetPhrase outPhrase(inPhrase);
+
+      if (IsFilteringStep()) {
+        if (!inputPartialTranslOpt.IsCompatible(targetPhrase, m_conflictFactors))
+          continue;
+      }
+
+      outPhrase.Merge(targetPhrase, m_newOutputFactors);
+      outPhrase.EvaluateInIsolation(inputPath.GetPhrase(), m_featuresToApply); // need to do this as all non-transcores would be screwed up
+
+      TranslationOption *newTransOpt = new TranslationOption(sourceWordsRange, outPhrase);
+      assert(newTransOpt != NULL);
+
+      newTransOpt->SetInputPath(inputPath);
+
+      outputPartialTranslOptColl.Add(newTransOpt );
+
+    }
+  } else if (sourceWordsRange.GetNumWordsCovered() == 1) {
+    // unknown handler
+    //toc->ProcessUnknownWord(sourceWordsRange.GetStartPos(), factorCollection);
+  }
+}
+
+void
+DecodeStepTranslation::
+ProcessInitialTranslation(InputType const& source,
+                          PartialTranslOptColl &outputPartialTranslOptColl,
+                          size_t startPos, size_t endPos,
+                          bool adhereTableLimit,
+                          InputPath const& inputPath,
+                          TargetPhraseCollection::shared_ptr phraseColl) const
+{
+  const PhraseDictionary* phraseDictionary = GetPhraseDictionaryFeature();
+  const size_t tableLimit = phraseDictionary->GetTableLimit();
+
+  const Range range(startPos, endPos);
+
+  if (phraseColl != NULL) {
+    IFVERBOSE(3) {
+      if(source.GetType() == SentenceInput)
+        TRACE_ERR("[" << source.GetSubString(range) << "; "
+                  << startPos << "-" << endPos << "]\n");
+      else
+        TRACE_ERR("[" << startPos << "-" << endPos << "]" << std::endl);
+    }
+
+    TargetPhraseCollection::const_iterator iterTargetPhrase, iterEnd;
+    iterEnd = (!adhereTableLimit || tableLimit == 0 || phraseColl->GetSize() < tableLimit) ? phraseColl->end() : phraseColl->begin() + tableLimit;
+
+    for (iterTargetPhrase = phraseColl->begin() ; iterTargetPhrase != iterEnd ; ++iterTargetPhrase) {
+      const TargetPhrase	&targetPhrase = **iterTargetPhrase;
+      TranslationOption *transOpt = new TranslationOption(range, targetPhrase);
+
+      transOpt->SetInputPath(inputPath);
+
+      outputPartialTranslOptColl.Add (transOpt);
+
+      VERBOSE(3,"\t" << targetPhrase << "\n");
+    }
+    VERBOSE(3,std::endl);
+  }
+}
+
+void
+DecodeStepTranslation::
+ProcessInitialTransLEGACY(InputType const& source,
+                          PartialTranslOptColl &outputPartialTranslOptColl,
+                          size_t startPos, size_t endPos,
+                          bool adhereTableLimit,
+                          InputPathList const& inputPathList) const
+{
+  const PhraseDictionary* phraseDictionary = GetPhraseDictionaryFeature();
+  const size_t tableLimit = phraseDictionary->GetTableLimit();
+
+  const Range range(startPos, endPos);
+  TargetPhraseCollectionWithSourcePhrase::shared_ptr phraseColl
+  = phraseDictionary->GetTargetPhraseCollectionLEGACY(source,range);
+
+  if (phraseColl != NULL) {
+    IFVERBOSE(3) {
+      if(source.GetType() == SentenceInput)
+        TRACE_ERR("[" << source.GetSubString(range) << "; "
+                  << startPos << "-" << endPos << "]\n");
+      else
+        TRACE_ERR("[" << startPos << "-" << endPos << "]" << std::endl);
+    }
+
+    const std::vector<Phrase> &sourcePhrases = phraseColl->GetSourcePhrases();
+
+    TargetPhraseCollection::const_iterator iterTargetPhrase, iterEnd;
+    std::vector<Phrase>::const_iterator iterSourcePhrase;
+    iterEnd = (!adhereTableLimit || tableLimit == 0 || phraseColl->GetSize() < tableLimit) ? phraseColl->end() : phraseColl->begin() + tableLimit;
+
+    for (iterTargetPhrase = phraseColl->begin(), iterSourcePhrase =  sourcePhrases.begin()
+         ; iterTargetPhrase != iterEnd
+         ; ++iterTargetPhrase, ++iterSourcePhrase) {
+      assert(iterSourcePhrase != sourcePhrases.end());
+
+      const TargetPhrase	&targetPhrase = **iterTargetPhrase;
+      const Phrase			&sourcePhrase = *iterSourcePhrase;
+
+      const InputPath &inputPath = GetInputPathLEGACY(targetPhrase, sourcePhrase, inputPathList);
+
+      TranslationOption *transOpt = new TranslationOption(range, targetPhrase);
+      transOpt->SetInputPath(inputPath);
+
+      outputPartialTranslOptColl.Add (transOpt);
+
+      VERBOSE(3,"\t" << targetPhrase << "\n");
+    }
+    VERBOSE(3,std::endl);
+  }
+}
+
+const InputPath &DecodeStepTranslation::GetInputPathLEGACY(
+  const TargetPhrase targetPhrase,
+  const Phrase sourcePhrase,
+  const InputPathList &inputPathList) const
+{
+  const Word &wordFromPt =  sourcePhrase.GetWord(0);
+
+  InputPathList::const_iterator iter;
+  for (iter = inputPathList.begin(); iter != inputPathList.end(); ++iter) {
+    const InputPath &inputPath = **iter;
+    const Phrase &phraseFromIP = inputPath.GetPhrase();
+
+    const Word *wordIP = NULL;
+    for (size_t i = 0; i < phraseFromIP.GetSize(); ++i) {
+      const Word &tempWord =  phraseFromIP.GetWord(i);
+      if (!tempWord.IsEpsilon()) {
+        wordIP = &tempWord;
+        break;
+      }
+    }
+
+    // const Range &range = inputPath.GetWordsRange();
+
+    if (wordIP && *wordIP == wordFromPt) {
+      return inputPath;
+    }
+  }
+
+  UTIL_THROW(util::Exception, "Input path not found");
+}
+
+void
+DecodeStepTranslation::
+ProcessLEGACY(TranslationOption const& in,
+              DecodeStep const& decodeStep,
+              PartialTranslOptColl &out,
+              TranslationOptionCollection *toc,
+              bool adhereTableLimit) const
+{
+  if (in.GetTargetPhrase().GetSize() == 0) {
+    // word deletion
+    out.Add(new TranslationOption(in));
+    return;
+  }
+
+  // normal trans step
+  Range const& srcRange = in.GetSourceWordsRange();
+  InputPath const& inputPath = in.GetInputPath();
+  PhraseDictionary const* pdict  = decodeStep.GetPhraseDictionaryFeature();
+  TargetPhrase const& inPhrase = in.GetTargetPhrase();
+  size_t const currSize = inPhrase.GetSize();
+  size_t const tableLimit = pdict->GetTableLimit();
+
+  TargetPhraseCollectionWithSourcePhrase::shared_ptr phraseColl
+  = pdict->GetTargetPhraseCollectionLEGACY(toc->GetSource(),srcRange);
+
+  if (phraseColl != NULL) {
+    TargetPhraseCollection::const_iterator iterTargetPhrase, iterEnd;
+    iterEnd = ((adhereTableLimit && tableLimit && phraseColl->GetSize() >= tableLimit)
+               ? phraseColl->begin() + tableLimit : phraseColl->end());
+
+    for (iterTargetPhrase = phraseColl->begin();
+         iterTargetPhrase != iterEnd;
+         ++iterTargetPhrase) {
+      TargetPhrase const& targetPhrase = **iterTargetPhrase;
+      if (targetPhrase.GetSize() != currSize ||
+          (IsFilteringStep() && !in.IsCompatible(targetPhrase, m_conflictFactors)))
+        continue;
+
+      TargetPhrase outPhrase(inPhrase);
+      outPhrase.Merge(targetPhrase, m_newOutputFactors);
+      outPhrase.EvaluateInIsolation(inputPath.GetPhrase(), m_featuresToApply); // need to do this as all non-transcores would be screwed up
+
+      TranslationOption *newTransOpt = new TranslationOption(srcRange, outPhrase);
+      assert(newTransOpt != NULL);
+
+      newTransOpt->SetInputPath(inputPath);
+
+      out.Add(newTransOpt);
+
+    }
+  }
+}
+}
+
+
+

mosesdecoder/moses/DecodeStepTranslation.h

@@ -0,0 +1,89 @@
+// $Id$
+
+/***********************************************************************
+Moses - factored phrase-based language decoder
+Copyright (C) 2006 University of Edinburgh
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Lesser General Public
+License as published by the Free Software Foundation; either
+version 2.1 of the License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public
+License along with this library; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+***********************************************************************/
+
+#ifndef moses_DecodeStepTranslation_h
+#define moses_DecodeStepTranslation_h
+
+#include "DecodeStep.h"
+#include "moses/TranslationModel/PhraseDictionary.h"
+#include "InputPath.h"
+
+namespace Moses
+{
+
+class PhraseDictionary;
+class TargetPhrase;
+class InputPath;
+
+//! subclass of DecodeStep for translation step
+class DecodeStepTranslation : public DecodeStep
+{
+public:
+  DecodeStepTranslation(); //! not implemented
+  DecodeStepTranslation(PhraseDictionary* phraseFeature,
+                        const DecodeStep* prev,
+                        const std::vector<FeatureFunction*> &features);
+
+
+  virtual void Process(const TranslationOption &inputPartialTranslOpt
+                       , const DecodeStep &decodeStep
+                       , PartialTranslOptColl &outputPartialTranslOptColl
+                       , TranslationOptionCollection *toc
+                       , bool adhereTableLimit
+                       , TargetPhraseCollection::shared_ptr phraseColl) const;
+
+
+  /*! initialize list of partial translation options by applying the first translation step
+  * Ideally, this function should be in DecodeStepTranslation class
+  */
+  void ProcessInitialTranslation(const InputType &source
+                                 , PartialTranslOptColl &outputPartialTranslOptColl
+                                 , size_t startPos, size_t endPos, bool adhereTableLimit
+                                 , const InputPath &inputPath
+                                 , TargetPhraseCollection::shared_ptr phraseColl) const;
+
+  // legacy
+  void
+  ProcessInitialTransLEGACY(InputType const& source,
+                            PartialTranslOptColl &outputPartialTranslOptColl,
+                            size_t startPos, size_t endPos,
+                            bool adhereTableLimit,
+                            InputPathList const& inputPathList) const;
+
+  void ProcessLEGACY(const TranslationOption &inputPartialTranslOpt
+                     , const DecodeStep &decodeStep
+                     , PartialTranslOptColl &outputPartialTranslOptColl
+                     , TranslationOptionCollection *toc
+                     , bool adhereTableLimit) const;
+
+private:
+  // I'm not sure whether this actually works or not for binary phrase table.
+  // The source phrase only appears to contain the 1st word, therefore, this function
+  // only compares the 1st word
+  const InputPath &GetInputPathLEGACY(const TargetPhrase targetPhrase,
+                                      const Phrase sourcePhrase,
+                                      const InputPathList &inputPathList) const;
+
+};
+
+
+}
+#endif

mosesdecoder/moses/Factor.cpp

@@ -0,0 +1,48 @@
+// $Id$
+
+/***********************************************************************
+Moses - factored phrase-based language decoder
+Copyright (C) 2006 University of Edinburgh
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Lesser General Public
+License as published by the Free Software Foundation; either
+version 2.1 of the License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public
+License along with this library; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+***********************************************************************/
+
+#include "Factor.h"
+
+#include <boost/functional/hash.hpp>
+
+using namespace std;
+
+namespace Moses
+{
+
+TO_STRING_BODY(Factor)
+
+// friend
+ostream& operator<<(ostream& out, const Factor& factor)
+{
+  out << factor.GetString();
+  return out;
+}
+
+size_t hash_value(const Factor& f)
+{
+  boost::hash<size_t> hasher;
+  return hasher(f.GetId());
+}
+
+}
+
+

mosesdecoder/moses/ForestInput.h

@@ -0,0 +1,88 @@
+// -*- c++ -*-
+#ifndef moses_ForestInput_h
+#define moses_ForestInput_h
+
+#include <string>
+#include <vector>
+
+#include <boost/shared_ptr.hpp>
+
+#include <util/string_piece.hh>
+
+#include "moses/Syntax/F2S/Forest.h"
+
+#include "Sentence.h"
+
+namespace Moses
+{
+class TranslationTask;
+class ForestInput : public Sentence
+{
+public:
+  friend std::ostream &operator<<(std::ostream&, const ForestInput &);
+
+  ForestInput(AllOptions::ptr const& opts) : Sentence(opts), m_rootVertex(NULL) {}
+
+  InputTypeEnum GetType() const {
+    return ForestInputType;
+  }
+
+  //! populate this InputType with data from in stream
+  virtual int
+  Read(std::istream& in);
+
+  //! Output debugging info to stream out
+  virtual void Print(std::ostream&) const;
+
+  //! create trans options specific to this InputType
+  virtual TranslationOptionCollection*
+  CreateTranslationOptionCollection() const;
+
+  boost::shared_ptr<const Syntax::F2S::Forest> GetForest() const {
+    return m_forest;
+  }
+
+  const Syntax::F2S::Forest::Vertex *GetRootVertex() const {
+    return m_rootVertex;
+  }
+
+private:
+  typedef Syntax::F2S::Forest Forest;
+
+  struct VertexSetHash {
+    std::size_t operator()(const Forest::Vertex *v) const {
+      std::size_t seed = 0;
+      boost::hash_combine(seed, v->pvertex.symbol);
+      boost::hash_combine(seed, v->pvertex.span.GetStartPos());
+      boost::hash_combine(seed, v->pvertex.span.GetEndPos());
+      return seed;
+    }
+  };
+
+  struct VertexSetPred {
+    bool operator()(const Forest::Vertex *v, const Forest::Vertex *w) const {
+      return v->pvertex == w->pvertex;
+    }
+  };
+
+  typedef boost::unordered_set<Forest::Vertex *, VertexSetHash,
+          VertexSetPred> VertexSet;
+
+  Forest::Vertex *AddOrDeleteVertex(Forest::Vertex *);
+
+  std::size_t FindMaxEnd(const Forest &);
+
+  void FindTopVertices(Forest &, std::vector<Forest::Vertex *> &);
+
+  void ParseHyperedgeLine(const std::string &);
+
+  Forest::Vertex *ParseVertex(const StringPiece &);
+
+  boost::shared_ptr<Forest> m_forest;
+  Forest::Vertex *m_rootVertex;
+  VertexSet m_vertexSet;
+};
+
+}  // namespace Moses
+
+#endif

mosesdecoder/moses/GenerationDictionary.h

@@ -0,0 +1,83 @@
+// $Id$
+
+/***********************************************************************
+Moses - factored phrase-based language decoder
+Copyright (C) 2006 University of Edinburgh
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Lesser General Public
+License as published by the Free Software Foundation; either
+version 2.1 of the License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public
+License along with this library; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+***********************************************************************/
+
+#ifndef moses_GenerationDictionary_h
+#define moses_GenerationDictionary_h
+
+#include <list>
+#include <stdexcept>
+#include <vector>
+#include <boost/unordered_map.hpp>
+#include "ScoreComponentCollection.h"
+#include "Phrase.h"
+#include "TypeDef.h"
+#include "moses/FF/DecodeFeature.h"
+
+namespace Moses
+{
+
+class FactorCollection;
+
+typedef boost::unordered_map < Word , ScoreComponentCollection > OutputWordCollection;
+// 1st = output phrase
+// 2nd = log probability (score)
+
+/** Implementation of a generation table in a trie.
+ */
+class GenerationDictionary : public DecodeFeature
+{
+  typedef boost::unordered_map<const Word* , OutputWordCollection, UnorderedComparer<Word>, UnorderedComparer<Word> > Collection;
+protected:
+  static std::vector<GenerationDictionary*> s_staticColl;
+
+  Collection m_collection;
+  // 1st = source
+  // 2nd = target
+  std::string						m_filePath;
+
+public:
+  static const std::vector<GenerationDictionary*>& GetColl() {
+    return s_staticColl;
+  }
+
+  GenerationDictionary(const std::string &line);
+  virtual ~GenerationDictionary();
+
+  //! load data file
+  void Load(AllOptions::ptr const& opts);
+
+  /** number of unique input entries in the generation table.
+  * NOT the number of lines in the generation table
+  */
+  size_t GetSize() const {
+    return m_collection.size();
+  }
+  /** returns a bag of output words, OutputWordCollection, for a particular input word.
+  *	Or NULL if the input word isn't found. The search function used is the WordComparer functor
+  */
+  const OutputWordCollection *FindWord(const Word &word) const;
+  void SetParameter(const std::string& key, const std::string& value);
+
+};
+
+
+}
+#endif

mosesdecoder/moses/HypothesisStackCubePruning.cpp

@@ -0,0 +1,313 @@
+// $Id$
+
+/***********************************************************************
+Moses - factored phrase-based language decoder
+Copyright (C) 2006 University of Edinburgh
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Lesser General Public
+License as published by the Free Software Foundation; either
+version 2.1 of the License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public
+License along with this library; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+***********************************************************************/
+
+#include <algorithm>
+#include <set>
+#include <queue>
+#include "HypothesisStackCubePruning.h"
+#include "TypeDef.h"
+#include "Util.h"
+#include "StaticData.h"
+#include "Manager.h"
+#include "util/exception.hh"
+
+using namespace std;
+
+namespace Moses
+{
+HypothesisStackCubePruning::HypothesisStackCubePruning(Manager& manager) :
+  HypothesisStack(manager)
+{
+  m_nBestIsEnabled = manager.options()->nbest.enabled;
+  m_bestScore = -std::numeric_limits<float>::infinity();
+  m_worstScore = -std::numeric_limits<float>::infinity();
+  m_deterministic = manager.options()->cube.deterministic_search;
+}
+
+/** remove all hypotheses from the collection */
+void HypothesisStackCubePruning::RemoveAll()
+{
+  // delete all bitmap accessors;
+  _BMType::iterator iter;
+  for (iter = m_bitmapAccessor.begin(); iter != m_bitmapAccessor.end(); ++iter) {
+    delete iter->second;
+  }
+}
+
+pair<HypothesisStackCubePruning::iterator, bool> HypothesisStackCubePruning::Add(Hypothesis *hypo)
+{
+  std::pair<iterator, bool> ret = m_hypos.insert(hypo);
+
+  if (ret.second) {
+    // equiv hypo doesn't exists
+    VERBOSE(3,"added hyp to stack");
+
+    // Update best score, if this hypothesis is new best
+    if (hypo->GetFutureScore() > m_bestScore) {
+      VERBOSE(3,", best on stack");
+      m_bestScore = hypo->GetFutureScore();
+      // this may also affect the worst score
+      if ( m_bestScore + m_beamWidth > m_worstScore )
+        m_worstScore = m_bestScore + m_beamWidth;
+    }
+
+    // Prune only if stack is twice as big as needed (lazy pruning)
+    VERBOSE(3,", now size " << m_hypos.size());
+    if (m_hypos.size() > 2*m_maxHypoStackSize-1) {
+      PruneToSize(m_maxHypoStackSize);
+    } else {
+      VERBOSE(3,std::endl);
+    }
+  }
+
+  return ret;
+}
+
+bool HypothesisStackCubePruning::AddPrune(Hypothesis *hypo)
+{
+  if (hypo->GetFutureScore() == - std::numeric_limits<float>::infinity()) {
+    m_manager.GetSentenceStats().AddDiscarded();
+    VERBOSE(3,"discarded, constraint" << std::endl);
+    delete hypo;
+    return false;
+  }
+
+  if (hypo->GetFutureScore() < m_worstScore) {
+    // too bad for stack. don't bother adding hypo into collection
+    m_manager.GetSentenceStats().AddDiscarded();
+    VERBOSE(3,"discarded, too bad for stack" << std::endl);
+    delete hypo;
+    return false;
+  }
+
+  // over threshold, try to add to collection
+  std::pair<iterator, bool> addRet = Add(hypo);
+  if (addRet.second) {
+    // nothing found. add to collection
+    return true;
+  }
+
+  // equiv hypo exists, recombine with other hypo
+  iterator &iterExisting = addRet.first;
+  assert(iterExisting != m_hypos.end());
+  Hypothesis *hypoExisting = *iterExisting;
+
+  m_manager.GetSentenceStats().AddRecombination(*hypo, **iterExisting);
+
+  // found existing hypo with same target ending.
+  // keep the best 1
+  if (hypo->GetFutureScore() > hypoExisting->GetFutureScore()) {
+    // incoming hypo is better than the one we have
+    VERBOSE(3,"better than matching hyp " << hypoExisting->GetId() << ", recombining, ");
+    if (m_nBestIsEnabled) {
+      hypo->AddArc(hypoExisting);
+      Detach(iterExisting);
+    } else {
+      Remove(iterExisting);
+    }
+
+    bool added = Add(hypo).second;
+    if (!added) {
+      iterExisting = m_hypos.find(hypo);
+      UTIL_THROW(util::Exception, "Should have added hypothesis " << **iterExisting);
+    }
+    return false;
+  } else {
+    // already storing the best hypo. discard current hypo
+    VERBOSE(3,"worse than matching hyp " << hypoExisting->GetId() << ", recombining" << std::endl)
+    if (m_nBestIsEnabled) {
+      hypoExisting->AddArc(hypo);
+    } else {
+      delete hypo;
+    }
+    return false;
+  }
+}
+
+void HypothesisStackCubePruning::AddInitial(Hypothesis *hypo)
+{
+  std::pair<iterator, bool> addRet = Add(hypo);
+  UTIL_THROW_IF2(!addRet.second,
+                 "Should have added hypothesis " << *hypo);
+
+  const Bitmap &bitmap = hypo->GetWordsBitmap();
+  AddBitmapContainer(bitmap, *this);
+}
+
+void HypothesisStackCubePruning::PruneToSize(size_t newSize)
+{
+  if ( newSize == 0) return; // no limit
+
+  if (m_hypos.size() > newSize) { // ok, if not over the limit
+    priority_queue<float> bestScores;
+
+    // push all scores to a heap
+    // (but never push scores below m_bestScore+m_beamWidth)
+    iterator iter = m_hypos.begin();
+    float score = 0;
+    while (iter != m_hypos.end()) {
+      Hypothesis *hypo = *iter;
+      score = hypo->GetFutureScore();
+      if (score > m_bestScore+m_beamWidth) {
+        bestScores.push(score);
+      }
+      ++iter;
+    }
+
+    // pop the top newSize scores (and ignore them, these are the scores of hyps that will remain)
+    //  ensure to never pop beyond heap size
+    size_t minNewSizeHeapSize = newSize > bestScores.size() ? bestScores.size() : newSize;
+    for (size_t i = 1 ; i < minNewSizeHeapSize ; i++)
+      bestScores.pop();
+
+    // and remember the threshold
+    float scoreThreshold = bestScores.top();
+
+    // delete all hypos under score threshold
+    iter = m_hypos.begin();
+    while (iter != m_hypos.end()) {
+      Hypothesis *hypo = *iter;
+      float score = hypo->GetFutureScore();
+      if (score < scoreThreshold) {
+        iterator iterRemove = iter++;
+        Remove(iterRemove);
+        m_manager.GetSentenceStats().AddPruning();
+      } else {
+        ++iter;
+      }
+    }
+    VERBOSE(3,", pruned to size " << size() << endl);
+
+    IFVERBOSE(3) {
+      TRACE_ERR("stack now contains: ");
+      for(iter = m_hypos.begin(); iter != m_hypos.end(); iter++) {
+        Hypothesis *hypo = *iter;
+        TRACE_ERR( hypo->GetId() << " (" << hypo->GetFutureScore() << ") ");
+      }
+      TRACE_ERR( endl);
+    }
+
+    // set the worstScore, so that newly generated hypotheses will not be added if worse than the worst in the stack
+    m_worstScore = scoreThreshold;
+  }
+}
+
+const Hypothesis *HypothesisStackCubePruning::GetBestHypothesis() const
+{
+  if (!m_hypos.empty()) {
+    const_iterator iter = m_hypos.begin();
+    Hypothesis *bestHypo = *iter;
+    while (++iter != m_hypos.end()) {
+      Hypothesis *hypo = *iter;
+      if (hypo->GetFutureScore() > bestHypo->GetFutureScore())
+        bestHypo = hypo;
+    }
+    return bestHypo;
+  }
+  return NULL;
+}
+
+vector<const Hypothesis*> HypothesisStackCubePruning::GetSortedList() const
+{
+  vector<const Hypothesis*> ret;
+  ret.reserve(m_hypos.size());
+  std::copy(m_hypos.begin(), m_hypos.end(), std::inserter(ret, ret.end()));
+  sort(ret.begin(), ret.end(), CompareHypothesisTotalScore());
+
+  return ret;
+}
+
+
+void HypothesisStackCubePruning::CleanupArcList()
+{
+  // only necessary if n-best calculations are enabled
+  if (!m_nBestIsEnabled) return;
+
+  iterator iter;
+  for (iter = m_hypos.begin() ; iter != m_hypos.end() ; ++iter) {
+    Hypothesis *mainHypo = *iter;
+    mainHypo->CleanupArcList(this->m_manager.options()->nbest.nbest_size, this->m_manager.options()->NBestDistinct());
+  }
+}
+
+void HypothesisStackCubePruning::SetBitmapAccessor(const Bitmap &newBitmap
+    , HypothesisStackCubePruning &stack
+    , const Range &/*range*/
+    , BitmapContainer &bitmapContainer
+    , const SquareMatrix &estimatedScores
+    , const TranslationOptionList &transOptList)
+{
+  BitmapContainer *bmContainer =   AddBitmapContainer(newBitmap, stack);
+  BackwardsEdge *edge = new BackwardsEdge(bitmapContainer
+                                          , *bmContainer
+                                          , transOptList
+                                          , estimatedScores
+                                          , m_manager.GetSource()
+                                          , m_deterministic);
+  bmContainer->AddBackwardsEdge(edge);
+}
+
+
+TO_STRING_BODY(HypothesisStackCubePruning);
+
+
+// friend
+std::ostream& operator<<(std::ostream& out, const HypothesisStackCubePruning& hypoColl)
+{
+  HypothesisStackCubePruning::const_iterator iter;
+
+  for (iter = hypoColl.begin() ; iter != hypoColl.end() ; ++iter) {
+    const Hypothesis &hypo = **iter;
+    out << hypo << endl;
+
+  }
+  return out;
+}
+
+void
+HypothesisStackCubePruning::AddHypothesesToBitmapContainers()
+{
+  HypothesisStackCubePruning::const_iterator iter;
+  for (iter = m_hypos.begin() ; iter != m_hypos.end() ; ++iter) {
+    Hypothesis *h = *iter;
+    const Bitmap &bitmap = h->GetWordsBitmap();
+    BitmapContainer *container = m_bitmapAccessor[&bitmap];
+    container->AddHypothesis(h);
+  }
+}
+
+BitmapContainer *HypothesisStackCubePruning::AddBitmapContainer(const Bitmap &bitmap, HypothesisStackCubePruning &stack)
+{
+  _BMType::iterator iter = m_bitmapAccessor.find(&bitmap);
+
+  BitmapContainer *bmContainer;
+  if (iter == m_bitmapAccessor.end()) {
+    bmContainer = new BitmapContainer(bitmap, stack, m_deterministic);
+    m_bitmapAccessor[&bitmap] = bmContainer;
+  } else {
+    bmContainer = iter->second;
+  }
+
+  return bmContainer;
+}
+
+}
+

mosesdecoder/moses/HypothesisStackCubePruning.h

@@ -0,0 +1,153 @@
+// $Id$
+
+/***********************************************************************
+Moses - factored phrase-based language decoder
+Copyright (C) 2006 University of Edinburgh
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Lesser General Public
+License as published by the Free Software Foundation; either
+version 2.1 of the License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public
+License along with this library; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+***********************************************************************/
+
+#ifndef moses_HypothesisStackCubePruning_h
+#define moses_HypothesisStackCubePruning_h
+
+#include <limits>
+#include <set>
+#include <boost/unordered_map.hpp>
+#include "Hypothesis.h"
+#include "BitmapContainer.h"
+#include "HypothesisStack.h"
+#include "Util.h"
+
+namespace Moses
+{
+
+class BitmapContainer;
+class TranslationOptionList;
+class Manager;
+
+//typedef boost::unordered_map<Bitmap, BitmapContainer*, UnorderedComparer<Bitmap>, UnorderedComparer<Bitmap> > _BMType;
+typedef boost::unordered_map<const Bitmap*, BitmapContainer*> _BMType;
+// can compare Bitmap* 'cos all bitmaps are created from bitmaps factory class. MUST ensure this is the case
+
+/** A stack for phrase-based decoding with cube-pruning. */
+class HypothesisStackCubePruning : public HypothesisStack
+{
+public:
+  friend std::ostream& operator<<(std::ostream&, const HypothesisStackCubePruning&);
+
+protected:
+  _BMType m_bitmapAccessor;
+
+  float m_bestScore; /**< score of the best hypothesis in collection */
+  float m_worstScore; /**< score of the worse hypthesis in collection */
+  float m_beamWidth; /**< minimum score due to threashold pruning */
+  size_t m_maxHypoStackSize; /**< maximum number of hypothesis allowed in this stack */
+  bool m_nBestIsEnabled; /**< flag to determine whether to keep track of old arcs */
+  bool m_deterministic; /**< flag to determine whether to sort hypotheses deterministically */
+
+  /** add hypothesis to stack. Prune if necessary.
+   * Returns false if equiv hypo exists in collection, otherwise returns true
+   */
+  std::pair<HypothesisStackCubePruning::iterator, bool> Add(Hypothesis *hypothesis);
+
+  /** destroy all instances of Hypothesis in this collection */
+  void RemoveAll();
+
+  BitmapContainer *AddBitmapContainer(const Bitmap &bitmap, HypothesisStackCubePruning &stack);
+
+public:
+  HypothesisStackCubePruning(Manager& manager);
+  ~HypothesisStackCubePruning() {
+    RemoveAll();
+    m_bitmapAccessor.clear();
+  }
+
+  /** adds the hypo, but only if within thresholds (beamThr, stackSize).
+  *	This function will recombine hypotheses silently!  There is no record
+  * (could affect n-best list generation...TODO)
+  * Call stack for adding hypothesis is
+  		AddPrune()
+  			Add()
+  				AddNoPrune()
+  */
+  bool AddPrune(Hypothesis *hypothesis);
+
+  void AddInitial(Hypothesis *hypo);
+
+  /** set maximum number of hypotheses in the collection
+   * \param maxHypoStackSize maximum number (typical number: 100)
+   */
+  inline void SetMaxHypoStackSize(size_t maxHypoStackSize) {
+    m_maxHypoStackSize = maxHypoStackSize;
+  }
+
+  inline size_t GetMaxHypoStackSize() const {
+    return m_maxHypoStackSize;
+  }
+
+  /** set beam threshold, hypotheses in the stack must not be worse than
+    * this factor times the best score to be allowed in the stack
+   * \param beamThreshold minimum factor (typical number: 0.03)
+   */
+  inline void SetBeamWidth(float beamWidth) {
+    m_beamWidth = beamWidth;
+  }
+
+  /** return score of the best hypothesis in the stack */
+  inline float GetBestScore() const {
+    return m_bestScore;
+  }
+
+  /** return worst score allowed for the stack */
+  inline float GetWorstScore() const {
+    return m_worstScore;
+  }
+
+  void AddHypothesesToBitmapContainers();
+
+  const _BMType& GetBitmapAccessor() const {
+    return m_bitmapAccessor;
+  }
+
+  void SetBitmapAccessor(const Bitmap &newBitmap
+                         , HypothesisStackCubePruning &stack
+                         , const Range &range
+                         , BitmapContainer &bitmapContainer
+                         , const SquareMatrix &estimatedScores
+                         , const TranslationOptionList &transOptList);
+
+  /** pruning, if too large.
+   * Pruning algorithm: find a threshold and delete all hypothesis below it.
+   * The threshold is chosen so that exactly newSize top items remain on the
+   * stack in fact, in situations where some of the hypothesis fell below
+   * m_beamWidth, the stack will contain less items.
+   * \param newSize maximum size */
+  void PruneToSize(size_t newSize);
+
+  //! return the hypothesis with best score. Used to get the translated at end of decoding
+  const Hypothesis *GetBestHypothesis() const;
+  //! return all hypothesis, sorted by descending score. Used in creation of N best list
+  std::vector<const Hypothesis*> GetSortedList() const;
+
+  /** make all arcs in point to the equiv hypothesis that contains them.
+  * Ie update doubly linked list be hypo & arcs
+  */
+  void CleanupArcList();
+
+  TO_STRING();
+};
+
+}
+#endif

mosesdecoder/moses/Incremental.h

@@ -0,0 +1,124 @@
+// -*- c++ -*-
+#pragma once
+
+#include "lm/word_index.hh"
+#include "search/applied.hh"
+#include "search/nbest.hh"
+
+#include "moses/ChartCellCollection.h"
+#include "moses/ChartParser.h"
+
+#include "BaseManager.h"
+
+#include <vector>
+#include <string>
+
+namespace Moses
+{
+class ScoreComponentCollection;
+class InputType;
+class LanguageModel;
+
+namespace Incremental
+{
+
+class Manager : public BaseManager
+{
+public:
+  Manager(ttasksptr const& ttask);
+
+  ~Manager();
+
+  template <class Model> void LMCallback(const Model &model, const std::vector<lm::WordIndex> &words);
+
+  void Decode();
+
+  const std::vector<search::Applied> &GetNBest() const;
+
+  // Call to get the same value as ProcessSentence returned.
+  const std::vector<search::Applied> &Completed() const {
+    return *completed_nbest_;
+  }
+
+  // output
+  void OutputBest(OutputCollector *collector) const;
+  void OutputNBest(OutputCollector *collector) const;
+  void OutputDetailedTranslationReport(OutputCollector *collector) const;
+  void OutputNBestList(OutputCollector *collector, const std::vector<search::Applied> &nbest, long translationId) const;
+  void OutputLatticeSamples(OutputCollector *collector) const {
+  }
+  void OutputAlignment(OutputCollector *collector) const {
+  }
+  void OutputDetailedTreeFragmentsTranslationReport(OutputCollector *collector) const;
+  void OutputWordGraph(OutputCollector *collector) const {
+  }
+  void OutputSearchGraph(OutputCollector *collector) const {
+  }
+  void OutputSearchGraphSLF() const {
+  }
+
+  void
+  OutputSearchGraphAsHypergraph
+  ( std::string const& fname, size_t const precision ) const
+  { }
+
+
+private:
+  template <class Model, class Best> search::History PopulateBest(const Model &model, const std::vector<lm::WordIndex> &words, Best &out);
+
+  ChartCellCollectionBase cells_;
+  ChartParser parser_;
+
+  // Only one of single_best_ or n_best_ will be used, but it was easier to do this than a template.
+  search::SingleBest single_best_;
+  // ProcessSentence returns a reference to a vector.  ProcessSentence
+  // doesn't have one, so this is populated and returned.
+  std::vector<search::Applied> backing_for_single_;
+
+  search::NBest n_best_;
+
+  const std::vector<search::Applied> *completed_nbest_;
+
+  // outputs
+  void OutputDetailedTranslationReport(
+    OutputCollector *collector,
+    const search::Applied *applied,
+    const Sentence &sentence,
+    long translationId) const;
+  void OutputTranslationOptions(std::ostream &out,
+                                ApplicationContext &applicationContext,
+                                const search::Applied *applied,
+                                const Sentence &sentence,
+                                long translationId) const;
+  void OutputTranslationOption(std::ostream &out,
+                               ApplicationContext &applicationContext,
+                               const search::Applied *applied,
+                               const Sentence &sentence,
+                               long translationId) const;
+  void ReconstructApplicationContext(const search::Applied *applied,
+                                     const Sentence &sentence,
+                                     ApplicationContext &context) const;
+  void OutputTreeFragmentsTranslationOptions(std::ostream &out,
+      ApplicationContext &applicationContext,
+      const search::Applied *applied,
+      const Sentence &sentence,
+      long translationId) const;
+  void OutputBestHypo(OutputCollector *collector, search::Applied applied, long translationId) const;
+  void OutputBestNone(OutputCollector *collector, long translationId) const;
+
+  void OutputUnknowns(OutputCollector *collector) const {
+  }
+  void CalcDecoderStatistics() const {
+  }
+
+};
+
+// Just get the phrase.
+void ToPhrase(const search::Applied final, Phrase &out);
+// Get the phrase and the features.
+void PhraseAndFeatures(const search::Applied final, Phrase &phrase, ScoreComponentCollection &features);
+
+
+} // namespace Incremental
+} // namespace Moses
+

mosesdecoder/moses/Jamfile

@@ -0,0 +1,143 @@
+# -*- jam -*-
+max-factors = [ option.get "max-factors" : 4 : 4 ] ;
+path-constant FACTOR-LOG : bin/factor.log ;
+update-if-changed $(FACTOR-LOG) $(max-factors) ;
+max-factors = <define>MAX_NUM_FACTORS=$(max-factors) <dependency>$(FACTOR-LOG) ;
+
+with-dlib = [ option.get "with-dlib" ] ;
+if $(with-dlib) {
+  dlib = <define>WITH_DLIB <include>$(with-dlib) ;
+} else {
+  dlib = ;
+}
+
+with-oxlm = [ option.get "with-oxlm" ] ; 
+if $(with-oxlm) {
+  oxlm = <cxxflags>-std=c++0x <define>LM_OXLM <include>$(with-oxlm)/src <include>$(with-oxlm)/third_party/eigen ;
+} else {
+  oxlm = ;
+}
+
+local classifier = ;
+if [ option.get "with-vw" ] {
+  classifier += ..//vw//classifier ;
+}
+
+alias headers : ../util//kenutil $(classifier) : : : $(max-factors) $(dlib) $(oxlm) ; 
+alias ThreadPool : ThreadPool.cpp ;
+alias Util : Util.cpp Timer.cpp ;
+
+if [ option.get "with-synlm" : no : yes ] = yes
+{
+  lib m ;
+  obj SyntacticLanguageModel.o : SyntacticLanguageModel.cpp headers : <include>$(TOP)/synlm/hhmm/rvtl/include <include>$(TOP)/synlm/hhmm/wsjparse/include ;
+  alias synlm : SyntacticLanguageModel.o m : : : <define>HAVE_SYNLM ;
+} else {
+  alias synlm ;
+}
+
+#This is a kludge to force rebuilding if different --with options are passed. 
+#Could have used features like <srilm>on but getting these to apply only to
+#linking was ugly and it still didn't trigger an install (since the install
+#path doesn't encode features).  It stores a file lm.log with the previous
+#options and forces a rebuild if the current options differ.  
+local current = ;
+for local i in srilm irstlm randlm {
+  local optval = [ option.get "with-$(i)" ] ;
+  if $(optval) {
+    current += "--with-$(i)=$(optval)" ;
+  }
+}
+current = $(current:J=" ") ;
+current ?= "" ;
+path-constant LM-LOG : bin/lm.log ;
+update-if-changed $(LM-LOG) $(current) ;
+
+obj FF_Factory.o : FF/Factory.cpp LM//macros headers ../lm//kenlm mmlib : <dependency>$(LM-LOG) ;
+
+# check if we have xmlrpc-c's abyss server available
+# if yes, include server capabilities in the moses executable
+# include $(TOP)/jam-files/server.jam ;
+
+if [ xmlrpc ] 
+{
+  echo "BUILDING MOSES SERVER!" ;
+  alias mserver : [ glob server/*.cpp ] ;
+}
+else 
+{
+  echo "NOT BUILDING MOSES SERVER!" ;
+  alias mserver ;
+}
+
+if [ option.get "with-mm" : no : yes ] = yes
+{
+  alias mmlib : 
+  $(TOP)/moses/TranslationModel/UG//mmsapt
+  $(TOP)/moses/TranslationModel/UG/generic//generic
+  $(TOP)/moses/TranslationModel/UG/mm//mm
+  ;
+} else {
+  alias mmlib ;
+}
+
+local with-vw = [ option.get "with-vw" ] ;
+if $(with-vw) {
+  alias vwfiles : [ glob FF/VW/*.cpp ] ;
+} else {
+  alias vwfiles ;
+}
+
+lib moses :
+[ glob 
+  *.cpp
+  parameters/*.cpp
+  Syntax/*.cpp
+  Syntax/F2S/*.cpp
+  Syntax/S2T/*.cpp
+  Syntax/S2T/Parsers/*.cpp
+  Syntax/S2T/Parsers/RecursiveCYKPlusParser/*.cpp
+  Syntax/S2T/Parsers/Scope3Parser/*.cpp
+  Syntax/T2S/*.cpp
+  TranslationModel/*.cpp
+  TranslationModel/fuzzy-match/*.cpp
+  TranslationModel/DynSAInclude/*.cpp
+  TranslationModel/RuleTable/*.cpp
+  TranslationModel/Scope3Parser/*.cpp
+  TranslationModel/CYKPlusParser/*.cpp
+  ../phrase-extract/PhraseOrientation.cpp
+  FF/*.cpp
+  FF/bilingual-lm/*.cpp
+  FF/OSM-Feature/*.cpp
+  FF/Dsg-Feature/*.cpp
+  FF/LexicalReordering/*.cpp
+  PP/*.cpp
+: #exceptions
+  ThreadPool.cpp
+  SyntacticLanguageModel.cpp
+  *Test.cpp Mock*.cpp FF/*Test.cpp
+  FF/Factory.cpp
+] 
+vwfiles synlm mmlib mserver headers 
+FF_Factory.o 
+LM//LM 
+TranslationModel/CompactPT//CompactPT 
+ThreadPool
+..//search 
+../util/double-conversion//double-conversion 
+../probingpt//probingpt
+..//z 
+../OnDiskPt//OnDiskPt 
+$(TOP)//boost_filesystem 
+$(TOP)//boost_iostreams 
+:
+<threading>single:<source>../util//rt
+;
+
+
+alias headers-to-install : [ glob-tree *.h ] ;
+
+import testing ;
+
+unit-test moses_test : [ glob *Test.cpp Mock*.cpp FF/*Test.cpp ] ..//boost_filesystem moses headers ..//z ../OnDiskPt//OnDiskPt ../probingpt//probingpt ..//boost_unit_test_framework ;
+

mosesdecoder/moses/LVoc.cpp

@@ -0,0 +1,7 @@
+#include<limits>
+#include "LVoc.h"
+
+//rather pointless file because LVoc is template all wee need here is the definitions of consts
+
+const LabelId InvalidLabelId = std::numeric_limits<LabelId>::max();
+const LabelId Epsilon        = InvalidLabelId-1;

mosesdecoder/moses/LVoc.h

@@ -0,0 +1,93 @@
+#ifndef moses_LVoc_h
+#define moses_LVoc_h
+
+#include<map>
+#include<vector>
+#include<iostream>
+#include<fstream>
+#include <sstream>
+
+typedef unsigned LabelId;
+extern const LabelId InvalidLabelId;
+extern const LabelId Epsilon;
+
+typedef std::vector<LabelId> IPhrase;
+
+/** class used in phrase-based binary phrase-table.
+ *  @todo vocab?
+ *  A = type of things to numberize, ie, std::string
+ *  B = map type to use, might consider using hash_map for better performance
+ */
+template<typename A,typename B=std::map<A,LabelId> >
+class LVoc
+{
+  typedef A Key;
+  typedef B M;
+  typedef std::vector<Key> V;
+  M m;
+  V data;
+public:
+  LVoc() {}
+
+  bool isKnown(const Key& k) const {
+    return m.find(k)!=m.end();
+  }
+  LabelId index(const Key& k) const {
+    typename M::const_iterator i=m.find(k);
+    return i!=m.end()? i->second : InvalidLabelId;
+  }
+  LabelId add(const Key& k) {
+    std::pair<typename M::iterator,bool> p
+    =m.insert(std::make_pair(k,data.size()));
+    if(p.second) data.push_back(k);
+    assert(static_cast<size_t>(p.first->second)<data.size());
+    return p.first->second;
+  }
+  Key const& symbol(LabelId i) const {
+    assert(static_cast<size_t>(i)<data.size());
+    return data[i];
+  }
+
+  typedef typename V::const_iterator const_iterator;
+  const_iterator begin() const {
+    return data.begin();
+  }
+  const_iterator end() const {
+    return data.end();
+  }
+
+  void Write(const std::string& fname) const {
+    std::ofstream out(fname.c_str());
+    // Little-known fact: ofstream tracks failures but does not, by default,
+    // report them.  You have to tell it to, or check for errors yourself.
+    out.exceptions(std::ifstream::failbit | std::ifstream::badbit);
+    Write(out);
+    // Make sure the file is flushed, so that any errors are reported.  If we
+    // flush implicitly in the destructor, it won't be able to throw
+    // exceptions.
+    out.close();
+  }
+  void Write(std::ostream& out) const {
+    for(int i=data.size()-1; i>=0; --i)
+      out<<i<<' '<<data[i]<<'\n';
+  }
+  void Read(const std::string& fname) {
+    std::ifstream in(fname.c_str());
+    Read(in);
+  }
+  void Read(std::istream& in) {
+    Key k;
+    size_t i;
+    std::string line;
+    while(getline(in,line)) {
+      std::istringstream is(line);
+      if(is>>i>>k) {
+        if(i>=data.size()) data.resize(i+1);
+        data[i]=k;
+        m[k]=i;
+      }
+    }
+  }
+};
+
+#endif

mosesdecoder/moses/Manager.cpp

@@ -0,0 +1,2016 @@
+// -*- mode: c++; indent-tabs-mode: nil; tab-width:2  -*-
+// vim:tabstop=2
+/***********************************************************************
+Moses - factored phrase-based language decoder
+Copyright (C) 2006 University of Edinburgh
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Lesser General Public
+License as published by the Free Software Foundation; either
+version 2.1 of the License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public
+License along with this library; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+***********************************************************************/
+#ifdef WIN32
+#include <hash_set>
+#else
+// #include <ext/hash_set>
+#endif
+
+#include <algorithm>
+#include <cmath>
+#include <limits>
+#include <map>
+#include <set>
+#include "Manager.h"
+#include "TypeDef.h"
+#include "Util.h"
+#include "TargetPhrase.h"
+#include "TrellisPath.h"
+#include "TrellisPathCollection.h"
+#include "TranslationOption.h"
+#include "TranslationOptionCollection.h"
+#include "Timer.h"
+#include "moses/OutputCollector.h"
+#include "moses/FF/DistortionScoreProducer.h"
+#include "moses/LM/Base.h"
+#include "moses/TranslationModel/PhraseDictionary.h"
+#include "moses/TranslationAnalysis.h"
+#include "moses/TranslationTask.h"
+#include "moses/HypergraphOutput.h"
+#include "moses/mbr.h"
+#include "moses/LatticeMBR.h"
+#include "moses/SearchNormal.h"
+#include "moses/SearchCubePruning.h"
+#include <boost/foreach.hpp>
+
+#ifdef HAVE_PROTOBUF
+#include "hypergraph.pb.h"
+#include "rule.pb.h"
+#endif
+
+#include "util/exception.hh"
+#include "util/random.hh"
+#include "util/string_stream.hh"
+
+using namespace std;
+
+namespace Moses
+{
+
+Manager::Manager(ttasksptr const& ttask)
+  : BaseManager(ttask)
+  , interrupted_flag(0)
+  , m_hypoId(0)
+{
+  boost::shared_ptr<InputType> source = ttask->GetSource();
+  m_transOptColl = source->CreateTranslationOptionCollection(ttask);
+
+  switch(options()->search.algo) {
+  case Normal:
+    m_search = new SearchNormal(*this, *m_transOptColl);
+    break;
+  case CubePruning:
+    m_search = new SearchCubePruning(*this, *m_transOptColl);
+    break;
+  default:
+    UTIL_THROW2("ERROR: search. Aborting\n");
+  }
+
+  StaticData::Instance().InitializeForInput(ttask);
+}
+
+Manager::~Manager()
+{
+  delete m_transOptColl;
+  delete m_search;
+  StaticData::Instance().CleanUpAfterSentenceProcessing(m_ttask.lock());
+}
+
+const InputType&
+Manager::GetSource() const
+{
+  return m_source ;
+}
+
+/**
+ * Main decoder loop that translates a sentence by expanding
+ * hypotheses stack by stack, until the end of the sentence.
+ */
+void Manager::Decode()
+{
+
+  //std::cerr << options().nbest.nbest_size << " "
+  //          << options().nbest.enabled << " " << std::endl;
+
+  // initialize statistics
+  ResetSentenceStats(m_source);
+  IFVERBOSE(2) {
+    GetSentenceStats().StartTimeTotal();
+  }
+
+  // check if alternate weight setting is used
+  // this is not thread safe! it changes StaticData
+  if (StaticData::Instance().GetHasAlternateWeightSettings()) {
+    if (m_source.GetSpecifiesWeightSetting()) {
+      StaticData::Instance().SetWeightSetting(m_source.GetWeightSetting());
+    } else {
+      StaticData::Instance().SetWeightSetting("default");
+    }
+  }
+
+  // get translation options
+  IFVERBOSE(1) {
+    GetSentenceStats().StartTimeCollectOpts();
+  }
+  m_transOptColl->CreateTranslationOptions();
+
+  // some reporting on how long this took
+  IFVERBOSE(1) {
+    GetSentenceStats().StopTimeCollectOpts();
+    TRACE_ERR("Line "<< m_source.GetTranslationId()
+              << ": Collecting options took "
+              << GetSentenceStats().GetTimeCollectOpts() << " seconds at "
+              << __FILE__ << " Line " << __LINE__ << endl);
+  }
+
+  // search for best translation with the specified algorithm
+  Timer searchTime;
+  searchTime.start();
+  m_search->Decode();
+  VERBOSE(1, "Line " << m_source.GetTranslationId()
+          << ": Search took " << searchTime << " seconds" << endl);
+  IFVERBOSE(2) {
+    GetSentenceStats().StopTimeTotal();
+    TRACE_ERR(GetSentenceStats());
+  }
+}
+
+/**
+ * Print all derivations in search graph. Note: The number of derivations is exponential in the sentence length
+ *
+ */
+
+void Manager::PrintAllDerivations(long translationId, ostream& outputStream) const
+{
+  const std::vector < HypothesisStack* > &hypoStackColl = m_search->GetHypothesisStacks();
+
+  vector<const Hypothesis*> sortedPureHypo = hypoStackColl.back()->GetSortedList();
+
+  if (sortedPureHypo.size() == 0)
+    return;
+
+  float remainingScore = 0;
+  vector<const TargetPhrase*> remainingPhrases;
+
+  // add all pure paths
+  vector<const Hypothesis*>::const_iterator iterBestHypo;
+  for (iterBestHypo = sortedPureHypo.begin()
+                      ; iterBestHypo != sortedPureHypo.end()
+       ; ++iterBestHypo) {
+    printThisHypothesis(translationId, *iterBestHypo, remainingPhrases, remainingScore, outputStream);
+    printDivergentHypothesis(translationId, *iterBestHypo, remainingPhrases, remainingScore, outputStream);
+  }
+}
+
+const TranslationOptionCollection* Manager::getSntTranslationOptions()
+{
+  return m_transOptColl;
+}
+
+void Manager::printDivergentHypothesis(long translationId, const Hypothesis* hypo, const vector <const TargetPhrase*> & remainingPhrases, float remainingScore , ostream& outputStream ) const
+{
+  //Backtrack from the predecessor
+  if (hypo->GetId()  > 0) {
+    vector <const TargetPhrase*> followingPhrases;
+    followingPhrases.push_back(& (hypo->GetCurrTargetPhrase()));
+    ///((Phrase) hypo->GetPrevHypo()->GetTargetPhrase());
+    followingPhrases.insert(followingPhrases.end()--, remainingPhrases.begin(), remainingPhrases.end());
+    printDivergentHypothesis(translationId, hypo->GetPrevHypo(), followingPhrases , remainingScore + hypo->GetScore() - hypo->GetPrevHypo()->GetScore(), outputStream);
+  }
+
+  //Process the arcs
+  const ArcList *pAL = hypo->GetArcList();
+  if (pAL) {
+    const ArcList &arcList = *pAL;
+    // every possible Arc to replace this edge
+    ArcList::const_iterator iterArc;
+    for (iterArc = arcList.begin() ; iterArc != arcList.end() ; ++iterArc) {
+      const Hypothesis *loserHypo = *iterArc;
+      const Hypothesis* loserPrevHypo = loserHypo->GetPrevHypo();
+      float arcScore = loserHypo->GetScore() - loserPrevHypo->GetScore();
+      vector <const TargetPhrase* > followingPhrases;
+      followingPhrases.push_back(&(loserHypo->GetCurrTargetPhrase()));
+      followingPhrases.insert(followingPhrases.end()--, remainingPhrases.begin(), remainingPhrases.end());
+      printThisHypothesis(translationId, loserPrevHypo, followingPhrases, remainingScore + arcScore, outputStream);
+      printDivergentHypothesis(translationId, loserPrevHypo, followingPhrases, remainingScore + arcScore, outputStream);
+    }
+  }
+}
+
+
+void
+Manager::
+printThisHypothesis(long translationId, const Hypothesis* hypo,
+                    const vector <const TargetPhrase*> & remainingPhrases,
+                    float remainingScore, ostream& outputStream) const
+{
+
+  outputStream << translationId << " ||| ";
+
+  //Yield of this hypothesis
+  hypo->ToStream(outputStream);
+  for (size_t p = 0; p < remainingPhrases.size(); ++p) {
+    const TargetPhrase * phrase = remainingPhrases[p];
+    size_t size = phrase->GetSize();
+    for (size_t pos = 0 ; pos < size ; pos++) {
+      const Factor *factor = phrase->GetFactor(pos, 0);
+      outputStream << *factor;
+      outputStream << " ";
+    }
+  }
+
+  outputStream << "||| " << hypo->GetScore() + remainingScore;
+  outputStream << endl;
+}
+
+
+
+
+/**
+ * After decoding, the hypotheses in the stacks and additional arcs
+ * form a search graph that can be mined for n-best lists.
+ * The heavy lifting is done in the TrellisPath and TrellisPathCollection
+ * this function controls this for one sentence.
+ *
+ * \param count the number of n-best translations to produce
+ * \param ret holds the n-best list that was calculated
+ */
+void Manager::CalcNBest(size_t count, TrellisPathList &ret, bool onlyDistinct) const
+{
+  if (count <= 0)
+    return;
+
+  const std::vector < HypothesisStack* > &hypoStackColl = m_search->GetHypothesisStacks();
+
+  vector<const Hypothesis*> sortedPureHypo = hypoStackColl.back()->GetSortedList();
+
+  if (sortedPureHypo.size() == 0)
+    return;
+
+  TrellisPathCollection contenders;
+
+  set<Phrase> distinctHyps;
+
+  // add all pure paths
+  vector<const Hypothesis*>::const_iterator iterBestHypo;
+  for (iterBestHypo = sortedPureHypo.begin()
+                      ; iterBestHypo != sortedPureHypo.end()
+       ; ++iterBestHypo) {
+    contenders.Add(new TrellisPath(*iterBestHypo));
+  }
+
+  // factor defines stopping point for distinct n-best list if too
+  // many candidates identical
+  size_t nBestFactor = options()->nbest.factor;
+  if (nBestFactor < 1) nBestFactor = 1000; // 0 = unlimited
+
+  // MAIN loop
+  for (size_t iteration = 0 ; (onlyDistinct ? distinctHyps.size() : ret.GetSize()) < count && contenders.GetSize() > 0 && (iteration < count * nBestFactor) ; iteration++) {
+    // get next best from list of contenders
+    TrellisPath *path = contenders.pop();
+    UTIL_THROW_IF2(path == NULL, "path is NULL");
+    // create deviations from current best
+    path->CreateDeviantPaths(contenders);
+    if(onlyDistinct) {
+      Phrase tgtPhrase = path->GetSurfacePhrase();
+      if (distinctHyps.insert(tgtPhrase).second) {
+        ret.Add(path);
+      } else {
+        delete path;
+        path = NULL;
+      }
+    } else {
+      ret.Add(path);
+    }
+
+
+    if(onlyDistinct) {
+      const size_t nBestFactor = options()->nbest.factor;
+      if (nBestFactor > 0)
+        contenders.Prune(count * nBestFactor);
+    } else {
+      contenders.Prune(count);
+    }
+  }
+}
+
+struct SGNReverseCompare {
+  bool operator() (const SearchGraphNode& s1, const SearchGraphNode& s2) const {
+    return s1.hypo->GetId() > s2.hypo->GetId();
+  }
+};
+
+/**
+  * Implements lattice sampling, as in Chatterjee & Cancedda, emnlp 2010
+  **/
+void Manager::CalcLatticeSamples(size_t count, TrellisPathList &ret) const
+{
+
+  vector<SearchGraphNode> searchGraph;
+  GetSearchGraph(searchGraph);
+
+  //Calculation of the sigmas of each hypothesis and edge. In C&C notation this is
+  //the "log of the cumulative unnormalized probability of all the paths in the
+  // lattice for the hypothesis to a final node"
+  typedef pair<int, int> Edge;
+  map<const Hypothesis*, float> sigmas;
+  map<Edge, float> edgeScores;
+  map<const Hypothesis*, set<const Hypothesis*> > outgoingHyps;
+  map<int,const Hypothesis*> idToHyp;
+  map<int,float> fscores;
+
+  //Iterating through the hypos in reverse order of id gives  a reverse
+  //topological order. We rely on the fact that hypo ids are given out
+  //sequentially, as the search proceeds.
+  //NB: Could just sort by stack.
+  sort(searchGraph.begin(), searchGraph.end(), SGNReverseCompare());
+
+  //first task is to fill in the outgoing hypos and edge scores.
+  for (vector<SearchGraphNode>::const_iterator i = searchGraph.begin();
+       i != searchGraph.end(); ++i) {
+    const Hypothesis* hypo = i->hypo;
+    idToHyp[hypo->GetId()] = hypo;
+    fscores[hypo->GetId()] = i->fscore;
+    if (hypo->GetId()) {
+      //back to  current
+      const Hypothesis* prevHypo = i->hypo->GetPrevHypo();
+      outgoingHyps[prevHypo].insert(hypo);
+      edgeScores[Edge(prevHypo->GetId(),hypo->GetId())] =
+        hypo->GetScore() - prevHypo->GetScore();
+    }
+    //forward from current
+    if (i->forward >= 0) {
+      map<int,const Hypothesis*>::const_iterator idToHypIter = idToHyp.find(i->forward);
+      UTIL_THROW_IF2(idToHypIter == idToHyp.end(),
+                     "Couldn't find hypothesis " << i->forward);
+      const Hypothesis* nextHypo = idToHypIter->second;
+      outgoingHyps[hypo].insert(nextHypo);
+      map<int,float>::const_iterator fscoreIter = fscores.find(nextHypo->GetId());
+      UTIL_THROW_IF2(fscoreIter == fscores.end(),
+                     "Couldn't find scores for hypothsis " << nextHypo->GetId());
+      edgeScores[Edge(hypo->GetId(),nextHypo->GetId())] =
+        i->fscore - fscoreIter->second;
+    }
+  }
+
+
+  //then run through again to calculate sigmas
+  for (vector<SearchGraphNode>::const_iterator i = searchGraph.begin();
+       i != searchGraph.end(); ++i) {
+
+    if (i->forward == -1) {
+      sigmas[i->hypo] = 0;
+    } else {
+      map<const Hypothesis*, set<const Hypothesis*> >::const_iterator outIter =
+        outgoingHyps.find(i->hypo);
+
+      UTIL_THROW_IF2(outIter == outgoingHyps.end(),
+                     "Couldn't find hypothesis " << i->hypo->GetId());
+      float sigma = 0;
+      for (set<const Hypothesis*>::const_iterator j = outIter->second.begin();
+           j != outIter->second.end(); ++j) {
+        map<const Hypothesis*, float>::const_iterator succIter = sigmas.find(*j);
+        UTIL_THROW_IF2(succIter == sigmas.end(),
+                       "Couldn't find hypothesis " << (*j)->GetId());
+        map<Edge,float>::const_iterator edgeScoreIter =
+          edgeScores.find(Edge(i->hypo->GetId(),(*j)->GetId()));
+        UTIL_THROW_IF2(edgeScoreIter == edgeScores.end(),
+                       "Couldn't find edge for hypothesis " << (*j)->GetId());
+        float term = edgeScoreIter->second + succIter->second; // Add sigma(*j)
+        if (sigma == 0) {
+          sigma = term;
+        } else {
+          sigma = log_sum(sigma,term);
+        }
+      }
+      sigmas[i->hypo] = sigma;
+    }
+  }
+
+  //The actual sampling!
+  const Hypothesis* startHypo = searchGraph.back().hypo;
+  UTIL_THROW_IF2(startHypo->GetId() != 0, "Expecting the start hypothesis ");
+  for (size_t i = 0; i < count; ++i) {
+    vector<const Hypothesis*> path;
+    path.push_back(startHypo);
+    while(1) {
+      map<const Hypothesis*, set<const Hypothesis*> >::const_iterator outIter =
+        outgoingHyps.find(path.back());
+      if (outIter == outgoingHyps.end() || !outIter->second.size()) {
+        //end of the path
+        break;
+      }
+      //score the possibles
+      vector<const Hypothesis*> candidates;
+      vector<float> candidateScores;
+      float scoreTotal = 0;
+      for (set<const Hypothesis*>::const_iterator j = outIter->second.begin();
+           j != outIter->second.end(); ++j) {
+        candidates.push_back(*j);
+        UTIL_THROW_IF2(sigmas.find(*j) == sigmas.end(),
+                       "Hypothesis " << (*j)->GetId() << " not found");
+        Edge edge(path.back()->GetId(),(*j)->GetId());
+        UTIL_THROW_IF2(edgeScores.find(edge) == edgeScores.end(),
+                       "Edge not found");
+        candidateScores.push_back(sigmas[*j]  + edgeScores[edge]);
+        if (scoreTotal == 0) {
+          scoreTotal = candidateScores.back();
+        } else {
+          scoreTotal = log_sum(candidateScores.back(), scoreTotal);
+        }
+      }
+
+      //normalise
+      transform(candidateScores.begin(), candidateScores.end(), candidateScores.begin(), bind2nd(minus<float>(),scoreTotal));
+      //copy(candidateScores.begin(),candidateScores.end(),ostream_iterator<float>(cerr," "));
+      //cerr << endl;
+
+      //draw the sample
+      const float frandom = log(util::rand_incl(0.0f, 1.0f));
+      size_t position = 1;
+      float sum = candidateScores[0];
+      for (; position < candidateScores.size() && sum < frandom; ++position) {
+        sum = log_sum(sum,candidateScores[position]);
+      }
+      //cerr << "Random: " << frandom << " Chose " << position-1 << endl;
+      const Hypothesis* chosen =  candidates[position-1];
+      path.push_back(chosen);
+    }
+    //cerr << "Path: " << endl;
+    //for (size_t j = 0; j < path.size(); ++j) {
+    // cerr << path[j]->GetId() <<  " " << path[j]->GetScoreBreakdown() << endl;
+    //}
+    //cerr << endl;
+
+    //Convert the hypos to TrellisPath
+    ret.Add(new TrellisPath(path));
+    //cerr << ret.at(ret.GetSize()-1).GetScoreBreakdown() << endl;
+  }
+
+}
+
+
+
+void Manager::CalcDecoderStatistics() const
+{
+  const Hypothesis *hypo = GetBestHypothesis();
+  if (hypo != NULL) {
+    GetSentenceStats().CalcFinalStats(*hypo);
+    IFVERBOSE(2) {
+      if (hypo != NULL) {
+        string buff;
+        string buff2;
+        TRACE_ERR( "Source and Target Units:"
+                   << hypo->GetInput());
+        buff2.insert(0,"] ");
+        buff2.insert(0,(hypo->GetCurrTargetPhrase()).ToString());
+        buff2.insert(0,":");
+        buff2.insert(0,(hypo->GetCurrSourceWordsRange()).ToString());
+        buff2.insert(0,"[");
+
+        hypo = hypo->GetPrevHypo();
+        while (hypo != NULL) {
+          //dont print out the empty final hypo
+          buff.insert(0,buff2);
+          buff2.clear();
+          buff2.insert(0,"] ");
+          buff2.insert(0,(hypo->GetCurrTargetPhrase()).ToString());
+          buff2.insert(0,":");
+          buff2.insert(0,(hypo->GetCurrSourceWordsRange()).ToString());
+          buff2.insert(0,"[");
+          hypo = hypo->GetPrevHypo();
+        }
+        TRACE_ERR( buff << endl);
+      }
+    }
+  }
+}
+
+void Manager::OutputWordGraph(std::ostream &outputWordGraphStream, const Hypothesis *hypo, size_t &linkId) const
+{
+
+  const Hypothesis *prevHypo = hypo->GetPrevHypo();
+
+
+  outputWordGraphStream << "J=" << linkId++
+                        << "\tS=" << prevHypo->GetId()
+                        << "\tE=" << hypo->GetId()
+                        << "\ta=";
+
+  // phrase table scores
+  const std::vector<PhraseDictionary*> &phraseTables = PhraseDictionary::GetColl();
+  std::vector<PhraseDictionary*>::const_iterator iterPhraseTable;
+  for (iterPhraseTable = phraseTables.begin() ; iterPhraseTable != phraseTables.end() ; ++iterPhraseTable) {
+    const PhraseDictionary *phraseTable = *iterPhraseTable;
+    vector<float> scores = hypo->GetScoreBreakdown().GetScoresForProducer(phraseTable);
+
+    outputWordGraphStream << scores[0];
+    vector<float>::const_iterator iterScore;
+    for (iterScore = ++scores.begin() ; iterScore != scores.end() ; ++iterScore) {
+      outputWordGraphStream << ", " << *iterScore;
+    }
+  }
+
+  // language model scores
+  outputWordGraphStream << "\tl=";
+
+  const std::vector<const StatefulFeatureFunction*> &statefulFFs = StatefulFeatureFunction::GetStatefulFeatureFunctions();
+  for (size_t i = 0; i < statefulFFs.size(); ++i) {
+    const StatefulFeatureFunction *ff = statefulFFs[i];
+    const LanguageModel *lm = static_cast<const LanguageModel*>(ff);
+
+    vector<float> scores = hypo->GetScoreBreakdown().GetScoresForProducer(lm);
+
+    outputWordGraphStream << scores[0];
+    vector<float>::const_iterator iterScore;
+    for (iterScore = ++scores.begin() ; iterScore != scores.end() ; ++iterScore) {
+      outputWordGraphStream << ", " << *iterScore;
+    }
+  }
+
+  // re-ordering
+  outputWordGraphStream << "\tr=";
+
+  const std::vector<FeatureFunction*> &ffs = FeatureFunction::GetFeatureFunctions();
+  std::vector<FeatureFunction*>::const_iterator iter;
+  for (iter = ffs.begin(); iter != ffs.end(); ++iter) {
+    const FeatureFunction *ff = *iter;
+
+    const DistortionScoreProducer *model = dynamic_cast<const DistortionScoreProducer*>(ff);
+    if (model) {
+      outputWordGraphStream << hypo->GetScoreBreakdown().GetScoreForProducer(model);
+    }
+  }
+
+  // output both source and target phrases in the word graph
+  outputWordGraphStream << "\tw=" << hypo->GetSourcePhraseStringRep()
+                        << "|" << hypo->GetCurrTargetPhrase();
+
+  outputWordGraphStream << endl;
+}
+
+// VN put back of OutputPassthroughInformation
+void Manager::OutputPassthroughInformation(std::ostream &out, const Hypothesis *hypo) const
+{
+  const std::string passthrough = hypo->GetManager().GetSource().GetPassthroughInformation();
+  out << passthrough;
+}
+// end of put back
+
+void Manager::GetOutputLanguageModelOrder( std::ostream &out, const Hypothesis *hypo ) const
+{
+  Phrase translation;
+  hypo->GetOutputPhrase(translation);
+  const std::vector<const StatefulFeatureFunction*> &statefulFFs = StatefulFeatureFunction::GetStatefulFeatureFunctions();
+  for (size_t i = 0; i < statefulFFs.size(); ++i) {
+    const StatefulFeatureFunction *ff = statefulFFs[i];
+    if (const LanguageModel *lm = dynamic_cast<const LanguageModel*>(ff)) {
+      lm->ReportHistoryOrder(out, translation);
+    }
+  }
+}
+
+void Manager::GetWordGraph(long translationId, std::ostream &outputWordGraphStream) const
+{
+  const StaticData &staticData = StaticData::Instance();
+  const PARAM_VEC *params;
+
+  string fileName;
+  bool outputNBest = false;
+  params = staticData.GetParameter().GetParam("output-word-graph");
+  if (params && params->size()) {
+    fileName = params->at(0);
+
+    if (params->size() == 2) {
+      outputNBest = Scan<bool>(params->at(1));
+    }
+  }
+
+  const std::vector < HypothesisStack* > &hypoStackColl = m_search->GetHypothesisStacks();
+
+  outputWordGraphStream << "VERSION=1.0" << endl
+                        << "UTTERANCE=" << translationId << endl;
+
+  size_t linkId = 0;
+  std::vector < HypothesisStack* >::const_iterator iterStack;
+  for (iterStack = ++hypoStackColl.begin() ; iterStack != hypoStackColl.end() ; ++iterStack) {
+    const HypothesisStack &stack = **iterStack;
+    HypothesisStack::const_iterator iterHypo;
+    for (iterHypo = stack.begin() ; iterHypo != stack.end() ; ++iterHypo) {
+      const Hypothesis *hypo = *iterHypo;
+      OutputWordGraph(outputWordGraphStream, hypo, linkId);
+
+      if (outputNBest) {
+        const ArcList *arcList = hypo->GetArcList();
+        if (arcList != NULL) {
+          ArcList::const_iterator iterArcList;
+          for (iterArcList = arcList->begin() ; iterArcList != arcList->end() ; ++iterArcList) {
+            const Hypothesis *loserHypo = *iterArcList;
+            OutputWordGraph(outputWordGraphStream, loserHypo, linkId);
+          }
+        }
+      } //if (outputNBest)
+    } //for (iterHypo
+  } // for (iterStack
+}
+
+void Manager::GetSearchGraph(vector<SearchGraphNode>& searchGraph) const
+{
+  std::map < int, bool > connected;
+  std::map < int, int > forward;
+  std::map < int, double > forwardScore;
+
+  // *** find connected hypotheses ***
+  std::vector< const Hypothesis *> connectedList;
+  GetConnectedGraph(&connected, &connectedList);
+
+  // ** compute best forward path for each hypothesis *** //
+
+  // forward cost of hypotheses on final stack is 0
+  const std::vector < HypothesisStack* > &hypoStackColl = m_search->GetHypothesisStacks();
+  const HypothesisStack &finalStack = *hypoStackColl.back();
+  HypothesisStack::const_iterator iterHypo;
+  for (iterHypo = finalStack.begin() ; iterHypo != finalStack.end() ; ++iterHypo) {
+    const Hypothesis *hypo = *iterHypo;
+    forwardScore[ hypo->GetId() ] = 0.0f;
+    forward[ hypo->GetId() ] = -1;
+  }
+
+  // compete for best forward score of previous hypothesis
+  std::vector < HypothesisStack* >::const_iterator iterStack;
+  for (iterStack = --hypoStackColl.end() ; iterStack != hypoStackColl.begin() ; --iterStack) {
+    const HypothesisStack &stack = **iterStack;
+    HypothesisStack::const_iterator iterHypo;
+    for (iterHypo = stack.begin() ; iterHypo != stack.end() ; ++iterHypo) {
+      const Hypothesis *hypo = *iterHypo;
+      if (connected.find( hypo->GetId() ) != connected.end()) {
+        // make a play for previous hypothesis
+        const Hypothesis *prevHypo = hypo->GetPrevHypo();
+        double fscore = forwardScore[ hypo->GetId() ] +
+                        hypo->GetScore() - prevHypo->GetScore();
+        if (forwardScore.find( prevHypo->GetId() ) == forwardScore.end()
+            || forwardScore.find( prevHypo->GetId() )->second < fscore) {
+          forwardScore[ prevHypo->GetId() ] = fscore;
+          forward[ prevHypo->GetId() ] = hypo->GetId();
+        }
+        // all arcs also make a play
+        const ArcList *arcList = hypo->GetArcList();
+        if (arcList != NULL) {
+          ArcList::const_iterator iterArcList;
+          for (iterArcList = arcList->begin() ; iterArcList != arcList->end() ; ++iterArcList) {
+            const Hypothesis *loserHypo = *iterArcList;
+            // make a play
+            const Hypothesis *loserPrevHypo = loserHypo->GetPrevHypo();
+            double fscore = forwardScore[ hypo->GetId() ] +
+                            loserHypo->GetScore() - loserPrevHypo->GetScore();
+            if (forwardScore.find( loserPrevHypo->GetId() ) == forwardScore.end()
+                || forwardScore.find( loserPrevHypo->GetId() )->second < fscore) {
+              forwardScore[ loserPrevHypo->GetId() ] = fscore;
+              forward[ loserPrevHypo->GetId() ] = loserHypo->GetId();
+            }
+          } // end for arc list
+        } // end if arc list empty
+      } // end if hypo connected
+    } // end for hypo
+  } // end for stack
+
+  // *** output all connected hypotheses *** //
+
+  connected[ 0 ] = true;
+  for (iterStack = hypoStackColl.begin() ; iterStack != hypoStackColl.end() ; ++iterStack) {
+    const HypothesisStack &stack = **iterStack;
+    HypothesisStack::const_iterator iterHypo;
+    for (iterHypo = stack.begin() ; iterHypo != stack.end() ; ++iterHypo) {
+      const Hypothesis *hypo = *iterHypo;
+      if (connected.find( hypo->GetId() ) != connected.end()) {
+        searchGraph.push_back(SearchGraphNode(hypo,NULL,forward[hypo->GetId()],
+                                              forwardScore[hypo->GetId()]));
+
+        const ArcList *arcList = hypo->GetArcList();
+        if (arcList != NULL) {
+          ArcList::const_iterator iterArcList;
+          for (iterArcList = arcList->begin() ; iterArcList != arcList->end() ; ++iterArcList) {
+            const Hypothesis *loserHypo = *iterArcList;
+            searchGraph.push_back(SearchGraphNode(loserHypo,hypo,
+                                                  forward[hypo->GetId()], forwardScore[hypo->GetId()]));
+          }
+        } // end if arcList empty
+      } // end if connected
+    } // end for iterHypo
+  } // end for iterStack
+
+}
+
+void Manager::OutputFeatureWeightsForSLF(std::ostream &outputSearchGraphStream) const
+{
+  outputSearchGraphStream.setf(std::ios::fixed);
+  outputSearchGraphStream.precision(6);
+
+  const vector<const StatelessFeatureFunction*>& slf  = StatelessFeatureFunction::GetStatelessFeatureFunctions();
+  const vector<const StatefulFeatureFunction*>& sff   = StatefulFeatureFunction::GetStatefulFeatureFunctions();
+  size_t featureIndex = 1;
+  for (size_t i = 0; i < sff.size(); ++i) {
+    featureIndex = OutputFeatureWeightsForSLF(featureIndex, sff[i], outputSearchGraphStream);
+  }
+  for (size_t i = 0; i < slf.size(); ++i) {
+    /*
+    if (slf[i]->GetScoreProducerWeightShortName() != "u" &&
+          slf[i]->GetScoreProducerWeightShortName() != "tm" &&
+          slf[i]->GetScoreProducerWeightShortName() != "I" &&
+          slf[i]->GetScoreProducerWeightShortName() != "g")
+    */
+    {
+      featureIndex = OutputFeatureWeightsForSLF(featureIndex, slf[i], outputSearchGraphStream);
+    }
+  }
+  const vector<PhraseDictionary*>& pds = PhraseDictionary::GetColl();
+  for( size_t i=0; i<pds.size(); i++ ) {
+    featureIndex = OutputFeatureWeightsForSLF(featureIndex, pds[i], outputSearchGraphStream);
+  }
+  const vector<GenerationDictionary*>& gds = GenerationDictionary::GetColl();
+  for( size_t i=0; i<gds.size(); i++ ) {
+    featureIndex = OutputFeatureWeightsForSLF(featureIndex, gds[i], outputSearchGraphStream);
+  }
+}
+
+void Manager::OutputFeatureValuesForSLF(const Hypothesis* hypo, bool zeros, std::ostream &outputSearchGraphStream) const
+{
+  outputSearchGraphStream.setf(std::ios::fixed);
+  outputSearchGraphStream.precision(6);
+
+  // outputSearchGraphStream << endl;
+  // outputSearchGraphStream << (*hypo) << endl;
+  // const ScoreComponentCollection& scoreCollection = hypo->GetScoreBreakdown();
+  // outputSearchGraphStream << scoreCollection << endl;
+
+  const vector<const StatelessFeatureFunction*>& slf =StatelessFeatureFunction::GetStatelessFeatureFunctions();
+  const vector<const StatefulFeatureFunction*>& sff = StatefulFeatureFunction::GetStatefulFeatureFunctions();
+  size_t featureIndex = 1;
+  for (size_t i = 0; i < sff.size(); ++i) {
+    featureIndex = OutputFeatureValuesForSLF(featureIndex, zeros, hypo, sff[i], outputSearchGraphStream);
+  }
+  for (size_t i = 0; i < slf.size(); ++i) {
+    /*
+    if (slf[i]->GetScoreProducerWeightShortName() != "u" &&
+          slf[i]->GetScoreProducerWeightShortName() != "tm" &&
+          slf[i]->GetScoreProducerWeightShortName() != "I" &&
+          slf[i]->GetScoreProducerWeightShortName() != "g")
+    */
+    {
+      featureIndex = OutputFeatureValuesForSLF(featureIndex, zeros, hypo, slf[i], outputSearchGraphStream);
+    }
+  }
+  const vector<PhraseDictionary*>& pds = PhraseDictionary::GetColl();
+  for( size_t i=0; i<pds.size(); i++ ) {
+    featureIndex = OutputFeatureValuesForSLF(featureIndex, zeros, hypo, pds[i], outputSearchGraphStream);
+  }
+  const vector<GenerationDictionary*>& gds = GenerationDictionary::GetColl();
+  for( size_t i=0; i<gds.size(); i++ ) {
+    featureIndex = OutputFeatureValuesForSLF(featureIndex, zeros, hypo, gds[i], outputSearchGraphStream);
+  }
+
+}
+
+void Manager::OutputFeatureValuesForHypergraph(const Hypothesis* hypo, std::ostream &outputSearchGraphStream) const
+{
+  outputSearchGraphStream.setf(std::ios::fixed);
+  outputSearchGraphStream.precision(6);
+  ScoreComponentCollection scores = hypo->GetScoreBreakdown();
+  const Hypothesis *prevHypo = hypo->GetPrevHypo();
+  if (prevHypo) {
+    scores.MinusEquals(prevHypo->GetScoreBreakdown());
+  }
+  scores.Save(outputSearchGraphStream, false);
+}
+
+
+size_t Manager::OutputFeatureWeightsForSLF(size_t index, const FeatureFunction* ff, std::ostream &outputSearchGraphStream) const
+{
+  size_t numScoreComps = ff->GetNumScoreComponents();
+  if (numScoreComps != 0) {
+    vector<float> values = StaticData::Instance().GetAllWeights().GetScoresForProducer(ff);
+    for (size_t i = 0; i < numScoreComps; ++i) {
+      outputSearchGraphStream << "# " << ff->GetScoreProducerDescription()
+                              << " "  << ff->GetScoreProducerDescription()
+                              << " "  << (i+1) << " of " << numScoreComps << endl
+                              << "x"  << (index+i) << "scale=" << values[i] << endl;
+    }
+    return index+numScoreComps;
+  } else {
+    cerr << "Sparse features are not supported when outputting HTK standard lattice format" << endl;
+    assert(false);
+    return 0;
+  }
+}
+
+size_t
+Manager::
+OutputFeatureValuesForSLF(size_t index, bool zeros, const Hypothesis* hypo,
+                          const FeatureFunction* ff, std::ostream &out) const
+{
+  const ScoreComponentCollection& scoreCollection = hypo->GetScoreBreakdown();
+  vector<float> featureValues = scoreCollection.GetScoresForProducer(ff);
+  size_t numScoreComps = featureValues.size();
+  for (size_t i = 0; i < numScoreComps; ++i) {
+    out << "x"  << (index+i) << "=" << ((zeros) ? 0.0 : featureValues[i]) << " ";
+  }
+  return index + numScoreComps;
+}
+
+/**! Output search graph in hypergraph format of Kenneth Heafield's lazy hypergraph decoder */
+void
+Manager::
+OutputSearchGraphAsHypergraph(std::ostream &outputSearchGraphStream) const
+{
+
+  VERBOSE(2,"Getting search graph to output as hypergraph for sentence " << m_source.GetTranslationId() << std::endl)
+
+  vector<SearchGraphNode> searchGraph;
+  GetSearchGraph(searchGraph);
+
+
+  map<int,int> mosesIDToHypergraphID;
+  // map<int,int> hypergraphIDToMosesID;
+  set<int> terminalNodes;
+  multimap<int,int> hypergraphIDToArcs;
+
+  VERBOSE(2,"Gathering information about search graph to output as hypergraph for sentence " << m_source.GetTranslationId() << std::endl)
+
+  long numNodes = 0;
+  long endNode = 0;
+  {
+    long hypergraphHypothesisID = 0;
+    for (size_t arcNumber = 0, size=searchGraph.size(); arcNumber < size; ++arcNumber) {
+
+      // Get an id number for the previous hypothesis
+      const Hypothesis *prevHypo = searchGraph[arcNumber].hypo->GetPrevHypo();
+      if (prevHypo!=NULL) {
+        int mosesPrevHypothesisID = prevHypo->GetId();
+        if (mosesIDToHypergraphID.count(mosesPrevHypothesisID) == 0) {
+          mosesIDToHypergraphID[mosesPrevHypothesisID] = hypergraphHypothesisID;
+          //	hypergraphIDToMosesID[hypergraphHypothesisID] = mosesPrevHypothesisID;
+          hypergraphHypothesisID += 1;
+        }
+      }
+
+      // Get an id number for this hypothesis
+      int mosesHypothesisID;
+      if (searchGraph[arcNumber].recombinationHypo) {
+        mosesHypothesisID = searchGraph[arcNumber].recombinationHypo->GetId();
+      } else {
+        mosesHypothesisID = searchGraph[arcNumber].hypo->GetId();
+      }
+
+      if (mosesIDToHypergraphID.count(mosesHypothesisID) == 0) {
+
+        mosesIDToHypergraphID[mosesHypothesisID] = hypergraphHypothesisID;
+        //      hypergraphIDToMosesID[hypergraphHypothesisID] = mosesHypothesisID;
+
+        bool terminalNode = (searchGraph[arcNumber].forward == -1);
+        if (terminalNode) {
+          // Final arc to end node, representing the end of the sentence </s>
+          terminalNodes.insert(hypergraphHypothesisID);
+        }
+
+        hypergraphHypothesisID += 1;
+      }
+
+      // Record that this arc ends at this node
+      hypergraphIDToArcs.insert(pair<int,int>(mosesIDToHypergraphID[mosesHypothesisID],arcNumber));
+
+    }
+
+    // Unique end node
+    endNode = hypergraphHypothesisID;
+    //    mosesIDToHypergraphID[hypergraphHypothesisID] = hypergraphHypothesisID;
+    numNodes = endNode + 1;
+
+  }
+
+
+  long numArcs = searchGraph.size() + terminalNodes.size();
+
+  //Header
+  outputSearchGraphStream << "# target ||| features ||| source-covered" << endl;
+
+  // Print number of nodes and arcs
+  outputSearchGraphStream << numNodes << " " << numArcs << endl;
+
+  VERBOSE(2,"Search graph to output as hypergraph for sentence " << m_source.GetTranslationId()
+          << " contains " << numArcs << " arcs and " << numNodes << " nodes" << std::endl)
+
+  VERBOSE(2,"Outputting search graph to output as hypergraph for sentence " << m_source.GetTranslationId() << std::endl)
+
+
+  for (int hypergraphHypothesisID=0; hypergraphHypothesisID < endNode; hypergraphHypothesisID+=1) {
+    if (hypergraphHypothesisID % 100000 == 0) {
+      VERBOSE(2,"Processed " << hypergraphHypothesisID << " of " << numNodes << " hypergraph nodes for sentence " << m_source.GetTranslationId() << std::endl);
+    }
+    //    int mosesID = hypergraphIDToMosesID[hypergraphHypothesisID];
+    size_t count = hypergraphIDToArcs.count(hypergraphHypothesisID);
+    //    VERBOSE(2,"Hypergraph node " << hypergraphHypothesisID << " has " << count << " incoming arcs" << std::endl)
+    if (count > 0) {
+      outputSearchGraphStream << "# node " << hypergraphHypothesisID << endl;
+      outputSearchGraphStream << count << "\n";
+
+      pair<multimap<int,int>::iterator, multimap<int,int>::iterator> range =
+        hypergraphIDToArcs.equal_range(hypergraphHypothesisID);
+      for (multimap<int,int>::iterator it=range.first; it!=range.second; ++it) {
+        int lineNumber = (*it).second;
+        const Hypothesis *thisHypo = searchGraph[lineNumber].hypo;
+        int mosesHypothesisID;// = thisHypo->GetId();
+        if (searchGraph[lineNumber].recombinationHypo) {
+          mosesHypothesisID = searchGraph[lineNumber].recombinationHypo->GetId();
+        } else {
+          mosesHypothesisID = searchGraph[lineNumber].hypo->GetId();
+        }
+        //	int actualHypergraphHypothesisID = mosesIDToHypergraphID[mosesHypothesisID];
+        UTIL_THROW_IF2(
+          (hypergraphHypothesisID != mosesIDToHypergraphID[mosesHypothesisID]),
+          "Error while writing search lattice as hypergraph for sentence " << m_source.GetTranslationId() << ". " <<
+          "Moses node " << mosesHypothesisID << " was expected to have hypergraph id " << hypergraphHypothesisID <<
+          ", but actually had hypergraph id " << mosesIDToHypergraphID[mosesHypothesisID] <<
+          ". There are " << numNodes << " nodes in the search lattice."
+        );
+
+        const Hypothesis *prevHypo = thisHypo->GetPrevHypo();
+        if (prevHypo==NULL) {
+          //	VERBOSE(2,"Hypergraph node " << hypergraphHypothesisID << " start of sentence" << std::endl)
+          outputSearchGraphStream << "<s> |||  ||| 0\n";
+        } else {
+          int startNode = mosesIDToHypergraphID[prevHypo->GetId()];
+          //	  VERBOSE(2,"Hypergraph node " << hypergraphHypothesisID << " has parent node " << startNode << std::endl)
+          UTIL_THROW_IF2(
+            (startNode >= hypergraphHypothesisID),
+            "Error while writing search lattice as hypergraph for sentence" << m_source.GetTranslationId() << ". " <<
+            "The nodes must be output in topological order. The code attempted to violate this restriction."
+          );
+
+          const TargetPhrase &targetPhrase = thisHypo->GetCurrTargetPhrase();
+          int targetWordCount = targetPhrase.GetSize();
+
+          outputSearchGraphStream << "[" << startNode << "] ";
+          for (int targetWordIndex=0; targetWordIndex<targetWordCount; targetWordIndex+=1) {
+            outputSearchGraphStream << targetPhrase.GetWord(targetWordIndex)[0]->GetString() << " ";
+          }
+          outputSearchGraphStream << " ||| ";
+          OutputFeatureValuesForHypergraph(thisHypo, outputSearchGraphStream);
+          outputSearchGraphStream << " ||| " << thisHypo->GetWordsBitmap().GetNumWordsCovered();
+          outputSearchGraphStream << "\n";
+        }
+      }
+    }
+  }
+
+  // Print node and arc(s) for end of sentence </s>
+  outputSearchGraphStream << "# node " << endNode << endl;
+  outputSearchGraphStream << terminalNodes.size() << "\n";
+  for (set<int>::iterator it=terminalNodes.begin(); it!=terminalNodes.end(); ++it) {
+    outputSearchGraphStream << "[" << (*it) << "] </s> |||  ||| " << GetSource().GetSize() << "\n";
+  }
+
+}
+
+
+/**! Output search graph in HTK standard lattice format (SLF) */
+void Manager::OutputSearchGraphAsSLF(long translationId, std::ostream &outputSearchGraphStream) const
+{
+
+  vector<SearchGraphNode> searchGraph;
+  GetSearchGraph(searchGraph);
+
+  long numArcs = 0;
+  long numNodes = 0;
+
+  map<int,int> nodes;
+  set<int> terminalNodes;
+
+  // Unique start node
+  nodes[0] = 0;
+
+  for (size_t arcNumber = 0; arcNumber < searchGraph.size(); ++arcNumber) {
+
+    int targetWordCount = searchGraph[arcNumber].hypo->GetCurrTargetPhrase().GetSize();
+    numArcs += targetWordCount;
+
+    int hypothesisID = searchGraph[arcNumber].hypo->GetId();
+    if (nodes.count(hypothesisID) == 0) {
+
+      numNodes += targetWordCount;
+      nodes[hypothesisID] = numNodes;
+      //numNodes += 1;
+
+      bool terminalNode = (searchGraph[arcNumber].forward == -1);
+      if (terminalNode) {
+        numArcs += 1;
+      }
+    }
+
+  }
+  numNodes += 1;
+
+  // Unique end node
+  nodes[numNodes] = numNodes;
+
+  outputSearchGraphStream << "UTTERANCE=Sentence_" << translationId << endl;
+  outputSearchGraphStream << "VERSION=1.1" << endl;
+  outputSearchGraphStream << "base=2.71828182845905" << endl;
+  outputSearchGraphStream << "NODES=" << (numNodes+1) << endl;
+  outputSearchGraphStream << "LINKS=" << numArcs  << endl;
+
+  OutputFeatureWeightsForSLF(outputSearchGraphStream);
+
+  for (size_t arcNumber = 0, lineNumber = 0; lineNumber < searchGraph.size(); ++lineNumber) {
+    const Hypothesis *thisHypo = searchGraph[lineNumber].hypo;
+    const Hypothesis *prevHypo = thisHypo->GetPrevHypo();
+    if (prevHypo) {
+
+      int startNode = nodes[prevHypo->GetId()];
+      int endNode   = nodes[thisHypo->GetId()];
+      bool terminalNode = (searchGraph[lineNumber].forward == -1);
+      const TargetPhrase &targetPhrase = thisHypo->GetCurrTargetPhrase();
+      int targetWordCount = targetPhrase.GetSize();
+
+      for (int targetWordIndex=0; targetWordIndex<targetWordCount; targetWordIndex+=1) {
+        int x = (targetWordCount-targetWordIndex);
+
+        outputSearchGraphStream <<  "J=" << arcNumber;
+
+        if (targetWordIndex==0) {
+          outputSearchGraphStream << " S=" << startNode;
+        } else {
+          outputSearchGraphStream << " S=" << endNode - x;
+        }
+
+        outputSearchGraphStream << " E=" << endNode - (x-1)
+                                << " W=" << targetPhrase.GetWord(targetWordIndex);
+
+        OutputFeatureValuesForSLF(thisHypo, (targetWordIndex>0), outputSearchGraphStream);
+
+        outputSearchGraphStream  << endl;
+
+        arcNumber += 1;
+      }
+
+      if (terminalNode && terminalNodes.count(endNode) == 0) {
+        terminalNodes.insert(endNode);
+        outputSearchGraphStream <<  "J="   << arcNumber
+                                << " S="   << endNode
+                                << " E="   << numNodes
+                                << endl;
+        arcNumber += 1;
+      }
+    }
+  }
+
+}
+
+
+void
+OutputSearchNode(AllOptions const& opts, long translationId,
+                 std::ostream &out,
+                 SearchGraphNode const& searchNode)
+{
+  const vector<FactorType> &outputFactorOrder = opts.output.factor_order;
+  bool extendedFormat = opts.output.SearchGraphExtended.size();
+  out << translationId;
+
+  // special case: initial hypothesis
+  if ( searchNode.hypo->GetId() == 0 ) {
+    out << " hyp=0 stack=0";
+    if (extendedFormat) {
+      out << " forward=" << searchNode.forward	<< " fscore=" << searchNode.fscore;
+    }
+    out << endl;
+    return;
+  }
+
+  const Hypothesis *prevHypo = searchNode.hypo->GetPrevHypo();
+
+  // output in traditional format
+  if (!extendedFormat) {
+    out << " hyp=" << searchNode.hypo->GetId()
+        << " stack=" << searchNode.hypo->GetWordsBitmap().GetNumWordsCovered()
+        << " back=" << prevHypo->GetId()
+        << " score=" << searchNode.hypo->GetScore()
+        << " transition=" << (searchNode.hypo->GetScore() - prevHypo->GetScore());
+
+    if (searchNode.recombinationHypo != NULL)
+      out << " recombined=" << searchNode.recombinationHypo->GetId();
+
+    out << " forward=" << searchNode.forward	<< " fscore=" << searchNode.fscore
+        << " covered=" << searchNode.hypo->GetCurrSourceWordsRange().GetStartPos()
+        << "-" << searchNode.hypo->GetCurrSourceWordsRange().GetEndPos()
+        << " out=" << searchNode.hypo->GetCurrTargetPhrase().GetStringRep(outputFactorOrder)
+        << endl;
+    return;
+  }
+
+  out << " hyp=" << searchNode.hypo->GetId();
+  out << " stack=" << searchNode.hypo->GetWordsBitmap().GetNumWordsCovered()
+      << " back=" << prevHypo->GetId()
+      << " score=" << searchNode.hypo->GetScore()
+      << " transition=" << (searchNode.hypo->GetScore() - prevHypo->GetScore());
+
+  if (searchNode.recombinationHypo != NULL)
+    out << " recombined=" << searchNode.recombinationHypo->GetId();
+
+  out << " forward=" << searchNode.forward	<< " fscore=" << searchNode.fscore
+      << " covered=" << searchNode.hypo->GetCurrSourceWordsRange().GetStartPos()
+      << "-" << searchNode.hypo->GetCurrSourceWordsRange().GetEndPos();
+
+  // Modified so that -osgx is a superset of -osg (GST Oct 2011)
+  ScoreComponentCollection scoreBreakdown = searchNode.hypo->GetScoreBreakdown();
+  scoreBreakdown.MinusEquals( prevHypo->GetScoreBreakdown() );
+  out << " scores=\"" << scoreBreakdown << "\""
+      << " out=\"" << searchNode.hypo->GetSourcePhraseStringRep()
+      << "|" << searchNode.hypo->GetCurrTargetPhrase().GetStringRep(outputFactorOrder) << "\"" << endl;
+}
+
+void Manager::GetConnectedGraph(
+  std::map< int, bool >* pConnected,
+  std::vector< const Hypothesis* >* pConnectedList) const
+{
+  std::map < int, bool >& connected = *pConnected;
+  std::vector< const Hypothesis *>& connectedList = *pConnectedList;
+
+  // start with the ones in the final stack
+  const std::vector < HypothesisStack* > &hypoStackColl
+  = m_search->GetHypothesisStacks();
+  const HypothesisStack &finalStack = *hypoStackColl.back();
+  HypothesisStack::const_iterator iterHypo;
+  for (iterHypo = finalStack.begin() ; iterHypo != finalStack.end() ; ++iterHypo) {
+    const Hypothesis *hypo = *iterHypo;
+    connected[ hypo->GetId() ] = true;
+    connectedList.push_back( hypo );
+  }
+  // move back from known connected hypotheses
+  for(size_t i=0; i<connectedList.size(); i++) {
+    const Hypothesis *hypo = connectedList[i];
+
+    // add back pointer
+    const Hypothesis *prevHypo = hypo->GetPrevHypo();
+    if (prevHypo && prevHypo->GetId() > 0 // don't add empty hypothesis
+        && connected.find( prevHypo->GetId() ) == connected.end()) { // don't add already added
+      connected[ prevHypo->GetId() ] = true;
+      connectedList.push_back( prevHypo );
+    }
+
+    // add arcs
+    const ArcList *arcList = hypo->GetArcList();
+    if (arcList != NULL) {
+      ArcList::const_iterator iterArcList;
+      for (iterArcList = arcList->begin() ; iterArcList != arcList->end() ; ++iterArcList) {
+        const Hypothesis *loserHypo = *iterArcList;
+        if (connected.find( loserHypo->GetId() ) == connected.end()) { // don't add already added
+          connected[ loserHypo->GetId() ] = true;
+          connectedList.push_back( loserHypo );
+        }
+      }
+    }
+  }
+}
+
+void Manager::GetWinnerConnectedGraph(
+  std::map< int, bool >* pConnected,
+  std::vector< const Hypothesis* >* pConnectedList) const
+{
+  std::map < int, bool >& connected = *pConnected;
+  std::vector< const Hypothesis *>& connectedList = *pConnectedList;
+
+  // start with the ones in the final stack
+  const std::vector < HypothesisStack* > &hypoStackColl = m_search->GetHypothesisStacks();
+  const HypothesisStack &finalStack = *hypoStackColl.back();
+  HypothesisStack::const_iterator iterHypo;
+  for (iterHypo = finalStack.begin() ; iterHypo != finalStack.end() ; ++iterHypo) {
+    const Hypothesis *hypo = *iterHypo;
+    connected[ hypo->GetId() ] = true;
+    connectedList.push_back( hypo );
+  }
+
+  // move back from known connected hypotheses
+  for(size_t i=0; i<connectedList.size(); i++) {
+    const Hypothesis *hypo = connectedList[i];
+
+    // add back pointer
+    const Hypothesis *prevHypo = hypo->GetPrevHypo();
+    if (prevHypo->GetId() > 0 // don't add empty hypothesis
+        && connected.find( prevHypo->GetId() ) == connected.end()) { // don't add already added
+      connected[ prevHypo->GetId() ] = true;
+      connectedList.push_back( prevHypo );
+    }
+
+    // add arcs
+    const ArcList *arcList = hypo->GetArcList();
+    if (arcList != NULL) {
+      ArcList::const_iterator iterArcList;
+      for (iterArcList = arcList->begin() ; iterArcList != arcList->end() ; ++iterArcList) {
+        const Hypothesis *loserHypo = *iterArcList;
+        if (connected.find( loserHypo->GetPrevHypo()->GetId() ) == connected.end() && loserHypo->GetPrevHypo()->GetId() > 0) { // don't add already added & don't add hyp 0
+          connected[ loserHypo->GetPrevHypo()->GetId() ] = true;
+          connectedList.push_back( loserHypo->GetPrevHypo() );
+        }
+      }
+    }
+  }
+}
+
+
+#ifdef HAVE_PROTOBUF
+
+void SerializeEdgeInfo(const Hypothesis* hypo, hgmert::Hypergraph_Edge* edge)
+{
+  hgmert::Rule* rule = edge->mutable_rule();
+  hypo->GetCurrTargetPhrase().WriteToRulePB(rule);
+  const Hypothesis* prev = hypo->GetPrevHypo();
+  // if the feature values are empty, they default to 0
+  if (!prev) return;
+  // score breakdown is an aggregate (forward) quantity, but the exported
+  // graph object just wants the feature values on the edges
+  const ScoreComponentCollection& scores = hypo->GetScoreBreakdown();
+  const ScoreComponentCollection& pscores = prev->GetScoreBreakdown();
+  for (unsigned int i = 0; i < scores.size(); ++i)
+    edge->add_feature_values((scores[i] - pscores[i]) * -1.0);
+}
+
+hgmert::Hypergraph_Node* GetHGNode(
+  const Hypothesis* hypo,
+  std::map< int, int>* i2hgnode,
+  hgmert::Hypergraph* hg,
+  int* hgNodeIdx)
+{
+  hgmert::Hypergraph_Node* hgnode;
+  std::map < int, int >::iterator idxi = i2hgnode->find(hypo->GetId());
+  if (idxi == i2hgnode->end()) {
+    *hgNodeIdx = ((*i2hgnode)[hypo->GetId()] = hg->nodes_size());
+    hgnode = hg->add_nodes();
+  } else {
+    *hgNodeIdx = idxi->second;
+    hgnode = hg->mutable_nodes(*hgNodeIdx);
+  }
+  return hgnode;
+}
+
+void Manager::SerializeSearchGraphPB(
+  long translationId,
+  std::ostream& outputStream) const
+{
+  using namespace hgmert;
+  std::map < int, bool > connected;
+  std::map < int, int > i2hgnode;
+  std::vector< const Hypothesis *> connectedList;
+  GetConnectedGraph(&connected, &connectedList);
+  connected[ 0 ] = true;
+  Hypergraph hg;
+  hg.set_is_sorted(false);
+  int num_feats = (*m_search->GetHypothesisStacks().back()->begin())->GetScoreBreakdown().size();
+  hg.set_num_features(num_feats);
+  StaticData::Instance().GetScoreIndexManager().SerializeFeatureNamesToPB(&hg);
+  Hypergraph_Node* goal = hg.add_nodes();  // idx=0 goal node must have idx 0
+  Hypergraph_Node* source = hg.add_nodes();  // idx=1
+  i2hgnode[-1] = 1; // source node
+  const std::vector < HypothesisStack* > &hypoStackColl = m_search->GetHypothesisStacks();
+  const HypothesisStack &finalStack = *hypoStackColl.back();
+  for (std::vector < HypothesisStack* >::const_iterator iterStack = hypoStackColl.begin();
+       iterStack != hypoStackColl.end() ; ++iterStack) {
+    const HypothesisStack &stack = **iterStack;
+    HypothesisStack::const_iterator iterHypo;
+
+    for (iterHypo = stack.begin() ; iterHypo != stack.end() ; ++iterHypo) {
+      const Hypothesis *hypo = *iterHypo;
+      bool is_goal = hypo->GetWordsBitmap().IsComplete();
+      if (connected.find( hypo->GetId() ) != connected.end()) {
+        int headNodeIdx;
+        Hypergraph_Node* headNode = GetHGNode(hypo, &i2hgnode, &hg, &headNodeIdx);
+        if (is_goal) {
+          Hypergraph_Edge* ge = hg.add_edges();
+          ge->set_head_node(0);  // goal
+          ge->add_tail_nodes(headNodeIdx);
+          ge->mutable_rule()->add_trg_words("[X,1]");
+        }
+        Hypergraph_Edge* edge = hg.add_edges();
+        SerializeEdgeInfo(hypo, edge);
+        edge->set_head_node(headNodeIdx);
+        const Hypothesis* prev = hypo->GetPrevHypo();
+        int tailNodeIdx = 1; // source
+        if (prev)
+          tailNodeIdx = i2hgnode.find(prev->GetId())->second;
+        edge->add_tail_nodes(tailNodeIdx);
+
+        const ArcList *arcList = hypo->GetArcList();
+        if (arcList != NULL) {
+          ArcList::const_iterator iterArcList;
+          for (iterArcList = arcList->begin() ; iterArcList != arcList->end() ; ++iterArcList) {
+            const Hypothesis *loserHypo = *iterArcList;
+            UTIL_THROW_IF2(!connected[loserHypo->GetId()],
+                           "Hypothesis " << loserHypo->GetId() << " is not connected");
+            Hypergraph_Edge* edge = hg.add_edges();
+            SerializeEdgeInfo(loserHypo, edge);
+            edge->set_head_node(headNodeIdx);
+            tailNodeIdx = i2hgnode.find(loserHypo->GetPrevHypo()->GetId())->second;
+            edge->add_tail_nodes(tailNodeIdx);
+          }
+        } // end if arcList empty
+      } // end if connected
+    } // end for iterHypo
+  } // end for iterStack
+  hg.SerializeToOstream(&outputStream);
+}
+#endif
+
+void
+Manager::
+OutputSearchGraph(long translationId, std::ostream &out) const
+{
+  vector<SearchGraphNode> searchGraph;
+  GetSearchGraph(searchGraph);
+  for (size_t i = 0; i < searchGraph.size(); ++i) {
+    OutputSearchNode(*options(),translationId,out,searchGraph[i]);
+  }
+}
+
+void
+Manager::
+GetForwardBackwardSearchGraph
+( std::map< int, bool >* pConnected,
+  std::vector<Hypothesis const* >* pConnectedList,
+  std::map<Hypothesis const*, set<Hypothesis const*> >* pOutgoingHyps,
+  vector< float>* pFwdBwdScores) const
+{
+  std::map < int, bool > &connected = *pConnected;
+  std::vector< const Hypothesis *>& connectedList = *pConnectedList;
+  std::map < int, int > forward;
+  std::map < int, double > forwardScore;
+
+  std::map < const Hypothesis*, set <const Hypothesis*> > & outgoingHyps
+  = *pOutgoingHyps;
+  vector< float> & estimatedScores = *pFwdBwdScores;
+
+  // *** find connected hypotheses ***
+  GetWinnerConnectedGraph(&connected, &connectedList);
+
+  // ** compute best forward path for each hypothesis *** //
+
+  // forward cost of hypotheses on final stack is 0
+  const std::vector < HypothesisStack* > &hypoStackColl
+  = m_search->GetHypothesisStacks();
+  const HypothesisStack &finalStack = *hypoStackColl.back();
+  HypothesisStack::const_iterator iterHypo;
+  for (iterHypo = finalStack.begin() ; iterHypo != finalStack.end() ; ++iterHypo) {
+    const Hypothesis *hypo = *iterHypo;
+    forwardScore[ hypo->GetId() ] = 0.0f;
+    forward[ hypo->GetId() ] = -1;
+  }
+
+  // compete for best forward score of previous hypothesis
+  std::vector < HypothesisStack* >::const_iterator iterStack;
+  for (iterStack = --hypoStackColl.end() ; iterStack != hypoStackColl.begin() ; --iterStack) {
+    const HypothesisStack &stack = **iterStack;
+    HypothesisStack::const_iterator iterHypo;
+    for (iterHypo = stack.begin() ; iterHypo != stack.end() ; ++iterHypo) {
+      const Hypothesis *hypo = *iterHypo;
+      if (connected.find( hypo->GetId() ) != connected.end()) {
+        // make a play for previous hypothesis
+        const Hypothesis *prevHypo = hypo->GetPrevHypo();
+        double fscore = forwardScore[ hypo->GetId() ] +
+                        hypo->GetScore() - prevHypo->GetScore();
+        if (forwardScore.find( prevHypo->GetId() ) == forwardScore.end()
+            || forwardScore.find( prevHypo->GetId() )->second < fscore) {
+          forwardScore[ prevHypo->GetId() ] = fscore;
+          forward[ prevHypo->GetId() ] = hypo->GetId();
+        }
+        //store outgoing info
+        outgoingHyps[prevHypo].insert(hypo);
+
+        // all arcs also make a play
+        const ArcList *arcList = hypo->GetArcList();
+        if (arcList != NULL) {
+          ArcList::const_iterator iterArcList;
+          for (iterArcList = arcList->begin() ; iterArcList != arcList->end() ; ++iterArcList) {
+            const Hypothesis *loserHypo = *iterArcList;
+            // make a play
+            const Hypothesis *loserPrevHypo = loserHypo->GetPrevHypo();
+            double fscore = forwardScore[ hypo->GetId() ] +
+                            loserHypo->GetScore() - loserPrevHypo->GetScore();
+            if (forwardScore.find( loserPrevHypo->GetId() ) == forwardScore.end()
+                || forwardScore.find( loserPrevHypo->GetId() )->second < fscore) {
+              forwardScore[ loserPrevHypo->GetId() ] = fscore;
+              forward[ loserPrevHypo->GetId() ] = loserHypo->GetId();
+            }
+            //store outgoing info
+            outgoingHyps[loserPrevHypo].insert(hypo);
+
+
+          } // end for arc list
+        } // end if arc list empty
+      } // end if hypo connected
+    } // end for hypo
+  } // end for stack
+
+  for (std::vector< const Hypothesis *>::iterator it = connectedList.begin(); it != connectedList.end(); ++it) {
+    float estimatedScore = (*it)->GetScore() + forwardScore[(*it)->GetId()];
+    estimatedScores.push_back(estimatedScore);
+  }
+}
+
+
+const Hypothesis *Manager::GetBestHypothesis() const
+{
+  return m_search->GetBestHypothesis();
+}
+
+int Manager::GetNextHypoId()
+{
+  GetSentenceStats().AddCreated(); // count created hypotheses
+  return m_hypoId++;
+}
+
+void Manager::ResetSentenceStats(const InputType& source)
+{
+  m_sentenceStats = std::auto_ptr<SentenceStats>(new SentenceStats(source));
+}
+SentenceStats& Manager::GetSentenceStats() const
+{
+  return *m_sentenceStats;
+
+}
+
+void Manager::OutputBest(OutputCollector *collector)  const
+{
+  long translationId = m_source.GetTranslationId();
+
+  Timer additionalReportingTime;
+
+  // apply decision rule and output best translation(s)
+  if (collector) {
+    ostringstream out;
+    ostringstream debug;
+    FixPrecision(debug,PRECISION);
+
+    // all derivations - send them to debug stream
+    if (options()->output.PrintAllDerivations) {
+      additionalReportingTime.start();
+      PrintAllDerivations(translationId, debug);
+      additionalReportingTime.stop();
+    }
+
+    Timer decisionRuleTime;
+    decisionRuleTime.start();
+
+    // MAP decoding: best hypothesis
+    const Hypothesis* bestHypo = NULL;
+    if (!options()->mbr.enabled) {
+      bestHypo = GetBestHypothesis();
+      if (bestHypo) {
+        if (options()->output.ReportHypoScore) {
+          out << bestHypo->GetFutureScore() << ' ';
+        }
+        if (options()->output.RecoverPath) {
+          bestHypo->OutputInput(out);
+          out << "||| ";
+        }
+
+        if (options()->output.PrintID) {
+          out << translationId << " ";
+        }
+
+        // VN : I put back the code for OutputPassthroughInformation
+        if (options()->output.PrintPassThrough) {
+          OutputPassthroughInformation(out, bestHypo);
+        }
+        // end of add back
+
+        if (options()->output.ReportSegmentation == 2) {
+          GetOutputLanguageModelOrder(out, bestHypo);
+        }
+        OutputSurface(out,*bestHypo, true);
+        if (options()->output.PrintAlignmentInfo) {
+          out << "||| ";
+          bestHypo->OutputAlignment(out, true);
+        }
+
+        IFVERBOSE(1) {
+          debug << "BEST TRANSLATION: " << *bestHypo << endl;
+        }
+      } else {
+        VERBOSE(1, "NO BEST TRANSLATION" << endl);
+      }
+
+      out << endl;
+    } // if (!staticData.UseMBR())
+
+    // MBR decoding (n-best MBR, lattice MBR, consensus)
+    else {
+      // we first need the n-best translations
+      size_t nBestSize = options()->mbr.size;
+      if (nBestSize <= 0) {
+        cerr << "ERROR: negative size for number of MBR candidate translations not allowed (option mbr-size)" << endl;
+        exit(1);
+      }
+      TrellisPathList nBestList;
+      CalcNBest(nBestSize, nBestList, true);
+      VERBOSE(2,"size of n-best: " << nBestList.GetSize() << " (" << nBestSize << ")" << endl);
+      IFVERBOSE(2) {
+        PrintUserTime("calculated n-best list for (L)MBR decoding");
+      }
+
+      // lattice MBR
+      if (options()->lmbr.enabled) {
+        if (options()->nbest.enabled) {
+          //lattice mbr nbest
+          vector<LatticeMBRSolution> solutions;
+          size_t n  = min(nBestSize, options()->nbest.nbest_size);
+          getLatticeMBRNBest(*this,nBestList,solutions,n);
+          OutputLatticeMBRNBest(m_latticeNBestOut, solutions, translationId);
+        } else {
+          //Lattice MBR decoding
+          vector<Word> mbrBestHypo = doLatticeMBR(*this,nBestList);
+          OutputBestHypo(mbrBestHypo, out);
+          IFVERBOSE(2) {
+            PrintUserTime("finished Lattice MBR decoding");
+          }
+        }
+      }
+
+      // consensus decoding
+      else if (options()->search.consensus) {
+        const TrellisPath &conBestHypo = doConsensusDecoding(*this,nBestList);
+        OutputBestHypo(conBestHypo, out);
+        OutputAlignment(m_alignmentOut, conBestHypo);
+        IFVERBOSE(2) {
+          PrintUserTime("finished Consensus decoding");
+        }
+      }
+
+      // n-best MBR decoding
+      else {
+        const TrellisPath &mbrBestHypo = doMBR(nBestList, *options());
+        OutputBestHypo(mbrBestHypo, out);
+        OutputAlignment(m_alignmentOut, mbrBestHypo);
+        IFVERBOSE(2) {
+          PrintUserTime("finished MBR decoding");
+        }
+      }
+    }
+
+    // report best translation to output collector
+    collector->Write(translationId,out.str(),debug.str());
+
+    decisionRuleTime.stop();
+    VERBOSE(1, "Line " << translationId << ": Decision rule took " << decisionRuleTime << " seconds total" << endl);
+  } // if (m_ioWrapper.GetSingleBestOutputCollector())
+
+}
+
+void Manager::OutputNBest(OutputCollector *collector) const
+{
+  if (collector == NULL) {
+    return;
+  }
+
+  if (options()->lmbr.enabled) {
+    if (options()->nbest.enabled) {
+      collector->Write(m_source.GetTranslationId(), m_latticeNBestOut.str());
+    }
+  } else {
+    TrellisPathList nBestList;
+    ostringstream out;
+    NBestOptions const& nbo = options()->nbest;
+    CalcNBest(nbo.nbest_size, nBestList, nbo.only_distinct);
+    OutputNBest(out, nBestList);
+    collector->Write(m_source.GetTranslationId(), out.str());
+  }
+
+}
+
+void
+Manager::
+OutputNBest(std::ostream& out, Moses::TrellisPathList const& nBestList) const
+{
+  NBestOptions const& nbo = options()->nbest;
+  bool reportAllFactors     = nbo.include_all_factors;
+  bool includeSegmentation  = nbo.include_segmentation;
+  bool includeWordAlignment = nbo.include_alignment_info;
+
+  TrellisPathList::const_iterator iter;
+  for (iter = nBestList.begin() ; iter != nBestList.end() ; ++iter) {
+    const TrellisPath &path = **iter;
+    const std::vector<const Hypothesis *> &edges = path.GetEdges();
+
+    // print the surface factor of the translation
+    out << m_source.GetTranslationId() << " ||| ";
+    for (int currEdge = (int)edges.size() - 1 ; currEdge >= 0 ; currEdge--) {
+      const Hypothesis &edge = *edges[currEdge];
+      OutputSurface(out, edge);
+    }
+    out << " |||";
+
+    // print scores with feature names
+    bool with_labels = options()->nbest.include_feature_labels;
+    path.GetScoreBreakdown()->OutputAllFeatureScores(out, with_labels);
+
+    // total
+    out << " ||| " << path.GetFutureScore();
+
+    //phrase-to-phrase segmentation
+    if (includeSegmentation) {
+      out << " |||";
+      for (int currEdge = (int)edges.size() - 2 ; currEdge >= 0 ; currEdge--) {
+        const Hypothesis &edge = *edges[currEdge];
+        const Range &sourceRange = edge.GetCurrSourceWordsRange();
+        Range targetRange = path.GetTargetWordsRange(edge);
+        out << " " << sourceRange.GetStartPos();
+        if (sourceRange.GetStartPos() < sourceRange.GetEndPos()) {
+          out << "-" << sourceRange.GetEndPos();
+        }
+        out<< "=" << targetRange.GetStartPos();
+        if (targetRange.GetStartPos() < targetRange.GetEndPos()) {
+          out<< "-" << targetRange.GetEndPos();
+        }
+      }
+    }
+
+    if (includeWordAlignment) {
+      out << " ||| ";
+      for (int currEdge = (int)edges.size() - 2 ; currEdge >= 0 ; currEdge--) {
+        const Hypothesis &edge = *edges[currEdge];
+        const Range &sourceRange = edge.GetCurrSourceWordsRange();
+        Range targetRange = path.GetTargetWordsRange(edge);
+        const int sourceOffset = sourceRange.GetStartPos();
+        const int targetOffset = targetRange.GetStartPos();
+        const AlignmentInfo &ai = edge.GetCurrTargetPhrase().GetAlignTerm();
+
+        OutputAlignment(out, ai, sourceOffset, targetOffset);
+
+      }
+    }
+
+    if (options()->output.RecoverPath) {
+      out << " ||| ";
+      OutputInput(out, edges[0]);
+    }
+
+    out << endl;
+  }
+
+  out << std::flush;
+}
+
+//////////////////////////////////////////////////////////////////////////
+/***
+ * print surface factor only for the given phrase
+ */
+void
+Manager::
+OutputSurface(std::ostream &out, Hypothesis const& edge, bool const recursive) const
+{
+  if (recursive && edge.GetPrevHypo()) {
+    OutputSurface(out,*edge.GetPrevHypo(), true);
+  }
+
+  std::vector<FactorType> outputFactorOrder = options()->output.factor_order;
+  UTIL_THROW_IF2(outputFactorOrder.size() == 0,
+                 "Must specific at least 1 output factor");
+
+  FactorType placeholderFactor = options()->input.placeholder_factor;
+  std::map<size_t, const Factor*> placeholders;
+  if (placeholderFactor != NOT_FOUND) {
+    // creates map of target position -> factor for placeholders
+    placeholders = GetPlaceholders(edge, placeholderFactor);
+  }
+
+  bool markUnknown = options()->unk.mark;
+  std::string const& fd = options()->output.factor_delimiter;
+
+  TargetPhrase const& phrase = edge.GetCurrTargetPhrase();
+  size_t size = phrase.GetSize();
+  for (size_t pos = 0 ; pos < size ; pos++) {
+    const Factor *factor = phrase.GetFactor(pos, outputFactorOrder[0]);
+    if (placeholders.size()) {
+      // do placeholders
+      std::map<size_t, const Factor*>::const_iterator iter = placeholders.find(pos);
+      if (iter != placeholders.end()) {
+        factor = iter->second;
+      }
+    }
+
+    UTIL_THROW_IF2(factor == NULL, "No factor 0 at position " << pos);
+
+    //preface surface form with UNK if marking unknowns
+    const Word &word = phrase.GetWord(pos);
+    if(markUnknown && word.IsOOV()) {
+      out << options()->unk.prefix;
+    }
+
+    out << *factor;
+    for (size_t i = 1 ; i < outputFactorOrder.size() ; i++) {
+      const Factor *factor = phrase.GetFactor(pos, outputFactorOrder[i]);
+      if (factor) out << fd << *factor;
+      //else        out << fd << UNKNOWN_FACTOR;
+    }
+
+    if(markUnknown && word.IsOOV()) {
+      out << options()->unk.suffix;
+    }
+
+    out << " ";
+
+  }
+
+  // trace ("report segmentation") option "-t" / "-tt"
+  int reportSegmentation = options()->output.ReportSegmentation;
+  if (reportSegmentation > 0 && phrase.GetSize() > 0) {
+    const Range &sourceRange = edge.GetCurrSourceWordsRange();
+    const int sourceStart = sourceRange.GetStartPos();
+    const int sourceEnd = sourceRange.GetEndPos();
+    out << "|" << sourceStart << "-" << sourceEnd;    // enriched "-tt"
+    if (reportSegmentation == 2) {
+      out << ",wa=";
+      const AlignmentInfo &ai = edge.GetCurrTargetPhrase().GetAlignTerm();
+      OutputAlignment(out, ai, 0, 0);
+      out << ",total=";
+      out << edge.GetScore() - edge.GetPrevHypo()->GetScore();
+      out << ",";
+      ScoreComponentCollection scoreBreakdown(edge.GetScoreBreakdown());
+      scoreBreakdown.MinusEquals(edge.GetPrevHypo()->GetScoreBreakdown());
+      bool with_labels = options()->nbest.include_feature_labels;
+      scoreBreakdown.OutputAllFeatureScores(out, with_labels);
+    }
+    out << "| ";
+  }
+}
+
+void
+Manager::
+OutputAlignment(ostream &out, const AlignmentInfo &ai,
+                size_t sourceOffset, size_t targetOffset) const
+{
+  typedef std::vector< const std::pair<size_t,size_t>* > AlignVec;
+  AlignVec alignments = ai.GetSortedAlignments(options()->output.WA_SortOrder);
+
+  AlignVec::const_iterator it;
+  for (it = alignments.begin(); it != alignments.end(); ++it) {
+    const std::pair<size_t,size_t> &alignment = **it;
+    out << alignment.first  + sourceOffset << "-"
+        << alignment.second + targetOffset << " ";
+  }
+
+}
+
+void
+Manager::
+OutputInput(std::ostream& os, const Hypothesis* hypo) const
+{
+  size_t len = hypo->GetInput().GetSize();
+  std::vector<const Phrase*> inp_phrases(len, 0);
+  OutputInput(inp_phrases, hypo);
+  for (size_t i=0; i<len; ++i)
+    if (inp_phrases[i]) os << *inp_phrases[i];
+}
+
+void Manager::OutputInput(std::vector<const Phrase*>& map, const Hypothesis* hypo) const
+{
+  if (hypo->GetPrevHypo()) {
+    OutputInput(map, hypo->GetPrevHypo());
+    map[hypo->GetCurrSourceWordsRange().GetStartPos()] = &hypo->GetTranslationOption().GetInputPath().GetPhrase();
+  }
+}
+
+std::map<size_t, const Factor*> Manager::GetPlaceholders(const Hypothesis &hypo, FactorType placeholderFactor) const
+{
+  const InputPath &inputPath = hypo.GetTranslationOption().GetInputPath();
+  const Phrase &inputPhrase = inputPath.GetPhrase();
+
+  std::map<size_t, const Factor*> ret;
+
+  for (size_t sourcePos = 0; sourcePos < inputPhrase.GetSize(); ++sourcePos) {
+    const Factor *factor = inputPhrase.GetFactor(sourcePos, placeholderFactor);
+    if (factor) {
+      TargetPhrase const& tp = hypo.GetTranslationOption().GetTargetPhrase();
+      std::set<size_t> targetPos = tp.GetAlignTerm().GetAlignmentsForSource(sourcePos);
+      UTIL_THROW_IF2(targetPos.size() != 1,
+                     "Placeholder should be aligned to 1, and only 1, word");
+      ret[*targetPos.begin()] = factor;
+    }
+  }
+
+  return ret;
+}
+
+void Manager::OutputLatticeSamples(OutputCollector *collector) const
+{
+  if (collector) {
+    TrellisPathList latticeSamples;
+    ostringstream out;
+    CalcLatticeSamples(options()->output.lattice_sample_size, latticeSamples);
+    OutputNBest(out,latticeSamples);
+    collector->Write(m_source.GetTranslationId(), out.str());
+  }
+
+}
+
+void Manager::OutputAlignment(OutputCollector *collector) const
+{
+  if (collector == NULL) {
+    return;
+  }
+
+  if (!m_alignmentOut.str().empty()) {
+    collector->Write(m_source.GetTranslationId(), m_alignmentOut.str());
+  } else {
+    std::vector<const Hypothesis *> edges;
+    const Hypothesis *currentHypo = GetBestHypothesis();
+    while (currentHypo) {
+      edges.push_back(currentHypo);
+      currentHypo = currentHypo->GetPrevHypo();
+    }
+    ostringstream out;
+    size_t targetOffset = 0;
+    BOOST_REVERSE_FOREACH(Hypothesis const* e, edges) {
+      const TargetPhrase &tp = e->GetCurrTargetPhrase();
+      size_t sourceOffset = e->GetCurrSourceWordsRange().GetStartPos();
+      OutputAlignment(out, tp.GetAlignTerm(), sourceOffset, targetOffset);
+      targetOffset += tp.GetSize();
+    }
+    out << std::endl; // Used by --alignment-output-file so requires endl
+    collector->Write(m_source.GetTranslationId(), out.str());
+
+  }
+}
+
+void
+Manager::
+OutputDetailedTranslationReport(OutputCollector *collector) const
+{
+  if (collector) {
+    ostringstream out;
+    FixPrecision(out,PRECISION);
+    TranslationAnalysis::PrintTranslationAnalysis(out, GetBestHypothesis());
+    collector->Write(m_source.GetTranslationId(),out.str());
+  }
+
+}
+
+void
+Manager::
+OutputUnknowns(OutputCollector *collector) const
+{
+  if (collector) {
+    long translationId = m_source.GetTranslationId();
+    const vector<const Phrase*>& unknowns = m_transOptColl->GetUnknownSources();
+    ostringstream out;
+    for (size_t i = 0; i < unknowns.size(); ++i) {
+      out << *(unknowns[i]);
+    }
+    out << endl;
+    collector->Write(translationId, out.str());
+  }
+
+}
+
+void
+Manager::
+OutputWordGraph(OutputCollector *collector) const
+{
+  if (collector) {
+    long translationId = m_source.GetTranslationId();
+    ostringstream out;
+    FixPrecision(out,PRECISION);
+    GetWordGraph(translationId, out);
+    collector->Write(translationId, out.str());
+  }
+}
+
+void
+Manager::
+OutputSearchGraph(OutputCollector *collector) const
+{
+  if (collector) {
+    long translationId = m_source.GetTranslationId();
+    ostringstream out;
+    FixPrecision(out,PRECISION);
+    OutputSearchGraph(translationId, out);
+    collector->Write(translationId, out.str());
+
+#ifdef HAVE_PROTOBUF
+    const StaticData &staticData = StaticData::Instance();
+    if (staticData.GetOutputSearchGraphPB()) {
+      ostringstream sfn;
+      sfn << staticData.GetParam("output-search-graph-pb")[0] << '/' << translationId << ".pb" << ends;
+      string fn = sfn.str();
+      VERBOSE(2, "Writing search graph to " << fn << endl);
+      fstream output(fn.c_str(), ios::trunc | ios::binary | ios::out);
+      SerializeSearchGraphPB(translationId, output);
+    }
+#endif
+  }
+
+}
+
+void Manager::OutputSearchGraphSLF() const
+{
+  // const StaticData &staticData = StaticData::Instance();
+  long translationId = m_source.GetTranslationId();
+
+  // Output search graph in HTK standard lattice format (SLF)
+  std::string const& slf = options()->output.SearchGraphSLF;
+  if (slf.size()) {
+    util::StringStream fileName;
+    fileName << slf << "/" << translationId << ".slf";
+    ofstream *file = new ofstream;
+    file->open(fileName.str().c_str());
+    if (file->is_open() && file->good()) {
+      ostringstream out;
+      FixPrecision(out,PRECISION);
+      OutputSearchGraphAsSLF(translationId, out);
+      *file << out.str();
+      file -> flush();
+    } else {
+      TRACE_ERR("Cannot output HTK standard lattice for line " << translationId << " because the output file is not open or not ready for writing" << endl);
+    }
+    delete file;
+  }
+
+}
+
+void Manager::OutputLatticeMBRNBest(std::ostream& out, const vector<LatticeMBRSolution>& solutions,long translationId) const
+{
+  for (vector<LatticeMBRSolution>::const_iterator si = solutions.begin(); si != solutions.end(); ++si) {
+    out << translationId;
+    out << " |||";
+    const vector<Word> mbrHypo = si->GetWords();
+    for (size_t i = 0 ; i < mbrHypo.size() ; i++) {
+      const Factor *factor = mbrHypo[i].GetFactor(options()->output.factor_order[0]);
+      if (i>0) out << " " << *factor;
+      else     out << *factor;
+    }
+    out << " |||";
+    out << " map: " << si->GetMapScore();
+    out << " w: " << mbrHypo.size();
+    const vector<float>& ngramScores = si->GetNgramScores();
+    for (size_t i = 0; i < ngramScores.size(); ++i) {
+      out << " " << ngramScores[i];
+    }
+    out << " ||| " << si->GetScore();
+
+    out << endl;
+  }
+}
+
+void
+Manager::
+OutputBestHypo(const std::vector<Word>&  mbrBestHypo, ostream& out) const
+{
+  FactorType f = options()->output.factor_order[0];
+  for (size_t i = 0 ; i < mbrBestHypo.size() ; i++) {
+    const Factor *factor = mbrBestHypo[i].GetFactor(f);
+    UTIL_THROW_IF2(factor == NULL, "No factor " << f << " at position " << i);
+    if (i) out << " ";
+    out << *factor;
+  }
+  out << endl;
+}
+
+void
+Manager::
+OutputBestHypo(const Moses::TrellisPath &path, std::ostream &out) const
+{
+  std::vector<const Hypothesis *> const& edges = path.GetEdges();
+  for (int currEdge = (int)edges.size() - 1 ; currEdge >= 0 ; currEdge--) {
+    Hypothesis const& edge = *edges[currEdge];
+    OutputSurface(out, edge);
+  }
+  out << endl;
+}
+
+void
+Manager::
+OutputAlignment(std::ostringstream &out, const TrellisPath &path) const
+{
+  WordAlignmentSort waso = options()->output.WA_SortOrder;
+  BOOST_REVERSE_FOREACH(Hypothesis const* e, path.GetEdges())
+  e->OutputAlignment(out, false);
+  // Hypothesis::OutputAlignment(out, path.GetEdges(), waso);
+  // Used by --alignment-output-file so requires endl
+  out << std::endl;
+}
+
+} // namespace

mosesdecoder/moses/MockHypothesis.h

@@ -0,0 +1,97 @@
+// -*- c++ -*-
+/***********************************************************************
+Moses - factored phrase-based language decoder
+Copyright (C) 2010 University of Edinburgh
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Lesser General Public
+License as published by the Free Software Foundation; either
+version 2.1 of the License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public
+License along with this library; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+***********************************************************************/
+
+#ifndef _MOCK_HYPOTHESIS_
+#define _MOCK_HYPOTHESIS_
+
+#include <memory>
+#include <vector>
+
+#include "moses/FF/UnknownWordPenaltyProducer.h"
+#include "moses/FF/DistortionScoreProducer.h"
+#include "moses/FF/WordPenaltyProducer.h"
+#include "Hypothesis.h"
+#include "Manager.h"
+#include "TranslationOption.h"
+
+namespace MosesTest
+{
+
+//
+// Construct a hypothesis with arbitrary source and target phrase
+// sequences. Useful for testing feature functions.
+//
+
+typedef std::pair<size_t,size_t> Alignment; //(first,last) in source
+
+class MockHypothesisGuard
+{
+public:
+  /** Creates a phrase-based hypothesis.
+  */
+  MockHypothesisGuard
+  ( const std::string& sourceSentence,
+    const std::vector<Alignment>& alignments,
+    const std::vector<std::string>& targetSegments);
+
+  Moses::Hypothesis* operator*() const {
+    return m_hypothesis;
+  }
+
+  /** Destroy the hypothesis chain */
+  ~MockHypothesisGuard();
+
+private:
+  Moses::TranslationOption m_initialTransOpt;
+  boost::shared_ptr<Moses::Sentence> m_sentence;
+  Moses::WordPenaltyProducer m_wp;
+  Moses::UnknownWordPenaltyProducer m_uwp;
+  Moses::DistortionScoreProducer m_dist;
+  boost::shared_ptr<Moses::Manager> m_manager;
+  boost::shared_ptr<Moses::TranslationTask> m_ttask;
+  Moses::Hypothesis* m_hypothesis;
+  std::vector<Moses::TargetPhrase> m_targetPhrases;
+  std::vector<Moses::TranslationOption*> m_toptions;
+};
+
+class HypothesisFixture
+{
+public:
+  HypothesisFixture();
+  const Moses::Hypothesis* empty() {
+    return **m_empty;
+  }
+  const Moses::Hypothesis* partial() {
+    return **m_partial;
+  }
+  const Moses::Hypothesis* full() {
+    return **m_full;
+  }
+
+private:
+  std::auto_ptr<MockHypothesisGuard> m_empty;
+  std::auto_ptr<MockHypothesisGuard> m_partial;
+  std::auto_ptr<MockHypothesisGuard> m_full;
+};
+
+
+}
+
+#endif

mosesdecoder/moses/OutputFileStream.h

@@ -0,0 +1,81 @@
+// $Id: InputFileStream.h 2939 2010-02-24 11:15:44Z jfouet $
+
+/***********************************************************************
+ Moses - factored phrase-based language decoder
+ Copyright (C) 2006 University of Edinburgh
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ ***********************************************************************/
+
+#pragma once
+
+#include <cstdlib>
+#include <fstream>
+#include <string>
+#include <iostream>
+#include <boost/iostreams/filtering_stream.hpp>
+
+namespace Moses
+{
+
+/** Version of std::ostream with transparent compression.
+ *
+ * Transparently compresses output when writing to a file whose name ends in
+ * ".gz".  Or, writes to stdout instead of a file when given a filename
+ * consisting of just a dash ("-").
+ */
+class OutputFileStream : public boost::iostreams::filtering_ostream
+{
+private:
+  /** File that needs flushing & closing when we close this stream.
+   *
+   * Is NULL when no file is opened, e.g. when writing to standard output.
+   */
+  std::ofstream *m_outFile;
+
+  /// Is this stream open?
+  bool m_open;
+
+public:
+  /** Create an unopened OutputFileStream.
+   *
+   * Until it's been opened, nothing can be done with this stream.
+   */
+  OutputFileStream();
+
+  /// Create an OutputFileStream, and open it by calling Open().
+  OutputFileStream(const std::string &filePath);
+  virtual ~OutputFileStream();
+
+  // TODO: Can we please just always throw an exception when this fails?
+  /** Open stream.
+   *
+   * If filePath is "-" (just a dash), this opens the stream for writing to
+   * standard output.  Otherwise, it opens the given file.  If the filename
+   * has the ".gz" suffix, output will be transparently compressed.
+   *
+   * Call Close() to close the file.
+   *
+   * Returns whether opening the file was successful.  It may also throw an
+   * exception on failure.
+   */
+  bool Open(const std::string &filePath);
+
+  /// Flush and close stream.  After this, the stream can be opened again.
+  void Close();
+};
+
+}
+

mosesdecoder/moses/PCNTools.h

@@ -0,0 +1,67 @@
+// $Id$
+
+/***********************************************************************
+Moses - factored phrase-based language decoder
+Copyright (C) 2006 University of Edinburgh
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Lesser General Public
+License as published by the Free Software Foundation; either
+version 2.1 of the License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public
+License along with this library; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+***********************************************************************/
+
+#ifndef moses_PCNTools
+#define moses_PCNTools
+
+#include <vector>
+#include <map>
+#include <string>
+#include <utility>
+#include <cstdlib>
+
+/** A couple of utilities to read .pcn files. A python-compatible format
+ * for encoding confusion networks and word lattices.
+ */
+namespace PCN
+{
+
+struct CNAlt {
+  CNAlt() {
+  }
+  CNAlt(const std::string &word,
+        const std::vector<float> &denseFeatures,
+        const std::map<std::string, float> &sparseFeatures,
+        size_t next)
+    :m_word(word)
+    ,m_denseFeatures(denseFeatures)
+    ,m_sparseFeatures(sparseFeatures)
+    ,m_next(next) {
+  }
+
+  std::string m_word;
+  std::vector<float> m_denseFeatures;
+  std::map<std::string, float> m_sparseFeatures;
+  size_t m_next;
+};
+
+//typedef std::pair<std::pair<std::string, std::vector<float> >, size_t> CNAlt;
+typedef std::vector<CNAlt> CNCol;
+typedef std::vector<CNCol> CN;
+
+/** Given a string ((('foo',0.1,1),('bar',0.9,2)),...) representation of a
+ * word lattice in PCN format, return a CN object representing the lattice
+ */
+CN parsePCN(const std::string& in);
+
+};
+
+#endif

mosesdecoder/moses/PDTAimp.cpp

@@ -0,0 +1,476 @@
+#include "PDTAimp.h"
+
+namespace Moses
+{
+
+PDTAimp::PDTAimp(PhraseDictionaryTreeAdaptor *p)
+  : m_dict(0),
+    m_obj(p),
+    useCache(1),
+    totalE(0),
+    distinctE(0)
+{
+  m_numInputScores = 0;
+  m_inputFeature = InputFeature::InstancePtr();
+
+  if (m_inputFeature) {
+    const PhraseDictionary *firstPt = PhraseDictionary::GetColl()[0];
+    if (firstPt == m_obj) {
+      m_numInputScores = m_inputFeature->GetNumScoreComponents();
+    }
+  }
+}
+
+PDTAimp::~PDTAimp()
+{
+  CleanUp();
+  delete m_dict;
+
+  if (StaticData::Instance().GetVerboseLevel() >= 2) {
+
+    TRACE_ERR("tgt candidates stats:  total="<<totalE<<";  distinct="
+              <<distinctE<<" ("<<distinctE/(0.01*totalE)<<");  duplicates="
+              <<totalE-distinctE<<" ("<<(totalE-distinctE)/(0.01*totalE)
+              <<")\n");
+
+    TRACE_ERR("\npath statistics\n");
+
+    if(path1Best.size()) {
+      TRACE_ERR("1-best:        ");
+      std::copy(path1Best.begin()+1,path1Best.end(),
+                std::ostream_iterator<size_t>(std::cerr," \t"));
+      TRACE_ERR("\n");
+    }
+    if(pathCN.size()) {
+      TRACE_ERR("CN (full):     ");
+      std::transform(pathCN.begin()+1
+                     ,pathCN.end()
+                     ,std::ostream_iterator<double>(std::cerr," \t")
+                     ,Exp);
+      TRACE_ERR("\n");
+    }
+    if(pathExplored.size()) {
+      TRACE_ERR("CN (explored): ");
+      std::copy(pathExplored.begin()+1,pathExplored.end(),
+                std::ostream_iterator<size_t>(std::cerr," \t"));
+      TRACE_ERR("\n");
+    }
+  }
+
+}
+
+void PDTAimp::CleanUp()
+{
+  assert(m_dict);
+  m_dict->FreeMemory();
+  // for(size_t i=0; i<m_tgtColls.size(); ++i) m_tgtColls[i].reset();
+  m_tgtColls.clear();
+  m_cache.clear();
+  m_rangeCache.clear();
+  uniqSrcPhr.clear();
+}
+
+TargetPhraseCollectionWithSourcePhrase::shared_ptr
+PDTAimp::GetTargetPhraseCollection(Phrase const &src) const
+{
+
+  assert(m_dict);
+
+  TargetPhraseCollectionWithSourcePhrase::shared_ptr ret;
+  if(src.GetSize()==0) return ret;
+
+  std::pair<MapSrc2Tgt::iterator,bool> piter;
+  if(useCache) {
+    piter=m_cache.insert(std::make_pair(src, ret));
+    if(!piter.second) return piter.first->second;
+  } else if (m_cache.size()) {
+    MapSrc2Tgt::const_iterator i=m_cache.find(src);
+    return (i!=m_cache.end() ? i->second : ret);
+  }
+
+  std::vector<std::string> srcString(src.GetSize());
+  // convert source Phrase into vector of strings
+  for(size_t i=0; i<srcString.size(); ++i) {
+    Factors2String(src.GetWord(i),srcString[i]);
+  }
+
+  // get target phrases in string representation
+  std::vector<StringTgtCand> cands;
+  std::vector<std::string> wacands;
+  m_dict->GetTargetCandidates(srcString,cands,wacands);
+  if(cands.empty()) {
+    return ret;
+  }
+
+  //TODO: Multiple models broken here
+  std::vector<float> weights = StaticData::Instance().GetWeights(m_obj);
+
+  std::vector<TargetPhrase> tCands;
+  tCands.reserve(cands.size());
+
+  std::vector<std::pair<float,size_t> > costs;
+  costs.reserve(cands.size());
+
+  std::vector<Phrase> sourcePhrases;
+  sourcePhrases.reserve(cands.size());
+
+
+  // convert into TargetPhrases
+  std::string fd = m_obj->options()->output.factor_delimiter;
+  for(size_t i=0; i<cands.size(); ++i) {
+    TargetPhrase targetPhrase(m_obj);
+
+    StringTgtCand::Tokens const& factorStrings=cands[i].tokens;
+    Scores const& probVector=cands[i].scores;
+
+    std::vector<float> scoreVector(probVector.size());
+    std::transform(probVector.begin(),probVector.end(),scoreVector.begin(),
+                   TransformScore);
+    std::transform(scoreVector.begin(),scoreVector.end(),scoreVector.begin(),
+                   FloorScore);
+
+    //sparse features.
+    //These are already in log-space
+    for (size_t j = 0; j < cands[i].fnames.size(); ++j) {
+      targetPhrase.GetScoreBreakdown().Assign(m_obj, *cands[i].fnames[j], cands[i].fvalues[j]);
+    }
+
+    CreateTargetPhrase(targetPhrase,factorStrings, fd, scoreVector, Scores(0),
+                       &wacands[i], &src);
+
+    costs.push_back(std::make_pair(-targetPhrase.GetFutureScore(),tCands.size()));
+    tCands.push_back(targetPhrase);
+
+    sourcePhrases.push_back(src);
+  }
+
+  ret = PruneTargetCandidates(tCands,costs, sourcePhrases);
+  if(ret->IsEmpty()) {
+    ret.reset();
+  } else {
+    if(useCache) piter.first->second = ret;
+    m_tgtColls.push_back(ret);
+  }
+  return ret;
+
+}
+
+void PDTAimp::Create(const std::vector<FactorType> &input
+                     , const std::vector<FactorType> &output
+                     , const std::string &filePath
+                     , const std::vector<float> &weight
+                    )
+{
+
+  // set my members
+  m_dict=new PhraseDictionaryTree();
+  m_input=input;
+  m_output=output;
+
+  const StaticData &staticData = StaticData::Instance();
+  m_dict->NeedAlignmentInfo(staticData.NeedAlignmentInfo());
+
+  std::string binFname=filePath+".binphr.idx";
+  if(!FileExists(binFname.c_str())) {
+    UTIL_THROW2( "bin ttable does not exist");
+    //TRACE_ERR( "bin ttable does not exist -> create it\n");
+    //InputFileStream in(filePath);
+    //m_dict->Create(in,filePath);
+  }
+  VERBOSE(1,"reading bin ttable\n");
+//		m_dict->Read(filePath);
+  bool res=m_dict->Read(filePath);
+  if (!res) {
+    std::cerr << "bin ttable was read in a wrong way\n";
+    exit(1);
+  }
+}
+
+
+void PDTAimp::CacheSource(ConfusionNet const& src)
+{
+  assert(m_dict);
+  const size_t srcSize=src.GetSize();
+
+  std::vector<size_t> exploredPaths(srcSize+1,0);
+  std::vector<double> exPathsD(srcSize+1,-1.0);
+
+  // collect some statistics
+  std::vector<size_t> cnDepths(srcSize,0);
+  for(size_t i=0; i<srcSize; ++i) cnDepths[i]=src[i].size();
+
+  for(size_t len=1; len<=srcSize; ++len)
+    for(size_t i=0; i<=srcSize-len; ++i) {
+      double pd=0.0;
+      for(size_t k=i; k<i+len; ++k)	pd+=log(1.0*cnDepths[k]);
+      exPathsD[len]=(exPathsD[len]>=0.0 ? addLogScale(pd,exPathsD[len]) : pd);
+    }
+
+  // update global statistics
+  if(pathCN.size()<=srcSize) pathCN.resize(srcSize+1,-1.0);
+  for(size_t len=1; len<=srcSize; ++len)
+    pathCN[len]=pathCN[len]>=0.0 ? addLogScale(pathCN[len],exPathsD[len]) : exPathsD[len];
+
+  if(path1Best.size()<=srcSize) path1Best.resize(srcSize+1,0);
+  for(size_t len=1; len<=srcSize; ++len) path1Best[len]+=srcSize-len+1;
+
+
+  if (StaticData::Instance().GetVerboseLevel() >= 2 && exPathsD.size()) {
+    TRACE_ERR("path stats for current CN: \nCN (full):     ");
+    std::transform(exPathsD.begin()+1
+                   ,exPathsD.end()
+                   ,std::ostream_iterator<double>(std::cerr," ")
+                   ,Exp);
+    TRACE_ERR("\n");
+  }
+
+  typedef std::map<StringTgtCand::Tokens,TScores> E2Costs;
+
+  std::map<Range,E2Costs> cov2cand;
+  std::vector<State> stack;
+  for(Position i=0 ; i < srcSize ; ++i)
+    stack.push_back(State(i, i, m_dict->GetRoot(), std::vector<float>(m_numInputScores,0.0)));
+
+  std::vector<float> weightTrans = StaticData::Instance().GetWeights(m_obj);
+  std::vector<float> weightInput = StaticData::Instance().GetWeights(m_inputFeature);
+  float weightWP = StaticData::Instance().GetWeightWordPenalty();
+
+  while(!stack.empty()) {
+    State curr(stack.back());
+    stack.pop_back();
+
+    UTIL_THROW_IF2(curr.end() >= srcSize, "Error");
+    const ConfusionNet::Column &currCol=src[curr.end()];
+    // in a given column, loop over all possibilities
+    for(size_t colidx=0; colidx<currCol.size(); ++colidx) {
+      const Word& w=currCol[colidx].first; // w=the i^th possibility in column colidx
+      std::string s;
+      Factors2String(w,s);
+      bool isEpsilon=(s=="" || s==EPSILON);
+
+      //assert that we have the right number of link params in this CN option
+      UTIL_THROW_IF2(currCol[colidx].second.denseScores.size() < m_numInputScores,
+                     "Incorrect number of input scores");
+
+      // do not start with epsilon (except at first position)
+      if(isEpsilon && curr.begin()==curr.end() && curr.begin()>0) continue;
+
+      // At a given node in the prefix tree, look to see if w defines an edge to
+      // another node (Extend).  Stay at the same node if w==EPSILON
+      PPtr nextP = (isEpsilon ? curr.ptr : m_dict->Extend(curr.ptr,s));
+
+      if(nextP) { // w is a word that should be considered
+        Range newRange(curr.begin(),curr.end()+src.GetColumnIncrement(curr.end(),colidx));
+
+        //add together the link scores from the current state and the new arc
+        float inputScoreSum = 0;
+        std::vector<float> newInputScores(m_numInputScores,0.0);
+        if (m_numInputScores) {
+          std::transform(currCol[colidx].second.denseScores.begin(), currCol[colidx].second.denseScores.end(),
+                         curr.GetScores().begin(),
+                         newInputScores.begin(),
+                         std::plus<float>());
+
+
+          //we need to sum up link weights (excluding realWordCount, which isn't in numLinkParams)
+          //if the sum is too low, then we won't expand this.
+          //TODO: dodgy! shouldn't we consider weights here? what about zero-weight params?
+          inputScoreSum = std::accumulate(newInputScores.begin(),newInputScores.begin()+m_numInputScores,0.0);
+        }
+
+        Phrase newSrc(curr.src);
+        if(!isEpsilon) newSrc.AddWord(w);
+        if(newRange.second<srcSize && inputScoreSum>LOWEST_SCORE) {
+          // if there is more room to grow, add a new state onto the queue
+          // to be explored that represents [begin, curEnd+)
+          stack.push_back(State(newRange,nextP,newInputScores));
+          stack.back().src=newSrc;
+        }
+
+        std::vector<StringTgtCand> tcands;
+        // now, look up the target candidates (aprx. TargetPhraseCollection) for
+        // the current path through the CN
+        m_dict->GetTargetCandidates(nextP,tcands);
+
+        if(newRange.second>=exploredPaths.size()+newRange.first)
+          exploredPaths.resize(newRange.second-newRange.first+1,0);
+        ++exploredPaths[newRange.second-newRange.first];
+
+        totalE+=tcands.size();
+
+        if(tcands.size()) {
+          E2Costs& e2costs=cov2cand[newRange];
+          Phrase const* srcPtr=uniqSrcPhr(newSrc);
+          for(size_t i=0; i<tcands.size(); ++i) {
+            //put input scores in first - already logged, just drop in directly
+            std::vector<float> transcores(m_obj->GetNumScoreComponents());
+            UTIL_THROW_IF2(transcores.size() != weightTrans.size(),
+                           "Incorrect number of translation scores");
+
+            //put in phrase table scores, logging as we insert
+            std::transform(tcands[i].scores.begin()
+                           ,tcands[i].scores.end()
+                           ,transcores.begin()
+                           ,TransformScore);
+
+
+            //tally up
+            float score=std::inner_product(transcores.begin(), transcores.end(), weightTrans.begin(), 0.0f);
+
+            // input feature
+            score +=std::inner_product(newInputScores.begin(), newInputScores.end(), weightInput.begin(), 0.0f);
+
+            //count word penalty
+            score-=tcands[i].tokens.size() * weightWP;
+
+            std::pair<E2Costs::iterator,bool> p=e2costs.insert(std::make_pair(tcands[i].tokens,TScores()));
+
+            if(p.second) ++distinctE;
+
+            TScores & scores=p.first->second;
+            if(p.second || scores.total<score) {
+              scores.total=score;
+              scores.transScore=transcores;
+              scores.inputScores=newInputScores;
+              scores.src=srcPtr;
+            }
+          }
+        }
+      }
+    }
+  } // end while(!stack.empty())
+
+
+  if (StaticData::Instance().GetVerboseLevel() >= 2 && exploredPaths.size()) {
+    TRACE_ERR("CN (explored): ");
+    std::copy(exploredPaths.begin()+1,exploredPaths.end(),
+              std::ostream_iterator<size_t>(std::cerr," "));
+    TRACE_ERR("\n");
+  }
+
+  if(pathExplored.size()<exploredPaths.size())
+    pathExplored.resize(exploredPaths.size(),0);
+  for(size_t len=1; len<=srcSize; ++len)
+    pathExplored[len]+=exploredPaths[len];
+
+
+  // m_rangeCache.resize(src.GetSize(),vTPC(src.GetSize(),0));
+  m_rangeCache.resize(src.GetSize(),vTPC(src.GetSize()));
+
+  for(std::map<Range,E2Costs>::const_iterator i=cov2cand.begin(); i!=cov2cand.end(); ++i) {
+    assert(i->first.first<m_rangeCache.size());
+    assert(i->first.second>0);
+    assert(static_cast<size_t>(i->first.second-1)<m_rangeCache[i->first.first].size());
+    assert(m_rangeCache[i->first.first][i->first.second-1]==0);
+
+    std::vector<TargetPhrase> tCands;
+    tCands.reserve(i->second.size());
+
+    std::vector<std::pair<float,size_t> > costs;
+    costs.reserve(i->second.size());
+
+    std::vector<Phrase> sourcePhrases;
+    sourcePhrases.reserve(i->second.size());
+
+    for(E2Costs::const_iterator j=i->second.begin(); j!=i->second.end(); ++j) {
+      TScores const & scores=j->second;
+      TargetPhrase targetPhrase(m_obj);
+      CreateTargetPhrase(targetPhrase
+                         , j ->first
+                         , m_obj->options()->output.factor_delimiter
+                         , scores.transScore
+                         , scores.inputScores
+                         , NULL
+                         , scores.src);
+      costs.push_back(std::make_pair(-targetPhrase.GetFutureScore(),tCands.size()));
+      tCands.push_back(targetPhrase);
+
+      sourcePhrases.push_back(*scores.src);
+
+      //std::cerr << i->first.first << "-" << i->first.second << ": " << targetPhrase << std::endl;
+    }
+
+    TargetPhraseCollectionWithSourcePhrase::shared_ptr
+    rv = PruneTargetCandidates(tCands, costs, sourcePhrases);
+
+    if(rv->IsEmpty())
+      rv.reset();
+    else {
+      m_rangeCache[i->first.first][i->first.second-1]=rv;
+      m_tgtColls.push_back(rv);
+    }
+  }
+  // free memory
+  m_dict->FreeMemory();
+}
+
+void PDTAimp::CreateTargetPhrase(TargetPhrase& targetPhrase,
+                                 StringTgtCand::Tokens const& factorStrings,
+                                 std::string const& factorDelimiter,
+                                 Scores const& transVector,
+                                 Scores const& inputVector,
+                                 const std::string *alignmentString,
+                                 Phrase const* srcPtr) const
+{
+  FactorCollection &factorCollection = FactorCollection::Instance();
+
+  for(size_t k=0; k<factorStrings.size(); ++k) {
+    util::TokenIter<util::MultiCharacter, false>
+    word(*factorStrings[k], factorDelimiter);
+    Word& w=targetPhrase.AddWord();
+    for(size_t l=0; l<m_output.size(); ++l, ++word) {
+      w[m_output[l]]= factorCollection.AddFactor(*word);
+    }
+  }
+
+  if (alignmentString) {
+    targetPhrase.SetAlignmentInfo(*alignmentString);
+  }
+
+  if (m_numInputScores) {
+    targetPhrase.GetScoreBreakdown().Assign(m_inputFeature, inputVector);
+  }
+
+  targetPhrase.GetScoreBreakdown().Assign(m_obj, transVector);
+  targetPhrase.EvaluateInIsolation(*srcPtr, m_obj->GetFeaturesToApply());
+}
+
+TargetPhraseCollectionWithSourcePhrase::shared_ptr
+PDTAimp::PruneTargetCandidates
+(const std::vector<TargetPhrase> & tCands,
+ std::vector<std::pair<float,size_t> >& costs,
+ const std::vector<Phrase> &sourcePhrases) const
+{
+  // convert into TargetPhraseCollection
+  UTIL_THROW_IF2(tCands.size() != sourcePhrases.size(),
+                 "Number of target phrases must equal number of source phrases");
+
+  TargetPhraseCollectionWithSourcePhrase::shared_ptr rv;
+  rv.reset(new TargetPhraseCollectionWithSourcePhrase);
+
+
+  // set limit to tableLimit or actual size, whatever is smaller
+  std::vector<std::pair<float,size_t> >::iterator nth =
+    costs.begin() + ((m_obj->m_tableLimit>0 && // 0 indicates no limit
+                      m_obj->m_tableLimit < costs.size()) ?
+                     m_obj->m_tableLimit : costs.size());
+
+  // find the nth phrase according to future cost
+  NTH_ELEMENT3(costs.begin(),nth ,costs.end());
+
+  // add n top phrases to the return list
+  for(std::vector<std::pair<float,size_t> >::iterator
+      it = costs.begin(); it != nth; ++it) {
+    size_t ind = it->second;
+    TargetPhrase *targetPhrase = new TargetPhrase(tCands[ind]);
+    const Phrase &sourcePhrase = sourcePhrases[ind];
+    rv->Add(targetPhrase, sourcePhrase);
+
+  }
+
+  return rv;
+}
+
+}
+
+

mosesdecoder/moses/Parameter.cpp

@@ -0,0 +1,1690 @@
+// $Id$
+
+/***********************************************************************
+Moses - factored phrase-based language decoder
+Copyright (C) 2006 University of Edinburgh
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Lesser General Public
+License as published by the Free Software Foundation; either
+version 2.1 of the License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public
+License along with this library; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+***********************************************************************/
+
+#include <ctime>
+#include <iostream>
+#include <iterator>
+#include <fstream>
+#include <sstream>
+#include <algorithm>
+#include <boost/algorithm/string/predicate.hpp>
+#include "Parameter.h"
+#include "Util.h"
+#include "InputFileStream.h"
+#include "StaticData.h"
+#include "util/string_stream.hh"
+#include "util/exception.hh"
+#include "util/random.hh"
+#include <boost/program_options.hpp>
+
+#ifdef HAVE_XMLRPC_C
+#include <xmlrpc_server.h>
+#endif
+
+using namespace std;
+using namespace boost::algorithm;
+namespace po = boost::program_options;
+
+namespace Moses
+{
+
+/** define allowed parameters */
+Parameter::Parameter()
+{
+  ///////////////////////////////////////////////////////////////////////////////////////
+  // general options
+  po::options_description main_opts("Main Options");
+  AddParam(main_opts,"config", "f", "location of the configuration file");
+  AddParam(main_opts,"input-file", "i", "location of the input file to be translated");
+
+  AddParam(main_opts,"verbose", "v", "verbosity level of the logging");
+  AddParam(main_opts,"version", "show version of Moses and libraries used");
+  AddParam(main_opts,"show-weights", "print feature weights and exit");
+  AddParam(main_opts,"time-out", "seconds after which is interrupted (-1=no time-out, default is -1)");
+  AddParam(main_opts,"segment-time-out", "seconds for single segment after which is interrupted (-1=no time-out, default is -1)");
+
+  ///////////////////////////////////////////////////////////////////////////////////////
+  // factorization options
+  po::options_description factor_opts("General Factorization Options");
+  AddParam(factor_opts,"factor-delimiter", "fd", "specify a different factor delimiter than the default");
+  // one should be able to specify different factor delimiters for intput and output
+  AddParam(factor_opts,"mapping", "description of decoding steps"); // whatever that means ...
+  AddParam(factor_opts,"placeholder-factor", "Which source factor to use to store the original text for placeholders. The factor must not be used by a translation or gen model");
+
+  ///////////////////////////////////////////////////////////////////////////////////////
+  // general search options
+  po::options_description search_opts("Search Options");
+  string desc = "Which search algorithm to use.\n";
+  desc += "0=normal stack (default)\n";
+  desc += "1=cube pruning\n";
+  desc += "3=chart (with cube pruning)\n";
+  desc += "4=stack with batched lm requests\n";
+  desc += "5=chart (with incremental search)\n";
+  desc += "6=string-to-tree\n";
+  desc += "7=tree-to-string\n";
+  desc += "8=tree-to-string (SCFG-based)\n";
+  desc += "9=forest-to-string";
+  AddParam(search_opts,"search-algorithm", desc);
+  AddParam(search_opts,"beam-threshold", "b", "threshold for threshold pruning");
+  AddParam(search_opts,"early-discarding-threshold", "edt", "threshold for constructing hypotheses based on estimate cost");
+  AddParam(search_opts,"stack", "s", "maximum stack size for histogram pruning. 0 = unlimited stack size");
+  AddParam(search_opts,"stack-diversity", "sd", "minimum number of hypothesis of each coverage in stack (default 0)");
+
+  // feature weight-related options
+  AddParam(search_opts,"weight-file", "wf", "feature weights file. Do *not* put weights for 'core' features in here - they go in moses.ini");
+  AddParam(search_opts,"weight", "weights for ALL models, 1 per line 'WeightName value'. Weight names can be repeated");
+
+  AddParam(search_opts,"feature-overwrite", "Override arguments in a particular feature function with a particular key. Format: -feature-overwrite \"FeatureName key=value\"");
+
+  po::options_description tune_opts("Options used in tuning.");
+  AddParam(tune_opts,"weight-overwrite", "special parameter for mert. All on 1 line. Overrides weights specified in 'weights' argument");
+  AddParam(tune_opts,"feature-add", "Add a feature function on the command line. Used by mira to add BLEU feature");
+  AddParam(tune_opts,"weight-add", "Add weight for FF if it doesn't exist, i.e weights here are added 1st, and can be override by the ini file or on the command line. Used to specify initial weights for FF that was also specified on the copmmand line");
+
+  // phrase table limitations:
+  AddParam(search_opts,"max-partial-trans-opt", "maximum number of partial translation options per input span (during mapping steps)");
+  AddParam(search_opts,"max-trans-opt-per-coverage", "maximum number of translation options per input span (after applying mapping steps)");
+  AddParam(search_opts,"max-phrase-length", "maximum phrase length (default 20)");
+  AddParam(search_opts,"translation-option-threshold", "tot", "threshold for translation options relative to best for input phrase");
+
+  // miscellaneous search options
+  AddParam(search_opts,"disable-discarding", "dd", "disable hypothesis discarding"); // ??? memory management? UG
+  AddParam(search_opts,"phrase-drop-allowed", "da", "if present, allow dropping of source words"); //da = drop any (word); see -du for comparison
+  AddParam(search_opts,"threads","th", "number of threads to use in decoding (defaults to single-threaded)");
+
+  // distortion options
+  po::options_description disto_opts("Distortion options");
+  AddParam(disto_opts,"distortion-limit", "dl", "distortion (reordering) limit in maximum number of words (0 = monotone, -1 = unlimited)");
+  AddParam(disto_opts,"monotone-at-punctuation", "mp", "do not reorder over punctuation");
+  AddParam(disto_opts,"early-distortion-cost", "edc", "include estimate of distortion cost yet to be incurred in the score [Moore & Quirk 2007]. Default is no");
+  AddParam(disto_opts,"distortion", "configurations for each factorized/lexicalized reordering model."); // zombie parameter?
+
+  // cube pruning
+  po::options_description cube_opts("Cube pruning options.");
+  AddParam(cube_opts,"cube-pruning-pop-limit", "cbp", "How many hypotheses should be popped for each stack. (default = 1000)");
+  AddParam(cube_opts,"cube-pruning-diversity", "cbd", "How many hypotheses should be created for each coverage. (default = 0)");
+  AddParam(cube_opts,"cube-pruning-lazy-scoring", "cbls", "Don't fully score a hypothesis until it is popped");
+  AddParam(cube_opts,"cube-pruning-deterministic-search", "cbds", "Break ties deterministically during search");
+
+  ///////////////////////////////////////////////////////////////////////////////////////
+  // minimum bayes risk decoding
+  po::options_description mbr_opts("Minimum Bayes Risk (MBR), Lattice MBR, and Consensus decoding");
+
+  AddParam(mbr_opts,"minimum-bayes-risk", "mbr", "use miminum Bayes risk to determine best translation");
+  AddParam(mbr_opts,"mbr-size", "number of translation candidates considered in MBR decoding (default 200)");
+  AddParam(mbr_opts,"mbr-scale", "scaling factor to convert log linear score probability in MBR decoding (default 1.0)");
+
+  AddParam(mbr_opts,"lminimum-bayes-risk", "lmbr", "use lattice miminum Bayes risk to determine best translation");
+  AddParam(mbr_opts,"consensus-decoding", "con", "use consensus decoding (De Nero et. al. 2009)");
+
+  po::options_description lmbr_opts("Options specific to Lattic MBR");
+  AddParam(lmbr_opts,"lmbr-p", "unigram precision value for lattice mbr");
+  AddParam(lmbr_opts,"lmbr-r", "ngram precision decay value for lattice mbr");
+  AddParam(lmbr_opts,"lmbr-thetas", "theta(s) for lattice mbr calculation");
+  AddParam(mbr_opts,"lmbr-map-weight", "weight given to map solution when doing lattice MBR (default 0)");
+  AddParam(mbr_opts,"lmbr-pruning-factor", "average number of nodes/word wanted in pruned lattice");
+  AddParam(mbr_opts,"lattice-hypo-set", "to use lattice as hypo set during lattice MBR");
+
+  ///////////////////////////////////////////////////////////////////////////////////////
+  // OOV handling options
+  po::options_description oov_opts("OOV Handling Options");
+  AddParam(oov_opts,"drop-unknown", "du", "drop unknown words instead of copying them");
+  AddParam(oov_opts,"mark-unknown", "mu", "mark unknown words in output");
+  AddParam(oov_opts,"unknown-word-prefix", "prefix to unknwon word when marked (default: 'UNK')");
+  AddParam(oov_opts,"unknown-word-suffix", "suffix to unknwon word when marked (default: '')");
+  AddParam(oov_opts,"lmodel-oov-feature", "add language model oov feature, one per model");
+  AddParam(oov_opts,"output-unknowns", "Output the unknown (OOV) words to the given file, one line per sentence");
+  AddParam(oov_opts,"always-create-direct-transopt", "Always create a translation that translates the source word ad-verbatim");
+
+  ///////////////////////////////////////////////////////////////////////////////////////
+  // input options
+  po::options_description input_opts("Input Format Options");
+  AddParam(input_opts,"input-factors", "list of factors in the input");
+  AddParam(input_opts,"inputtype", "text (0), confusion network (1), word lattice (2), tree (3) (default = 0)");
+  AddParam(input_opts,"xml-input", "xi", "allows markup of input with desired translations and probabilities. values can be 'pass-through' (default), 'inclusive', 'exclusive', 'constraint', 'ignore'");
+  AddParam(input_opts,"xml-brackets", "xb", "specify strings to be used as xml tags opening and closing, e.g. \"{{ }}\" (default \"< >\"). Avoid square brackets because of configuration file format. Valid only with text input mode" );
+  AddParam(input_opts,"start-translation-id", "Id of 1st input. Default = 0");
+  AddParam(input_opts,"alternate-weight-setting", "aws", "alternate set of weights to used per xml specification");
+
+  ///////////////////////////////////////////////////////////////////////////////////////
+  // output options
+  po::options_description output_opts("Output Options");
+  AddParam(output_opts,"report-all-factors", "report all factors in output, not just first");
+  AddParam(output_opts,"output-factors", "list if factors in the output");
+  AddParam(output_opts,"print-id", "prefix translations with id. Default if false");
+  AddParam(output_opts,"print-passthrough", "output the sgml tag <passthrough> without any computation on that. Default is false");
+  AddParam(output_opts,"print-passthrough-in-n-best", "output the sgml tag <passthrough> without any computation on that in each entry of the n-best-list. Default is false");
+  AddParam(output_opts,"output-factors", "list of factors in the output");
+  AddParam(output_opts,"print-all-derivations", "to print all derivations in search graph");
+  AddParam(output_opts,"translation-details", "T", "for each best hypothesis, report translation details to the given file");
+
+  AddParam(output_opts,"output-hypo-score", "Output the hypo score to stdout with the output string. For search error analysis. Default is false");
+  AddParam(output_opts,"output-word-graph", "owg", "Output stack info as word graph. Takes filename, 0=only hypos in stack, 1=stack + nbest hypos");
+  AddParam(output_opts,"tree-translation-details", "Ttree", "for each hypothesis, report translation details with tree fragment info to given file");
+  AddParam(output_opts,"print-alignment-info", "Output word-to-word alignment to standard out, separated from translation by |||. Word-to-word alignments are takne from the phrase table if any. Default is false");
+  AddParam(output_opts,"alignment-output-file", "print output word alignments into given file");
+  AddParam(output_opts,"sort-word-alignment", "Sort word alignments for more consistent display. 0=no sort (default), 1=target order");
+  AddParam(output_opts,"report-segmentation", "t", "report phrase segmentation in the output");
+  AddParam(output_opts,"report-segmentation-enriched", "tt", "report phrase segmentation in the output with additional information");
+
+  // translation-all-details was introduced in the context of DIMwid: Decoder Inspection for Moses (using Widgets)
+  // see here: https://ufal.mff.cuni.cz/pbml/100/art-kurtz-seemann-braune-maletti.pdf
+  AddParam(output_opts,"translation-all-details", "Tall", "for all hypotheses, report translation details to the given file");
+
+  po::options_description osg_opts("Options for outputting search graphs");
+  AddParam(osg_opts,"output-search-graph", "osg", "Output connected hypotheses of search into specified filename");
+  AddParam(osg_opts,"output-search-graph-extended", "osgx", "Output connected hypotheses of search into specified filename, in extended format");
+  AddParam(osg_opts,"unpruned-search-graph", "usg", "When outputting chart search graph, do not exclude dead ends. Note: stack pruning may have eliminated some hypotheses");
+  AddParam(osg_opts,"output-search-graph-slf", "slf", "Output connected hypotheses of search into specified directory, one file per sentence, in HTK standard lattice format (SLF) - the flag should be followed by a directory name, which must exist");
+  AddParam(output_opts,"include-lhs-in-search-graph", "lhssg", "When outputting chart search graph, include the label of the LHS of the rule (useful when using syntax)");
+#ifdef HAVE_PROTOBUF
+  AddParam(osg_opts,"output-search-graph-pb", "pb", "Write phrase lattice to protocol buffer objects in the specified path.");
+#endif
+  AddParam(osg_opts,"output-search-graph-hypergraph", "DEPRECATED! Output connected hypotheses of search into specified directory, one file per sentence, in a hypergraph format (see Kenneth Heafield's lazy hypergraph decoder). This flag is followed by 3 values: 'true (gz|txt|bz) directory-name'");
+
+  ///////////////////////////////////////////////////////////////////////////////////////
+  // nbest-options
+  po::options_description nbest_opts("N-best Options");
+  AddParam(nbest_opts,"n-best-list", "file and size of n-best-list to be generated; specify - as the file in order to write to STDOUT");
+  // AddParam(nbest_opts,"n-best-list-file", "file of n-best-list to be generated; specify - as the file in order to write to STDOUT");
+  // AddParam(nbest_opts,"n-best-list-size", "size of n-best-list to be generated; specify - as the file in order to write to STDOUT");
+  AddParam(nbest_opts,"labeled-n-best-list", "print out labels for each weight type in n-best list. default is true");
+  AddParam(nbest_opts,"n-best-trees", "Write n-best target-side trees to n-best-list");
+  AddParam(nbest_opts,"n-best-factor", "factor to compute the maximum number of contenders (=factor*nbest-size). value 0 means infinity, i.e. no threshold. default is 0");
+  AddParam(nbest_opts,"report-all-factors-in-n-best", "Report all factors in n-best-lists. Default is false");
+  AddParam(nbest_opts,"lattice-samples", "generate samples from lattice, in same format as nbest list. Uses the file and size arguments, as in n-best-list");
+  AddParam(nbest_opts,"include-segmentation-in-n-best", "include phrasal segmentation in the n-best list. default is false");
+  AddParam(nbest_opts,"print-alignment-info-in-n-best",
+           "Include word-to-word alignment in the n-best list. Word-to-word alignments are taken from the phrase table if any. Default is false");
+
+  ///////////////////////////////////////////////////////////////////////////////////////
+  // server options
+  po::options_description server_opts("Moses Server Options");
+  AddParam(server_opts,"server", "Run moses as a translation server.");
+  AddParam(server_opts,"daemon", "Run moses as a translation server in the background.");
+  AddParam(server_opts,"server-port", "Port for moses server");
+  AddParam(server_opts,"server-log", "Log destination for moses server");
+  AddParam(server_opts,"serial", "Run server in serial mode, processing only one request at a time.");
+
+  AddParam(server_opts,"server-maxconn",
+           "Max. No of simultaneous HTTP transactions allowed by the server.");
+  AddParam(server_opts,"server-maxconn-backlog",
+           "Max. No. of requests the OS will queue if the server is busy.");
+  AddParam(server_opts,"server-keepalive-maxconn",
+           "Max. No. of requests the server will accept on a single TCP connection.");
+  AddParam(server_opts,"server-keepalive-timeout",
+           "Max. number of seconds the server will keep a persistent connection alive.");
+  AddParam(server_opts,"server-timeout",
+           "Max. number of seconds the server will wait for a client to submit a request once a connection has been established.");
+  // session timeout and session cache size are for moses translation session handling
+  // they have nothing to do with the abyss server (but relate to the moses server)
+  AddParam(server_opts,"session-timeout",
+           "Timeout for sessions, e.g. '2h30m' or 1d (=24h)");
+  AddParam(server_opts,"session-cache-size", string("Max. number of sessions cached.")
+           +"Least recently used session is dumped first.");
+
+  po::options_description irstlm_opts("IRSTLM Options");
+  AddParam(irstlm_opts,"clean-lm-cache",
+           "clean language model caches after N translations (default N=1)");
+
+  po::options_description chart_opts("Chart Decoding Options");
+  AddParam(chart_opts,"max-chart-span", "maximum num. of source word chart rules can consume (default 10)");
+  AddParam(chart_opts,"non-terminals", "list of non-term symbols, space separated");
+  AddParam(chart_opts,"rule-limit", "a little like table limit. But for chart decoding rules. Default is DEFAULT_MAX_TRANS_OPT_SIZE");
+  AddParam(chart_opts,"source-label-overlap", "What happens if a span already has a label. 0=add more. 1=replace. 2=discard. Default is 0");
+  AddParam(chart_opts,"unknown-lhs", "file containing target lhs of unknown words. 1 per line: LHS prob");
+
+  po::options_description misc_opts("Miscellaneous Options");
+  AddParam(misc_opts,"mira", "do mira training");
+  AddParam(misc_opts,"description", "Source language, target language, description");
+  AddParam(misc_opts,"no-cache", "Disable all phrase-table caching. Default = false (ie. enable caching)");
+  AddParam(misc_opts,"default-non-term-for-empty-range-only", "Don't add [X] to all ranges, just ranges where there isn't a source non-term. Default = false (ie. add [X] everywhere)");
+  AddParam(misc_opts,"s2t-parsing-algorithm", "Which S2T parsing algorithm to use. 0=recursive CYK+, 1=scope-3 (default = 0)");
+
+  //AddParam(o,"continue-partial-translation", "cpt", "start from nonempty hypothesis");
+  AddParam(misc_opts,"decoding-graph-backoff", "dpb", "only use subsequent decoding paths for unknown spans of given length");
+  AddParam(misc_opts,"references", "Reference file(s) - used for bleu score feature");
+  AddParam(misc_opts,"recover-input-path", "r", "(conf net/word lattice only) - recover input path corresponding to the best translation");
+  AddParam(misc_opts,"link-param-count", "Number of parameters on word links when using confusion networks or lattices (default = 1)");
+  AddParam(misc_opts,"feature-name-overwrite", "Override feature name (NOT arguments). Eg. SRILM-->KENLM, PhraseDictionaryMemory-->PhraseDictionaryScope3");
+
+  AddParam(misc_opts,"feature", "All the feature functions should be here");
+  AddParam(misc_opts,"context-string",
+           "A (tokenized) string containing context words for context-sensitive translation.");
+  AddParam(misc_opts,"context-weights", "A key-value map for context-sensitive translation.");
+  AddParam(misc_opts,"context-window",
+           "Context window (in words) for context-sensitive translation: {+|-|+-}<number>.");
+
+  // Compact phrase table and reordering table.
+  po::options_description cpt_opts("Options when using compact phrase and reordering tables.");
+  AddParam(cpt_opts,"minphr-memory", "Load phrase table in minphr format into memory");
+  AddParam(cpt_opts,"minlexr-memory", "Load lexical reordering table in minlexr format into memory");
+
+  po::options_description spe_opts("Simulated Post-editing Options");
+  AddParam(spe_opts,"spe-src", "Simulated post-editing. Source filename");
+  AddParam(spe_opts,"spe-trg", "Simulated post-editing. Target filename");
+  AddParam(spe_opts,"spe-aln", "Simulated post-editing. Alignment filename");
+
+  ///////////////////////////////////////////////////////////////////////////////////////
+  // DEPRECATED options
+  po::options_description deprec_opts("Deprecated Options");
+  AddParam(deprec_opts,"link-param-count", "DEPRECATED. DO NOT USE. Number of parameters on word links when using confusion networks or lattices (default = 1)");
+  AddParam(deprec_opts,"weight-slm", "slm", "DEPRECATED. DO NOT USE. weight(s) for syntactic language model");
+  AddParam(deprec_opts,"weight-bl", "bl", "DEPRECATED. DO NOT USE. weight for bleu score feature");
+  AddParam(deprec_opts,"weight-d", "d", "DEPRECATED. DO NOT USE. weight(s) for distortion (reordering components)");
+  AddParam(deprec_opts,"weight-dlm", "dlm", "DEPRECATED. DO NOT USE. weight for discriminative LM feature function (on top of sparse weights)");
+  AddParam(deprec_opts,"weight-lr", "lr", "DEPRECATED. DO NOT USE. weight(s) for lexicalized reordering, if not included in weight-d");
+  AddParam(deprec_opts,"weight-generation", "g", "DEPRECATED. DO NOT USE. weight(s) for generation components");
+  AddParam(deprec_opts,"weight-i", "I", "DEPRECATED. DO NOT USE. weight(s) for word insertion - used for parameters from confusion network and lattice input links");
+  AddParam(deprec_opts,"weight-l", "lm", "DEPRECATED. DO NOT USE. weight(s) for language models");
+  AddParam(deprec_opts,"weight-lex", "lex", "DEPRECATED. DO NOT USE. weight for global lexical model");
+  AddParam(deprec_opts,"weight-glm", "glm", "DEPRECATED. DO NOT USE. weight for global lexical feature, sparse producer");
+  AddParam(deprec_opts,"weight-wt", "wt", "DEPRECATED. DO NOT USE. weight for word translation feature");
+  AddParam(deprec_opts,"weight-pp", "pp", "DEPRECATED. DO NOT USE. weight for phrase pair feature");
+  AddParam(deprec_opts,"weight-pb", "pb", "DEPRECATED. DO NOT USE. weight for phrase boundary feature");
+  AddParam(deprec_opts,"weight-t", "tm", "DEPRECATED. DO NOT USE. weights for translation model components");
+  AddParam(deprec_opts,"weight-p", "w", "DEPRECATED. DO NOT USE. weight for phrase penalty");
+  AddParam(deprec_opts,"weight-w", "w", "DEPRECATED. DO NOT USE. weight for word penalty");
+  AddParam(deprec_opts,"weight-u", "u", "DEPRECATED. DO NOT USE. weight for unknown word penalty");
+  AddParam(deprec_opts,"weight-e", "e", "DEPRECATED. DO NOT USE. weight for word deletion");
+  AddParam(deprec_opts,"text-type", "DEPRECATED. DO NOT USE. should be one of dev/devtest/test, used for domain adaptation features");
+  AddParam(deprec_opts,"input-scores", "DEPRECATED. DO NOT USE. 2 numbers on 2 lines - [1] of scores on each edge of a confusion network or lattice input (default=1). [2] Number of 'real' word scores (0 or 1. default=0)");
+  AddParam(deprec_opts,"dlm-model", "DEPRECATED. DO NOT USE. Order, factor and vocabulary file for discriminative LM. Use * for filename to indicate unlimited vocabulary.");
+  AddParam(deprec_opts,"generation-file", "DEPRECATED. DO NOT USE. location and properties of the generation table");
+  AddParam(deprec_opts,"global-lexical-file", "gl", "DEPRECATED. DO NOT USE. discriminatively trained global lexical translation model file");
+  AddParam(deprec_opts,"glm-feature", "DEPRECATED. DO NOT USE. discriminatively trained global lexical translation feature, sparse producer");
+  AddParam(deprec_opts,"lmodel-file", "DEPRECATED. DO NOT USE. location and properties of the language models");
+  AddParam(deprec_opts,"lmodel-dub", "DEPRECATED. DO NOT USE. dictionary upper bounds of language models");
+#ifdef HAVE_SYNLM
+  AddParam(deprec_opts,"slmodel-file", "DEPRECATED. DO NOT USE. location of the syntactic language model file(s)");
+  AddParam(deprec_opts,"slmodel-factor", "DEPRECATED. DO NOT USE. factor to use with syntactic language model");
+  AddParam(deprec_opts,"slmodel-beam", "DEPRECATED. DO NOT USE. beam width to use with syntactic language model's parser");
+#endif
+  AddParam(deprec_opts,"ttable-file", "DEPRECATED. DO NOT USE. location and properties of the translation tables");
+  AddParam(deprec_opts,"phrase-pair-feature", "DEPRECATED. DO NOT USE. Source and target factors for phrase pair feature");
+  AddParam(deprec_opts,"phrase-boundary-source-feature", "DEPRECATED. DO NOT USE. Source factors for phrase boundary feature");
+  AddParam(deprec_opts,"phrase-boundary-target-feature", "DEPRECATED. DO NOT USE. Target factors for phrase boundary feature");
+  AddParam(deprec_opts,"phrase-length-feature", "DEPRECATED. DO NOT USE. Count features for source length, target length, both of each phrase");
+  AddParam(deprec_opts,"target-word-insertion-feature", "DEPRECATED. DO NOT USE. Count feature for each unaligned target word");
+  AddParam(deprec_opts,"source-word-deletion-feature", "DEPRECATED. DO NOT USE. Count feature for each unaligned source word");
+  AddParam(deprec_opts,"word-translation-feature", "DEPRECATED. DO NOT USE. Count feature for word translation according to word alignment");
+
+  po::options_description zombie_opts("Zombie Options");
+  AddParam(zombie_opts,"distortion-file", "source factors (0 if table independent of source), target factors, location of the factorized/lexicalized reordering tables");
+
+
+  mbr_opts.add(lmbr_opts);
+  search_opts.add(cube_opts);
+  search_opts.add(mbr_opts);
+  search_opts.add(disto_opts);
+  search_opts.add(chart_opts);
+
+  input_opts.add(spe_opts);
+
+  output_opts.add(nbest_opts);
+  output_opts.add(osg_opts);
+
+  m_options.add(main_opts);
+  m_options.add(server_opts);
+  m_options.add(input_opts);
+  m_options.add(search_opts);
+  m_options.add(output_opts);
+  m_options.add(oov_opts);
+  m_options.add(factor_opts);
+  m_options.add(cpt_opts);
+  m_options.add(irstlm_opts);
+  m_options.add(tune_opts);
+  m_options.add(misc_opts);
+  m_options.add(deprec_opts);
+  m_options.add(zombie_opts);
+
+}
+
+Parameter::~Parameter()
+{
+}
+
+const PARAM_VEC *Parameter::GetParam(const std::string &paramName) const
+{
+  PARAM_MAP::const_iterator iter = m_setting.find( paramName );
+  if (iter == m_setting.end()) {
+    return NULL;
+  } else {
+    return &iter->second;
+  }
+
+}
+
+/** initialize a parameter, sub of constructor */
+void
+Parameter::
+AddParam(po::options_description& optgroup,
+         string const& paramName,
+         string const& description)
+{
+  m_valid[paramName] = true;
+  m_description[paramName] = description;
+  optgroup.add_options()(paramName.c_str(), description.c_str());
+}
+
+/** initialize a parameter (including abbreviation), sub of constructor */
+void
+Parameter::
+AddParam(po::options_description& optgroup,
+         string const& paramName,
+         string const& abbrevName,
+         string const& description)
+{
+  m_valid[paramName] = true;
+  m_valid[abbrevName] = true;
+  m_abbreviation[paramName] = abbrevName;
+  m_fullname[abbrevName] = paramName;
+  m_description[paramName] = description;
+  string optname = paramName;
+  if (abbrevName.size() == 1) {
+    optname += string(",")+abbrevName;
+    // m_confusable[abbrevName[0]].insert(paramName);
+  }
+  optgroup.add_options()(optname.c_str(),description.c_str());
+}
+
+/** print descriptions of all parameters */
+void
+Parameter::
+Explain()
+{
+  cerr << "Usage:" << endl;
+  cerr << m_options << endl;
+  // for(PARAM_STRING::const_iterator iterParam = m_description.begin();
+  //     iterParam != m_description.end(); iterParam++)
+  //   {
+  //     const string paramName = iterParam->first;
+  //     const string paramDescription = iterParam->second;
+  //     cerr <<  "\t-" << paramName;
+  //     PARAM_STRING::const_iterator iterAbbr = m_abbreviation.find( paramName );
+  //     if ( iterAbbr != m_abbreviation.end() )
+  // 	cerr <<  " (" << iterAbbr->second << ")";
+  //     cerr <<  ": " << paramDescription << endl;
+  //   }
+}
+
+/** check whether an item on the command line is a switch or a value
+ * \param token token on the command line to checked **/
+
+bool
+Parameter::
+isOption(const char* token)
+{
+  if (! token) return false;
+  std::string tokenString(token);
+  size_t length = tokenString.size();
+  if (length <= 1) return false;
+  if (!starts_with(tokenString, "-")) return false;
+  if (tokenString.substr(1,1).find_first_not_of("0123456789") == 0) return true;
+  return false;
+}
+
+/** load all parameters from the configuration file and the command line switches */
+bool
+Parameter::
+LoadParam(const string &filePath)
+{
+  const char *argv[] = {"executable", "-f", filePath.c_str() };
+  return LoadParam(3, (char const**) argv);
+}
+
+/// Print out version information about the things that went into this
+/// executable.
+void show_version()
+{
+  std::cout << "\nMoses code version (git tag or commit hash):\n   "
+            << MOSES_VERSION_ID << std::endl
+            << "Libraries used:" << std::endl
+            << "      Boost  version "
+            << BOOST_VERSION / 100000     << "."  // major version
+            << BOOST_VERSION / 100 % 1000 << "."  // minor version
+            << BOOST_VERSION % 100                // patch level
+            << std::endl;
+#ifdef HAVE_XMLRPC_C
+  unsigned int major, minor, point;
+  xmlrpc_server_version(&major, &minor, &point);
+  std::cout << "   Xmlrpc-c  version "
+            << major << "." << minor << "." << point << std::endl;
+#endif
+#ifdef HAVE_CMPH
+  // there's no easy way to determine the cmph version at compile time
+  std::cout << "       CMPH (version unknown)" << std::endl;
+#endif
+
+#ifdef MMT_VERSION_ID
+  std::cout << string(20,'-')
+            << "\nMMT extras version: " << MMT_VERSION_ID << std::endl;
+#endif
+}
+
+/** load all parameters from the configuration file and the command line switches */
+bool
+Parameter::
+LoadParam(int argc, char const* xargv[])
+{
+  // legacy parameter handling: all parameters are expected
+  // to start with a single dash
+  char const* argv[argc+1];
+  for (int i = 0; i < argc; ++i) {
+    argv[i] = xargv[i];
+    if (strlen(argv[i]) > 2 && argv[i][0] == '-' && argv[i][1] == '-')
+      ++argv[i];
+    if (!strcmp(argv[i],"-version")) {
+      show_version();
+      exit(0);
+    }
+  }
+
+
+
+  // config file (-f) arg mandatory
+  string configPath;
+  if ( (configPath = FindParam("-f", argc, argv)) == ""
+       && (configPath = FindParam("-config", argc, argv)) == "") {
+    PrintCredit();
+    Explain();
+    PrintFF();
+
+    cerr << endl;
+    cerr << "No configuration file was specified.  Use -config or -f";
+    cerr << endl;
+    return false;
+  } else {
+    if (!ReadConfigFile(configPath)) {
+      std::cerr << "Could not read " << configPath;
+      return false;
+    }
+  }
+
+  // overwrite parameters with values from switches
+  for(PARAM_STRING::const_iterator iterParam = m_description.begin();
+      iterParam != m_description.end(); iterParam++) {
+    const string paramName = iterParam->first;
+    OverwriteParam("-" + paramName, paramName, argc, argv);
+  }
+
+  // ... also shortcuts
+  for(PARAM_STRING::const_iterator iterParam = m_abbreviation.begin();
+      iterParam != m_abbreviation.end(); iterParam++) {
+    const string paramName = iterParam->first;
+    const string paramShortName = iterParam->second;
+    OverwriteParam("-" + paramShortName, paramName, argc, argv);
+  }
+
+  AddFeaturesCmd();
+
+  // logging of parameters that were set in either config or switch
+  int verbose = 1;
+  if (m_setting.find("verbose") != m_setting.end() &&
+      m_setting["verbose"].size() > 0)
+    verbose = Scan<int>(m_setting["verbose"][0]);
+  if (verbose >= 1) { // only if verbose
+    TRACE_ERR( "Defined parameters (per moses.ini or switch):" << endl);
+    for(PARAM_MAP::const_iterator iterParam = m_setting.begin() ;
+        iterParam != m_setting.end(); iterParam++) {
+      TRACE_ERR( "\t" << iterParam->first << ": ");
+      for ( size_t i = 0; i < iterParam->second.size(); i++ )
+        TRACE_ERR( iterParam->second[i] << " ");
+      TRACE_ERR( endl);
+    }
+  }
+
+  // don't mix old and new format
+  if ((GetParam("feature") || GetParam("weight"))
+      && (GetParam("weight-slm") || GetParam("weight-bl") || GetParam("weight-d") ||
+          GetParam("weight-dlm") || GetParam("weight-lrl") || GetParam("weight-generation") ||
+          GetParam("weight-i") || GetParam("weight-l") || GetParam("weight-lex") ||
+          GetParam("weight-glm") || GetParam("weight-wt") || GetParam("weight-pp") ||
+          GetParam("weight-pb") || GetParam("weight-t") || GetParam("weight-w") ||
+          GetParam("weight-p") ||
+          GetParam("weight-u") || GetParam("weight-e") ||
+          GetParam("dlm-mode") || GetParam("generation-file") || GetParam("global-lexical-file") ||
+          GetParam("glm-feature") || GetParam("lmodel-file") || GetParam("lmodel-dub") ||
+          GetParam("slmodel-file") || GetParam("slmodel-factor") ||
+          GetParam("slmodel-beam") || GetParam("ttable-file") || GetParam("phrase-pair-feature") ||
+          GetParam("phrase-boundary-source-feature") || GetParam("phrase-boundary-target-feature") || GetParam("phrase-length-feature") ||
+          GetParam("target-word-insertion-feature") || GetParam("source-word-deletion-feature") || GetParam("word-translation-feature")
+         )
+     ) {
+    UTIL_THROW(util::Exception, "Don't mix old and new ini file format");
+  }
+
+  // convert old weights args to new format
+  if (GetParam("feature") == NULL) {
+    ConvertWeightArgs();
+  }
+  CreateWeightsMap();
+  WeightOverwrite();
+
+  // check for illegal parameters
+  bool noErrorFlag = true;
+  for (int i = 0 ; i < argc ; i++) {
+    if (isOption(argv[i])) {
+      string paramSwitch = (string) argv[i];
+      string paramName = paramSwitch.substr(1);
+      if (m_valid.find(paramName) == m_valid.end()) {
+        std::cerr << "illegal switch: " << paramSwitch;
+        noErrorFlag = false;
+      }
+    }
+  }
+
+  //Save("/tmp/moses.ini.new");
+
+  // check if parameters make sense
+  return Validate() && noErrorFlag;
+}
+
+void
+Parameter::
+AddFeaturesCmd()
+{
+  const PARAM_VEC *params = GetParam("feature-add");
+  if (params) {
+    PARAM_VEC::const_iterator iter;
+    for (iter = params->begin(); iter != params->end(); ++iter) {
+      const string &line = *iter;
+      AddFeature(line);
+    }
+
+    m_setting.erase("feature-add");
+  }
+}
+
+std::vector<float>
+Parameter::
+GetWeights(const std::string &name)
+{
+  std::vector<float> ret = m_weights[name];
+
+  // cerr << "WEIGHT " << name << "=";
+  // for (size_t i = 0; i < ret.size(); ++i) {
+  //   cerr << ret[i] << ",";
+  // }
+  // cerr << endl;
+  return ret;
+}
+
+void
+Parameter::
+SetWeight(const std::string &name, size_t ind, float weight)
+{
+  PARAM_VEC &newWeights = m_setting["weight"];
+  string line = name + SPrint(ind) + "= " + SPrint(weight);
+  newWeights.push_back(line);
+}
+
+void Parameter::SetWeight(const std::string &name, size_t ind, const vector<float> &weights)
+{
+  PARAM_VEC &newWeights = m_setting["weight"];
+  string line = name + SPrint(ind) + "=";
+
+  for (size_t i = 0; i < weights.size(); ++i) {
+    line += " " + SPrint(weights[i]);
+  }
+  newWeights.push_back(line);
+}
+
+void
+Parameter::
+AddWeight(const std::string &name, size_t ind,
+          const std::vector<float> &weights)
+{
+  PARAM_VEC &newWeights = m_setting["weight"];
+
+  string sought = name + SPrint(ind) + "=";
+  for (size_t i = 0; i < newWeights.size(); ++i) {
+    string &line = newWeights[i];
+    if (line.find(sought) == 0) {
+      // found existing weight, most likely to be input weights. Append to this line
+      for (size_t i = 0; i < weights.size(); ++i) {
+        line += " " + SPrint(weights[i]);
+      }
+      return;
+    }
+  }
+
+  // nothing found. Just set
+  SetWeight(name, ind, weights);
+}
+
+void
+Parameter::
+ConvertWeightArgsSingleWeight(const string &oldWeightName, const string &newWeightName)
+{
+  size_t ind = 0;
+  PARAM_MAP::iterator iterMap;
+
+  iterMap = m_setting.find(oldWeightName);
+  if (iterMap != m_setting.end()) {
+    const PARAM_VEC &weights = iterMap->second;
+    for (size_t i = 0; i < weights.size(); ++i) {
+      SetWeight(newWeightName, ind, Scan<float>(weights[i]));
+    }
+
+    m_setting.erase(iterMap);
+  }
+}
+
+void
+Parameter::
+ConvertWeightArgsPhraseModel(const string &oldWeightName)
+{
+  const PARAM_VEC *params;
+
+  // process input weights 1st
+  params = GetParam("weight-i");
+  if (params) {
+    vector<float> inputWeights = Scan<float>(*params);
+    PARAM_VEC &numInputScores = m_setting["input-scores"];
+    if (inputWeights.size() == 1) {
+      UTIL_THROW_IF2(numInputScores.size() != 0, "No [input-scores] section allowed");
+      numInputScores.push_back("1");
+      numInputScores.push_back("0");
+    } else if (inputWeights.size() == 2) {
+      UTIL_THROW_IF2(numInputScores.size() != 0, "No [input-scores] section allowed");
+      numInputScores.push_back("1");
+      numInputScores.push_back("1");
+    }
+
+    SetWeight("PhraseDictionaryBinary", 0, inputWeights);
+  }
+
+  // convert actually pt feature
+  VERBOSE(2,"Creating phrase table features" << endl);
+
+  size_t numInputScores = 0;
+  size_t numRealWordsInInput = 0;
+  map<string, size_t> ptIndices;
+
+  params = GetParam("input-scores");
+  if (params) {
+    numInputScores = Scan<size_t>(params->at(0));
+
+    if (params->size() > 1) {
+      numRealWordsInInput = Scan<size_t>(params->at(1));
+    }
+  }
+
+  // load phrase translation tables
+  params = GetParam("ttable-file");
+  if (params) {
+    // weights
+    const vector<string> translationVector = *params;
+
+    vector<size_t>  maxTargetPhrase;
+    params = GetParam("ttable-limit");
+    if (params) {
+      maxTargetPhrase = Scan<size_t>(*params);
+    }
+
+    if(maxTargetPhrase.size() == 1 && translationVector.size() > 1) {
+      VERBOSE(1, "Using uniform ttable-limit of " << maxTargetPhrase[0] << " for all translation tables." << endl);
+      for(size_t i = 1; i < translationVector.size(); i++)
+        maxTargetPhrase.push_back(maxTargetPhrase[0]);
+    } else if(maxTargetPhrase.size() != 1 && maxTargetPhrase.size() < translationVector.size()) {
+      std::cerr << "You specified " << translationVector.size() << " translation tables, but only " << maxTargetPhrase.size() << " ttable-limits.";
+      return;
+    }
+
+    // MAIN LOOP
+    const PARAM_VEC &oldWeights = m_setting[oldWeightName];
+
+    size_t currOldInd = 0;
+    for(size_t currDict = 0 ; currDict < translationVector.size(); currDict++) {
+      util::StringStream ptLine;
+
+      vector<string> token = Tokenize(translationVector[currDict]);
+
+      if(currDict == 0 && token.size() == 4) {
+        std::cerr << "Phrase table specification in old 4-field format. No longer supported";
+        return;
+      }
+      UTIL_THROW_IF2(token.size() < 5, "Phrase table must have at least 5 scores");
+
+      int implementation = Scan<int>(token[0]);
+
+      string ptType;
+      switch (implementation) {
+      case 0: // Memory
+        ptType = "PhraseDictionaryMemory";
+        break;
+      case 1: // Binary
+        ptType = "PhraseDictionaryBinary";
+        break;
+      case 2: // OnDisk
+        ptType = "PhraseDictionaryOnDisk";
+        break;
+      case 6: // SCFG
+        ptType = "PhraseDictionaryMemory";
+        break;
+      case 12: // Compact
+        ptType = "PhraseDictionaryCompact";
+        break;
+      case 8: // SuffixArray
+        ptType = "PhraseDictionarySuffixArray";
+        break;
+      case 14: // DSuffixArray
+        ptType = "PhraseDictionaryDynSuffixArray";
+        break;
+      case 15: // DCacheBased:
+        ptType = "PhraseDictionaryDynamicCacheBased";
+        break;
+      case 16: // CachePT:
+        ptType = "PhraseDictionaryCache";
+        break;
+      default:
+        break;
+      }
+
+      size_t ptInd;
+      if (ptIndices.find(ptType) == ptIndices.end()) {
+        ptIndices[ptType] = 0;
+        ptInd = 0;
+      } else {
+        ptInd = ++ptIndices[ptType];
+      }
+
+      // weights
+      size_t numFFInd = (token.size() == 4) ? 2 : 3;
+      size_t numFF = Scan<size_t>(token[numFFInd]);
+
+      vector<float> weights(numFF);
+      for (size_t currFF = 0; currFF < numFF; ++currFF) {
+        UTIL_THROW_IF2(currOldInd >= oldWeights.size(),
+                       "Errors converting old phrase-table weights to new weights");
+        float weight = Scan<float>(oldWeights[currOldInd]);
+        weights[currFF] = weight;
+
+        ++currOldInd;
+      }
+
+      // cerr << weights.size() << " PHRASE TABLE WEIGHTS "
+      // << __FILE__ << ":" << __LINE__ << endl;
+      AddWeight(ptType, ptInd, weights);
+
+      // actual pt
+      ptLine << ptType << " ";
+      ptLine << "input-factor=" << token[1] << " ";
+      ptLine << "output-factor=" << token[2] << " ";
+      ptLine << "path=" << token[4] << " ";
+
+      //characteristics of the phrase table
+
+      vector<FactorType>  input   = Tokenize<FactorType>(token[1], ",")
+                                    ,output  = Tokenize<FactorType>(token[2], ",");
+      size_t numScoreComponent = Scan<size_t>(token[3]);
+      string filePath= token[4];
+
+      if(currDict==0) {
+        // only the 1st pt. THis is shit
+        // TODO. find what the assumptions made by confusion network about phrase table output which makes
+        // it only work with binary file. This is a hack
+        numScoreComponent += numInputScores + numRealWordsInInput;
+      }
+
+      ptLine << "num-features=" << numScoreComponent << " ";
+      ptLine << "table-limit=" << maxTargetPhrase[currDict] << " ";
+
+      if (implementation == 8 || implementation == 14) {
+        ptLine << "target-path=" << token[5] << " ";
+        ptLine << "alignment-path=" << token[6] << " ";
+      }
+
+      AddFeature(ptLine.str());
+    } // for(size_t currDict = 0 ; currDict < translationVector.size(); currDict++) {
+  } // if (GetParam("ttable-file").size() > 0) {
+
+  m_setting.erase("weight-i");
+  m_setting.erase(oldWeightName);
+  m_setting.erase("ttable-file");
+  m_setting.erase("ttable-limit");
+
+}
+
+void
+Parameter::
+AddFeature(const std::string &line)
+{
+  PARAM_VEC &features = m_setting["feature"];
+  features.push_back(line);
+}
+
+void
+Parameter::
+ConvertWeightArgsDistortion()
+{
+  const string oldWeightName = "weight-d";
+  const string oldLexReordingName = "distortion-file";
+
+  // distortion / lex distortion
+  const PARAM_VEC *oldWeights = GetParam(oldWeightName);
+
+  if (oldWeights) {
+    const PARAM_VEC *searchAlgo = GetParam("search-algorithm");
+    if (searchAlgo == NULL ||
+        (searchAlgo->size() > 0
+         && (Trim(searchAlgo->at(0)) == "0" || Trim(searchAlgo->at(0)) == "1")
+        )
+       ) {
+      // phrase-based. Add distance distortion to list of features
+      AddFeature("Distortion");
+      SetWeight("Distortion", 0, Scan<float>(oldWeights->at(0)));
+    }
+
+    // everything but the last is lex reordering model
+
+    size_t currOldInd = 1;
+    const PARAM_VEC *lextable = GetParam(oldLexReordingName);
+
+    for (size_t indTable = 0; lextable && indTable < lextable->size(); ++indTable) {
+      const string &line = lextable->at(indTable);
+      vector<string> toks = Tokenize(line);
+
+      size_t numFF = Scan<size_t>(toks[2]);
+
+      vector<float> weights(numFF);
+      for (size_t currFF = 0; currFF < numFF; ++currFF) {
+        UTIL_THROW_IF2(oldWeights && currOldInd >= oldWeights->size(),
+                       "Errors converting old distortion weights to new weights");
+        float weight = Scan<float>(oldWeights->at(currOldInd));
+        weights[currFF] = weight;
+
+        ++currOldInd;
+      }
+      SetWeight("LexicalReordering", indTable, weights);
+
+      util::StringStream strme;
+      strme << "LexicalReordering "
+            << "type=" << toks[1] << " ";
+
+      vector<FactorType> factors = Tokenize<FactorType>(toks[0], "-");
+      UTIL_THROW_IF2(factors.size() != 2,
+                     "Error in old factor specification for lexicalized reordering model: "
+                     << toks[0]);
+      strme << "input-factor=" << factors[0]
+            << " output-factor=" << factors[1] << " ";
+
+      strme << "num-features=" << toks[2] << " ";
+      strme << "path=" << toks[3];
+
+      AddFeature(strme.str());
+    }
+  }
+
+  m_setting.erase(oldWeightName);
+  m_setting.erase(oldLexReordingName);
+
+}
+
+void
+Parameter::
+ConvertWeightArgsLM()
+{
+  const string oldWeightName = "weight-l";
+  const string oldFeatureName = "lmodel-file";
+  const PARAM_VEC *params;
+
+  bool isChartDecoding = true;
+
+  params = GetParam("search-algorithm");
+  if (params == NULL ||
+      (params->size() > 0
+       && (Trim(params->at(0)) == "0" || Trim(params->at(0)) == "1")
+      )
+     ) {
+    isChartDecoding = false;
+  }
+
+  vector<int> oovWeights;
+  params = GetParam("lmodel-oov-feature");
+  if (params) {
+    oovWeights = Scan<int>(*params);
+  }
+
+  PARAM_MAP::iterator iterMap;
+
+  iterMap = m_setting.find(oldWeightName);
+  if (iterMap != m_setting.end()) {
+
+    size_t currOldInd = 0;
+    const PARAM_VEC &weights = iterMap->second;
+    const PARAM_VEC &models = m_setting[oldFeatureName];
+    for (size_t lmIndex = 0; lmIndex < models.size(); ++lmIndex) {
+      const string &line = models[lmIndex];
+      vector<string> modelToks = Tokenize(line);
+
+      int lmType = Scan<int>(modelToks[0]);
+
+      string newFeatureName;
+      switch (lmType) {
+      case 0:
+        newFeatureName = "SRILM";
+        break;
+      case 1:
+        newFeatureName = "IRSTLM";
+        break;
+      case 8:
+      case 9:
+        newFeatureName = "KENLM";
+        break;
+      default:
+        UTIL_THROW2("Unkown language model type id:"  << lmType);
+      }
+
+      size_t numFF = 1;
+      if (oovWeights.size() > lmIndex)
+        numFF += oovWeights[lmIndex];
+
+      vector<float> weightsLM(numFF);
+      for (size_t currFF = 0; currFF < numFF; ++currFF) {
+        UTIL_THROW_IF2(currOldInd >= weights.size(),
+                       "Errors converting old LM weights to new weights");
+        weightsLM[currFF] = Scan<float>(weights[currOldInd]);
+        if (isChartDecoding) {
+          weightsLM[currFF] = UntransformLMScore(weightsLM[currFF]);
+        }
+
+        ++currOldInd;
+      }
+
+      SetWeight(newFeatureName, lmIndex, weightsLM);
+
+      string featureLine = newFeatureName + " "
+                           + "factor=" + modelToks[1] + " "  // factor
+                           + "order="  + modelToks[2] + " " // order
+                           + "num-features=" + SPrint(numFF) + " ";
+      if (lmType == 9) {
+        featureLine += "load=lazy ";
+      }
+
+      if(oovWeights.size() > lmIndex)
+        featureLine += "oov-feature=1 ";
+
+      featureLine += "path=" + modelToks[3]; // file
+
+      AddFeature(featureLine);
+    } // for (size_t lmIndex = 0; lmIndex < models.size(); ++lmIndex) {
+
+    m_setting.erase(iterMap);
+  }
+
+  m_setting.erase(oldFeatureName);
+}
+
+void
+Parameter::
+ConvertWeightArgsGeneration(const std::string &oldWeightName, const std::string &newWeightName)
+{
+  string oldFeatureName = "generation-file";
+
+  // distortion / lex distortion
+  PARAM_VEC &oldWeights = m_setting[oldWeightName];
+
+  if (oldWeights.size() > 0) {
+    size_t currOldInd = 0;
+    PARAM_VEC &models = m_setting[oldFeatureName];
+
+    for (size_t indTable = 0; indTable < models.size(); ++indTable) {
+      string &line = models[indTable];
+      vector<string> modelToks = Tokenize(line);
+
+      size_t numFF = Scan<size_t>(modelToks[2]);
+
+      vector<float> weights(numFF);
+      for (size_t currFF = 0; currFF < numFF; ++currFF) {
+        UTIL_THROW_IF2(currOldInd >= oldWeights.size(),
+                       "Errors converting old generation weights to new weights");
+        float weight = Scan<float>(oldWeights[currOldInd]);
+        weights[currFF] = weight;
+
+        ++currOldInd;
+      }
+      SetWeight(newWeightName, indTable, weights);
+
+      util::StringStream strme;
+      strme << "Generation "
+            << "input-factor=" << modelToks[0] << " "
+            << "output-factor=" << modelToks[1] << " "
+            << "num-features=" << modelToks[2] << " "
+            << "path=" << modelToks[3];
+      AddFeature(strme.str());
+    }
+  }
+
+  m_setting.erase(oldWeightName);
+  m_setting.erase(oldFeatureName);
+}
+
+void
+Parameter::
+ConvertWeightArgsWordPenalty()
+{
+  const std::string oldWeightName = "weight-w";
+  const std::string newWeightName = "WordPenalty";
+
+  bool isChartDecoding = true;
+  const PARAM_VEC *searchAlgo = GetParam("search-algorithm");
+  if (searchAlgo == NULL ||
+      (searchAlgo->size() > 0
+       && (Trim(searchAlgo->at(0)) == "0" || Trim(searchAlgo->at(0)) == "1")
+      )
+     ) {
+    isChartDecoding = false;
+  }
+
+  PARAM_MAP::iterator iterMap;
+
+  iterMap = m_setting.find(oldWeightName);
+  if (iterMap != m_setting.end()) {
+    const PARAM_VEC &weights = iterMap->second;
+    for (size_t i = 0; i < weights.size(); ++i) {
+      float weight = Scan<float>(weights[i]);
+      if (isChartDecoding) {
+        weight *= 0.434294482;
+      }
+      SetWeight(newWeightName, i, weight);
+    }
+
+    m_setting.erase(iterMap);
+  }
+
+}
+
+void
+Parameter::
+ConvertPhrasePenalty()
+{
+  string oldWeightName = "weight-p";
+  const PARAM_VEC *params = GetParam(oldWeightName);
+  if (params) {
+    UTIL_THROW_IF2(params->size() != 1,
+                   "There should be only 1 phrase-penalty weight");
+    float weight = Scan<float>(params->at(0));
+    AddFeature("PhrasePenalty");
+    SetWeight("PhrasePenalty", 0, weight);
+
+    m_setting.erase(oldWeightName);
+  }
+}
+
+void
+Parameter::
+ConvertWeightArgs()
+{
+  // can't handle discr LM. must do it manually 'cos of bigram/n-gram split
+  UTIL_THROW_IF2( m_setting.count("weight-dlm") != 0,
+                  "Can't handle discr LM. must do it manually 'cos of bigram/n-gram split");
+
+  // check that old & new format aren't mixed
+  if (m_setting.count("weight") &&
+      (m_setting.count("weight-i") || m_setting.count("weight-t") || m_setting.count("weight-w") ||
+       m_setting.count("weight-l") || m_setting.count("weight-u") || m_setting.count("weight-lex") ||
+       m_setting.count("weight-generation") || m_setting.count("weight-lr") || m_setting.count("weight-d")
+      )) {
+    cerr << "Do not mix old and new format for specify weights";
+  }
+
+  ConvertWeightArgsWordPenalty();
+  ConvertWeightArgsLM();
+  ConvertWeightArgsSingleWeight("weight-slm", "SyntacticLM");
+  ConvertWeightArgsSingleWeight("weight-u", "UnknownWordPenalty");
+  ConvertWeightArgsGeneration("weight-generation", "Generation");
+  ConvertWeightArgsDistortion();
+
+  // don't know or can't be bothered converting these weights
+  ConvertWeightArgsSingleWeight("weight-lr", "LexicalReordering");
+  ConvertWeightArgsSingleWeight("weight-bl", "BleuScoreFeature");
+  ConvertWeightArgsSingleWeight("weight-glm", "GlobalLexicalModel");
+  ConvertWeightArgsSingleWeight("weight-wt", "WordTranslationFeature");
+  ConvertWeightArgsSingleWeight("weight-pp", "PhrasePairFeature");
+  ConvertWeightArgsSingleWeight("weight-pb", "PhraseBoundaryFeature");
+
+  ConvertWeightArgsSingleWeight("weight-e", "WordDeletion"); // TODO Can't find real name
+  ConvertWeightArgsSingleWeight("weight-lex", "GlobalLexicalReordering"); // TODO Can't find real name
+
+  ConvertPhrasePenalty();
+
+  AddFeature("WordPenalty");
+  AddFeature("UnknownWordPenalty");
+
+  ConvertWeightArgsPhraseModel("weight-t");
+
+}
+
+void
+Parameter::
+CreateWeightsMap()
+{
+  CreateWeightsMap(m_setting["weight-add"]);
+  CreateWeightsMap(m_setting["weight"]);
+}
+
+void
+Parameter::
+CreateWeightsMap(const PARAM_VEC &vec)
+{
+  for (size_t i = 0; i < vec.size(); ++i) {
+    const string &line = vec[i];
+    vector<string> toks = Tokenize(line);
+    UTIL_THROW_IF2(toks.size() < 2,
+                   "Error in format of weights: " << line);
+
+    string name = toks[0];
+    name = name.substr(0, name.size() - 1);
+
+    vector<float> weights(toks.size() - 1);
+    for (size_t i = 1; i < toks.size(); ++i) {
+      float weight = Scan<float>(toks[i]);
+      weights[i - 1] = weight;
+    }
+    m_weights[name] = weights;
+  }
+}
+
+void
+Parameter::
+WeightOverwrite()
+{
+  PARAM_VEC &vec = m_setting["weight-overwrite"];
+
+  if (vec.size() == 0)
+    return;
+
+  // should only be on 1 line
+  UTIL_THROW_IF2(vec.size() != 1,
+                 "weight-overwrite should only be on 1 line");
+
+  string name("");
+  vector<float> weights;
+  vector<string> toks = Tokenize(vec[0]);
+  size_t cnt = 0;
+  const std::vector<float>* oldWeights = NULL;
+  for (size_t i = 0; i < toks.size(); ++i) {
+    const string &tok = toks[i];
+
+    if (ends_with(tok, "=")) {
+      // start of new feature
+
+      if (name != "") {
+        // save previous ff
+        m_weights[name] = weights;
+        weights.clear();
+      }
+
+      name = tok.substr(0, tok.size() - 1);
+      std::map<std::string, std::vector<float> >::const_iterator found = m_weights.find(name);
+      if (found!=m_weights.end()) {
+        oldWeights = &(found->second);
+      } else {
+        oldWeights = NULL;
+      }
+      cnt = 0;
+    } else {
+      // a weight for curr ff
+      if (toks[i] == "x") {
+        UTIL_THROW_IF2(!oldWeights || cnt>=oldWeights->size(),
+                       "Keeping previous weight failed in weight-overwrite");
+        weights.push_back(oldWeights->at(cnt));
+      } else {
+        float weight = Scan<float>(toks[i]);
+        weights.push_back(weight);
+      }
+      ++cnt;
+    }
+  }
+
+  if (name != "") {
+    m_weights[name] = weights;
+  }
+
+}
+
+/** check that parameter settings make sense */
+bool
+Parameter::
+Validate()
+{
+  bool noErrorFlag = true;
+
+  PARAM_MAP::const_iterator iterParams;
+  for (iterParams = m_setting.begin(); iterParams != m_setting.end(); ++iterParams) {
+    const std::string &key = iterParams->first;
+
+    if (m_valid.find(key) == m_valid.end()) {
+      std::cerr << "Unknown parameter " << key;
+      noErrorFlag = false;
+    }
+  }
+
+  if (m_setting["lmodel-dub"].size() > 0) {
+    if (m_setting["lmodel-file"].size() != m_setting["lmodel-dub"].size()) {
+      std::cerr << "Config and parameters specify "
+                << static_cast<int>(m_setting["lmodel-file"].size())
+                << " language model files (lmodel-file), but "
+                << static_cast<int>(m_setting["lmodel-dub"].size())
+                << " LM upperbounds (lmodel-dub)"
+                << endl;
+      noErrorFlag = false;
+    }
+  }
+
+  // do files exist?
+
+  // input file
+  if (noErrorFlag && m_setting["input-file"].size() == 1) {
+    noErrorFlag = FileExists(m_setting["input-file"][0]);
+    if (!noErrorFlag) {
+      std::cerr << endl << "Input file " << m_setting["input-file"][0] << " does not exist";
+    }
+  }
+  // generation tables
+  if (noErrorFlag) {
+    std::vector<std::string> ext;
+    //raw tables in either un compressed or compressed form
+    ext.push_back("");
+    ext.push_back(".gz");
+    noErrorFlag = FilesExist("generation-file", 3, ext);
+  }
+  // distortion
+  if (noErrorFlag) {
+    std::vector<std::string> ext;
+    //raw tables in either un compressed or compressed form
+    ext.push_back("");
+    ext.push_back(".gz");
+    //prefix tree format
+    ext.push_back(".binlexr.idx");
+    //prefix tree format
+    ext.push_back(".minlexr");
+    noErrorFlag = FilesExist("distortion-file", 3, ext);
+  }
+  return noErrorFlag;
+}
+
+/** check whether a file exists */
+bool
+Parameter::
+FilesExist(const string &paramName, int fieldNo,
+           std::vector<std::string> const& extensions)
+{
+  typedef std::vector<std::string> StringVec;
+  StringVec::const_iterator iter;
+
+  PARAM_MAP::const_iterator iterParam = m_setting.find(paramName);
+  if (iterParam == m_setting.end()) {
+    // no param. therefore nothing to check
+    return true;
+  }
+  const StringVec &pathVec = (*iterParam).second;
+  for (iter = pathVec.begin() ; iter != pathVec.end() ; ++iter) {
+    StringVec vec = Tokenize(*iter);
+
+    size_t tokenizeIndex;
+    if (fieldNo == -1)
+      tokenizeIndex = vec.size() - 1;
+    else
+      tokenizeIndex = static_cast<size_t>(fieldNo);
+
+    if (tokenizeIndex >= vec.size()) {
+      std::cerr << "Expected at least " << (tokenizeIndex+1) << " tokens per entry in '"
+                << paramName << "', but only found "
+                << vec.size();
+      return false;
+    }
+    const string &pathStr = vec[tokenizeIndex];
+
+    bool fileFound=0;
+    for(size_t i=0; i<extensions.size() && !fileFound; ++i) {
+      fileFound|=FileExists(pathStr + extensions[i]);
+    }
+    if(!fileFound) {
+      std::cerr << "File " << pathStr << " does not exist";
+      return false;
+    }
+  }
+  return true;
+}
+
+/** look for a switch in arg, update parameter */
+// TODO arg parsing like this does not belong in the library, it belongs
+// in moses-cmd
+string
+Parameter::
+FindParam(const string &paramSwitch, int argc, char const* argv[])
+{
+  for (int i = 0 ; i < argc ; i++) {
+    if (string(argv[i]) == paramSwitch) {
+      if (i+1 < argc) {
+        return argv[i+1];
+      } else {
+        std::cerr << "Option " << paramSwitch << " requires a parameter!";
+        // TODO return some sort of error, not the empty string
+      }
+    }
+  }
+  return "";
+}
+
+/** update parameter settings with command line switches
+ * \param paramSwitch (potentially short) name of switch
+ * \param paramName full name of parameter
+ * \param argc number of arguments on command line
+ * \param argv values of paramters on command line */
+void
+Parameter::
+OverwriteParam(const string &paramSwitch, const string &paramName,
+               int argc, char const* argv[])
+{
+  int startPos = -1;
+  for (int i = 0 ; i < argc ; i++) {
+    if (string(argv[i]) == paramSwitch) {
+      startPos = i+1;
+      break;
+    }
+  }
+  if (startPos < 0)
+    return;
+
+  int index = 0;
+  m_setting[paramName]; // defines the parameter, important for boolean switches
+  while (startPos < argc && (!isOption(argv[startPos]))) {
+    if (m_setting[paramName].size() > (size_t)index)
+      m_setting[paramName][index] = argv[startPos];
+    else
+      m_setting[paramName].push_back(argv[startPos]);
+    index++;
+    startPos++;
+  }
+}
+
+
+/** read parameters from a configuration file */
+bool
+Parameter::
+ReadConfigFile(const string &filePath )
+{
+  InputFileStream inFile(filePath);
+  string line, paramName;
+  while(getline(inFile, line)) {
+    // comments
+    size_t comPos = line.find_first_of("#");
+    if (comPos != string::npos)
+      line = line.substr(0, comPos);
+    // trim leading and trailing spaces/tabs
+    line = Trim(line);
+
+    if (line.size() == 0) {
+      // blank line. do nothing.
+    } else if (line[0]=='[') {
+      // new parameter
+      for (size_t currPos = 0 ; currPos < line.size() ; currPos++) {
+        if (line[currPos] == ']') {
+          paramName = line.substr(1, currPos - 1);
+          break;
+        }
+      }
+    } else {
+      // add value to parameter
+      m_setting[paramName].push_back(line);
+    }
+  }
+  return true;
+}
+
+struct Credit {
+  string name, contact, currentPursuits, areaResponsibility;
+  int sortId;
+
+  Credit(string name, string contact, string currentPursuits, string areaResponsibility) {
+    this->name								= name							;
+    this->contact							= contact						;
+    this->currentPursuits			= currentPursuits		;
+    this->areaResponsibility	= areaResponsibility;
+    this->sortId							= util::rand_excl(1000);
+  }
+
+  bool operator<(const Credit &other) const {
+    /*
+    if (areaResponsibility.size() != 0 && other.areaResponsibility.size() ==0)
+    	return true;
+    if (areaResponsibility.size() == 0 && other.areaResponsibility.size() !=0)
+    	return false;
+
+    return name < other.name;
+    */
+    return sortId < other.sortId;
+  }
+
+};
+
+std::ostream& operator<<(std::ostream &os, const Credit &credit)
+{
+  os << credit.name;
+  if (credit.contact != "")
+    os << "\t   contact: " << credit.contact;
+  if (credit.currentPursuits != "")
+    os << "   " << credit.currentPursuits;
+  if (credit.areaResponsibility != "")
+    os << "   I'll answer question on: " << credit.areaResponsibility;
+  return os;
+}
+
+void
+Parameter::
+PrintCredit()
+{
+  vector<Credit> everyone;
+  srand ( time(NULL) );
+
+  everyone.push_back(Credit("Nicola Bertoldi"
+                            , "911"
+                            , ""
+                            , "scripts & other stuff"));
+  everyone.push_back(Credit("Ondrej Bojar"
+                            , ""
+                            , "czech this out!"
+                            , ""));
+  everyone.push_back(Credit("Chris Callison-Burch"
+                            , "anytime, anywhere"
+                            , "international playboy"
+                            , ""));
+  everyone.push_back(Credit("Alexandra Constantin"
+                            , ""
+                            , "eu sunt varza"
+                            , ""));
+  everyone.push_back(Credit("Brooke Cowan"
+                            , "brooke@csail.mit.edu"
+                            , "if you're going to san francisco, be sure to wear a flower in your hair"
+                            , ""));
+  everyone.push_back(Credit("Chris Dyer"
+                            , "can't. i'll be out driving my mustang"
+                            , "driving my mustang"
+                            , ""));
+  everyone.push_back(Credit("Marcello Federico"
+                            , "federico at itc at it"
+                            , "Researcher at ITC-irst, Trento, Italy"
+                            , "IRST language model"));
+  everyone.push_back(Credit("Evan Herbst"
+                            , "Small college in upstate New York"
+                            , ""
+                            , ""));
+  everyone.push_back(Credit("Philipp Koehn"
+                            , "only between 2 and 4am"
+                            , ""
+                            , "Nothing fazes this dude"));
+  everyone.push_back(Credit("Christine Moran"
+                            , "weird building at MIT"
+                            , ""
+                            , ""));
+  everyone.push_back(Credit("Wade Shen"
+                            , "via morse code"
+                            , "buying another laptop"
+                            , ""));
+  everyone.push_back(Credit("Richard Zens"
+                            , "richard at aachen dot de"
+                            , ""
+                            , "ambiguous source input, confusion networks, confusing source code"));
+  everyone.push_back(Credit("Hieu Hoang", "http://www.hoang.co.uk/hieu/"
+                            , "phd student at Edinburgh Uni. Original Moses developer"
+                            , "general queries/ flames on Moses."));
+
+  sort(everyone.begin(), everyone.end());
+
+
+  cerr <<  "Moses - A beam search decoder for phrase-based statistical machine translation models" << endl
+       << "Copyright (C) 2006 University of Edinburgh" << endl << endl
+
+       << "This library is free software; you can redistribute it and/or" << endl
+       << "modify it under the terms of the GNU Lesser General Public" << endl
+       << "License as published by the Free Software Foundation; either" << endl
+       << "version 2.1 of the License, or (at your option) any later version." << endl << endl
+
+       << "This library is distributed in the hope that it will be useful," << endl
+       << "but WITHOUT ANY WARRANTY; without even the implied warranty of" << endl
+       << "MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU" << endl
+       << "Lesser General Public License for more details." << endl << endl
+
+       << "You should have received a copy of the GNU Lesser General Public" << endl
+       << "License along with this library; if not, write to the Free Software" << endl
+       << "Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA" << endl << endl
+       << "***********************************************************************" << endl << endl
+       << "Built on " << __DATE__ << " at " __TIME__ << endl << endl
+       << "WHO'S FAULT IS THIS GODDAM SOFTWARE:" << endl;
+
+  ostream_iterator<Credit> out(cerr, "\n");
+  copy(everyone.begin(), everyone.end(), out);
+  cerr <<  endl << endl;
+}
+
+/** update parameter settings with command line switches
+ * \param paramName full name of parameter
+ * \param values inew values for paramName */
+void
+Parameter::
+OverwriteParam(const string &paramName, PARAM_VEC values)
+{
+  VERBOSE(2,"Overwriting parameter " << paramName);
+
+  m_setting[paramName]; // defines the parameter, important for boolean switches
+  if (m_setting[paramName].size() > 1) {
+    VERBOSE(2," (the parameter had " << m_setting[paramName].size() << " previous values)");
+    UTIL_THROW_IF2(m_setting[paramName].size() != values.size(),
+                   "Number of weight override for " << paramName
+                   << " is not the same as the original number of weights");
+  } else {
+    VERBOSE(2," (the parameter does not have previous values)");
+    m_setting[paramName].resize(values.size());
+  }
+  VERBOSE(2," with the following values:");
+  int i=0;
+  for (PARAM_VEC::iterator iter = values.begin(); iter != values.end() ; iter++, i++) {
+    m_setting[paramName][i] = *iter;
+    VERBOSE(2, " " << *iter);
+  }
+  VERBOSE(2, std::endl);
+}
+
+void
+Parameter::
+PrintFF() const
+{
+  StaticData::Instance().GetFeatureRegistry().PrintFF();
+}
+
+std::set<std::string>
+Parameter::
+GetWeightNames() const
+{
+  std::set<std::string> ret;
+  std::map<std::string, std::vector<float> >::const_iterator iter;
+  for (iter = m_weights.begin(); iter != m_weights.end(); ++iter) {
+    const string &key = iter->first;
+    ret.insert(key);
+  }
+  return ret;
+}
+
+void
+Parameter::
+Save(const std::string path)
+{
+  ofstream file;
+  file.open(path.c_str());
+
+  PARAM_MAP::const_iterator iterOuter;
+  for (iterOuter = m_setting.begin(); iterOuter != m_setting.end(); ++iterOuter) {
+    const std::string &sectionName = iterOuter->first;
+    file << "[" << sectionName << "]" << endl;
+
+    const PARAM_VEC &values = iterOuter->second;
+
+    PARAM_VEC::const_iterator iterInner;
+    for (iterInner = values.begin(); iterInner != values.end(); ++iterInner) {
+      const std::string &value = *iterInner;
+      file << value << endl;
+    }
+
+    file << endl;
+  }
+
+
+  file.close();
+}
+
+template<>
+void
+Parameter::
+SetParameter<bool>(bool &parameter, std::string const& parameterName,
+                   bool const& defaultValue) const
+{
+  const PARAM_VEC *params = GetParam(parameterName);
+
+  // default value if nothing is specified
+  parameter = defaultValue;
+  if (params == NULL) {
+    return;
+  }
+
+  // if parameter is just specified as, e.g. "-parameter" set it true
+  if (params->size() == 0) {
+    parameter = true;
+  }
+  // if paramter is specified "-parameter true" or "-parameter false"
+  else if (params->size() == 1) {
+    parameter = Scan<bool>( params->at(0));
+  }
+}
+
+void
+Parameter::
+SetParameter(bool& var, std::string const& name)
+{
+  SetParameter(var,name,false);
+}
+
+} // namespace
+
+

mosesdecoder/moses/Parameter.h

@@ -0,0 +1,173 @@
+/// $Id$
+
+/***********************************************************************
+Moses - factored phrase-based language decoder
+Copyright (C) 2006 University of Edinburgh
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Lesser General Public
+License as published by the Free Software Foundation; either
+version 2.1 of the License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public
+License along with this library; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+***********************************************************************/
+
+#ifndef moses_Parameter_h
+#define moses_Parameter_h
+
+#include <string>
+#include <set>
+#include <map>
+#include <vector>
+#include "TypeDef.h"
+#include "Util.h"
+#include <boost/program_options.hpp>
+
+namespace Moses
+{
+typedef std::vector<std::string>            PARAM_VEC;
+typedef std::map<std::string, PARAM_VEC >   PARAM_MAP;
+typedef std::map<std::string, bool>         PARAM_BOOL;
+typedef std::map<std::string, std::string > PARAM_STRING;
+
+/** Handles parameter values set in config file or on command line.
+ * Process raw parameter data (names and values as strings) for StaticData
+ * to parse; to get useful values, see StaticData.
+ */
+class Parameter
+{
+  typedef boost::program_options::options_description options_description;
+  typedef boost::program_options::value_semantic value_semantic;
+protected:
+  PARAM_MAP m_setting;
+  PARAM_BOOL m_valid;
+  PARAM_STRING m_abbreviation;
+  PARAM_STRING m_description;
+  PARAM_STRING m_fullname;
+  // std::map<char,std::set<std::string> > m_confusable;
+  // stores long parameter names that start with a letter that is also a short option.
+  options_description m_options;
+
+  std::map<std::string, std::vector<float> >  m_weights;
+
+  std::string FindParam(const std::string &paramSwitch, int argc, char const* argv[]);
+  void OverwriteParam(const std::string &paramSwitch, const std::string &paramName,
+                      int argc, char const* argv[]);
+  bool ReadConfigFile(const std::string &filePath );
+  bool FilesExist(const std::string &paramName, int fieldNo, std::vector<std::string> const& fileExtension=std::vector<std::string>(1,""));
+  bool isOption(const char* token);
+  bool Validate();
+
+  void
+  AddParam(options_description& optgroup,
+           value_semantic const* optvalue,
+           std::string const& paramName,
+           std::string const& description);
+
+  void
+  AddParam(options_description& optgroup,
+           std::string const &paramName,
+           std::string const &description);
+
+  void
+  AddParam(options_description& optgroup,
+           value_semantic const* optvalue,
+           std::string const& paramName,
+           std::string const& abbrevName,
+           std::string const& description);
+
+  void
+  AddParam(options_description& optgroup,
+           std::string const& paramName,
+           std::string const& abbrevName,
+           std::string const& description);
+
+  void PrintCredit();
+  void PrintFF() const;
+
+  void SetWeight(const std::string &name, size_t ind, float weight);
+  void SetWeight(const std::string &name, size_t ind, const std::vector<float> &weights);
+  void AddWeight(const std::string &name, size_t ind, const std::vector<float> &weights);
+  void ConvertWeightArgs();
+  void ConvertWeightArgsSingleWeight(const std::string &oldWeightName, const std::string &newWeightName);
+  void ConvertWeightArgsPhraseModel(const std::string &oldWeightName);
+  void ConvertWeightArgsLM();
+  void ConvertWeightArgsDistortion();
+  void ConvertWeightArgsGeneration(const std::string &oldWeightName, const std::string &newWeightName);
+  void ConvertWeightArgsPhrasePenalty();
+  void ConvertWeightArgsWordPenalty();
+  void ConvertPhrasePenalty();
+  void CreateWeightsMap();
+  void CreateWeightsMap(const PARAM_VEC &vec);
+  void WeightOverwrite();
+  void AddFeature(const std::string &line);
+  void AddFeaturesCmd();
+
+
+public:
+  Parameter();
+  ~Parameter();
+  bool LoadParam(int argc, char const* argv[]);
+  bool LoadParam(const std::string &filePath);
+  void Explain();
+
+  /** return a vector of strings holding the whitespace-delimited values on the ini-file line corresponding to the given parameter name */
+  const PARAM_VEC *GetParam(const std::string &paramName) const;
+
+  /** check if parameter is defined (either in moses.ini or as switch) */
+  bool isParamSpecified(const std::string &paramName) const {
+    return  m_setting.find( paramName ) != m_setting.end();
+  }
+
+  void OverwriteParam(const std::string &paramName, PARAM_VEC values);
+
+  std::vector<float> GetWeights(const std::string &name);
+  const std::map<std::string, std::vector<float> > &GetAllWeights() const {
+    return m_weights;
+  }
+  std::set<std::string> GetWeightNames() const;
+
+  const PARAM_MAP &GetParams() const {
+    return m_setting;
+  }
+
+  void Save(const std::string path);
+
+  template<typename T>
+  void SetParameter(T &var, const std::string &name, const T &defaultValue) const {
+    const PARAM_VEC *params = GetParam(name);
+    if (params && params->size()) {
+      var = Scan<T>( params->at(0));
+    } else {
+      var = defaultValue;
+    }
+  }
+
+  void SetParameter(bool& var, std::string const& name);
+
+  bool SetBooleanSwitch(bool& val, std::string const name) {
+    // issues a warning if format is wrong
+    const PARAM_VEC *params = GetParam(name);
+    val = (params && params->size());
+    if (val && params->size() != 1) {
+      TRACE_ERR("ERROR: wrong format for switch -" << name);
+      return false;
+    }
+    return true;
+  }
+
+};
+
+template<>
+void Parameter::SetParameter<bool>(bool &var, const std::string &name, const bool &defaultValue) const;
+
+}
+
+#endif

mosesdecoder/moses/Phrase.h

@@ -0,0 +1,244 @@
+// -*- mode: c++; indent-tabs-mode: nil; tab-width:2  -*-
+// vim:tabstop=2
+
+/***********************************************************************
+Moses - factored phrase-based language decoder
+Copyright (C) 2006 University of Edinburgh
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Lesser General Public
+License as published by the Free Software Foundation; either
+version 2.1 of the License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public
+License along with this library; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+***********************************************************************/
+
+#ifndef moses_Phrase_h
+#define moses_Phrase_h
+
+#include <iostream>
+#include <vector>
+#include <list>
+#include <string>
+
+#include <boost/functional/hash.hpp>
+
+#include "Word.h"
+#include "Util.h"
+
+#include "util/string_piece.hh"
+#include "util/exception.hh"
+#include "parameters/AllOptions.h"
+
+namespace Moses
+{
+class FactorMask;
+class Range;
+class ContextScope;
+
+/** Representation of a phrase, ie. a contiguous number of words.
+ *  Wrapper for vector of words
+ */
+class Phrase
+{
+  friend std::ostream& operator<<(std::ostream&, const Phrase&);
+  // private:
+protected:
+  std::vector<Word>			m_words;
+
+public:
+
+  virtual bool HasScope() const {
+    return false;
+  }
+
+  virtual SPTR<ContextScope> GetScope() const {
+    return SPTR<ContextScope>();
+  }
+
+
+  /** No longer does anything as not using mem pool for Phrase class anymore */
+  static void InitializeMemPool();
+  static void FinalizeMemPool();
+
+  /** create empty phrase
+  */
+  Phrase();
+  explicit Phrase(size_t reserveSize);
+  /** create phrase from vectors of words	*/
+  explicit Phrase(const std::vector< const Word* > &mergeWords);
+
+  /* This isn't a swap function because classes inherit from Phrase and might
+   * not override swap, which would be bad.
+   */
+  void SwapWords(Phrase &other) {
+    swap(m_words, other.m_words);
+  }
+
+  /** destructor */
+  virtual ~Phrase();
+
+  /**
+   * Fills phrase with words from format string, typically from phrase table or sentence input
+   *
+   * \param factorOrder  factor types of each element in 2D string vector
+   * \param phraseString formatted input string to parse
+   * \param lhs          returns the non-terminal Word for the left-hand side of an SCFG rule, may be NULL for phrase-based
+  */
+  void CreateFromString(FactorDirection direction,
+                        const std::vector<FactorType> &factorOrder,
+                        const StringPiece &phraseString,
+                        Word **lhs);
+
+  /**	copy factors from the other phrase to this phrase.
+  	IsCompatible() must be run beforehand to ensure incompatible factors aren't overwritten
+  */
+  void MergeFactors(const Phrase &copy);
+  //! copy a single factor (specified by factorType)
+  void MergeFactors(const Phrase &copy, FactorType factorType);
+  //! copy all factors specified in factorVec and none others
+  void MergeFactors(const Phrase &copy, const std::vector<FactorType>& factorVec);
+
+  /** compare 2 phrases to ensure no factors are lost if the phrases are merged
+  *	must run IsCompatible() to ensure incompatible factors aren't being overwritten
+  */
+  bool IsCompatible(const Phrase &inputPhrase) const;
+  bool IsCompatible(const Phrase &inputPhrase, FactorType factorType) const;
+  bool IsCompatible(const Phrase &inputPhrase, const std::vector<FactorType>& factorVec) const;
+
+  //! number of words
+  inline size_t GetSize() const {
+    return m_words.size();
+  }
+
+  //! word at a particular position
+  inline const Word &GetWord(size_t pos) const {
+    return m_words[pos];
+  }
+  inline Word &GetWord(size_t pos) {
+    return m_words[pos];
+  }
+
+  inline Word &Front() {
+    return m_words[0];
+  }
+
+  inline Word &Back() {
+    return m_words[GetSize() - 1];
+  }
+
+  inline const Word &Front() const {
+    return m_words[0];
+  }
+
+  inline const Word &Back() const {
+    return m_words[GetSize() - 1];
+  }
+
+  //! particular factor at a particular position
+  inline const Factor *GetFactor(size_t pos, FactorType factorType) const {
+    const Word &ptr = m_words[pos];
+    return ptr[factorType];
+  }
+  inline void SetFactor(size_t pos, FactorType factorType, const Factor *factor) {
+    Word &ptr = m_words[pos];
+    ptr[factorType] = factor;
+  }
+
+  size_t GetNumTerminals() const;
+  size_t GetNumNonTerminals() const {
+    return GetSize() - GetNumTerminals();
+  }
+
+  //! whether the 2D vector is a substring of this phrase
+  bool Contains(const std::vector< std::vector<std::string> > &subPhraseVector
+                , const std::vector<FactorType> &inputFactor) const;
+
+  size_t Find(const Phrase &sought, int maxUnknown) const;
+
+  //! create an empty word at the end of the phrase
+  Word &AddWord();
+  //! create copy of input word at the end of the phrase
+  void AddWord(const Word &newWord) {
+    AddWord() = newWord;
+  }
+
+  /** appends a phrase at the end of current phrase **/
+  void Append(const Phrase &endPhrase);
+  void PrependWord(const Word &newWord);
+
+  void Clear() {
+    m_words.clear();
+  }
+
+  void RemoveWord(size_t pos) {
+    UTIL_THROW_IF2(pos >= m_words.size(),
+                   "Referencing position " << pos << " out of bound");
+    m_words.erase(m_words.begin() + pos);
+  }
+
+  void InitStartEndWord();
+
+  //! create new phrase class that is a substring of this phrase
+  Phrase GetSubString(const Range &range) const;
+  Phrase GetSubString(const Range &range, FactorType factorType) const;
+
+  //! return a string rep of the phrase;
+  // w/ factors delimited by FactorDelimiter
+  std::string
+  GetStringRep(std::vector<FactorType> const& factorsToPrint,
+               AllOptions const* opts=NULL) const;
+
+  TO_STRING();
+
+
+  int Compare(const Phrase &other) const;
+
+  /** transitive comparison between 2 phrases
+   *		used to insert & find phrase in dictionary
+   */
+  bool operator< (const Phrase &compare) const {
+    return Compare(compare) < 0;
+  }
+
+  size_t hash() const;
+
+  bool operator==(const Phrase &compare) const;
+  bool operator!=(const Phrase &compare) const {
+    return ! (*this == compare);
+  }
+
+  void OnlyTheseFactors(const FactorMask &factors);
+
+};
+
+inline size_t hash_value(const Phrase& phrase)
+{
+  return phrase.hash();
+}
+
+struct PhrasePtrComparator {
+  inline bool operator()(const Phrase* lhs, const Phrase* rhs) const {
+    return *lhs == *rhs;
+  }
+};
+
+struct PhrasePtrHasher {
+  inline size_t operator()(const Phrase* phrase) const {
+    size_t seed = 0;
+    boost::hash_combine(seed,*phrase);
+    return seed;
+  }
+
+};
+
+}
+
+#endif

mosesdecoder/moses/PrefixTree.h

@@ -0,0 +1,339 @@
+// $Id$
+
+/* ---------------------------------------------------------------- */
+/* Copyright 2005 (c) by RWTH Aachen - Lehrstuhl fuer Informatik VI */
+/* Richard Zens                                                     */
+/* ---------------------------------------------------------------- */
+#ifndef moses_PrefixTree_h
+#define moses_PrefixTree_h
+
+#include <vector>
+#include <algorithm>
+#include <deque>
+#include "Util.h"
+#include "FilePtr.h"
+#include "File.h"
+
+namespace Moses
+{
+
+/** @todo How is this used in the pb binary phrase table?
+ */
+template<typename T,typename D>
+class PrefixTreeSA
+{
+public:
+  typedef T Key;
+  typedef D Data;
+
+  typedef PrefixTreeSA<T,D> Self;
+  typedef std::vector<T> VT;
+  typedef std::vector<Self*> VP;
+  typedef std::vector<D> VD;
+
+  VT keys;
+  VP ptr;
+  VD data;
+
+  static Data def;
+
+public:
+  PrefixTreeSA() {}
+
+  ~PrefixTreeSA() {
+    for(size_t i=0; i<ptr.size(); ++i) delete ptr[i];
+  }
+
+  static const Data& getDefault() {
+    return def;
+  }
+  static void setDefault(const Data& x) {
+    def=x;
+  }
+
+
+  // insert sequence
+  template<typename fwiter> Data& insert(fwiter b,fwiter e) {
+    typename VT::iterator i=std::lower_bound(keys.begin(),keys.end(),*b);
+    typename VT::iterator kb=keys.begin();
+    size_t pos=std::distance(kb,i);
+
+    if(i==keys.end() || *i!=*b) {
+      keys.insert(i,*b);
+      data.insert(data.begin()+pos,def);
+
+      Self *self = NULL;
+      ptr.insert(ptr.begin()+pos, self);
+    }
+    if(++b!=e) {
+      if(!ptr[pos]) ptr[pos]=new Self;
+      return ptr[pos]->insert(b,e);
+    } else return data[pos];
+  }
+  // insert container
+  template<typename cont> Data& insert(const cont& c) {
+    return insert(c.begin(),c.end());
+  }
+
+  size_t size() const {
+    return keys.size();
+  }
+  const Key& getKey(size_t i) const {
+    return keys[i];
+  }
+  const Data& getData(size_t i) const {
+    return data[i];
+  }
+  const Self* getPtr(size_t i) const {
+    return ptr[i];
+  }
+
+  size_t findKey(const Key& k) const {
+    typename VT::const_iterator i=std::lower_bound(keys.begin(),keys.end(),k);
+    if(i==keys.end() || *i!=k) return keys.size();
+    return std::distance(keys.begin(),i);
+  }
+
+  // find sequence
+  template<typename fwiter> const Data* findPtr(fwiter b,fwiter e) const {
+    size_t pos=findKey(*b);
+    if(pos==keys.size()) return 0;
+    if(++b==e) return &data[pos];
+    if(ptr[pos]) return ptr[pos]->findPtr(b,e);
+    else return 0;
+  }
+  // find container
+  template<typename cont> const Data* findPtr(const cont& c) const {
+    return findPtr(c.begin(),c.end());
+  }
+
+
+  // find sequence
+  template<typename fwiter> const Data& find(fwiter b,fwiter e) const {
+    if(const Data* p=findPtr(b,e)) return *p;
+    else return def;
+  }
+
+  // find container
+  template<typename cont> const Data& find(const cont& c) const {
+    return find(c.begin(),c.end());
+  }
+
+  void shrink() {
+    ShrinkToFit(keys);
+    ShrinkToFit(ptr);
+    ShrinkToFit(data);
+  }
+
+};
+template<typename T,typename D> D PrefixTreeSA<T,D>::def;
+
+/////////////////////////////////////////////////////////////////////////////
+
+/** @todo How is this used in the pb binary phrase table?
+ */
+template<typename T,typename D>
+class PrefixTreeF
+{
+public:
+  typedef T Key;
+  typedef D Data;
+private:
+  typedef PrefixTreeF<Key,Data> Self;
+public:
+  typedef FilePtr<Self> Ptr;
+private:
+  typedef std::vector<Key> VK;
+  typedef std::vector<Data> VD;
+  typedef std::vector<Ptr> VP;
+
+  VK keys;
+  VD data;
+  VP ptr;
+
+  static Data def;
+
+  OFF_T startPos;
+  FILE* f;
+public:
+
+  PrefixTreeF(FILE* f_=0) : f(f_) {
+    if(f) read();
+  }
+
+  ~PrefixTreeF() {
+    free();
+  }
+
+  void read() {
+    startPos=fTell(f);
+    fReadVector(f,keys);
+    fReadVector(f,data);
+    ptr.clear();
+    ptr.resize(keys.size());
+    std::vector<OFF_T> rawOffs(keys.size());
+    size_t bytes_read = fread(&rawOffs[0], sizeof(OFF_T), keys.size(), f);
+    UTIL_THROW_IF2(bytes_read != keys.size(), "Read error at " << HERE);
+    for(size_t i=0; i<ptr.size(); ++i)
+      if (rawOffs[i]) ptr[i].set(f, rawOffs[i]);
+  }
+
+  void free() {
+    for(typename VP::iterator i=ptr.begin(); i!=ptr.end(); ++i) i->free();
+  }
+
+  void reserve(size_t s) {
+    keys.reserve(s);
+    data.reserve(s);
+    ptr.reserve(s);
+  }
+
+  template<typename fwiter>
+  void changeData(fwiter b,fwiter e,const Data& d) {
+    typename VK::const_iterator i=std::lower_bound(keys.begin(),keys.end(),*b);
+    if(i==keys.end() || *i!=*b) {
+      TRACE_ERR("ERROR: key not found in changeData!\n");
+      return;
+    }
+    typename VK::const_iterator kb=keys.begin();
+    size_t pos=std::distance(kb,i);
+    if(++b==e) {
+      OFF_T p=startPos+keys.size()*sizeof(Key)+2*sizeof(unsigned)+pos*sizeof(Data);
+      TRACE_ERR("elem found at pos "<<p<<" old val: "<<data[pos]<<"  startpos: "<<startPos<<"\n");
+      if(data[pos]!=d) {
+        data[pos]=d;
+        fSeek(f,p);
+        fWrite(f,d);
+      }
+      return;
+    }
+    if(ptr[pos]) ptr[pos]->changeData(b,e,d);
+    else {
+      TRACE_ERR("ERROR: seg not found!in changeData\n");
+    }
+  }
+
+
+  void create(const PrefixTreeSA<Key,Data>& psa,const std::string& fname) {
+    FILE* f=fOpen(fname.c_str(),"wb");
+    create(psa,f);
+    fclose(f);
+  }
+
+  void create(const PrefixTreeSA<Key,Data>& psa,FILE* f,int verbose=0) {
+    setDefault(psa.getDefault());
+
+    typedef std::pair<const PrefixTreeSA<Key,Data>*,OFF_T> P;
+    typedef std::deque<P> Queue;
+
+    Queue queue;
+
+    queue.push_back(P(&psa,fTell(f)));
+    bool isFirst=1;
+    size_t ns=1;
+    while(queue.size()) {
+      if(verbose && queue.size()>ns) {
+        TRACE_ERR("stack size in PF create: "<<queue.size()<<"\n");
+        while(ns<queue.size()) ns*=2;
+      }
+      const P& pp=queue.back();
+      const PrefixTreeSA<Key,Data>& p=*pp.first;
+      OFF_T pos=pp.second;
+      queue.pop_back();
+
+      if(!isFirst) {
+        OFF_T curr=fTell(f);
+        fSeek(f,pos);
+        fWrite(f,curr);
+        fSeek(f,curr);
+      } else isFirst=0;
+
+      size_t s=0;
+      s+=fWriteVector(f,p.keys);
+      s+=fWriteVector(f,p.data);
+
+      for(size_t i=0; i<p.ptr.size(); ++i) {
+        if(p.ptr[i])
+          queue.push_back(P(p.ptr[i],fTell(f)));
+        OFF_T ppos=0;
+        s+=fWrite(f,ppos);
+      }
+    }
+  }
+
+  size_t size() const {
+    return keys.size();
+  }
+  const Key& getKey(size_t i) const {
+    return keys[i];
+  }
+  const Data& getData(size_t i) const {
+    return data[i];
+  }
+  const Self* getPtr(size_t i) const {
+    return ptr[i];
+  }
+
+  size_t findKey(const Key& k) const {
+    typename VK::const_iterator i=std::lower_bound(keys.begin(),keys.end(),k);
+    if(i==keys.end() || *i!=k) return keys.size();
+    return std::distance(keys.begin(),i);
+  }
+
+  Ptr const* findKeyPtr(const Key& k) const {
+    size_t pos=findKey(k);
+    return (pos<keys.size() ? &ptr[pos] : 0);
+  }
+
+  // find sequence
+  template<typename fwiter> const Data* findPtr(fwiter b,fwiter e) const {
+    typename VK::const_iterator i=std::lower_bound(keys.begin(),keys.end(),*b);
+    if(i==keys.end() || *i!=*b) return 0;
+    size_t pos=std::distance(keys.begin(),i);
+    if(++b==e) return &data[pos];
+    if(ptr[pos]) return ptr[pos]->findPtr(b,e);
+    else return 0;
+  }
+  // find container
+  template<typename cont> const Data* findPtr(const cont& c) const {
+    return findPtr(c.begin(),c.end());
+  }
+
+
+  // find sequence
+  template<typename fwiter> const Data& find(fwiter b,fwiter e) const {
+    if(const Data* p=findPtr(b,e)) return *p;
+    else return def;
+  } //return (p?*p:def);}
+
+  // find container
+  template<typename cont> const Data& find(const cont& c) const {
+    return find(c.begin(),c.end());
+  }
+
+  static void setDefault(const Data& d) {
+    def=d;
+  }
+  static const Data& getDefault() {
+    return def;
+  }
+
+
+  void print(std::ostream& out,const std::string s="") const {
+
+    out<<s<<"startpos: "<<startPos<<"  size: "<<keys.size()<<"\n";
+    for(size_t i=0; i<keys.size(); ++i) {
+      out<<s<<i<<" - "<<keys[i]<<" "<<data[i]<<"\n";
+    }
+    for(size_t i=0; i<ptr.size(); ++i)
+      if(ptr[i])
+        ptr[i]->print(out,s+"  ");
+  }
+
+
+};
+template<typename T,typename D> D PrefixTreeF<T,D>::def;
+
+}
+
+#endif

mosesdecoder/moses/Range.h

@@ -0,0 +1,107 @@
+// $Id$
+
+/***********************************************************************
+Moses - factored phrase-based language decoder
+Copyright (C) 2006 University of Edinburgh
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Lesser General Public
+License as published by the Free Software Foundation; either
+version 2.1 of the License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public
+License along with this library; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+***********************************************************************/
+
+#ifndef moses_WordsRange_h
+#define moses_WordsRange_h
+
+#include <iostream>
+#include <boost/functional/hash.hpp>
+#include "TypeDef.h"
+#include "Util.h"
+#include "util/exception.hh"
+
+#ifdef WIN32
+#undef max
+#endif
+
+namespace Moses
+{
+
+/***
+ * Efficient version of Bitmap for contiguous ranges
+ */
+class Range
+{
+  friend std::ostream& operator << (std::ostream& out, const Range& range);
+
+  // m_endPos is inclusive
+  size_t m_startPos, m_endPos;
+public:
+  inline explicit Range() {}
+  inline Range(size_t startPos, size_t endPos) : m_startPos(startPos), m_endPos(endPos) {}
+  inline Range(const Range &copy)
+    : m_startPos(copy.GetStartPos())
+    , m_endPos(copy.GetEndPos()) {
+  }
+
+  inline size_t GetStartPos() const {
+    return m_startPos;
+  }
+  inline size_t GetEndPos() const {
+    return m_endPos;
+  }
+
+  //! count of words translated
+  inline size_t GetNumWordsCovered() const {
+    return (m_startPos == NOT_FOUND) ? 0 : m_endPos - m_startPos + 1;
+  }
+
+  //! transitive comparison
+  inline bool operator<(const Range& x) const {
+    return (m_startPos<x.m_startPos
+            || (m_startPos==x.m_startPos && m_endPos<x.m_endPos));
+  }
+
+  // equality operator
+  inline bool operator==(const Range& x) const {
+    return (m_startPos==x.m_startPos && m_endPos==x.m_endPos);
+  }
+  // Whether two word ranges overlap or not
+  inline bool Overlap(const Range& x) const {
+
+    if ( x.m_endPos < m_startPos || x.m_startPos > m_endPos) return false;
+
+    return true;
+  }
+
+  inline size_t GetNumWordsBetween(const Range& x) const {
+    UTIL_THROW_IF2(Overlap(x), "Overlapping ranges");
+
+    if (x.m_endPos < m_startPos) {
+      return m_startPos - x.m_endPos - 1;
+    }
+
+    return x.m_startPos - m_endPos - 1;
+  }
+
+
+  TO_STRING();
+};
+
+inline size_t hash_value(const Range& range)
+{
+  size_t  seed = range.GetStartPos();
+  boost::hash_combine(seed, range.GetEndPos());
+  return seed;
+}
+
+}
+#endif

mosesdecoder/moses/ReorderingConstraint.cpp

@@ -0,0 +1,260 @@
+// $Id$
+// vim:tabstop=2
+
+/***********************************************************************
+Moses - factored phrase-based language decoder
+Copyright (C) 2008 University of Edinburgh
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Lesser General Public
+License as published by the Free Software Foundation; either
+version 2.1 of the License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public
+License along with this library; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+***********************************************************************/
+
+#include "ReorderingConstraint.h"
+#include "InputType.h"
+#include "StaticData.h"
+#include "Bitmap.h"
+
+namespace Moses
+{
+
+//! allocate memory for reordering walls
+void ReorderingConstraint::InitializeWalls(size_t size)
+{
+  m_size = size;
+  m_wall      = (bool*)   malloc(sizeof(bool) * size);
+  m_localWall = (size_t*) malloc(sizeof(size_t) * size);
+
+  for (size_t pos = 0 ; pos < m_size ; pos++) {
+    m_wall[pos] = false;
+    m_localWall[pos] = NOT_A_ZONE;
+  }
+}
+
+
+//! set value at a particular position
+void ReorderingConstraint::SetWall( size_t pos, bool value )
+{
+  VERBOSE(3,"SETTING reordering wall at position " << pos << std::endl);
+  m_wall[pos] = value;
+  m_active = true;
+}
+
+//! has to be called to localized walls
+void ReorderingConstraint::FinalizeWalls()
+{
+  for(size_t z = 0; z < m_zone.size(); z++ ) {
+    const size_t startZone = m_zone[z].first;
+    const size_t endZone = m_zone[z].second;// note: wall after endZone is not local
+    for( size_t pos = startZone; pos < endZone; pos++ ) {
+      if (m_wall[ pos ]) {
+        m_localWall[ pos ] = z;
+        m_wall[ pos ] = false;
+        VERBOSE(3,"SETTING local wall " << pos << std::endl);
+      }
+      // enforce that local walls only apply to innermost zone
+      else if (m_localWall[ pos ] != NOT_A_ZONE) {
+        size_t assigned_z = m_localWall[ pos ];
+        if ((m_zone[assigned_z].first < startZone) ||
+            (m_zone[assigned_z].second > endZone)) {
+          m_localWall[ pos ] = z;
+        }
+      }
+    }
+  }
+}
+
+//! set walls based on "-monotone-at-punctuation" flag
+void ReorderingConstraint::SetMonotoneAtPunctuation( const Phrase &sentence )
+{
+  for( size_t i=0; i<sentence.GetSize(); i++ ) {
+    const Word& word = sentence.GetWord(i);
+    if (word[0]->GetString() == "," ||
+        word[0]->GetString() == "." ||
+        word[0]->GetString() == "!" ||
+        word[0]->GetString() == "?" ||
+        word[0]->GetString() == ":" ||
+        word[0]->GetString() == ";" ||
+        word[0]->GetString() == "\"") {
+      // set wall before and after punc, but not at sentence start, end
+      if (i>0 && i<m_size-1) SetWall( i, true );
+      if (i>1)               SetWall( i-1, true );
+    }
+  }
+}
+
+//! set a reordering zone (once entered, need to finish)
+void ReorderingConstraint::SetZone( size_t startPos, size_t endPos )
+{
+  VERBOSE(3,"SETTING zone " << startPos << "-" << endPos << std::endl);
+  std::pair<size_t,size_t> newZone;
+  newZone.first = startPos;
+  newZone.second = endPos;
+  m_zone.push_back( newZone );
+  m_active = true;
+}
+
+//! check if the current hypothesis extension violates reordering constraints
+bool ReorderingConstraint::Check( const Bitmap &bitmap, size_t startPos, size_t endPos ) const
+{
+  // nothing to be checked, we are done
+  if (! IsActive() ) return true;
+
+  VERBOSE(3,"Check " << bitmap << " " << startPos << "-" << endPos);
+
+  // check walls
+  size_t firstGapPos = bitmap.GetFirstGapPos();
+  // filling first gap -> no wall violation possible
+  if (firstGapPos != startPos) {
+    // if there is a wall before the last word,
+    // we created a gap while moving through wall
+    // -> violation
+    for( size_t pos = firstGapPos; pos < endPos; pos++ ) {
+      if( GetWall( pos ) ) {
+        VERBOSE(3," hitting wall " << pos << std::endl);
+        return false;
+      }
+    }
+  }
+
+  // monotone -> no violation possible
+  size_t lastPos = bitmap.GetLastPos();
+  if ((lastPos == NOT_FOUND && startPos == 0) || // nothing translated
+      (firstGapPos > lastPos &&  // no gaps
+       firstGapPos == startPos)) { // translating first empty word
+    VERBOSE(3," montone, fine." << std::endl);
+    return true;
+  }
+
+  // check zones
+  for(size_t z = 0; z < m_zone.size(); z++ ) {
+    const size_t startZone = m_zone[z].first;
+    const size_t endZone = m_zone[z].second;
+
+    // fine, if translation has not reached zone yet and phrase outside zone
+    if (lastPos < startZone && ( endPos < startZone || startPos > endZone ) ) {
+      continue;
+    }
+
+    // already completely translated zone, no violations possible
+    if (firstGapPos > endZone) {
+      continue;
+    }
+
+    // some words are translated beyond the start
+    // let's look closer if some are in the zone
+    size_t numWordsInZoneTranslated = 0;
+    if (lastPos >= startZone) {
+      for(size_t pos = startZone; pos <= endZone; pos++ ) {
+        if( bitmap.GetValue( pos ) ) {
+          numWordsInZoneTranslated++;
+        }
+      }
+    }
+
+    // all words in zone translated, no violation possible
+    if (numWordsInZoneTranslated == endZone-startZone+1) {
+      continue;
+    }
+
+    // flag if this is an active zone
+    bool activeZone = (numWordsInZoneTranslated > 0);
+
+    // fine, if zone completely untranslated and phrase outside zone
+    if (!activeZone && ( endPos < startZone || startPos > endZone ) ) {
+      continue;
+    }
+
+    // violation, if phrase completely outside active zone
+    if (activeZone && ( endPos < startZone || startPos > endZone ) ) {
+      VERBOSE(3," outside active zone" << std::endl);
+      return false;
+    }
+
+    // ok, this is what we know now:
+    // * the phrase is in the zone (at least partially)
+    // * either zone is already active, or it becomes active now
+
+
+    // check, if we are setting us up for a dead end due to distortion limits
+
+    // size_t distortionLimit = (size_t)StaticData::Instance().GetMaxDistortion();
+    size_t distortionLimit = m_max_distortion;
+    if (startPos != firstGapPos && endZone-firstGapPos >= distortionLimit) {
+      VERBOSE(3," dead end due to distortion limit" << std::endl);
+      return false;
+    }
+
+    // let us check on phrases that are partially outside
+
+    // phrase overlaps at the beginning, always ok
+    if (startPos <= startZone) {
+      continue;
+    }
+
+    // phrase goes beyond end, has to fill zone completely
+    if (endPos > endZone) {
+      if (endZone-startPos+1 < // num. words filled in by phrase
+          endZone-startZone+1-numWordsInZoneTranslated) { // num. untranslated
+        VERBOSE(3," overlap end, but not completing" << std::endl);
+        return false;
+      } else {
+        continue;
+      }
+    }
+
+    // now we are down to phrases that are completely inside the zone
+    // we have to check local walls
+    bool seenUntranslatedBeforeStartPos = false;
+    for(size_t pos = startZone; pos < endZone && pos < endPos; pos++ ) {
+      // be careful when there is a gap before phrase
+      if( !bitmap.GetValue( pos ) // untranslated word
+          && pos < startPos ) {   // before startPos
+        seenUntranslatedBeforeStartPos = true;
+      }
+      if( seenUntranslatedBeforeStartPos && GetLocalWall( pos, z ) ) {
+        VERBOSE(3," local wall violation" << std::endl);
+        return false;
+      }
+    }
+
+    // passed all checks for this zone, on to the next one
+  }
+
+  // passed all checks, no violations
+  VERBOSE(3," fine." << std::endl);
+  return true;
+}
+
+std::ostream& operator<<(std::ostream& out, const ReorderingConstraint &obj)
+{
+  out << "Zones:";
+  for (size_t i = 0; i < obj.m_zone.size(); ++i) {
+    const std::pair<size_t,size_t> &zone1 = obj.m_zone[i];
+    out << zone1.first << "-" << zone1.second << " ";
+  }
+
+  out << "Walls:";
+  for (size_t i = 0; i < obj.m_size; ++i) {
+    out << obj.m_wall[i];
+  }
+
+  out << " Local walls:";
+  for (size_t i = 0; i < obj.m_size; ++i) {
+    out << obj.m_localWall[i] << " ";
+  }
+
+  return out;
+}
+
+}

mosesdecoder/moses/ReorderingConstraint.h

@@ -0,0 +1,113 @@
+// -*- mode: c++; indent-tabs-mode: nil; tab-width: 2 -*-
+// $Id$
+// vim:tabstop=2
+
+/***********************************************************************
+Moses - factored phrase-based language decoder
+Copyright (C) 2008 University of Edinburgh
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Lesser General Public
+License as published by the Free Software Foundation; either
+version 2.1 of the License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public
+License along with this library; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+***********************************************************************/
+
+#ifndef moses_ReorderingConstraint_h
+#define moses_ReorderingConstraint_h
+
+//#include <malloc.h>
+#include <limits>
+#include <vector>
+#include <iostream>
+#include <cstring>
+#include <cmath>
+#include "TypeDef.h"
+#include "Word.h"
+#include "Phrase.h"
+
+namespace Moses
+{
+
+class InputType;
+class Bitmap;
+
+#define NOT_A_ZONE 999999999
+/** A list of zones and walls to limit which reordering can occur
+ */
+class ReorderingConstraint
+{
+  friend std::ostream& operator<<(std::ostream& out, const ReorderingConstraint &obj);
+protected:
+  // const size_t m_size; /**< number of words in sentence */
+  size_t m_size; /**< number of words in sentence */
+  bool *m_wall;	/**< flag for each word if it is a wall */
+  size_t *m_localWall;	/**< flag for each word if it is a local wall */
+  std::vector< std::pair<size_t,size_t> > m_zone; /** zones that limit reordering */
+  bool   m_active; /**< flag indicating, if there are any active constraints */
+  int m_max_distortion;
+public:
+
+  //! create ReorderingConstraint of length size and initialise to zero
+  ReorderingConstraint(int max_distortion)
+    : m_wall(NULL)
+    , m_localWall(NULL)
+    , m_active(false)
+    , m_max_distortion(max_distortion)
+  {}
+
+  //! destructer
+  ~ReorderingConstraint() {
+    if (m_wall != NULL) free(m_wall);
+    if (m_localWall != NULL) free(m_localWall);
+  }
+
+  //! allocate memory for memory for a sentence of a given size
+  void InitializeWalls(size_t size);
+
+  //! changes walls in zones into local walls
+  void FinalizeWalls();
+
+  //! set value at a particular position
+  void SetWall( size_t pos, bool value );
+
+  //! whether a word has been translated at a particular position
+  bool GetWall(size_t pos) const {
+    return m_wall[pos];
+  }
+
+  //! whether a word has been translated at a particular position
+  bool GetLocalWall(size_t pos, size_t zone ) const {
+    return (m_localWall[pos] == zone);
+  }
+
+  //! set a zone
+  void SetZone( size_t startPos, size_t endPos );
+
+  //! returns the vector of zones
+  std::vector< std::pair<size_t,size_t> > & GetZones() {
+    return m_zone;
+  }
+
+  //! set the reordering walls based on punctuation in the sentence
+  void SetMonotoneAtPunctuation( const Phrase & sentence );
+
+  //! check if all constraints are fulfilled -> all find
+  bool Check( const Bitmap &bitmap, size_t start, size_t end ) const;
+
+  //! checks if reordering constraints will be enforced
+  bool IsActive() const {
+    return m_active;
+  }
+};
+
+}
+#endif

mosesdecoder/moses/ScoreComponentCollectionTest.cpp

@@ -0,0 +1,184 @@
+/***********************************************************************
+Moses - factored phrase-based language decoder
+Copyright (C) 2010 University of Edinburgh
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Lesser General Public
+License as published by the Free Software Foundation; either
+version 2.1 of the License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public
+License along with this library; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+***********************************************************************/
+
+#include <stdexcept>
+
+#include <boost/test/unit_test.hpp>
+
+#include "moses/FF/StatelessFeatureFunction.h"
+#include "ScoreComponentCollection.h"
+
+using namespace Moses;
+using namespace std;
+
+BOOST_AUTO_TEST_SUITE(scc)
+
+class MockStatelessFeatureFunction : public StatelessFeatureFunction
+{
+public:
+  MockStatelessFeatureFunction(size_t n, const string &line) :
+    StatelessFeatureFunction(n, line) {}
+  void EvaluateWhenApplied(const Hypothesis&, ScoreComponentCollection*) const {}
+  void EvaluateWhenApplied(const ChartHypothesis&, ScoreComponentCollection*) const {}
+  void EvaluateWithSourceContext(const InputType &input
+                                 , const InputPath &inputPath
+                                 , const TargetPhrase &targetPhrase
+                                 , const StackVec *stackVec
+                                 , ScoreComponentCollection &scoreBreakdown
+                                 , ScoreComponentCollection *estimatedScores) const {
+  }
+
+  void EvaluateTranslationOptionListWithSourceContext(const InputType &input
+      , const TranslationOptionList &translationOptionList) const {
+  }
+  void EvaluateInIsolation(const Phrase &source
+                           , const TargetPhrase &targetPhrase
+                           , ScoreComponentCollection &scoreBreakdown
+                           , ScoreComponentCollection &estimatedScores) const {
+  }
+
+};
+
+class MockSingleFeature : public MockStatelessFeatureFunction
+{
+public:
+  MockSingleFeature(): MockStatelessFeatureFunction(1, "MockSingle") {}
+
+  bool IsUseable(const FactorMask &mask) const {
+    return true;
+  }
+};
+
+class MockMultiFeature : public MockStatelessFeatureFunction
+{
+public:
+  MockMultiFeature(): MockStatelessFeatureFunction(5, "MockMulti") {}
+
+  bool IsUseable(const FactorMask &mask) const {
+    return true;
+  }
+
+};
+
+class MockSparseFeature : public MockStatelessFeatureFunction
+{
+public:
+  MockSparseFeature(): MockStatelessFeatureFunction(0, "MockSparse") {}
+
+  bool IsUseable(const FactorMask &mask) const {
+    return true;
+  }
+};
+
+
+
+struct MockProducers {
+  MockProducers() {
+    FeatureFunction::Register(&single);
+    FeatureFunction::Register(&multi);
+    FeatureFunction::Register(&sparse);
+  }
+
+  MockSingleFeature single;
+  MockMultiFeature multi;
+  MockSparseFeature sparse;
+};
+
+BOOST_FIXTURE_TEST_CASE(ctor, MockProducers)
+{
+  ScoreComponentCollection scc;
+  BOOST_CHECK_EQUAL(scc.GetScoreForProducer(&single),0);
+  float expected[] =  {0,0,0,0,0};
+  std::vector<float> actual= scc.GetScoresForProducer(&multi);
+  BOOST_CHECK_EQUAL_COLLECTIONS(expected, expected+5, actual.begin(), actual.begin()+5);
+}
+
+BOOST_FIXTURE_TEST_CASE(plusequals, MockProducers)
+{
+  float arr1[] = {1,2,3,4,5};
+  float arr2[] = {2,4,6,8,10};
+  std::vector<float> vec1(arr1,arr1+5);
+  std::vector<float> vec2(arr2,arr2+5);
+
+  ScoreComponentCollection scc;
+  scc.PlusEquals(&single, 3.4f);
+  BOOST_CHECK_EQUAL(scc.GetScoreForProducer(&single), 3.4f);
+  scc.PlusEquals(&multi,vec1);
+  std::vector<float> actual = scc.GetScoresForProducer(&multi);
+  BOOST_CHECK_EQUAL_COLLECTIONS(vec1.begin(),vec1.end()
+                                ,actual.begin(), actual.end());
+  scc.PlusEquals(&multi,vec1);
+  actual = scc.GetScoresForProducer(&multi);
+  BOOST_CHECK_EQUAL_COLLECTIONS(vec2.begin(),vec2.end(),
+                                actual.begin(), actual.end());
+
+  BOOST_CHECK_EQUAL(scc.GetScoreForProducer(&single), 3.4f);
+}
+
+BOOST_FIXTURE_TEST_CASE(sparse_feature, MockProducers)
+{
+  ScoreComponentCollection scc;
+  scc.Assign(&sparse, "first", 1.3f);
+  scc.Assign(&sparse, "second", 2.1f);
+  BOOST_CHECK_EQUAL( scc.GetScoreForProducer(&sparse,"first"), 1.3f);
+  BOOST_CHECK_EQUAL( scc.GetScoreForProducer(&sparse,"second"), 2.1f);
+  BOOST_CHECK_EQUAL( scc.GetScoreForProducer(&sparse,"third"), 0.0f);
+  scc.Assign(&sparse, "first", -1.9f);
+  BOOST_CHECK_EQUAL( scc.GetScoreForProducer(&sparse,"first"), -1.9f);
+  scc.PlusEquals(&sparse, StringPiece("first"), -1.9f);
+  BOOST_CHECK_EQUAL( scc.GetScoreForProducer(&sparse,"first"), -3.8f);
+}
+
+/*
+ Doesn't work because of the static registration of ScoreProducers
+ in ScoreComponentCollection.
+BOOST_FIXTURE_TEST_CASE(save, MockProducers)
+{
+  ScoreComponentCollection scc;
+  scc.Assign(&sparse, "first", 1.1f);
+  scc.Assign(&single, 0.25f);
+  float arr[] = {1,2.1,3,4,5};
+  std::vector<float> vec1(arr,arr+5);
+  scc.Assign(&multi,vec1);
+  ostringstream out;
+  scc.Save(out);
+  cerr << out.str() << endl;
+  istringstream in (out.str());
+  string line;
+  getline(in,line);
+  BOOST_CHECK_EQUAL(line, "MockSingle:4_1 0.25");
+  getline(in,line);
+  BOOST_CHECK_EQUAL(line, "MockMulti:4_1 1");
+  getline(in,line);
+  BOOST_CHECK_EQUAL(line, "MockMulti:4_2 2.1");
+  getline(in,line);
+  BOOST_CHECK_EQUAL(line, "MockMulti:4_3 3");
+  getline(in,line);
+  BOOST_CHECK_EQUAL(line, "MockMulti:4_4 4");
+  getline(in,line);
+  BOOST_CHECK_EQUAL(line, "MockMulti:4_5 5");
+  getline(in,line);
+  BOOST_CHECK_EQUAL(line,"MockSparse:4_first 1.1");
+  BOOST_CHECK(!getline(in,line));
+}
+*/
+
+
+BOOST_AUTO_TEST_SUITE_END()
+

mosesdecoder/moses/Search.cpp

@@ -0,0 +1,50 @@
+#include "Manager.h"
+#include "SearchCubePruning.h"
+#include "SearchNormal.h"
+#include "InputType.h"
+#include "util/exception.hh"
+
+namespace Moses
+{
+
+Search::Search(Manager& manager)
+  : m_manager(manager)
+  , m_source(manager.GetSource())
+  , m_options(*manager.options())
+  , m_inputPath()
+  , m_initialTransOpt()
+  , m_bitmaps(manager.GetSource().GetSize(), manager.GetSource().m_sourceCompleted)
+  , interrupted_flag(0)
+{
+  m_initialTransOpt.SetInputPath(m_inputPath);
+  m_timer.start();
+}
+
+bool
+Search::
+out_of_time()
+{
+  int const& timelimit = m_options.search.timeout;
+  if (timelimit > 0) {
+    double elapsed_time = GetUserTime();
+    if (elapsed_time > timelimit) {
+      VERBOSE(1,"Decoding is out of time (" << elapsed_time << ","
+              << timelimit << ")" << std::endl);
+      interrupted_flag = 1;
+      return true;
+    }
+  }
+  int const& segment_timelimit = m_options.search.segment_timeout;
+  if (segment_timelimit > 0) {
+    double elapsed_time = m_timer.get_elapsed_time();
+    if (elapsed_time > segment_timelimit) {
+      VERBOSE(1,"Decoding for segment is out of time (" << elapsed_time << ","
+              << segment_timelimit << ")" << std::endl);
+      interrupted_flag = 1;
+      return true;
+    }
+  }
+  return false;
+}
+
+}

mosesdecoder/moses/Search.h

@@ -0,0 +1,57 @@
+#ifndef moses_Search_h
+#define moses_Search_h
+
+#include <vector>
+#include "TypeDef.h"
+#include "TranslationOption.h"
+#include "Phrase.h"
+#include "InputPath.h"
+#include "Bitmaps.h"
+#include "Timer.h"
+
+namespace Moses
+{
+
+class HypothesisStack;
+class Hypothesis;
+class InputType;
+class TranslationOptionCollection;
+class Manager;
+class Phrase;
+
+/** Base search class used in the phrase-based decoder.
+ *
+ * Actual search class that implement the cube pruning algorithm (SearchCubePruning)
+ * or standard beam search (SearchNormal) should inherits from this class, and
+ * override pure virtual functions.
+ */
+class Search
+{
+public:
+  virtual const std::vector<HypothesisStack*>& GetHypothesisStacks() const = 0;
+  virtual const Hypothesis *GetBestHypothesis() const = 0;
+
+  //! Decode the sentence according to the specified search algorithm.
+  virtual void Decode() = 0;
+
+  explicit Search(Manager& manager);
+  virtual ~Search() {}
+
+protected:
+  Manager& m_manager;
+  const InputType &m_source;
+  AllOptions const& m_options;
+
+  InputPath m_inputPath; // for initial hypo
+  TranslationOption m_initialTransOpt; /**< used to seed 1st hypo */
+  Bitmaps m_bitmaps;
+
+  /** flag indicating that decoder ran out of time (see switch -time-out) */
+  size_t interrupted_flag;
+
+  Timer m_timer;
+  bool out_of_time();
+};
+
+}
+#endif

mosesdecoder/moses/SearchCubePruning.h

@@ -0,0 +1,48 @@
+#ifndef moses_SearchCubePruning_h
+#define moses_SearchCubePruning_h
+
+#include <vector>
+#include "Search.h"
+#include "HypothesisStackCubePruning.h"
+#include "SentenceStats.h"
+
+namespace Moses
+{
+
+class InputType;
+class TranslationOptionCollection;
+
+/** Functions and variables you need to decoder an input using the phrase-based decoder with cube-pruning
+ *  Instantiated by the Manager class
+ */
+class SearchCubePruning: public Search
+{
+protected:
+  std::vector < HypothesisStack* > m_hypoStackColl; /**< stacks to store hypotheses (partial translations) */
+  // no of elements = no of words in source + 1
+  const TranslationOptionCollection &m_transOptColl; /**< pre-computed list of translation options for the phrases in this sentence */
+
+  //! go thru all bitmaps in 1 stack & create backpointers to bitmaps in the stack
+  void CreateForwardTodos(HypothesisStackCubePruning &stack);
+  //! create a back pointer to this bitmap, with edge that has this words range translation
+  void CreateForwardTodos(const Bitmap &bitmap, const Range &range, BitmapContainer &bitmapContainer);
+  bool CheckDistortion(const Bitmap &bitmap, const Range &range) const;
+
+  void PrintBitmapContainerGraph();
+
+public:
+  SearchCubePruning(Manager& manager, const TranslationOptionCollection &transOptColl);
+  ~SearchCubePruning();
+
+  void Decode();
+
+  void OutputHypoStackSize();
+  void OutputHypoStack(int stack);
+
+  virtual const std::vector < HypothesisStack* >& GetHypothesisStacks() const;
+  virtual const Hypothesis *GetBestHypothesis() const;
+};
+
+
+}
+#endif

mosesdecoder/moses/SearchNormal.cpp

@@ -0,0 +1,423 @@
+#include "Manager.h"
+#include "Timer.h"
+#include "SearchNormal.h"
+#include "SentenceStats.h"
+
+#include <boost/foreach.hpp>
+
+using namespace std;
+
+namespace Moses
+{
+/**
+ * Organizing main function
+ *
+ * /param source input sentence
+ * /param transOptColl collection of translation options to be used for this sentence
+ */
+SearchNormal::
+SearchNormal(Manager& manager, const TranslationOptionCollection &transOptColl)
+  : Search(manager)
+  , m_hypoStackColl(manager.GetSource().GetSize() + 1)
+  , m_transOptColl(transOptColl)
+{
+  VERBOSE(1, "Translating: " << m_source << endl);
+
+  // initialize the stacks: create data structure and set limits
+  std::vector < HypothesisStackNormal >::iterator iterStack;
+  for (size_t ind = 0 ; ind < m_hypoStackColl.size() ; ++ind) {
+    HypothesisStackNormal *sourceHypoColl = new HypothesisStackNormal(m_manager);
+    sourceHypoColl->SetMaxHypoStackSize(this->m_options.search.stack_size,
+                                        this->m_options.search.stack_diversity);
+    sourceHypoColl->SetBeamWidth(this->m_options.search.beam_width);
+    m_hypoStackColl[ind] = sourceHypoColl;
+  }
+}
+
+SearchNormal::~SearchNormal()
+{
+  RemoveAllInColl(m_hypoStackColl);
+}
+
+
+bool
+SearchNormal::
+ProcessOneStack(HypothesisStack* hstack)
+{
+  if (this->out_of_time()) return false;
+  SentenceStats &stats = m_manager.GetSentenceStats();
+  HypothesisStackNormal &sourceHypoColl
+  = *static_cast<HypothesisStackNormal*>(hstack);
+
+  // the stack is pruned before processing (lazy pruning):
+  VERBOSE(3,"processing hypothesis from next stack");
+  IFVERBOSE(2) stats.StartTimeStack();
+  sourceHypoColl.PruneToSize(m_options.search.stack_size);
+  VERBOSE(3,std::endl);
+  sourceHypoColl.CleanupArcList();
+  IFVERBOSE(2)  stats.StopTimeStack();
+
+  // go through each hypothesis on the stack and try to expand it
+  // BOOST_FOREACH(Hypothesis* h, sourceHypoColl)
+  HypothesisStackNormal::const_iterator h;
+  for (h = sourceHypoColl.begin(); h != sourceHypoColl.end(); ++h)
+    ProcessOneHypothesis(**h);
+  return true;
+}
+
+
+/**
+ * Main decoder loop that translates a sentence by expanding
+ * hypotheses stack by stack, until the end of the sentence.
+ */
+void SearchNormal::Decode()
+{
+  // initial seed hypothesis: nothing translated, no words produced
+  const Bitmap &initBitmap = m_bitmaps.GetInitialBitmap();
+  Hypothesis *hypo = new Hypothesis(m_manager, m_source, m_initialTransOpt, initBitmap, m_manager.GetNextHypoId());
+
+  m_hypoStackColl[0]->AddPrune(hypo);
+
+  // go through each stack
+  BOOST_FOREACH(HypothesisStack* hstack, m_hypoStackColl) {
+    if (!ProcessOneStack(hstack)) return;
+    IFVERBOSE(2) OutputHypoStackSize();
+    actual_hypoStack = static_cast<HypothesisStackNormal*>(hstack);
+  }
+}
+
+
+/** Find all translation options to expand one hypothesis, trigger expansion
+ * this is mostly a check for overlap with already covered words, and for
+ * violation of reordering limits.
+ * \param hypothesis hypothesis to be expanded upon
+ */
+void
+SearchNormal::
+ProcessOneHypothesis(const Hypothesis &hypothesis)
+{
+  // since we check for reordering limits, its good to have that limit handy
+  bool isWordLattice = m_source.GetType() == WordLatticeInput;
+
+  const Bitmap &hypoBitmap = hypothesis.GetWordsBitmap();
+  const size_t hypoFirstGapPos = hypoBitmap.GetFirstGapPos();
+  size_t const sourceSize = m_source.GetSize();
+
+  ReorderingConstraint const&
+  ReoConstraint = m_source.GetReorderingConstraint();
+
+  // no limit of reordering: only check for overlap
+  if (m_options.reordering.max_distortion < 0) {
+
+    for (size_t startPos = hypoFirstGapPos ; startPos < sourceSize ; ++startPos) {
+      TranslationOptionList const* tol;
+      size_t endPos = startPos;
+      for (tol = m_transOptColl.GetTranslationOptionList(startPos, endPos);
+           tol && endPos < sourceSize;
+           tol = m_transOptColl.GetTranslationOptionList(startPos, ++endPos)) {
+        if (tol->size() == 0
+            || hypoBitmap.Overlap(Range(startPos, endPos))
+            || !ReoConstraint.Check(hypoBitmap, startPos, endPos)) {
+          continue;
+        }
+
+        //TODO: does this method include incompatible WordLattice hypotheses?
+        ExpandAllHypotheses(hypothesis, startPos, endPos);
+      }
+    }
+    return; // done with special case (no reordering limit)
+  }
+
+  // There are reordering limits. Make sure they are not violated.
+
+  Range prevRange = hypothesis.GetCurrSourceWordsRange();
+  for (size_t startPos = hypoFirstGapPos ; startPos < sourceSize ; ++startPos) {
+
+    // don't bother expanding phrases if the first position is already taken
+    if(hypoBitmap.GetValue(startPos)) continue;
+
+    size_t maxSize = sourceSize - startPos;
+    size_t maxSizePhrase = m_options.search.max_phrase_length;
+    maxSize = (maxSize < maxSizePhrase) ? maxSize : maxSizePhrase;
+    size_t closestLeft = hypoBitmap.GetEdgeToTheLeftOf(startPos);
+
+    if (isWordLattice) {
+      // first question: is there a path from the closest translated word to the left
+      // of the hypothesized extension to the start of the hypothesized extension?
+      // long version:
+      // - is there anything to our left?
+      // - is it farther left than where we're starting anyway?
+      // - can we get to it?
+
+      // closestLeft is exclusive: a value of 3 means 2 is covered, our
+      // arc is currently ENDING at 3 and can start at 3 implicitly
+      if (closestLeft != 0 && closestLeft != startPos
+          && !m_source.CanIGetFromAToB(closestLeft, startPos))
+        continue;
+
+      if (prevRange.GetStartPos() != NOT_FOUND &&
+          prevRange.GetStartPos() > startPos &&
+          !m_source.CanIGetFromAToB(startPos, prevRange.GetStartPos()))
+        continue;
+    }
+
+    Range currentStartRange(startPos, startPos);
+    if(m_source.ComputeDistortionDistance(prevRange, currentStartRange)
+        > m_options.reordering.max_distortion)
+      continue;
+
+    TranslationOptionList const* tol;
+    size_t endPos = startPos;
+    for (tol = m_transOptColl.GetTranslationOptionList(startPos, endPos);
+         tol && endPos < sourceSize;
+         tol = m_transOptColl.GetTranslationOptionList(startPos, ++endPos)) {
+      Range extRange(startPos, endPos);
+      if (tol->size() == 0
+          || hypoBitmap.Overlap(extRange)
+          || !ReoConstraint.Check(hypoBitmap, startPos, endPos)
+          || (isWordLattice && !m_source.IsCoveragePossible(extRange))) {
+        continue;
+      }
+
+      // ask second question here: we already know we can get to our
+      // starting point from the closest thing to the left. We now ask the
+      // follow up: can we get from our end to the closest thing on the
+      // right?
+      //
+      // long version: is anything to our right? is it farther
+      // right than our (inclusive) end? can our end reach it?
+      bool isLeftMostEdge = (hypoFirstGapPos == startPos);
+
+      size_t closestRight = hypoBitmap.GetEdgeToTheRightOf(endPos);
+      if (isWordLattice) {
+        if (closestRight != endPos
+            && ((closestRight + 1) < sourceSize)
+            && !m_source.CanIGetFromAToB(endPos + 1, closestRight + 1)) {
+          continue;
+        }
+      }
+
+      if (isLeftMostEdge) {
+        // any length extension is okay if starting at left-most edge
+        ExpandAllHypotheses(hypothesis, startPos, endPos);
+      } else { // starting somewhere other than left-most edge, use caution
+        // the basic idea is this: we would like to translate a phrase
+        // starting from a position further right than the left-most
+        // open gap. The distortion penalty for the following phrase
+        // will be computed relative to the ending position of the
+        // current extension, so we ask now what its maximum value will
+        // be (which will always be the value of the hypothesis starting
+        // at the left-most edge).  If this value is less than the
+        // distortion limit, we don't allow this extension to be made.
+        Range bestNextExtension(hypoFirstGapPos, hypoFirstGapPos);
+
+        if (m_source.ComputeDistortionDistance(extRange, bestNextExtension)
+            > m_options.reordering.max_distortion) continue;
+
+        // everything is fine, we're good to go
+        ExpandAllHypotheses(hypothesis, startPos, endPos);
+      }
+    }
+  }
+}
+
+
+/**
+ * Expand a hypothesis given a list of translation options
+ * \param hypothesis hypothesis to be expanded upon
+ * \param startPos first word position of span covered
+ * \param endPos last word position of span covered
+ */
+
+void
+SearchNormal::
+ExpandAllHypotheses(const Hypothesis &hypothesis, size_t startPos, size_t endPos)
+{
+  // early discarding: check if hypothesis is too bad to build
+  // this idea is explained in (Moore&Quirk, MT Summit 2007)
+  float expectedScore = 0.0f;
+
+  const Bitmap &sourceCompleted = hypothesis.GetWordsBitmap();
+  float estimatedScore = m_transOptColl.GetEstimatedScores().CalcEstimatedScore( sourceCompleted, startPos, endPos );
+
+  const Range &hypoRange = hypothesis.GetCurrSourceWordsRange();
+  //cerr << "DOING " << sourceCompleted << " [" << hypoRange.GetStartPos() << " " << hypoRange.GetEndPos() << "]"
+  //		  " [" << startPos << " " << endPos << "]" << endl;
+
+  if (m_options.search.UseEarlyDiscarding()) {
+    // expected score is based on score of current hypothesis
+    expectedScore = hypothesis.GetScore();
+
+    // add new future score estimate
+    expectedScore += estimatedScore;
+  }
+
+  // loop through all translation options
+  const TranslationOptionList* tol
+  = m_transOptColl.GetTranslationOptionList(startPos, endPos);
+  if (!tol || tol->size() == 0) return;
+
+  // Create new bitmap
+  const TranslationOption &transOpt = **tol->begin();
+  const Range &nextRange = transOpt.GetSourceWordsRange();
+  const Bitmap &nextBitmap = m_bitmaps.GetBitmap(sourceCompleted, nextRange);
+
+  TranslationOptionList::const_iterator iter;
+  for (iter = tol->begin() ; iter != tol->end() ; ++iter) {
+    const TranslationOption &transOpt = **iter;
+    ExpandHypothesis(hypothesis, transOpt, expectedScore, estimatedScore, nextBitmap);
+  }
+}
+
+/**
+ * Expand one hypothesis with a translation option.
+ * this involves initial creation, scoring and adding it to the proper stack
+ * \param hypothesis hypothesis to be expanded upon
+ * \param transOpt translation option (phrase translation)
+ *        that is applied to create the new hypothesis
+ * \param expectedScore base score for early discarding
+ *        (base hypothesis score plus future score estimation)
+ */
+void SearchNormal::ExpandHypothesis(const Hypothesis &hypothesis,
+                                    const TranslationOption &transOpt,
+                                    float expectedScore,
+                                    float estimatedScore,
+                                    const Bitmap &bitmap)
+{
+  SentenceStats &stats = m_manager.GetSentenceStats();
+
+  Hypothesis *newHypo;
+  if (! m_options.search.UseEarlyDiscarding()) {
+    // simple build, no questions asked
+    IFVERBOSE(2) {
+      stats.StartTimeBuildHyp();
+    }
+    newHypo = new Hypothesis(hypothesis, transOpt, bitmap, m_manager.GetNextHypoId());
+    IFVERBOSE(2) {
+      stats.StopTimeBuildHyp();
+    }
+    if (newHypo==NULL) return;
+
+    IFVERBOSE(2) {
+      m_manager.GetSentenceStats().StartTimeOtherScore();
+    }
+    newHypo->EvaluateWhenApplied(estimatedScore);
+    IFVERBOSE(2) {
+      m_manager.GetSentenceStats().StopTimeOtherScore();
+
+      // TODO: these have been meaningless for a while.
+      // At least since commit 67fb5c
+      // should now be measured in SearchNormal.cpp:254 instead, around CalcFutureScore2()
+      // CalcFutureScore2() also called in BackwardsEdge::Initialize().
+      //
+      // however, CalcFutureScore2() should be quick
+      // since it uses dynamic programming results in SquareMatrix
+      m_manager.GetSentenceStats().StartTimeEstimateScore();
+      m_manager.GetSentenceStats().StopTimeEstimateScore();
+    }
+  } else
+    // early discarding: check if hypothesis is too bad to build
+  {
+    // worst possible score may have changed -> recompute
+    size_t wordsTranslated = hypothesis.GetWordsBitmap().GetNumWordsCovered() + transOpt.GetSize();
+    float allowedScore = m_hypoStackColl[wordsTranslated]->GetWorstScore();
+    if (m_options.search.stack_diversity) {
+      WordsBitmapID id = hypothesis.GetWordsBitmap().GetIDPlus(transOpt.GetStartPos(), transOpt.GetEndPos());
+      float allowedScoreForBitmap = m_hypoStackColl[wordsTranslated]->GetWorstScoreForBitmap( id );
+      allowedScore = std::min( allowedScore, allowedScoreForBitmap );
+    }
+    allowedScore += m_options.search.early_discarding_threshold;
+
+    // add expected score of translation option
+    expectedScore += transOpt.GetFutureScore();
+
+    // check if transOpt score push it already below limit
+    if (expectedScore < allowedScore) {
+      IFVERBOSE(2) {
+        stats.AddNotBuilt();
+      }
+      return;
+    }
+
+    // build the hypothesis without scoring
+    IFVERBOSE(2) {
+      stats.StartTimeBuildHyp();
+    }
+    newHypo = new Hypothesis(hypothesis, transOpt, bitmap, m_manager.GetNextHypoId());
+    if (newHypo==NULL) return;
+    IFVERBOSE(2) {
+      stats.StopTimeBuildHyp();
+    }
+
+    // ... and check if that is below the limit
+    if (expectedScore < allowedScore) {
+      IFVERBOSE(2) {
+        stats.AddEarlyDiscarded();
+      }
+      delete newHypo;
+      return;
+    }
+
+  }
+
+  // logging for the curious
+  IFVERBOSE(3) {
+    newHypo->PrintHypothesis();
+  }
+
+  // add to hypothesis stack
+  size_t wordsTranslated = newHypo->GetWordsBitmap().GetNumWordsCovered();
+  IFVERBOSE(2) {
+    stats.StartTimeStack();
+  }
+  m_hypoStackColl[wordsTranslated]->AddPrune(newHypo);
+  IFVERBOSE(2) {
+    stats.StopTimeStack();
+  }
+}
+
+const std::vector < HypothesisStack* >& SearchNormal::GetHypothesisStacks() const
+{
+  return m_hypoStackColl;
+}
+
+/**
+ * Find best hypothesis on the last stack.
+ * This is the end point of the best translation, which can be traced back from here
+ */
+const Hypothesis *SearchNormal::GetBestHypothesis() const
+{
+  if (interrupted_flag == 0) {
+    const HypothesisStackNormal &hypoColl = *static_cast<HypothesisStackNormal*>(m_hypoStackColl.back());
+    return hypoColl.GetBestHypothesis();
+  } else {
+    const HypothesisStackNormal &hypoColl = *actual_hypoStack;
+    return hypoColl.GetBestHypothesis();
+  }
+}
+
+/**
+ * Logging of hypothesis stack sizes
+ */
+void SearchNormal::OutputHypoStackSize()
+{
+  std::vector < HypothesisStack* >::const_iterator iterStack = m_hypoStackColl.begin();
+  TRACE_ERR( "Stack sizes: " << (int)(*iterStack)->size());
+  for (++iterStack; iterStack != m_hypoStackColl.end() ; ++iterStack) {
+    TRACE_ERR( ", " << (int)(*iterStack)->size());
+  }
+  TRACE_ERR( endl);
+}
+
+void SearchNormal::OutputHypoStack()
+{
+  // all stacks
+  int i = 0;
+  vector < HypothesisStack* >::iterator iterStack;
+  for (iterStack = m_hypoStackColl.begin() ; iterStack != m_hypoStackColl.end() ; ++iterStack) {
+    HypothesisStackNormal &hypoColl = *static_cast<HypothesisStackNormal*>(*iterStack);
+    TRACE_ERR( "Stack " << i++ << ": " << endl << hypoColl << endl);
+  }
+}
+
+}

mosesdecoder/moses/SquareMatrix.cpp

@@ -0,0 +1,127 @@
+// $Id$
+// vim:tabstop=2
+
+/***********************************************************************
+Moses - factored phrase-based language decoder
+Copyright (C) 2006 University of Edinburgh
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Lesser General Public
+License as published by the Free Software Foundation; either
+version 2.1 of the License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public
+License along with this library; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+***********************************************************************/
+
+#include <string>
+#include <iostream>
+#include "SquareMatrix.h"
+#include "TypeDef.h"
+#include "Util.h"
+
+using namespace std;
+
+namespace Moses
+{
+void SquareMatrix::InitTriangle(float val)
+{
+  for(size_t row=0; row < m_size; row++) {
+    for(size_t col=row; col<m_size; col++) {
+      SetScore(row, col, -numeric_limits<float>::infinity());
+    }
+  }
+}
+
+/**
+ * Calculate future score estimate for a given coverage bitmap
+ *
+ * /param bitmap coverage bitmap
+ */
+
+float SquareMatrix::CalcEstimatedScore( Bitmap const &bitmap ) const
+{
+  const size_t notInGap= numeric_limits<size_t>::max();
+  size_t startGap = notInGap;
+  float estimatedScore = 0.0f;
+  for(size_t currPos = 0 ; currPos < bitmap.GetSize() ; currPos++) {
+    // start of a new gap?
+    if(bitmap.GetValue(currPos) == false && startGap == notInGap) {
+      startGap = currPos;
+    }
+    // end of a gap?
+    else if(bitmap.GetValue(currPos) == true && startGap != notInGap) {
+      estimatedScore += GetScore(startGap, currPos - 1);
+      startGap = notInGap;
+    }
+  }
+  // coverage ending with gap?
+  if (startGap != notInGap) {
+    estimatedScore += GetScore(startGap, bitmap.GetSize() - 1);
+  }
+
+  return estimatedScore;
+}
+
+/**
+ * Calculare future score estimate for a given coverage bitmap
+ * and an additional span that is also covered. This function is used
+ * to compute future score estimates for hypotheses that we may want
+ * build, but first want to check.
+ *
+ * Note: this function is implemented a bit more complex than
+ * the basic one (w/o additional phrase) for speed reasons,
+ * which is probably overkill.
+ *
+ * /param bitmap coverage bitmap
+ * /param startPos start of the span that is added to the coverage
+ * /param endPos end of the span that is added to the coverage
+ */
+
+float SquareMatrix::CalcEstimatedScore( Bitmap const &bitmap, size_t startPos, size_t endPos ) const
+{
+  const size_t notInGap= numeric_limits<size_t>::max();
+  float estimatedScore = 0.0f;
+  size_t startGap = bitmap.GetFirstGapPos();
+  if (startGap == NOT_FOUND) return estimatedScore; // everything filled
+
+  // start loop at first gap
+  size_t startLoop = startGap+1;
+  if (startPos == startGap) { // unless covered by phrase
+    startGap = notInGap;
+    startLoop = endPos+1; // -> postpone start
+  }
+
+  size_t lastCovered = bitmap.GetLastPos();
+  if (endPos > lastCovered || lastCovered == NOT_FOUND) lastCovered = endPos;
+
+  for(size_t currPos = startLoop; currPos <= lastCovered ; currPos++) {
+    // start of a new gap?
+    if(startGap == notInGap && bitmap.GetValue(currPos) == false && (currPos < startPos || currPos > endPos)) {
+      startGap = currPos;
+    }
+    // end of a gap?
+    else if(startGap != notInGap && (bitmap.GetValue(currPos) == true || (startPos <= currPos && currPos <= endPos))) {
+      estimatedScore += GetScore(startGap, currPos - 1);
+      startGap = notInGap;
+    }
+  }
+  // coverage ending with gap?
+  if (lastCovered != bitmap.GetSize() - 1) {
+    estimatedScore += GetScore(lastCovered+1, bitmap.GetSize() - 1);
+  }
+
+  return estimatedScore;
+}
+
+TO_STRING_BODY(SquareMatrix);
+
+}
+
+

mosesdecoder/moses/StaticData.cpp

@@ -0,0 +1,966 @@
+// -*- mode: c++; indent-tabs-mode: nil; tab-width: 2 -*-
+// $Id$
+// vim:tabstop=2
+
+/***********************************************************************
+Moses - factored phrase-based language decoder
+Copyright (C) 2006 University of Edinburgh
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Lesser General Public
+License as published by the Free Software Foundation; either
+version 2.1 of the License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public
+License along with this library; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+***********************************************************************/
+
+#include <string>
+#include <boost/algorithm/string/predicate.hpp>
+
+#include "moses/FF/Factory.h"
+#include "TypeDef.h"
+#include "moses/FF/WordPenaltyProducer.h"
+#include "moses/FF/UnknownWordPenaltyProducer.h"
+#include "moses/FF/InputFeature.h"
+#include "moses/FF/DynamicCacheBasedLanguageModel.h"
+#include "moses/TranslationModel/PhraseDictionaryDynamicCacheBased.h"
+
+#include "DecodeStepTranslation.h"
+#include "DecodeStepGeneration.h"
+#include "GenerationDictionary.h"
+#include "StaticData.h"
+#include "Util.h"
+#include "FactorCollection.h"
+#include "Timer.h"
+#include "TranslationOption.h"
+#include "DecodeGraph.h"
+#include "InputFileStream.h"
+#include "ScoreComponentCollection.h"
+#include "DecodeGraph.h"
+#include "TranslationModel/PhraseDictionary.h"
+#include "TranslationModel/PhraseDictionaryTreeAdaptor.h"
+
+#ifdef WITH_THREADS
+#include <boost/thread.hpp>
+#endif
+#ifdef HAVE_CMPH
+#include "moses/TranslationModel/CompactPT/PhraseDictionaryCompact.h"
+#endif
+#if defined HAVE_CMPH
+#include "moses/TranslationModel/CompactPT/LexicalReorderingTableCompact.h"
+#endif
+
+using namespace std;
+using namespace boost::algorithm;
+
+namespace Moses
+{
+StaticData StaticData::s_instance;
+
+StaticData::StaticData()
+  : m_options(new AllOptions)
+  , m_requireSortingAfterSourceContext(false)
+  , m_currentWeightSetting("default")
+  , m_treeStructure(NULL)
+  , m_coordSpaceNextID(1)
+{
+  Phrase::InitializeMemPool();
+}
+
+StaticData::~StaticData()
+{
+  RemoveAllInColl(m_decodeGraphs);
+  Phrase::FinalizeMemPool();
+}
+
+bool StaticData::LoadDataStatic(Parameter *parameter, const std::string &execPath)
+{
+  s_instance.SetExecPath(execPath);
+  return s_instance.LoadData(parameter);
+}
+
+void
+StaticData
+::initialize_features()
+{
+  std::map<std::string, std::string> featureNameOverride = OverrideFeatureNames();
+  // all features
+  map<string, int> featureIndexMap;
+
+  const PARAM_VEC* params = m_parameter->GetParam("feature");
+  for (size_t i = 0; params && i < params->size(); ++i) {
+    const string &line = Trim(params->at(i));
+    VERBOSE(1,"line=" << line << endl);
+    if (line.empty())
+      continue;
+
+    vector<string> toks = Tokenize(line);
+
+    string &feature = toks[0];
+    std::map<std::string, std::string>::const_iterator iter
+    = featureNameOverride.find(feature);
+    if (iter == featureNameOverride.end()) {
+      // feature name not override
+      m_registry.Construct(feature, line);
+    } else {
+      // replace feature name with new name
+      string newName = iter->second;
+      feature = newName;
+      string newLine = Join(" ", toks);
+      m_registry.Construct(newName, newLine);
+    }
+  }
+
+  NoCache();
+  OverrideFeatures();
+
+}
+
+bool
+StaticData
+::ini_output_options()
+{
+  // verbose level
+  m_parameter->SetParameter(m_verboseLevel, "verbose", (size_t) 1);
+  m_parameter->SetParameter<string>(m_outputUnknownsFile,
+                                    "output-unknowns", "");
+  return true;
+}
+
+// threads, timeouts, etc.
+bool
+StaticData
+::ini_performance_options()
+{
+  const PARAM_VEC *params;
+
+  m_threadCount = 1;
+  params = m_parameter->GetParam("threads");
+  if (params && params->size()) {
+    if (params->at(0) == "all") {
+#ifdef WITH_THREADS
+      m_threadCount = boost::thread::hardware_concurrency();
+      if (!m_threadCount) {
+        std::cerr << "-threads all specified but Boost doesn't know how many cores there are";
+        return false;
+      }
+#else
+      std::cerr << "-threads all specified but moses not built with thread support";
+      return false;
+#endif
+    } else {
+      m_threadCount = Scan<int>(params->at(0));
+      if (m_threadCount < 1) {
+        std::cerr << "Specify at least one thread.";
+        return false;
+      }
+#ifndef WITH_THREADS
+      if (m_threadCount > 1) {
+        std::cerr << "Error: Thread count of " << params->at(0)
+                  << " but moses not built with thread support";
+        return false;
+      }
+#endif
+    }
+  }
+  return true;
+}
+
+bool StaticData::LoadData(Parameter *parameter)
+{
+  m_parameter = parameter;
+
+  const PARAM_VEC *params;
+
+  m_options->init(*parameter);
+  if (is_syntax(m_options->search.algo))
+    m_options->syntax.LoadNonTerminals(*parameter, FactorCollection::Instance());
+
+  if (is_syntax(m_options->search.algo))
+    LoadChartDecodingParameters();
+
+  // ORDER HERE MATTERS, SO DON'T CHANGE IT UNLESS YOU KNOW WHAT YOU ARE DOING!
+  // input, output
+
+  m_parameter->SetParameter<string>(m_factorDelimiter, "factor-delimiter", "|");
+  m_parameter->SetParameter<size_t>(m_lmcache_cleanup_threshold, "clean-lm-cache", 1);
+
+  m_bookkeeping_options.init(*parameter);
+  if (!ini_output_options()) return false;
+
+  // threading etc.
+  if (!ini_performance_options()) return false;
+
+  // FEATURE FUNCTION INITIALIZATION HAPPENS HERE ===============================
+
+  // set class-specific default parameters
+#if defined HAVE_CMPH
+  LexicalReorderingTableCompact::SetStaticDefaultParameters(*parameter);
+  PhraseDictionaryCompact::SetStaticDefaultParameters(*parameter);
+#endif
+
+  initialize_features();
+
+  if (m_parameter->GetParam("show-weights") == NULL)
+    LoadFeatureFunctions();
+
+  LoadDecodeGraphs();
+
+  // sanity check that there are no weights without an associated FF
+  if (!CheckWeights()) return false;
+
+  //Load extra feature weights
+  string weightFile;
+  m_parameter->SetParameter<string>(weightFile, "weight-file", "");
+  if (!weightFile.empty()) {
+    ScoreComponentCollection extraWeights;
+    if (!extraWeights.Load(weightFile)) {
+      std::cerr << "Unable to load weights from " << weightFile;
+      return false;
+    }
+    m_allWeights.PlusEquals(extraWeights);
+  }
+
+  //Load sparse features from config (overrules weight file)
+  LoadSparseWeightsFromConfig();
+
+  // load alternate weight settings
+  //
+  // When and where are these used??? [UG]
+  //
+  // Update: Just checked the manual. The config file is NOT the right
+  // place to do this. [UG]
+  //
+  // <TODO>
+  // * Eliminate alternate-weight-setting. Alternate weight settings should
+  //   be provided with the input, not in the config file.
+  // </TODO>
+  params = m_parameter->GetParam("alternate-weight-setting");
+  if (params && params->size() && !LoadAlternateWeightSettings())
+    return false;
+
+  return true;
+}
+
+void StaticData::SetWeight(const FeatureFunction* sp, float weight)
+{
+  m_allWeights.Resize();
+  m_allWeights.Assign(sp,weight);
+}
+
+void StaticData::SetWeights(const FeatureFunction* sp,
+                            const std::vector<float>& weights)
+{
+  m_allWeights.Resize();
+  m_allWeights.Assign(sp,weights);
+}
+
+void StaticData::LoadNonTerminals()
+{
+  string defaultNonTerminals;
+  m_parameter->SetParameter<string>(defaultNonTerminals, "non-terminals", "X");
+
+  FactorCollection &factorCollection = FactorCollection::Instance();
+
+  m_inputDefaultNonTerminal.SetIsNonTerminal(true);
+  const Factor *sourceFactor = factorCollection.AddFactor(Input, 0, defaultNonTerminals, true);
+  m_inputDefaultNonTerminal.SetFactor(0, sourceFactor);
+
+  m_outputDefaultNonTerminal.SetIsNonTerminal(true);
+  const Factor *targetFactor = factorCollection.AddFactor(Output, 0, defaultNonTerminals, true);
+  m_outputDefaultNonTerminal.SetFactor(0, targetFactor);
+
+  // for unknown words
+  const PARAM_VEC *params = m_parameter->GetParam("unknown-lhs");
+  if (params == NULL || params->size() == 0) {
+    UnknownLHSEntry entry(defaultNonTerminals, 0.0f);
+    m_unknownLHS.push_back(entry);
+  } else {
+    const string &filePath = params->at(0);
+
+    InputFileStream inStream(filePath);
+    string line;
+    while(getline(inStream, line)) {
+      vector<string> tokens = Tokenize(line);
+      UTIL_THROW_IF2(tokens.size() != 2,
+                     "Incorrect unknown LHS format: " << line);
+      UnknownLHSEntry entry(tokens[0], Scan<float>(tokens[1]));
+      m_unknownLHS.push_back(entry);
+      // const Factor *targetFactor =
+      factorCollection.AddFactor(Output, 0, tokens[0], true);
+    }
+
+  }
+
+}
+
+void StaticData::LoadChartDecodingParameters()
+{
+  LoadNonTerminals();
+
+  // source label overlap
+  m_parameter->SetParameter(m_sourceLabelOverlap, "source-label-overlap",
+                            SourceLabelOverlapAdd);
+
+}
+
+void StaticData::LoadDecodeGraphs()
+{
+  vector<string> mappingVector;
+  vector<size_t> maxChartSpans;
+
+  const PARAM_VEC *params;
+
+  params = m_parameter->GetParam("mapping");
+  if (params && params->size()) {
+    mappingVector = *params;
+  } else {
+    mappingVector.assign(1,"0 T 0");
+  }
+
+  params = m_parameter->GetParam("max-chart-span");
+  if (params && params->size()) {
+    maxChartSpans = Scan<size_t>(*params);
+  }
+
+  vector<string> toks = Tokenize(mappingVector[0]);
+  if (toks.size() == 3) {
+    // eg 0 T 0
+    LoadDecodeGraphsOld(mappingVector, maxChartSpans);
+  } else if (toks.size() == 2) {
+    if (toks[0] == "T" || toks[0] == "G") {
+      // eg. T 0
+      LoadDecodeGraphsOld(mappingVector, maxChartSpans);
+    } else {
+      // eg. 0 TM1
+      LoadDecodeGraphsNew(mappingVector, maxChartSpans);
+    }
+  } else {
+    UTIL_THROW(util::Exception, "Malformed mapping");
+  }
+}
+
+void
+StaticData::
+LoadDecodeGraphsOld(const vector<string> &mappingVector,
+                    const vector<size_t> &maxChartSpans)
+{
+  const vector<PhraseDictionary*>& pts = PhraseDictionary::GetColl();
+  const vector<GenerationDictionary*>& gens = GenerationDictionary::GetColl();
+
+  const std::vector<FeatureFunction*> *featuresRemaining
+  = &FeatureFunction::GetFeatureFunctions();
+  DecodeStep *prev = 0;
+  size_t prevDecodeGraphInd = 0;
+
+  for(size_t i=0; i<mappingVector.size(); i++) {
+    vector<string>	token		= Tokenize(mappingVector[i]);
+    size_t decodeGraphInd;
+    DecodeType decodeType;
+    size_t index;
+    if (token.size() == 2) {
+      // eg. T 0
+      decodeGraphInd = 0;
+      decodeType = token[0] == "T" ? Translate : Generate;
+      index = Scan<size_t>(token[1]);
+    } else if (token.size() == 3) {
+      // eg. 0 T 0
+      // For specifying multiple translation model
+      decodeGraphInd = Scan<size_t>(token[0]);
+      //the vectorList index can only increment by one
+      UTIL_THROW_IF2(decodeGraphInd != prevDecodeGraphInd
+                     && decodeGraphInd != prevDecodeGraphInd + 1,
+                     "Malformed mapping");
+      if (decodeGraphInd > prevDecodeGraphInd) {
+        prev = NULL;
+      }
+
+      if (prevDecodeGraphInd < decodeGraphInd) {
+        featuresRemaining = &FeatureFunction::GetFeatureFunctions();
+      }
+
+      decodeType = token[1] == "T" ? Translate : Generate;
+      index = Scan<size_t>(token[2]);
+    } else {
+      UTIL_THROW(util::Exception, "Malformed mapping");
+    }
+
+    DecodeStep* decodeStep = NULL;
+    switch (decodeType) {
+    case Translate:
+      if(index>=pts.size()) {
+        util::StringStream strme;
+        strme << "No phrase dictionary with index "
+              << index << " available!";
+        UTIL_THROW(util::Exception, strme.str());
+      }
+      decodeStep = new DecodeStepTranslation(pts[index], prev, *featuresRemaining);
+      break;
+    case Generate:
+      if(index>=gens.size()) {
+        util::StringStream strme;
+        strme << "No generation dictionary with index "
+              << index << " available!";
+        UTIL_THROW(util::Exception, strme.str());
+      }
+      decodeStep = new DecodeStepGeneration(gens[index], prev, *featuresRemaining);
+      break;
+    default:
+      UTIL_THROW(util::Exception, "Unknown decode step");
+      break;
+    }
+
+    featuresRemaining = &decodeStep->GetFeaturesRemaining();
+
+    UTIL_THROW_IF2(decodeStep == NULL, "Null decode step");
+    if (m_decodeGraphs.size() < decodeGraphInd + 1) {
+      DecodeGraph *decodeGraph;
+      if (is_syntax(m_options->search.algo)) {
+        size_t maxChartSpan;
+        if (decodeGraphInd < maxChartSpans.size()) {
+          maxChartSpan = maxChartSpans[decodeGraphInd];
+          VERBOSE(1,"max-chart-span: " << maxChartSpans[decodeGraphInd] << endl);
+        } else {
+          maxChartSpan = DEFAULT_MAX_CHART_SPAN;
+        }
+        decodeGraph = new DecodeGraph(m_decodeGraphs.size(), maxChartSpan);
+      } else {
+        decodeGraph = new DecodeGraph(m_decodeGraphs.size());
+      }
+
+      m_decodeGraphs.push_back(decodeGraph); // TODO max chart span
+    }
+
+    m_decodeGraphs[decodeGraphInd]->Add(decodeStep);
+    prev = decodeStep;
+    prevDecodeGraphInd = decodeGraphInd;
+  }
+
+  // set maximum n-gram size for backoff approach to decoding paths
+  // default is always use subsequent paths (value = 0)
+  // if specified, record maxmimum unseen n-gram size
+  const vector<string> *backoffVector = m_parameter->GetParam("decoding-graph-backoff");
+  for(size_t i=0; i<m_decodeGraphs.size() && backoffVector && i<backoffVector->size(); i++) {
+    DecodeGraph &decodeGraph = *m_decodeGraphs[i];
+
+    if (i < backoffVector->size()) {
+      decodeGraph.SetBackoff(Scan<size_t>(backoffVector->at(i)));
+    }
+  }
+}
+
+void StaticData::LoadDecodeGraphsNew(const std::vector<std::string> &mappingVector, const std::vector<size_t> &maxChartSpans)
+{
+  const std::vector<FeatureFunction*> *featuresRemaining = &FeatureFunction::GetFeatureFunctions();
+  DecodeStep *prev = 0;
+  size_t prevDecodeGraphInd = 0;
+
+  for(size_t i=0; i<mappingVector.size(); i++) {
+    vector<string>	token		= Tokenize(mappingVector[i]);
+    size_t decodeGraphInd;
+
+    decodeGraphInd = Scan<size_t>(token[0]);
+    //the vectorList index can only increment by one
+    UTIL_THROW_IF2(decodeGraphInd != prevDecodeGraphInd
+                   && decodeGraphInd != prevDecodeGraphInd + 1,
+                   "Malformed mapping");
+    if (decodeGraphInd > prevDecodeGraphInd) {
+      prev = NULL;
+    }
+
+    if (prevDecodeGraphInd < decodeGraphInd) {
+      featuresRemaining = &FeatureFunction::GetFeatureFunctions();
+    }
+
+    FeatureFunction &ff = FeatureFunction::FindFeatureFunction(token[1]);
+
+    DecodeStep* decodeStep = NULL;
+    if (typeid(ff) == typeid(PhraseDictionary)) {
+      decodeStep = new DecodeStepTranslation(&static_cast<PhraseDictionary&>(ff), prev, *featuresRemaining);
+    } else if (typeid(ff) == typeid(GenerationDictionary)) {
+      decodeStep = new DecodeStepGeneration(&static_cast<GenerationDictionary&>(ff), prev, *featuresRemaining);
+    } else {
+      UTIL_THROW(util::Exception, "Unknown decode step");
+    }
+
+    featuresRemaining = &decodeStep->GetFeaturesRemaining();
+
+    UTIL_THROW_IF2(decodeStep == NULL, "Null decode step");
+    if (m_decodeGraphs.size() < decodeGraphInd + 1) {
+      DecodeGraph *decodeGraph;
+      if (is_syntax(m_options->search.algo)) {
+        size_t maxChartSpan = (decodeGraphInd < maxChartSpans.size()) ? maxChartSpans[decodeGraphInd] : DEFAULT_MAX_CHART_SPAN;
+        VERBOSE(1,"max-chart-span: " << maxChartSpans[decodeGraphInd] << endl);
+        decodeGraph = new DecodeGraph(m_decodeGraphs.size(), maxChartSpan);
+      } else {
+        decodeGraph = new DecodeGraph(m_decodeGraphs.size());
+      }
+
+      m_decodeGraphs.push_back(decodeGraph); // TODO max chart span
+    }
+
+    m_decodeGraphs[decodeGraphInd]->Add(decodeStep);
+    prev = decodeStep;
+    prevDecodeGraphInd = decodeGraphInd;
+  }
+
+  // set maximum n-gram size for backoff approach to decoding paths
+  // default is always use subsequent paths (value = 0)
+  // if specified, record maxmimum unseen n-gram size
+  const vector<string> *backoffVector = m_parameter->GetParam("decoding-graph-backoff");
+  for(size_t i=0; i<m_decodeGraphs.size() && backoffVector && i<backoffVector->size(); i++) {
+    DecodeGraph &decodeGraph = *m_decodeGraphs[i];
+
+    if (i < backoffVector->size()) {
+      decodeGraph.SetBackoff(Scan<size_t>(backoffVector->at(i)));
+    }
+  }
+
+}
+
+void StaticData::ReLoadBleuScoreFeatureParameter(float weight)
+{
+  //loop over ScoreProducers to update weights of BleuScoreFeature
+  const std::vector<FeatureFunction*> &producers = FeatureFunction::GetFeatureFunctions();
+  for(size_t i=0; i<producers.size(); ++i) {
+    FeatureFunction *ff = producers[i];
+    std::string ffName = ff->GetScoreProducerDescription();
+
+    if (ffName == "BleuScoreFeature") {
+      SetWeight(ff, weight);
+      break;
+    }
+  }
+}
+
+// ScoreComponentCollection StaticData::GetAllWeightsScoreComponentCollection() const {}
+// in ScoreComponentCollection.h
+
+void StaticData::SetExecPath(const std::string &path)
+{
+  // NOT TESTED
+  size_t pos = path.rfind("/");
+  if (pos !=  string::npos) {
+    m_binPath = path.substr(0, pos);
+  }
+  VERBOSE(1,m_binPath << endl);
+}
+
+const string &StaticData::GetBinDirectory() const
+{
+  return m_binPath;
+}
+
+float StaticData::GetWeightWordPenalty() const
+{
+  float weightWP = GetWeight(&WordPenaltyProducer::Instance());
+  return weightWP;
+}
+
+void
+StaticData::
+InitializeForInput(ttasksptr const& ttask) const
+{
+  const std::vector<FeatureFunction*> &producers
+  = FeatureFunction::GetFeatureFunctions();
+  for(size_t i=0; i<producers.size(); ++i) {
+    FeatureFunction &ff = *producers[i];
+    if (! IsFeatureFunctionIgnored(ff)) {
+      Timer iTime;
+      iTime.start();
+      ff.InitializeForInput(ttask);
+      VERBOSE(3,"InitializeForInput( " << ff.GetScoreProducerDescription()
+              << " )" << "= " << iTime << endl);
+    }
+  }
+}
+
+void
+StaticData::
+CleanUpAfterSentenceProcessing(ttasksptr const& ttask) const
+{
+  const std::vector<FeatureFunction*> &producers
+  = FeatureFunction::GetFeatureFunctions();
+  for(size_t i=0; i<producers.size(); ++i) {
+    FeatureFunction &ff = *producers[i];
+    if (! IsFeatureFunctionIgnored(ff)) {
+      ff.CleanUpAfterSentenceProcessing(ttask);
+    }
+  }
+}
+
+void StaticData::LoadFeatureFunctions()
+{
+  const std::vector<FeatureFunction*> &ffs = FeatureFunction::GetFeatureFunctions();
+  std::vector<FeatureFunction*>::const_iterator iter;
+  for (iter = ffs.begin(); iter != ffs.end(); ++iter) {
+    FeatureFunction *ff = *iter;
+    bool doLoad = true;
+
+    if (ff->RequireSortingAfterSourceContext()) {
+      m_requireSortingAfterSourceContext = true;
+    }
+
+    if (dynamic_cast<PhraseDictionary*>(ff)) {
+      doLoad = false;
+    }
+
+    if (doLoad) {
+      VERBOSE(1, "Loading " << ff->GetScoreProducerDescription() << endl);
+      ff->Load(options());
+    }
+  }
+
+  const std::vector<PhraseDictionary*> &pts = PhraseDictionary::GetColl();
+  for (size_t i = 0; i < pts.size(); ++i) {
+    PhraseDictionary *pt = pts[i];
+    VERBOSE(1, "Loading " << pt->GetScoreProducerDescription() << endl);
+    pt->Load(options());
+  }
+
+  CheckLEGACYPT();
+}
+
+bool StaticData::CheckWeights() const
+{
+  set<string> weightNames = m_parameter->GetWeightNames();
+  set<string> featureNames;
+
+  const std::vector<FeatureFunction*> &ffs = FeatureFunction::GetFeatureFunctions();
+  for (size_t i = 0; i < ffs.size(); ++i) {
+    const FeatureFunction &ff = *ffs[i];
+    const string &descr = ff.GetScoreProducerDescription();
+    featureNames.insert(descr);
+
+    set<string>::iterator iter = weightNames.find(descr);
+    if (iter == weightNames.end()) {
+      cerr << "Can't find weights for feature function " << descr << endl;
+    } else {
+      weightNames.erase(iter);
+    }
+  }
+
+  //sparse features
+  if (!weightNames.empty()) {
+    set<string>::iterator iter;
+    for (iter = weightNames.begin(); iter != weightNames.end(); ) {
+      string fname = (*iter).substr(0, (*iter).find("_"));
+      VERBOSE(1,fname << "\n");
+      if (featureNames.find(fname) != featureNames.end()) {
+        weightNames.erase(iter++);
+      } else {
+        ++iter;
+      }
+    }
+  }
+
+  if (!weightNames.empty()) {
+    cerr << "The following weights have no feature function. "
+         << "Maybe incorrectly spelt weights: ";
+    set<string>::iterator iter;
+    for (iter = weightNames.begin(); iter != weightNames.end(); ++iter) {
+      cerr << *iter << ",";
+    }
+    return false;
+  }
+
+  return true;
+}
+
+
+void StaticData::LoadSparseWeightsFromConfig()
+{
+  set<string> featureNames;
+  const std::vector<FeatureFunction*> &ffs = FeatureFunction::GetFeatureFunctions();
+  for (size_t i = 0; i < ffs.size(); ++i) {
+    const FeatureFunction &ff = *ffs[i];
+    const string &descr = ff.GetScoreProducerDescription();
+    featureNames.insert(descr);
+  }
+
+  const std::map<std::string, std::vector<float> > &weights = m_parameter->GetAllWeights();
+  std::map<std::string, std::vector<float> >::const_iterator iter;
+  for (iter = weights.begin(); iter != weights.end(); ++iter) {
+    // this indicates that it is sparse feature
+    if (featureNames.find(iter->first) == featureNames.end()) {
+      UTIL_THROW_IF2(iter->second.size() != 1, "ERROR: only one weight per sparse feature allowed: " << iter->first);
+      m_allWeights.Assign(iter->first, iter->second[0]);
+    }
+  }
+
+}
+
+
+/**! Read in settings for alternative weights */
+bool StaticData::LoadAlternateWeightSettings()
+{
+  if (m_threadCount > 1) {
+    cerr << "ERROR: alternative weight settings currently not supported with multi-threading.";
+    return false;
+  }
+
+  vector<string> weightSpecification;
+  const PARAM_VEC *params = m_parameter->GetParam("alternate-weight-setting");
+  if (params && params->size()) {
+    weightSpecification = *params;
+  }
+
+  // get mapping from feature names to feature functions
+  map<string,FeatureFunction*> nameToFF;
+  const std::vector<FeatureFunction*> &ffs = FeatureFunction::GetFeatureFunctions();
+  for (size_t i = 0; i < ffs.size(); ++i) {
+    nameToFF[ ffs[i]->GetScoreProducerDescription() ] = ffs[i];
+  }
+
+  // copy main weight setting as default
+  m_weightSetting["default"] = new ScoreComponentCollection( m_allWeights );
+
+  // go through specification in config file
+  string currentId = "";
+  bool hasErrors = false;
+  for (size_t i=0; i<weightSpecification.size(); ++i) {
+
+    // identifier line (with optional additional specifications)
+    if (weightSpecification[i].find("id=") == 0) {
+      vector<string> tokens = Tokenize(weightSpecification[i]);
+      vector<string> args = Tokenize(tokens[0], "=");
+      currentId = args[1];
+      VERBOSE(1,"alternate weight setting " << currentId << endl);
+      UTIL_THROW_IF2(m_weightSetting.find(currentId) != m_weightSetting.end(),
+                     "Duplicate alternate weight id: " << currentId);
+      m_weightSetting[ currentId ] = new ScoreComponentCollection;
+
+      // other specifications
+      for(size_t j=1; j<tokens.size(); j++) {
+        vector<string> args = Tokenize(tokens[j], "=");
+        // sparse weights
+        if (args[0] == "weight-file") {
+          if (args.size() != 2) {
+            std::cerr << "One argument should be supplied for weight-file";
+            return false;
+          }
+          ScoreComponentCollection extraWeights;
+          if (!extraWeights.Load(args[1])) {
+            std::cerr << "Unable to load weights from " << args[1];
+            return false;
+          }
+          m_weightSetting[ currentId ]->PlusEquals(extraWeights);
+        }
+        // ignore feature functions
+        else if (args[0] == "ignore-ff") {
+          set< string > *ffNameSet = new set< string >;
+          m_weightSettingIgnoreFF[ currentId ] = *ffNameSet;
+          vector<string> featureFunctionName = Tokenize(args[1], ",");
+          for(size_t k=0; k<featureFunctionName.size(); k++) {
+            // check if a valid nane
+            map<string,FeatureFunction*>::iterator ffLookUp = nameToFF.find(featureFunctionName[k]);
+            if (ffLookUp == nameToFF.end()) {
+              cerr << "ERROR: alternate weight setting " << currentId
+                   << " specifies to ignore feature function " << featureFunctionName[k]
+                   << " but there is no such feature function" << endl;
+              hasErrors = true;
+            } else {
+              m_weightSettingIgnoreFF[ currentId ].insert( featureFunctionName[k] );
+            }
+          }
+        }
+      }
+    }
+
+    // weight lines
+    else {
+      UTIL_THROW_IF2(currentId.empty(), "No alternative weights specified");
+      vector<string> tokens = Tokenize(weightSpecification[i]);
+      UTIL_THROW_IF2(tokens.size() < 2
+                     , "Incorrect format for alternate weights: " << weightSpecification[i]);
+
+      // get name and weight values
+      string name = tokens[0];
+      name = name.substr(0, name.size() - 1); // remove trailing "="
+      vector<float> weights(tokens.size() - 1);
+      for (size_t i = 1; i < tokens.size(); ++i) {
+        float weight = Scan<float>(tokens[i]);
+        weights[i - 1] = weight;
+      }
+
+      // check if a valid nane
+      map<string,FeatureFunction*>::iterator ffLookUp = nameToFF.find(name);
+      if (ffLookUp == nameToFF.end()) {
+        cerr << "ERROR: alternate weight setting " << currentId
+             << " specifies weight(s) for " << name
+             << " but there is no such feature function" << endl;
+        hasErrors = true;
+      } else {
+        m_weightSetting[ currentId ]->Assign( nameToFF[name], weights);
+      }
+    }
+  }
+  UTIL_THROW_IF2(hasErrors, "Errors loading alternate weights");
+  return true;
+}
+
+void StaticData::NoCache()
+{
+  bool noCache;
+  m_parameter->SetParameter(noCache, "no-cache", false );
+
+  if (noCache) {
+    const std::vector<PhraseDictionary*> &pts = PhraseDictionary::GetColl();
+    for (size_t i = 0; i < pts.size(); ++i) {
+      PhraseDictionary &pt = *pts[i];
+      pt.SetParameter("cache-size", "0");
+    }
+  }
+}
+
+std::map<std::string, std::string>
+StaticData
+::OverrideFeatureNames()
+{
+  std::map<std::string, std::string> ret;
+
+  const PARAM_VEC *params = m_parameter->GetParam("feature-name-overwrite");
+  if (params && params->size()) {
+    UTIL_THROW_IF2(params->size() != 1, "Only provide 1 line in the section [feature-name-overwrite]");
+    vector<string> toks = Tokenize(params->at(0));
+    UTIL_THROW_IF2(toks.size() % 2 != 0, "Format of -feature-name-overwrite must be [old-name new-name]*");
+
+    for (size_t i = 0; i < toks.size(); i += 2) {
+      const string &oldName = toks[i];
+      const string &newName = toks[i+1];
+      ret[oldName] = newName;
+    }
+  }
+
+  // FIXME Does this make sense for F2S?  Perhaps it should be changed once
+  // FIXME the pipeline uses RuleTable consistently.
+  SearchAlgorithm algo = m_options->search.algo;
+  if (algo == SyntaxS2T || algo == SyntaxT2S ||
+      algo == SyntaxT2S_SCFG || algo == SyntaxF2S) {
+    // Automatically override PhraseDictionary{Memory,Scope3}.  This will
+    // have to change if the FF parameters diverge too much in the future,
+    // but for now it makes switching between the old and new decoders much
+    // more convenient.
+    ret["PhraseDictionaryMemory"] = "RuleTable";
+    ret["PhraseDictionaryScope3"] = "RuleTable";
+  }
+
+  return ret;
+}
+
+void StaticData::OverrideFeatures()
+{
+  const PARAM_VEC *params = m_parameter->GetParam("feature-overwrite");
+  for (size_t i = 0; params && i < params->size(); ++i) {
+    const string &str = params->at(i);
+    vector<string> toks = Tokenize(str);
+    UTIL_THROW_IF2(toks.size() <= 1, "Incorrect format for feature override: " << str);
+
+    FeatureFunction &ff = FeatureFunction::FindFeatureFunction(toks[0]);
+
+    for (size_t j = 1; j < toks.size(); ++j) {
+      const string &keyValStr = toks[j];
+      vector<string> keyVal = Tokenize(keyValStr, "=");
+      UTIL_THROW_IF2(keyVal.size() != 2, "Incorrect format for parameter override: " << keyValStr);
+
+      VERBOSE(1, "Override " << ff.GetScoreProducerDescription() << " "
+              << keyVal[0] << "=" << keyVal[1] << endl);
+
+      ff.SetParameter(keyVal[0], keyVal[1]);
+
+    }
+  }
+
+}
+
+void StaticData::CheckLEGACYPT()
+{
+  const std::vector<PhraseDictionary*> &pts = PhraseDictionary::GetColl();
+  for (size_t i = 0; i < pts.size(); ++i) {
+    const PhraseDictionary *phraseDictionary = pts[i];
+    if (dynamic_cast<const PhraseDictionaryTreeAdaptor*>(phraseDictionary) != NULL) {
+      m_useLegacyPT = true;
+      return;
+    }
+  }
+
+  m_useLegacyPT = false;
+}
+
+
+void StaticData::ResetWeights(const std::string &denseWeights, const std::string &sparseFile)
+{
+  m_allWeights = ScoreComponentCollection();
+
+  // dense weights
+  string name("");
+  vector<float> weights;
+  vector<string> toks = Tokenize(denseWeights);
+  for (size_t i = 0; i < toks.size(); ++i) {
+    const string &tok = toks[i];
+
+    if (ends_with(tok, "=")) {
+      // start of new feature
+
+      if (name != "") {
+        // save previous ff
+        const FeatureFunction &ff = FeatureFunction::FindFeatureFunction(name);
+        m_allWeights.Assign(&ff, weights);
+        weights.clear();
+      }
+
+      name = tok.substr(0, tok.size() - 1);
+    } else {
+      // a weight for curr ff
+      float weight = Scan<float>(toks[i]);
+      weights.push_back(weight);
+    }
+  }
+
+  const FeatureFunction &ff = FeatureFunction::FindFeatureFunction(name);
+  m_allWeights.Assign(&ff, weights);
+
+  // sparse weights
+  InputFileStream sparseStrme(sparseFile);
+  string line;
+  while (getline(sparseStrme, line)) {
+    vector<string> toks = Tokenize(line);
+    UTIL_THROW_IF2(toks.size() != 2, "Incorrect sparse weight format. Should be FFName_spareseName weight");
+
+    vector<string> names = Tokenize(toks[0], "_");
+    UTIL_THROW_IF2(names.size() != 2, "Incorrect sparse weight name. Should be FFName_spareseName");
+
+    const FeatureFunction &ff = FeatureFunction::FindFeatureFunction(names[0]);
+    m_allWeights.Assign(&ff, names[1], Scan<float>(toks[1]));
+  }
+}
+
+size_t StaticData::GetCoordSpace(string space) const
+{
+  map<string const, size_t>::const_iterator m = m_coordSpaceMap.find(space);
+  if(m == m_coordSpaceMap.end()) {
+    return 0;
+  }
+  return m->second;
+}
+
+size_t StaticData::MapCoordSpace(string space)
+{
+  map<string const, size_t>::const_iterator m = m_coordSpaceMap.find(space);
+  if (m != m_coordSpaceMap.end()) {
+    return m->second;
+  }
+  size_t id = m_coordSpaceNextID;
+  m_coordSpaceNextID += 1;
+  m_coordSpaceMap[space] = id;
+  return id;
+}
+
+} // namespace