Add files using upload-large-folder tool

mosesdecoder/moses/AlignmentInfo.cpp

@@ -0,0 +1,170 @@
+/***********************************************************************
+ Moses - statistical machine translation system
+ Copyright (C) 2006-2011 University of Edinburgh
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+***********************************************************************/
+#include <algorithm>
+#include <set>
+#include "AlignmentInfo.h"
+#include "TypeDef.h"
+#include "StaticData.h"
+#include "Util.h"
+#include "util/exception.hh"
+
+namespace Moses
+{
+
+AlignmentInfo::AlignmentInfo(const std::set<std::pair<size_t,size_t> > &pairs)
+  : m_collection(pairs)
+{
+  BuildNonTermIndexMaps();
+}
+
+AlignmentInfo::AlignmentInfo(const std::vector<unsigned char> &aln)
+{
+  assert(aln.size()%2==0);
+  for (size_t i = 0; i < aln.size(); i+= 2)
+    m_collection.insert(std::make_pair(size_t(aln[i]),size_t(aln[i+1])));
+  BuildNonTermIndexMaps();
+}
+
+AlignmentInfo::AlignmentInfo(const std::string &str)
+{
+  std::vector<std::string> points = Tokenize(str, " ");
+  std::vector<std::string>::const_iterator iter;
+  for (iter = points.begin(); iter != points.end(); iter++) {
+    std::vector<size_t> point = Tokenize<size_t>(*iter, "-");
+    UTIL_THROW_IF2(point.size() != 2, "Bad format of word alignment point: " << *iter);
+    Add(point[0], point[1]);
+  }
+}
+
+void AlignmentInfo::BuildNonTermIndexMaps()
+{
+  if (m_collection.empty()) {
+    return;
+  }
+  const_iterator p = begin();
+  size_t maxIndex = p->second;
+  for (++p;  p != end(); ++p) {
+    if (p->second > maxIndex) {
+      maxIndex = p->second;
+    }
+  }
+  m_nonTermIndexMap.resize(maxIndex+1, NOT_FOUND);
+  m_nonTermIndexMap2.resize(maxIndex+1, NOT_FOUND);
+  size_t i = 0;
+  for (p = begin(); p != end(); ++p) {
+    if (m_nonTermIndexMap[p->second] != NOT_FOUND) {
+      // 1-to-many. Definitely a set of terminals. Don't bother storing 1-to-1 index map
+      m_nonTermIndexMap.clear();
+      m_nonTermIndexMap2.clear();
+      return;
+    }
+    m_nonTermIndexMap[p->second] = i++;
+    m_nonTermIndexMap2[p->second] = p->first;
+  }
+}
+
+std::set<size_t> AlignmentInfo::GetAlignmentsForSource(size_t sourcePos) const
+{
+  std::set<size_t> ret;
+  CollType::const_iterator iter;
+  for (iter = begin(); iter != end(); ++iter) {
+    // const std::pair<size_t,size_t> &align = *iter;
+    if (iter->first == sourcePos) {
+      ret.insert(iter->second);
+    }
+  }
+  return ret;
+}
+
+std::set<size_t> AlignmentInfo::GetAlignmentsForTarget(size_t targetPos) const
+{
+  std::set<size_t> ret;
+  CollType::const_iterator iter;
+  for (iter = begin(); iter != end(); ++iter) {
+    // const std::pair<size_t,size_t> &align = *iter;
+    if (iter->second == targetPos) {
+      ret.insert(iter->first);
+    }
+  }
+  return ret;
+}
+
+
+bool
+compare_target(std::pair<size_t,size_t> const* a,
+               std::pair<size_t,size_t> const* b)
+{
+  if(a->second < b->second)  return true;
+  if(a->second == b->second) return (a->first < b->first);
+  return false;
+}
+
+
+std::vector< const std::pair<size_t,size_t>* >
+AlignmentInfo::
+GetSortedAlignments(WordAlignmentSort SortOrder) const
+{
+  std::vector< const std::pair<size_t,size_t>* > ret;
+
+  CollType::const_iterator iter;
+  for (iter = m_collection.begin(); iter != m_collection.end(); ++iter) {
+    const std::pair<size_t,size_t> &alignPair = *iter;
+    ret.push_back(&alignPair);
+  }
+
+  switch (SortOrder) {
+  case NoSort:
+    break;
+
+  case TargetOrder:
+    std::sort(ret.begin(), ret.end(), compare_target);
+    break;
+
+  default:
+    UTIL_THROW(util::Exception, "Unknown word alignment sort option: "
+               << SortOrder);
+  }
+
+  return ret;
+
+}
+
+std::vector<size_t> AlignmentInfo::GetSourceIndex2PosMap() const
+{
+  std::set<size_t> sourcePoses;
+
+  CollType::const_iterator iter;
+  for (iter = m_collection.begin(); iter != m_collection.end(); ++iter) {
+    size_t sourcePos = iter->first;
+    sourcePoses.insert(sourcePos);
+  }
+  std::vector<size_t> ret(sourcePoses.begin(), sourcePoses.end());
+  return ret;
+}
+
+std::ostream& operator<<(std::ostream &out, const AlignmentInfo &alignmentInfo)
+{
+  AlignmentInfo::const_iterator iter;
+  for (iter = alignmentInfo.begin(); iter != alignmentInfo.end(); ++iter) {
+    out << iter->first << "-" << iter->second << " ";
+  }
+  return out;
+}
+
+}

mosesdecoder/moses/BitmapContainer.h

@@ -0,0 +1,275 @@
+// $Id$
+// vim:tabstop=2
+/***********************************************************************
+Moses - factored phrase-based language decoder
+Copyright (C) 2006 University of Edinburgh
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Lesser General Public
+License as published by the Free Software Foundation; either
+version 2.1 of the License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public
+License along with this library; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+***********************************************************************/
+
+#ifndef moses_BitmapContainer_h
+#define moses_BitmapContainer_h
+
+#include <queue>
+#include <set>
+#include <vector>
+
+#include "Hypothesis.h"
+#include "HypothesisStackCubePruning.h"
+#include "SquareMatrix.h"
+#include "TranslationOption.h"
+#include "TypeDef.h"
+#include "Bitmap.h"
+
+#include <boost/unordered_set.hpp>
+
+namespace Moses
+{
+
+class BitmapContainer;
+class BackwardsEdge;
+class Hypothesis;
+class HypothesisStackCubePruning;
+class HypothesisQueueItem;
+class QueueItemOrderer;
+class TranslationOptionList;
+
+typedef std::vector< Hypothesis* > HypothesisSet;
+typedef std::set< BackwardsEdge* > BackwardsEdgeSet;
+typedef std::priority_queue< HypothesisQueueItem*, std::vector< HypothesisQueueItem* >, QueueItemOrderer> HypothesisQueue;
+
+////////////////////////////////////////////////////////////////////////////////
+// Hypothesis Priority Queue Code
+////////////////////////////////////////////////////////////////////////////////
+
+//! 1 item in the priority queue for stack decoding (phrase-based)
+class HypothesisQueueItem
+{
+private:
+  size_t m_hypothesis_pos, m_translation_pos;
+  Hypothesis *m_hypothesis;
+  BackwardsEdge *m_edge;
+  boost::shared_ptr<TargetPhrase> m_target_phrase;
+
+  HypothesisQueueItem();
+
+public:
+  HypothesisQueueItem(const size_t hypothesis_pos
+                      , const size_t translation_pos
+                      , Hypothesis *hypothesis
+                      , BackwardsEdge *edge
+                      , const TargetPhrase *target_phrase = NULL)
+    : m_hypothesis_pos(hypothesis_pos)
+    , m_translation_pos(translation_pos)
+    , m_hypothesis(hypothesis)
+    , m_edge(edge) {
+    if (target_phrase != NULL) {
+      m_target_phrase.reset(new TargetPhrase(*target_phrase));
+    }
+  }
+
+  ~HypothesisQueueItem() {
+  }
+
+  int GetHypothesisPos() {
+    return m_hypothesis_pos;
+  }
+
+  int GetTranslationPos() {
+    return m_translation_pos;
+  }
+
+  Hypothesis *GetHypothesis() {
+    return m_hypothesis;
+  }
+
+  BackwardsEdge *GetBackwardsEdge() {
+    return m_edge;
+  }
+
+  boost::shared_ptr<TargetPhrase> GetTargetPhrase() {
+    return m_target_phrase;
+  }
+};
+
+//! Allows comparison of two HypothesisQueueItem objects by the corresponding scores.
+class QueueItemOrderer
+{
+public:
+  bool operator()(HypothesisQueueItem* itemA, HypothesisQueueItem* itemB) const {
+    float scoreA = itemA->GetHypothesis()->GetFutureScore();
+    float scoreB = itemB->GetHypothesis()->GetFutureScore();
+
+    if (scoreA < scoreB) {
+      return true;
+    } else if (scoreA > scoreB) {
+      return false;
+    } else {
+      // Equal scores: break ties by comparing target phrases (if they exist)
+      // *Important*: these are pointers to copies of the target phrases from the
+      // hypotheses.  This class is used to keep priority queues ordered in the
+      // background, so comparisons made as those data structures are cleaned up
+      // may occur *after* the target phrases in hypotheses have been cleaned up,
+      // leading to segfaults if relying on hypotheses to provide target phrases.
+      boost::shared_ptr<TargetPhrase> phrA = itemA->GetTargetPhrase();
+      boost::shared_ptr<TargetPhrase> phrB = itemB->GetTargetPhrase();
+      if (!phrA || !phrB) {
+        // Fallback: scoreA < scoreB == false, non-deterministic sort
+        return false;
+      }
+      return (phrA->Compare(*phrB) > 0);
+    }
+  }
+};
+
+////////////////////////////////////////////////////////////////////////////////
+// Hypothesis Orderer Code
+////////////////////////////////////////////////////////////////////////////////
+// Allows to compare two Hypothesis objects by the corresponding scores.
+////////////////////////////////////////////////////////////////////////////////
+
+class HypothesisScoreOrderer
+{
+private:
+  bool m_deterministic;
+
+public:
+  HypothesisScoreOrderer(const bool deterministic = false)
+    : m_deterministic(deterministic) {}
+
+  bool operator()(const Hypothesis* hypoA, const Hypothesis* hypoB) const {
+
+    float scoreA = hypoA->GetFutureScore();
+    float scoreB = hypoB->GetFutureScore();
+
+    if (scoreA > scoreB) {
+      return true;
+    } else if (scoreA < scoreB) {
+      return false;
+    } else {
+      if (m_deterministic) {
+        // Equal scores: break ties by comparing target phrases
+        return (hypoA->GetCurrTargetPhrase().Compare(hypoB->GetCurrTargetPhrase()) < 0);
+      }
+      // Fallback: scoreA > scoreB == false, non-deterministic sort
+      return false;
+    }
+  }
+};
+
+////////////////////////////////////////////////////////////////////////////////
+// Backwards Edge Code
+////////////////////////////////////////////////////////////////////////////////
+// Encodes an edge pointing to a BitmapContainer.
+////////////////////////////////////////////////////////////////////////////////
+
+class BackwardsEdge
+{
+private:
+  friend class BitmapContainer;
+  bool m_initialized;
+
+  const BitmapContainer &m_prevBitmapContainer;
+  BitmapContainer &m_parent;
+  const TranslationOptionList &m_translations;
+  const SquareMatrix &m_estimatedScores;
+  float m_estimatedScore;
+
+  bool m_deterministic;
+
+  std::vector< const Hypothesis* > m_hypotheses;
+  boost::unordered_set< int > m_seenPosition;
+
+  // We don't want to instantiate "empty" objects.
+  BackwardsEdge();
+
+  Hypothesis *CreateHypothesis(const Hypothesis &hypothesis, const TranslationOption &transOpt);
+  bool SeenPosition(const size_t x, const size_t y);
+  void SetSeenPosition(const size_t x, const size_t y);
+
+protected:
+  void Initialize();
+
+public:
+  BackwardsEdge(const BitmapContainer &prevBitmapContainer
+                , BitmapContainer &parent
+                , const TranslationOptionList &translations
+                , const SquareMatrix &estimatedScores
+                , const InputType& source
+                , const bool deterministic = false);
+  ~BackwardsEdge();
+
+  bool GetInitialized();
+  const BitmapContainer &GetBitmapContainer() const;
+  int GetDistortionPenalty();
+  void PushSuccessors(const size_t x, const size_t y);
+};
+
+////////////////////////////////////////////////////////////////////////////////
+// Bitmap Container Code
+////////////////////////////////////////////////////////////////////////////////
+// A BitmapContainer encodes an ordered set of hypotheses and a set of edges
+// pointing to the "generating" BitmapContainers.  It also stores a priority
+// queue that contains expanded hypotheses from the connected edges.
+////////////////////////////////////////////////////////////////////////////////
+
+class BitmapContainer
+{
+private:
+  const Bitmap &m_bitmap;
+  HypothesisStackCubePruning &m_stack;
+  HypothesisSet m_hypotheses;
+  BackwardsEdgeSet m_edges;
+  HypothesisQueue m_queue;
+  size_t m_numStackInsertions;
+  bool m_deterministic;
+
+  // We always require a corresponding bitmap to be supplied.
+  BitmapContainer();
+  BitmapContainer(const BitmapContainer &);
+public:
+  BitmapContainer(const Bitmap &bitmap
+                  , HypothesisStackCubePruning &stack
+                  , bool deterministic = false);
+
+  // The destructor will also delete all the edges that are
+  // connected to this BitmapContainer.
+  ~BitmapContainer();
+
+  void Enqueue(int hypothesis_pos, int translation_pos, Hypothesis *hypothesis, BackwardsEdge *edge);
+  HypothesisQueueItem *Dequeue(bool keepValue=false);
+  HypothesisQueueItem *Top() const;
+  size_t Size();
+  bool Empty() const;
+
+  const Bitmap &GetWordsBitmap() const {
+    return m_bitmap;
+  }
+
+  const HypothesisSet &GetHypotheses() const;
+  size_t GetHypothesesSize() const;
+  const BackwardsEdgeSet &GetBackwardsEdges();
+
+  void InitializeEdges();
+  void ProcessBestHypothesis();
+  void EnsureMinStackHyps(const size_t minNumHyps);
+  void AddHypothesis(Hypothesis *hypothesis);
+  void AddBackwardsEdge(BackwardsEdge *edge);
+  void SortHypotheses();
+};
+
+}
+
+#endif

mosesdecoder/moses/ChartCell.cpp

@@ -0,0 +1,233 @@
+// $Id$
+// vim:tabstop=2
+/***********************************************************************
+ Moses - factored phrase-based language decoder
+ Copyright (C) 2010 Hieu Hoang
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ ***********************************************************************/
+
+#include <algorithm>
+#include "ChartCell.h"
+#include "ChartCellCollection.h"
+#include "HypergraphOutput.h"
+#include "RuleCubeQueue.h"
+#include "RuleCube.h"
+#include "Range.h"
+#include "Util.h"
+#include "ChartTranslationOptions.h"
+#include "ChartTranslationOptionList.h"
+#include "ChartManager.h"
+#include "util/exception.hh"
+
+using namespace std;
+
+namespace Moses
+{
+
+ChartCellBase::ChartCellBase(size_t startPos, size_t endPos) :
+  m_coverage(startPos, endPos),
+  m_targetLabelSet(m_coverage) {}
+
+ChartCellBase::~ChartCellBase() {}
+
+/** Constructor
+ * \param startPos endPos range of this cell
+ * \param manager pointer back to the manager
+ */
+ChartCell::ChartCell(size_t startPos, size_t endPos, ChartManager &manager) :
+  ChartCellBase(startPos, endPos), m_manager(manager)
+{
+  m_nBestIsEnabled = manager.options()->nbest.enabled;
+}
+
+ChartCell::~ChartCell() {}
+
+/** Add the given hypothesis to the cell.
+ *  Returns true if added, false if not. Maybe it already exists in the collection or score falls below threshold etc.
+ *  This function just calls the corresponding AddHypothesis() in ChartHypothesisCollection
+ *  \param hypo Hypothesis to be added
+ */
+bool ChartCell::AddHypothesis(ChartHypothesis *hypo)
+{
+  const Word &targetLHS = hypo->GetTargetLHS();
+  MapType::iterator m = m_hypoColl.find(targetLHS);
+  if (m == m_hypoColl.end()) {
+    std::pair<Word, ChartHypothesisCollection>
+    e(targetLHS, ChartHypothesisCollection(*m_manager.options()));
+    m = m_hypoColl.insert(e).first;
+  }
+  return m->second.AddHypothesis(hypo, m_manager);
+}
+
+/** Prune each collection in this cell to a particular size */
+void ChartCell::PruneToSize()
+{
+  MapType::iterator iter;
+  for (iter = m_hypoColl.begin(); iter != m_hypoColl.end(); ++iter) {
+    ChartHypothesisCollection &coll = iter->second;
+    coll.PruneToSize(m_manager);
+  }
+}
+
+/** Decoding at span level: fill chart cell with hypotheses
+ *  (implementation of cube pruning)
+ * \param transOptList list of applicable rules to create hypotheses for the cell
+ * \param allChartCells entire chart - needed to look up underlying hypotheses
+ */
+void ChartCell::Decode(const ChartTranslationOptionList &transOptList
+                       , const ChartCellCollection &allChartCells)
+{
+  // priority queue for applicable rules with selected hypotheses
+  RuleCubeQueue queue(m_manager);
+
+  // add all trans opt into queue. using only 1st child node.
+  for (size_t i = 0; i < transOptList.GetSize(); ++i) {
+    const ChartTranslationOptions &transOpt = transOptList.Get(i);
+    RuleCube *ruleCube = new RuleCube(transOpt, allChartCells, m_manager);
+    queue.Add(ruleCube);
+  }
+
+  // pluck things out of queue and add to hypo collection
+  const size_t popLimit = m_manager.options()->cube.pop_limit;
+  for (size_t numPops = 0; numPops < popLimit && !queue.IsEmpty(); ++numPops) {
+    ChartHypothesis *hypo = queue.Pop();
+    AddHypothesis(hypo);
+  }
+}
+
+//! call SortHypotheses() in each hypo collection in this cell
+void ChartCell::SortHypotheses()
+{
+  UTIL_THROW_IF2(!m_targetLabelSet.Empty(), "Already sorted");
+
+  MapType::iterator iter;
+  for (iter = m_hypoColl.begin(); iter != m_hypoColl.end(); ++iter) {
+    ChartHypothesisCollection &coll = iter->second;
+
+    if (coll.GetSize()) {
+      coll.SortHypotheses();
+      m_targetLabelSet.AddConstituent(iter->first, &coll.GetSortedHypotheses());
+    }
+  }
+}
+
+/** Return the highest scoring hypothesis out of all the  hypo collection in this cell */
+const ChartHypothesis *ChartCell::GetBestHypothesis() const
+{
+  const ChartHypothesis *ret = NULL;
+  float bestScore = -std::numeric_limits<float>::infinity();
+
+  MapType::const_iterator iter;
+  for (iter = m_hypoColl.begin(); iter != m_hypoColl.end(); ++iter) {
+    const HypoList &sortedList = iter->second.GetSortedHypotheses();
+    if (sortedList.size() > 0) {
+      const ChartHypothesis *hypo = sortedList[0];
+      if (hypo->GetFutureScore() > bestScore) {
+        bestScore = hypo->GetFutureScore();
+        ret = hypo;
+      }
+    }
+  }
+
+  return ret;
+}
+
+//! call CleanupArcList() in each hypo collection in this cell
+void ChartCell::CleanupArcList()
+{
+  // only necessary if n-best calculations are enabled
+  if (!m_nBestIsEnabled) return;
+
+  MapType::iterator iter;
+  for (iter = m_hypoColl.begin(); iter != m_hypoColl.end(); ++iter) {
+    ChartHypothesisCollection &coll = iter->second;
+    coll.CleanupArcList();
+  }
+}
+
+//! debug info - size of each hypo collection in this cell
+void ChartCell::OutputSizes(std::ostream &out) const
+{
+  MapType::const_iterator iter;
+  for (iter = m_hypoColl.begin(); iter != m_hypoColl.end(); ++iter) {
+    const Word &targetLHS = iter->first;
+    const ChartHypothesisCollection &coll = iter->second;
+
+    out << targetLHS << "=" << coll.GetSize() << " ";
+  }
+}
+
+//! debug info - total number of hypos in all hypo collection in this cell
+size_t ChartCell::GetSize() const
+{
+  size_t ret = 0;
+  MapType::const_iterator iter;
+  for (iter = m_hypoColl.begin(); iter != m_hypoColl.end(); ++iter) {
+    const ChartHypothesisCollection &coll = iter->second;
+
+    ret += coll.GetSize();
+  }
+
+  return ret;
+}
+
+const HypoList *ChartCell::GetAllSortedHypotheses() const
+{
+  HypoList *ret = new HypoList();
+
+  MapType::const_iterator iter;
+  for (iter = m_hypoColl.begin(); iter != m_hypoColl.end(); ++iter) {
+    const ChartHypothesisCollection &coll = iter->second;
+    const HypoList &list = coll.GetSortedHypotheses();
+    std::copy(list.begin(), list.end(), std::inserter(*ret, ret->end()));
+  }
+  return ret;
+}
+
+//! call WriteSearchGraph() for each hypo collection
+void ChartCell::WriteSearchGraph(const ChartSearchGraphWriter& writer, const std::map<unsigned, bool> &reachable) const
+{
+  MapType::const_iterator iterOutside;
+  for (iterOutside = m_hypoColl.begin(); iterOutside != m_hypoColl.end(); ++iterOutside) {
+    const ChartHypothesisCollection &coll = iterOutside->second;
+    coll.WriteSearchGraph(writer, reachable);
+  }
+}
+
+std::ostream& operator<<(std::ostream &out, const ChartCell &cell)
+{
+  ChartCell::MapType::const_iterator iterOutside;
+  for (iterOutside = cell.m_hypoColl.begin(); iterOutside != cell.m_hypoColl.end(); ++iterOutside) {
+    const Word &targetLHS = iterOutside->first;
+    cerr << targetLHS << ":" << endl;
+
+    const ChartHypothesisCollection &coll = iterOutside->second;
+    cerr << coll;
+  }
+
+  /*
+  ChartCell::HCType::const_iterator iter;
+  for (iter = cell.m_hypos.begin(); iter != cell.m_hypos.end(); ++iter)
+  {
+  	const ChartHypothesis &hypo = **iter;
+  	out << hypo << endl;
+  }
+   */
+
+  return out;
+}
+
+} // namespace

mosesdecoder/moses/ChartCellLabelSet.h

@@ -0,0 +1,147 @@
+/***********************************************************************
+ Moses - statistical machine translation system
+ Copyright (C) 2006-2011 University of Edinburgh
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ ***********************************************************************/
+
+#pragma once
+
+#include "ChartCellLabel.h"
+#include "NonTerminal.h"
+#include "moses/FactorCollection.h"
+
+#include <boost/functional/hash.hpp>
+#include <boost/unordered_map.hpp>
+#include <boost/version.hpp>
+
+namespace Moses
+{
+
+class ChartHypothesisCollection;
+
+/** @todo I have no idea what's in here
+ */
+class ChartCellLabelSet
+{
+private:
+
+  typedef std::vector<ChartCellLabel*> MapType;
+
+public:
+  typedef MapType::const_iterator const_iterator;
+  typedef MapType::iterator iterator;
+
+  ChartCellLabelSet(const Range &coverage)
+    : m_coverage(coverage)
+    , m_map(FactorCollection::Instance().GetNumNonTerminals(), NULL)
+    , m_size(0) { }
+
+  ~ChartCellLabelSet() {
+    RemoveAllInColl(m_map);
+  }
+
+  // TODO: skip empty elements when iterating, or deprecate this
+  const_iterator begin() const {
+    return m_map.begin();
+  }
+  const_iterator end() const {
+    return m_map.end();
+  }
+
+  iterator mutable_begin() {
+    return m_map.begin();
+  }
+  iterator mutable_end() {
+    return m_map.end();
+  }
+
+  void AddWord(const Word &w) {
+    size_t idx = w[0]->GetId();
+    if (! ChartCellExists(idx)) {
+      m_size++;
+
+
+      m_map[idx] = new ChartCellLabel(m_coverage, w);
+    }
+  }
+
+  // Stack is a HypoList or whatever the search algorithm uses.
+  void AddConstituent(const Word &w, const HypoList *stack) {
+    size_t idx = w[0]->GetId();
+    if (ChartCellExists(idx)) {
+      ChartCellLabel::Stack & s = m_map[idx]->MutableStack();
+      s.cube = stack;
+    } else {
+      ChartCellLabel::Stack s;
+      s.cube = stack;
+      m_size++;
+      m_map[idx] = new ChartCellLabel(m_coverage, w, s);
+    }
+  }
+
+  // grow vector if necessary
+  bool ChartCellExists(size_t idx) {
+    try {
+      if (m_map.at(idx) != NULL) {
+        return true;
+      }
+    } catch (const std::out_of_range& oor) {
+      m_map.resize(FactorCollection::Instance().GetNumNonTerminals(), NULL);
+    }
+    return false;
+  }
+
+  bool Empty() const {
+    return m_size == 0;
+  }
+
+  size_t GetSize() const {
+    return m_size;
+  }
+
+  const ChartCellLabel *Find(const Word &w) const {
+    size_t idx = w[0]->GetId();
+    try {
+      return m_map.at(idx);
+    } catch (const std::out_of_range& oor) {
+      return NULL;
+    }
+  }
+
+  const ChartCellLabel *Find(size_t idx) const {
+    try {
+      return m_map.at(idx);
+    } catch (const std::out_of_range& oor) {
+      return NULL;
+    }
+  }
+
+  ChartCellLabel::Stack &FindOrInsert(const Word &w) {
+    size_t idx = w[0]->GetId();
+    if (! ChartCellExists(idx)) {
+      m_size++;
+      m_map[idx] = new ChartCellLabel(m_coverage, w);
+    }
+    return m_map[idx]->MutableStack();
+  }
+
+private:
+  const Range &m_coverage;
+  MapType m_map;
+  size_t m_size;
+};
+
+}

mosesdecoder/moses/ChartHypothesisCollection.h

@@ -0,0 +1,108 @@
+// $Id$
+// vim:tabstop=2
+/***********************************************************************
+ Moses - factored phrase-based language decoder
+ Copyright (C) 2010 Hieu Hoang
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ ***********************************************************************/
+#pragma once
+
+#include <set>
+#include "ChartHypothesis.h"
+#include "RuleCube.h"
+
+
+namespace Moses
+{
+
+class ChartSearchGraphWriter;
+struct AllOptions;
+
+//! functor to compare (chart) hypotheses by (descending) score
+class ChartHypothesisScoreOrderer
+{
+public:
+  bool operator()(const ChartHypothesis* hypoA, const ChartHypothesis* hypoB) const {
+    return hypoA->GetFutureScore() > hypoB->GetFutureScore();
+  }
+};
+
+/** Contains a set of unique hypos that have the same HS non-term.
+  * ie. 1 of these for each target LHS in each cell
+  */
+class ChartHypothesisCollection
+{
+  friend std::ostream& operator<<(std::ostream&, const ChartHypothesisCollection&);
+
+protected:
+  //typedef std::set<ChartHypothesis*, ChartHypothesisRecombinationOrderer> HCType;
+  typedef boost::unordered_set< ChartHypothesis*, UnorderedComparer<ChartHypothesis>, UnorderedComparer<ChartHypothesis> > HCType;
+  HCType m_hypos;
+  HypoList m_hyposOrdered;
+
+  float m_bestScore; /**< score of the best hypothesis in collection */
+  float m_beamWidth; /**< minimum score due to threashold pruning */
+  size_t m_maxHypoStackSize; /**< maximum number of hypothesis allowed in this stack */
+  bool m_nBestIsEnabled; /**< flag to determine whether to keep track of old arcs */
+
+  std::pair<HCType::iterator, bool> Add(ChartHypothesis *hypo, ChartManager &manager);
+
+public:
+  typedef HCType::iterator iterator;
+  typedef HCType::const_iterator const_iterator;
+  //! iterators
+  const_iterator begin() const {
+    return m_hypos.begin();
+  }
+  const_iterator end() const {
+    return m_hypos.end();
+  }
+
+  ChartHypothesisCollection(AllOptions const& opts);
+  ~ChartHypothesisCollection();
+  bool AddHypothesis(ChartHypothesis *hypo, ChartManager &manager);
+
+  void Detach(const HCType::iterator &iter);
+  void Remove(const HCType::iterator &iter);
+
+  void PruneToSize(ChartManager &manager);
+
+  size_t GetSize() const {
+    return m_hypos.size();
+  }
+  size_t GetHypo() const {
+    return m_hypos.size();
+  }
+
+  void SortHypotheses();
+  void CleanupArcList();
+
+  //! return vector of hypothesis that has been sorted by score
+  const HypoList &GetSortedHypotheses() const {
+    return m_hyposOrdered;
+  }
+
+  //! return the best total score of all hypos in this collection
+  float GetBestScore() const {
+    return m_bestScore;
+  }
+
+  void WriteSearchGraph(const ChartSearchGraphWriter& writer, const std::map<unsigned,bool> &reachable) const;
+
+};
+
+} // namespace
+

mosesdecoder/moses/ChartManager.cpp

@@ -0,0 +1,867 @@
+// $Id$
+// vim:tabstop=2
+/***********************************************************************
+ Moses - factored phrase-based language decoder
+ Copyright (C) 2010 Hieu Hoang
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ ***********************************************************************/
+
+#include <cstdio>
+#include "ChartManager.h"
+#include "ChartCell.h"
+#include "ChartHypothesis.h"
+#include "ChartKBestExtractor.h"
+#include "ChartTranslationOptions.h"
+#include "HypergraphOutput.h"
+#include "StaticData.h"
+#include "DecodeStep.h"
+#include "TreeInput.h"
+#include "moses/FF/StatefulFeatureFunction.h"
+#include "moses/FF/WordPenaltyProducer.h"
+#include "moses/OutputCollector.h"
+#include "moses/ChartKBestExtractor.h"
+#include "moses/HypergraphOutput.h"
+#include "moses/TranslationTask.h"
+
+using namespace std;
+
+namespace Moses
+{
+
+/* constructor. Initialize everything prior to decoding a particular sentence.
+ * \param source the sentence to be decoded
+ * \param system which particular set of models to use.
+ */
+ChartManager::ChartManager(ttasksptr const& ttask)
+  : BaseManager(ttask)
+  , m_hypoStackColl(m_source, *this)
+  , m_start(clock())
+  , m_hypothesisId(0)
+  , m_parser(ttask, m_hypoStackColl)
+  , m_translationOptionList(ttask->options()->syntax.rule_limit, m_source)
+{ }
+
+ChartManager::~ChartManager()
+{
+  clock_t end = clock();
+  float et = (end - m_start);
+  et /= (float)CLOCKS_PER_SEC;
+  VERBOSE(1, "Translation took " << et << " seconds" << endl);
+
+}
+
+//! decode the sentence. This contains the main laps. Basically, the CKY++ algorithm
+void ChartManager::Decode()
+{
+
+  VERBOSE(1,"Translating: " << m_source << endl);
+
+  ResetSentenceStats(m_source);
+
+  VERBOSE(2,"Decoding: " << endl);
+  //ChartHypothesis::ResetHypoCount();
+
+  AddXmlChartOptions();
+
+  // MAIN LOOP
+  size_t size = m_source.GetSize();
+  for (int startPos = size-1; startPos >= 0; --startPos) {
+    for (size_t width = 1; width <= size-startPos; ++width) {
+      size_t endPos = startPos + width - 1;
+      Range range(startPos, endPos);
+
+      // create trans opt
+      m_translationOptionList.Clear();
+      m_parser.Create(range, m_translationOptionList);
+      m_translationOptionList.ApplyThreshold(options()->search.trans_opt_threshold);
+
+      const InputPath &inputPath = m_parser.GetInputPath(range);
+      m_translationOptionList.EvaluateWithSourceContext(m_source, inputPath);
+
+      // decode
+      ChartCell &cell = m_hypoStackColl.Get(range);
+      cell.Decode(m_translationOptionList, m_hypoStackColl);
+
+      m_translationOptionList.Clear();
+      cell.PruneToSize();
+      cell.CleanupArcList();
+      cell.SortHypotheses();
+    }
+  }
+
+  IFVERBOSE(1) {
+
+    for (size_t startPos = 0; startPos < size; ++startPos) {
+      cerr.width(3);
+      cerr << startPos << " ";
+    }
+    cerr << endl;
+    for (size_t width = 1; width <= size; width++) {
+      for( size_t space = 0; space < width-1; space++ ) {
+        cerr << "  ";
+      }
+      for (size_t startPos = 0; startPos <= size-width; ++startPos) {
+        Range range(startPos, startPos+width-1);
+        cerr.width(3);
+        cerr << m_hypoStackColl.Get(range).GetSize() << " ";
+      }
+      cerr << endl;
+    }
+  }
+}
+
+/** add specific translation options and hypotheses according to the XML override translation scheme.
+ *  Doesn't seem to do anything about walls and zones.
+ *  @todo check walls & zones. Check that the implementation doesn't leak, xml options sometimes does if you're not careful
+ */
+void ChartManager::AddXmlChartOptions()
+{
+  const std::vector <ChartTranslationOptions*> xmlChartOptionsList
+  = m_source.GetXmlChartTranslationOptions();
+  IFVERBOSE(2) {
+    cerr << "AddXmlChartOptions " << xmlChartOptionsList.size() << endl;
+  }
+  if (xmlChartOptionsList.size() == 0) return;
+
+  typedef std::vector<ChartTranslationOptions*>::const_iterator citer;
+  for(citer i = xmlChartOptionsList.begin(); i != xmlChartOptionsList.end(); ++i) {
+    ChartTranslationOptions* opt = *i;
+
+    const Range &range = opt->GetSourceWordsRange();
+
+    RuleCubeItem* item = new RuleCubeItem( *opt, m_hypoStackColl );
+    ChartHypothesis* hypo = new ChartHypothesis(*opt, *item, *this);
+    hypo->EvaluateWhenApplied();
+
+
+    ChartCell &cell = m_hypoStackColl.Get(range);
+    cell.AddHypothesis(hypo);
+  }
+}
+
+//! get best complete translation from the top chart cell.
+const ChartHypothesis *ChartManager::GetBestHypothesis() const
+{
+  size_t size = m_source.GetSize();
+
+  if (size == 0) // empty source
+    return NULL;
+  else {
+    Range range(0, size-1);
+    const ChartCell &lastCell = m_hypoStackColl.Get(range);
+    return lastCell.GetBestHypothesis();
+  }
+}
+
+/** Calculate the n-best paths through the output hypergraph.
+ * Return the list of paths with the variable ret
+ * \param n how may paths to return
+ * \param ret return argument
+ * \param onlyDistinct whether to check for distinct output sentence or not (default - don't check, just return top n-paths)
+ */
+void ChartManager::CalcNBest(
+  std::size_t n,
+  std::vector<boost::shared_ptr<ChartKBestExtractor::Derivation> > &nBestList,
+  bool onlyDistinct) const
+{
+  nBestList.clear();
+  if (n == 0 || m_source.GetSize() == 0) {
+    return;
+  }
+
+  // Get the list of top-level hypotheses, sorted by score.
+  Range range(0, m_source.GetSize()-1);
+  const ChartCell &lastCell = m_hypoStackColl.Get(range);
+  boost::scoped_ptr<const std::vector<const ChartHypothesis*> > topLevelHypos(
+    lastCell.GetAllSortedHypotheses());
+  if (!topLevelHypos) {
+    return;
+  }
+
+  ChartKBestExtractor extractor;
+
+  if (!onlyDistinct) {
+    // Return the n-best list as is, including duplicate translations.
+    extractor.Extract(*topLevelHypos, n, nBestList);
+    return;
+  }
+
+  // Determine how many derivations to extract.  If the n-best list is
+  // restricted to distinct translations then this limit should be bigger
+  // than n.  The n-best factor determines how much bigger the limit should be,
+  // with 0 being 'unlimited.'  This actually sets a large-ish limit in case
+  // too many translations are identical.
+  const std::size_t nBestFactor = options()->nbest.factor;
+  std::size_t numDerivations = (nBestFactor == 0) ? n*1000 : n*nBestFactor;
+
+  // Extract the derivations.
+  ChartKBestExtractor::KBestVec bigList;
+  bigList.reserve(numDerivations);
+  extractor.Extract(*topLevelHypos, numDerivations, bigList);
+
+  // Copy derivations into nBestList, skipping ones with repeated translations.
+  std::set<Phrase> distinct;
+  for (ChartKBestExtractor::KBestVec::const_iterator p = bigList.begin();
+       nBestList.size() < n && p != bigList.end(); ++p) {
+    boost::shared_ptr<ChartKBestExtractor::Derivation> derivation = *p;
+    Phrase translation = ChartKBestExtractor::GetOutputPhrase(*derivation);
+    if (distinct.insert(translation).second) {
+      nBestList.push_back(derivation);
+    }
+  }
+}
+
+void ChartManager::WriteSearchGraph(const ChartSearchGraphWriter& writer) const
+{
+
+  size_t size = m_source.GetSize();
+
+  // which hypotheses are reachable?
+  std::map<unsigned,bool> reachable;
+  Range fullRange(0, size-1);
+  const ChartCell &lastCell = m_hypoStackColl.Get(fullRange);
+  const ChartHypothesis *hypo = lastCell.GetBestHypothesis();
+
+  if (hypo == NULL) {
+    // no hypothesis
+    return;
+  }
+  size_t winners = 0;
+  size_t losers = 0;
+
+  FindReachableHypotheses( hypo, reachable, &winners, &losers);
+  writer.WriteHeader(winners, losers);
+
+  for (size_t width = 1; width <= size; ++width) {
+    for (size_t startPos = 0; startPos <= size-width; ++startPos) {
+      size_t endPos = startPos + width - 1;
+      Range range(startPos, endPos);
+      TRACE_ERR(" " << range << "=");
+
+      const ChartCell &cell = m_hypoStackColl.Get(range);
+      cell.WriteSearchGraph(writer, reachable);
+    }
+  }
+}
+
+void ChartManager::FindReachableHypotheses(
+  const ChartHypothesis *hypo, std::map<unsigned,bool> &reachable, size_t* winners, size_t* losers) const
+{
+  // do not recurse, if already visited
+  if (reachable.find(hypo->GetId()) != reachable.end()) {
+    return;
+  }
+
+  // recurse
+  reachable[ hypo->GetId() ] = true;
+  if (hypo->GetWinningHypothesis() == hypo) {
+    (*winners)++;
+  } else {
+    (*losers)++;
+  }
+  const std::vector<const ChartHypothesis*> &previous = hypo->GetPrevHypos();
+  for(std::vector<const ChartHypothesis*>::const_iterator i = previous.begin(); i != previous.end(); ++i) {
+    FindReachableHypotheses( *i, reachable, winners, losers );
+  }
+
+  // also loop over recombined hypotheses (arcs)
+  const ChartArcList *arcList = hypo->GetArcList();
+  if (arcList) {
+    ChartArcList::const_iterator iterArc;
+    for (iterArc = arcList->begin(); iterArc != arcList->end(); ++iterArc) {
+      const ChartHypothesis &arc = **iterArc;
+      FindReachableHypotheses( &arc, reachable, winners, losers );
+    }
+  }
+}
+
+void
+ChartManager::
+OutputSearchGraphAsHypergraph(std::ostream& out) const
+{
+  ChartSearchGraphWriterHypergraph writer(options(), &out);
+  WriteSearchGraph(writer);
+}
+
+void ChartManager::OutputSearchGraphMoses(std::ostream &outputSearchGraphStream) const
+{
+  ChartSearchGraphWriterMoses writer(options(), &outputSearchGraphStream,
+                                     m_source.GetTranslationId());
+  WriteSearchGraph(writer);
+}
+
+void ChartManager::OutputBest(OutputCollector *collector) const
+{
+  const ChartHypothesis *bestHypo = GetBestHypothesis();
+  if (collector && bestHypo) {
+    const size_t translationId = m_source.GetTranslationId();
+    const ChartHypothesis *bestHypo = GetBestHypothesis();
+    OutputBestHypo(collector, bestHypo, translationId);
+  }
+}
+
+void ChartManager::OutputNBest(OutputCollector *collector) const
+{
+  size_t nBestSize = options()->nbest.nbest_size;
+  if (nBestSize > 0) {
+    const size_t translationId = m_source.GetTranslationId();
+
+    VERBOSE(2,"WRITING " << nBestSize << " TRANSLATION ALTERNATIVES TO "
+            << options()->nbest.output_file_path << endl);
+    std::vector<boost::shared_ptr<ChartKBestExtractor::Derivation> > nBestList;
+    CalcNBest(nBestSize, nBestList, options()->nbest.only_distinct);
+    OutputNBestList(collector, nBestList, translationId);
+    IFVERBOSE(2) {
+      PrintUserTime("N-Best Hypotheses Generation Time:");
+    }
+  }
+
+}
+
+void ChartManager::OutputNBestList(OutputCollector *collector,
+                                   const ChartKBestExtractor::KBestVec &nBestList,
+                                   long translationId) const
+{
+  std::ostringstream out;
+
+  if (collector->OutputIsCout()) {
+    // Set precision only if we're writing the n-best list to cout.  This is to
+    // preserve existing behaviour, but should probably be done either way.
+    FixPrecision(out);
+  }
+
+  NBestOptions const& nbo = options()->nbest;
+  bool includeWordAlignment = nbo.include_alignment_info;
+  bool PrintNBestTrees = nbo.print_trees;
+
+  for (ChartKBestExtractor::KBestVec::const_iterator p = nBestList.begin();
+       p != nBestList.end(); ++p) {
+    const ChartKBestExtractor::Derivation &derivation = **p;
+
+    // get the derivation's target-side yield
+    Phrase outputPhrase = ChartKBestExtractor::GetOutputPhrase(derivation);
+
+    // delete <s> and </s>
+    UTIL_THROW_IF2(outputPhrase.GetSize() < 2,
+                   "Output phrase should have contained at least 2 words (beginning and end-of-sentence)");
+    outputPhrase.RemoveWord(0);
+    outputPhrase.RemoveWord(outputPhrase.GetSize() - 1);
+
+    // print the translation ID, surface factors, and scores
+    out << translationId << " ||| ";
+    OutputSurface(out, outputPhrase); // , outputFactorOrder, false);
+    out << " ||| ";
+    boost::shared_ptr<ScoreComponentCollection> scoreBreakdown = ChartKBestExtractor::GetOutputScoreBreakdown(derivation);
+    bool with_labels = options()->nbest.include_feature_labels;
+    scoreBreakdown->OutputAllFeatureScores(out, with_labels);
+    out << " ||| " << derivation.score;
+
+    // optionally, print word alignments
+    if (includeWordAlignment) {
+      out << " ||| ";
+      Alignments align;
+      OutputAlignmentNBest(align, derivation, 0);
+      for (Alignments::const_iterator q = align.begin(); q != align.end();
+           ++q) {
+        out << q->first << "-" << q->second << " ";
+      }
+    }
+
+    // optionally, print tree
+    if (PrintNBestTrees) {
+      TreePointer tree = ChartKBestExtractor::GetOutputTree(derivation);
+      out << " ||| " << tree->GetString();
+    }
+
+    out << std::endl;
+  }
+
+  assert(collector);
+  collector->Write(translationId, out.str());
+}
+
+size_t ChartManager::CalcSourceSize(const Moses::ChartHypothesis *hypo) const
+{
+  size_t ret = hypo->GetCurrSourceRange().GetNumWordsCovered();
+  const std::vector<const ChartHypothesis*> &prevHypos = hypo->GetPrevHypos();
+  for (size_t i = 0; i < prevHypos.size(); ++i) {
+    size_t childSize = prevHypos[i]->GetCurrSourceRange().GetNumWordsCovered();
+    ret -= (childSize - 1);
+  }
+  return ret;
+}
+
+size_t ChartManager::OutputAlignmentNBest(
+  Alignments &retAlign,
+  const Moses::ChartKBestExtractor::Derivation &derivation,
+  size_t startTarget) const
+{
+  const ChartHypothesis &hypo = derivation.edge.head->hypothesis;
+
+  size_t totalTargetSize = 0;
+  size_t startSource = hypo.GetCurrSourceRange().GetStartPos();
+
+  const TargetPhrase &tp = hypo.GetCurrTargetPhrase();
+
+  size_t thisSourceSize = CalcSourceSize(&hypo);
+
+  // position of each terminal word in translation rule, irrespective of alignment
+  // if non-term, number is undefined
+  vector<size_t> sourceOffsets(thisSourceSize, 0);
+  vector<size_t> targetOffsets(tp.GetSize(), 0);
+
+  const AlignmentInfo &aiNonTerm = hypo.GetCurrTargetPhrase().GetAlignNonTerm();
+  vector<size_t> sourceInd2pos = aiNonTerm.GetSourceIndex2PosMap();
+  const AlignmentInfo::NonTermIndexMap &targetPos2SourceInd = aiNonTerm.GetNonTermIndexMap();
+
+  UTIL_THROW_IF2(sourceInd2pos.size() != derivation.subderivations.size(),
+                 "Error");
+
+  size_t targetInd = 0;
+  for (size_t targetPos = 0; targetPos < tp.GetSize(); ++targetPos) {
+    if (tp.GetWord(targetPos).IsNonTerminal()) {
+      UTIL_THROW_IF2(targetPos >= targetPos2SourceInd.size(), "Error");
+      size_t sourceInd = targetPos2SourceInd[targetPos];
+      size_t sourcePos = sourceInd2pos[sourceInd];
+
+      const Moses::ChartKBestExtractor::Derivation &subderivation =
+        *derivation.subderivations[sourceInd];
+
+      // calc source size
+      size_t sourceSize = subderivation.edge.head->hypothesis.GetCurrSourceRange().GetNumWordsCovered();
+      sourceOffsets[sourcePos] = sourceSize;
+
+      // calc target size.
+      // Recursively look thru child hypos
+      size_t currStartTarget = startTarget + totalTargetSize;
+      size_t targetSize = OutputAlignmentNBest(retAlign, subderivation,
+                          currStartTarget);
+      targetOffsets[targetPos] = targetSize;
+
+      totalTargetSize += targetSize;
+      ++targetInd;
+    } else {
+      ++totalTargetSize;
+    }
+  }
+
+  // convert position within translation rule to absolute position within
+  // source sentence / output sentence
+  ShiftOffsets(sourceOffsets, startSource);
+  ShiftOffsets(targetOffsets, startTarget);
+
+  // get alignments from this hypo
+  const AlignmentInfo &aiTerm = hypo.GetCurrTargetPhrase().GetAlignTerm();
+
+  // add to output arg, offsetting by source & target
+  AlignmentInfo::const_iterator iter;
+  for (iter = aiTerm.begin(); iter != aiTerm.end(); ++iter) {
+    const std::pair<size_t,size_t> &align = *iter;
+    size_t relSource = align.first;
+    size_t relTarget = align.second;
+    size_t absSource = sourceOffsets[relSource];
+    size_t absTarget = targetOffsets[relTarget];
+
+    pair<size_t, size_t> alignPoint(absSource, absTarget);
+    pair<Alignments::iterator, bool> ret = retAlign.insert(alignPoint);
+    UTIL_THROW_IF2(!ret.second, "Error");
+  }
+
+  return totalTargetSize;
+}
+
+void ChartManager::OutputAlignment(OutputCollector *collector) const
+{
+  if (collector == NULL) {
+    return;
+  }
+
+  ostringstream out;
+
+  const ChartHypothesis *hypo = GetBestHypothesis();
+  if (hypo) {
+    Alignments retAlign;
+    OutputAlignment(retAlign, hypo, 0);
+
+    // output alignments
+    Alignments::const_iterator iter;
+    for (iter = retAlign.begin(); iter != retAlign.end(); ++iter) {
+      const pair<size_t, size_t> &alignPoint = *iter;
+      out << alignPoint.first << "-" << alignPoint.second << " ";
+    }
+  }
+  out << endl;
+
+  collector->Write(m_source.GetTranslationId(), out.str());
+
+}
+
+size_t ChartManager::OutputAlignment(Alignments &retAlign,
+                                     const Moses::ChartHypothesis *hypo,
+                                     size_t startTarget) const
+{
+  size_t totalTargetSize = 0;
+  size_t startSource = hypo->GetCurrSourceRange().GetStartPos();
+
+  const TargetPhrase &tp = hypo->GetCurrTargetPhrase();
+
+  size_t thisSourceSize = CalcSourceSize(hypo);
+
+  // position of each terminal word in translation rule, irrespective of alignment
+  // if non-term, number is undefined
+  vector<size_t> sourceOffsets(thisSourceSize, 0);
+  vector<size_t> targetOffsets(tp.GetSize(), 0);
+
+  const vector<const ChartHypothesis*> &prevHypos = hypo->GetPrevHypos();
+
+  const AlignmentInfo &aiNonTerm = hypo->GetCurrTargetPhrase().GetAlignNonTerm();
+  vector<size_t> sourceInd2pos = aiNonTerm.GetSourceIndex2PosMap();
+  const AlignmentInfo::NonTermIndexMap &targetPos2SourceInd = aiNonTerm.GetNonTermIndexMap();
+
+  UTIL_THROW_IF2(sourceInd2pos.size() != prevHypos.size(), "Error");
+
+  size_t targetInd = 0;
+  for (size_t targetPos = 0; targetPos < tp.GetSize(); ++targetPos) {
+    if (tp.GetWord(targetPos).IsNonTerminal()) {
+      UTIL_THROW_IF2(targetPos >= targetPos2SourceInd.size(), "Error");
+      size_t sourceInd = targetPos2SourceInd[targetPos];
+      size_t sourcePos = sourceInd2pos[sourceInd];
+
+      const ChartHypothesis *prevHypo = prevHypos[sourceInd];
+
+      // calc source size
+      size_t sourceSize = prevHypo->GetCurrSourceRange().GetNumWordsCovered();
+      sourceOffsets[sourcePos] = sourceSize;
+
+      // calc target size.
+      // Recursively look thru child hypos
+      size_t currStartTarget = startTarget + totalTargetSize;
+      size_t targetSize = OutputAlignment(retAlign, prevHypo, currStartTarget);
+      targetOffsets[targetPos] = targetSize;
+
+      totalTargetSize += targetSize;
+      ++targetInd;
+    } else {
+      ++totalTargetSize;
+    }
+  }
+
+  // convert position within translation rule to absolute position within
+  // source sentence / output sentence
+  ShiftOffsets(sourceOffsets, startSource);
+  ShiftOffsets(targetOffsets, startTarget);
+
+  // get alignments from this hypo
+  const AlignmentInfo &aiTerm = hypo->GetCurrTargetPhrase().GetAlignTerm();
+
+  // add to output arg, offsetting by source & target
+  AlignmentInfo::const_iterator iter;
+  for (iter = aiTerm.begin(); iter != aiTerm.end(); ++iter) {
+    const std::pair<size_t,size_t> &align = *iter;
+    size_t relSource = align.first;
+    size_t relTarget = align.second;
+    size_t absSource = sourceOffsets[relSource];
+    size_t absTarget = targetOffsets[relTarget];
+
+    pair<size_t, size_t> alignPoint(absSource, absTarget);
+    pair<Alignments::iterator, bool> ret = retAlign.insert(alignPoint);
+    UTIL_THROW_IF2(!ret.second, "Error");
+
+  }
+
+  return totalTargetSize;
+}
+
+void ChartManager::OutputDetailedTranslationReport(OutputCollector *collector) const
+{
+  if (collector) {
+    OutputDetailedTranslationReport(collector,
+                                    GetBestHypothesis(),
+                                    static_cast<const Sentence&>(m_source),
+                                    m_source.GetTranslationId());
+  }
+}
+
+void ChartManager::OutputDetailedTranslationReport(
+  OutputCollector *collector,
+  const ChartHypothesis *hypo,
+  const Sentence &sentence,
+  long translationId) const
+{
+  if (hypo == NULL) {
+    return;
+  }
+  std::ostringstream out;
+  ApplicationContext applicationContext;
+
+  OutputTranslationOptions(out, applicationContext, hypo, sentence, translationId);
+  collector->Write(translationId, out.str());
+
+  //DIMw
+  if (options()->output.detailed_all_transrep_filepath.size()) {
+    const Sentence &sentence = static_cast<const Sentence &>(m_source);
+    size_t nBestSize = options()->nbest.nbest_size;
+    std::vector<boost::shared_ptr<ChartKBestExtractor::Derivation> > nBestList;
+    CalcNBest(nBestSize, nBestList, options()->nbest.only_distinct);
+    OutputDetailedAllTranslationReport(collector, nBestList, sentence, translationId);
+  }
+
+}
+
+void ChartManager::OutputTranslationOptions(std::ostream &out,
+    ApplicationContext &applicationContext,
+    const ChartHypothesis *hypo,
+    const Sentence &sentence,
+    long translationId) const
+{
+  if (hypo != NULL) {
+    OutputTranslationOption(out, applicationContext, hypo, sentence, translationId);
+    out << std::endl;
+  }
+
+  // recursive
+  const std::vector<const ChartHypothesis*> &prevHypos = hypo->GetPrevHypos();
+  std::vector<const ChartHypothesis*>::const_iterator iter;
+  for (iter = prevHypos.begin(); iter != prevHypos.end(); ++iter) {
+    const ChartHypothesis *prevHypo = *iter;
+    OutputTranslationOptions(out, applicationContext, prevHypo, sentence, translationId);
+  }
+}
+
+void ChartManager::OutputTranslationOption(std::ostream &out,
+    ApplicationContext &applicationContext,
+    const ChartHypothesis *hypo,
+    const Sentence &sentence,
+    long translationId) const
+{
+  ReconstructApplicationContext(*hypo, sentence, applicationContext);
+  out << "Trans Opt " << translationId
+      << " " << hypo->GetCurrSourceRange()
+      << ": ";
+  WriteApplicationContext(out, applicationContext);
+  out << ": " << hypo->GetCurrTargetPhrase().GetTargetLHS()
+      << "->" << hypo->GetCurrTargetPhrase()
+      << " " << hypo->GetFutureScore() << hypo->GetScoreBreakdown();
+}
+
+// Given a hypothesis and sentence, reconstructs the 'application context' --
+// the source RHS symbols of the SCFG rule that was applied, plus their spans.
+void ChartManager::ReconstructApplicationContext(const ChartHypothesis &hypo,
+    const Sentence &sentence,
+    ApplicationContext &context) const
+{
+  context.clear();
+  const std::vector<const ChartHypothesis*> &prevHypos = hypo.GetPrevHypos();
+  std::vector<const ChartHypothesis*>::const_iterator p = prevHypos.begin();
+  std::vector<const ChartHypothesis*>::const_iterator end = prevHypos.end();
+  const Range &span = hypo.GetCurrSourceRange();
+  size_t i = span.GetStartPos();
+  while (i <= span.GetEndPos()) {
+    if (p == end || i < (*p)->GetCurrSourceRange().GetStartPos()) {
+      // Symbol is a terminal.
+      const Word &symbol = sentence.GetWord(i);
+      context.push_back(std::make_pair(symbol, Range(i, i)));
+      ++i;
+    } else {
+      // Symbol is a non-terminal.
+      const Word &symbol = (*p)->GetTargetLHS();
+      const Range &range = (*p)->GetCurrSourceRange();
+      context.push_back(std::make_pair(symbol, range));
+      i = range.GetEndPos()+1;
+      ++p;
+    }
+  }
+}
+
+void ChartManager::OutputUnknowns(OutputCollector *collector) const
+{
+  if (collector) {
+    long translationId = m_source.GetTranslationId();
+    const std::vector<Phrase*> &oovs = GetParser().GetUnknownSources();
+
+    std::ostringstream out;
+    for (std::vector<Phrase*>::const_iterator p = oovs.begin();
+         p != oovs.end(); ++p) {
+      out << **p;
+    }
+    out << std::endl;
+    collector->Write(translationId, out.str());
+  }
+
+}
+
+void ChartManager::OutputDetailedTreeFragmentsTranslationReport(OutputCollector *collector) const
+{
+  const ChartHypothesis *hypo = GetBestHypothesis();
+  if (collector == NULL || hypo == NULL) {
+    return;
+  }
+
+  std::ostringstream out;
+  ApplicationContext applicationContext;
+
+  const Sentence &sentence = static_cast<const Sentence &>(m_source);
+  const size_t translationId = m_source.GetTranslationId();
+
+  OutputTreeFragmentsTranslationOptions(out, applicationContext, hypo, sentence, translationId);
+
+  //Tree of full sentence
+  const StatefulFeatureFunction* treeStructure;
+  treeStructure = StaticData::Instance().GetTreeStructure();
+  if (treeStructure != NULL) {
+    const vector<const StatefulFeatureFunction*>& sff = StatefulFeatureFunction::GetStatefulFeatureFunctions();
+    for( size_t i=0; i<sff.size(); i++ ) {
+      if (sff[i] == treeStructure) {
+        const TreeState* tree = static_cast<const TreeState*>(hypo->GetFFState(i));
+        out << "Full Tree " << translationId << ": " << tree->GetTree()->GetString() << "\n";
+        break;
+      }
+    }
+  }
+
+  collector->Write(translationId, out.str());
+
+}
+
+void ChartManager::OutputTreeFragmentsTranslationOptions(std::ostream &out,
+    ApplicationContext &applicationContext,
+    const ChartHypothesis *hypo,
+    const Sentence &sentence,
+    long translationId) const
+{
+
+  if (hypo != NULL) {
+    OutputTranslationOption(out, applicationContext, hypo, sentence, translationId);
+
+    const TargetPhrase &currTarPhr = hypo->GetCurrTargetPhrase();
+
+    out << " ||| ";
+    if (const PhraseProperty *property = currTarPhr.GetProperty("Tree")) {
+      out << " " << *property->GetValueString();
+    } else {
+      out << " " << "noTreeInfo";
+    }
+    out << std::endl;
+  }
+
+  // recursive
+  const std::vector<const ChartHypothesis*> &prevHypos = hypo->GetPrevHypos();
+  std::vector<const ChartHypothesis*>::const_iterator iter;
+  for (iter = prevHypos.begin(); iter != prevHypos.end(); ++iter) {
+    const ChartHypothesis *prevHypo = *iter;
+    OutputTreeFragmentsTranslationOptions(out, applicationContext, prevHypo, sentence, translationId);
+  }
+}
+
+void ChartManager::OutputSearchGraph(OutputCollector *collector) const
+{
+  if (collector) {
+    long translationId = m_source.GetTranslationId();
+    std::ostringstream out;
+    OutputSearchGraphMoses( out);
+    collector->Write(translationId, out.str());
+  }
+}
+
+//DIMw
+void ChartManager::OutputDetailedAllTranslationReport(
+  OutputCollector *collector,
+  const std::vector<boost::shared_ptr<Moses::ChartKBestExtractor::Derivation> > &nBestList,
+  const Sentence &sentence,
+  long translationId) const
+{
+  std::ostringstream out;
+  ApplicationContext applicationContext;
+
+  const ChartCellCollection& cells = GetChartCellCollection();
+  size_t size = GetSource().GetSize();
+  for (size_t width = 1; width <= size; ++width) {
+    for (size_t startPos = 0; startPos <= size-width; ++startPos) {
+      size_t endPos = startPos + width - 1;
+      Range range(startPos, endPos);
+      const ChartCell& cell = cells.Get(range);
+      const HypoList* hyps = cell.GetAllSortedHypotheses();
+      out << "Chart Cell [" << startPos << ".." << endPos << "]" << endl;
+      HypoList::const_iterator iter;
+      size_t c = 1;
+      for (iter = hyps->begin(); iter != hyps->end(); ++iter) {
+        out << "----------------Item " << c++ << " ---------------------"
+            << endl;
+        OutputTranslationOptions(out, applicationContext, *iter,
+                                 sentence, translationId);
+      }
+    }
+  }
+  collector->Write(translationId, out.str());
+}
+
+void ChartManager::OutputBestHypo(OutputCollector *collector, const ChartHypothesis *hypo, long translationId) const
+{
+  if (!collector)
+    return;
+  std::ostringstream out;
+  FixPrecision(out);
+  if (hypo != NULL) {
+    VERBOSE(1,"BEST TRANSLATION: " << *hypo << endl);
+    VERBOSE(3,"Best path: ");
+    Backtrack(hypo);
+    VERBOSE(3,"0" << std::endl);
+
+    if (options()->output.ReportHypoScore) {
+      out << hypo->GetFutureScore() << " ";
+    }
+
+    if (options()->output.RecoverPath) {
+      out << "||| ";
+    }
+    Phrase outPhrase(ARRAY_SIZE_INCR);
+    hypo->GetOutputPhrase(outPhrase);
+
+    // delete 1st & last
+    UTIL_THROW_IF2(outPhrase.GetSize() < 2,
+                   "Output phrase should have contained at least 2 words (beginning and end-of-sentence)");
+
+    outPhrase.RemoveWord(0);
+    outPhrase.RemoveWord(outPhrase.GetSize() - 1);
+
+    string output = outPhrase.GetStringRep(options()->output.factor_order);
+    out << output << endl;
+  } else {
+    VERBOSE(1, "NO BEST TRANSLATION" << endl);
+
+    if (options()->output.ReportHypoScore) {
+      out << "0 ";
+    }
+
+    out << endl;
+  }
+  collector->Write(translationId, out.str());
+}
+
+void ChartManager::Backtrack(const ChartHypothesis *hypo) const
+{
+  const vector<const ChartHypothesis*> &prevHypos = hypo->GetPrevHypos();
+
+  vector<const ChartHypothesis*>::const_iterator iter;
+  for (iter = prevHypos.begin(); iter != prevHypos.end(); ++iter) {
+    const ChartHypothesis *prevHypo = *iter;
+
+    VERBOSE(3,prevHypo->GetId() << " <= ");
+    Backtrack(prevHypo);
+  }
+}
+
+} // namespace Moses

mosesdecoder/moses/ChartParser.h

@@ -0,0 +1,99 @@
+// -*- c++ -*-
+// $Id$
+// vim:tabstop=2
+/***********************************************************************
+ Moses - factored phrase-based language decoder
+ Copyright (C) 2010 Hieu Hoang
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ ***********************************************************************/
+
+#pragma once
+
+#include <list>
+#include <vector>
+#include "Range.h"
+#include "StackVec.h"
+#include "InputPath.h"
+#include "TargetPhraseCollection.h"
+namespace Moses
+{
+
+class ChartParserCallback;
+class ChartRuleLookupManager;
+class InputType;
+class Sentence;
+class ChartCellCollectionBase;
+class Word;
+class Phrase;
+// class TargetPhraseCollection;
+class DecodeGraph;
+
+class ChartParserUnknown
+{
+  ttaskwptr m_ttask;
+public:
+  ChartParserUnknown(ttasksptr const& ttask);
+  ~ChartParserUnknown();
+
+  void Process(const Word &sourceWord, const Range &range, ChartParserCallback &to);
+
+  const std::vector<Phrase*> &GetUnknownSources() const {
+    return m_unksrcs;
+  }
+
+private:
+  std::vector<Phrase*> m_unksrcs;
+  std::list<TargetPhraseCollection::shared_ptr> m_cacheTargetPhraseCollection;
+  AllOptions::ptr const& options() const;
+};
+
+class ChartParser
+{
+  ttaskwptr m_ttask;
+public:
+  ChartParser(ttasksptr const& ttask, ChartCellCollectionBase &cells);
+  ~ChartParser();
+
+  void Create(const Range &range, ChartParserCallback &to);
+
+  //! the sentence being decoded
+  //const Sentence &GetSentence() const;
+  long GetTranslationId() const;
+  size_t GetSize() const;
+  const InputPath &GetInputPath(size_t startPos, size_t endPos) const;
+  const InputPath &GetInputPath(const Range &range) const;
+  const std::vector<Phrase*> &GetUnknownSources() const {
+    return m_unknown.GetUnknownSources();
+  }
+
+  AllOptions::ptr const& options() const;
+
+private:
+  ChartParserUnknown m_unknown;
+  std::vector <DecodeGraph*> m_decodeGraphList;
+  std::vector<ChartRuleLookupManager*> m_ruleLookupManagers;
+  InputType const& m_source; /**< source sentence to be translated */
+
+  typedef std::vector< std::vector<InputPath*> > InputPathMatrix;
+  InputPathMatrix	m_inputPathMatrix;
+
+  void CreateInputPaths(const InputType &input);
+  InputPath &GetInputPath(size_t startPos, size_t endPos);
+
+};
+
+}
+

mosesdecoder/moses/ChartTranslationOptionList.h

@@ -0,0 +1,90 @@
+/***********************************************************************
+Moses - factored phrase-based language decoder
+Copyright (C) 2006 University of Edinburgh
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Lesser General Public
+License as published by the Free Software Foundation; either
+version 2.1 of the License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public
+License along with this library; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+***********************************************************************/
+
+#pragma once
+
+#include "ChartTranslationOptions.h"
+#include "ChartParserCallback.h"
+#include "StackVec.h"
+
+#include <vector>
+
+namespace Moses
+{
+
+class TargetPhraseCollection;
+class Range;
+class InputType;
+class InputPath;
+class ChartCellLabel;
+
+//! a vector of translations options for a specific range, in a specific sentence
+class ChartTranslationOptionList : public ChartParserCallback
+{
+  friend std::ostream& operator<<(std::ostream&, const ChartTranslationOptionList&);
+
+public:
+  ChartTranslationOptionList(size_t ruleLimit, const InputType &input);
+  ~ChartTranslationOptionList();
+
+  const ChartTranslationOptions &Get(size_t i) const {
+    return *m_collection[i];
+  }
+
+  //! number of translation options
+  size_t GetSize() const {
+    return m_size;
+  }
+
+  void Add(const TargetPhraseCollection &, const StackVec &,
+           const Range &);
+
+  void AddPhraseOOV(TargetPhrase &phrase, std::list<TargetPhraseCollection::shared_ptr > &waste_memory, const Range &range);
+
+  bool Empty() const {
+    return m_size == 0;
+  }
+
+  float GetBestScore(const ChartCellLabel *chartCell) const;
+
+  void Clear();
+  void ApplyThreshold(float threshold);
+  void EvaluateWithSourceContext(const InputType &input, const InputPath &inputPath);
+
+private:
+  typedef std::vector<ChartTranslationOptions*> CollType;
+
+  struct ScoreThresholdPred {
+    ScoreThresholdPred(float threshold) : m_thresholdScore(threshold) {}
+    bool operator()(const ChartTranslationOptions *option)  {
+      return option->GetEstimateOfBestScore() >= m_thresholdScore;
+    }
+    float m_thresholdScore;
+  };
+
+  void SwapTranslationOptions(size_t a, size_t b);
+
+  CollType m_collection;
+  size_t m_size;
+  float m_scoreThreshold;
+  const size_t m_ruleLimit;
+
+};
+
+}

mosesdecoder/moses/Factor.h

@@ -0,0 +1,100 @@
+// $Id$
+
+/***********************************************************************
+Moses - factored phrase-based language decoder
+Copyright (C) 2006 University of Edinburgh
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Lesser General Public
+License as published by the Free Software Foundation; either
+version 2.1 of the License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public
+License along with this library; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+***********************************************************************/
+
+#pragma once
+
+#include <ostream>
+#include <string>
+#include "TypeDef.h"
+#include "Util.h"
+#include "util/string_piece.hh"
+
+namespace Moses
+{
+
+struct FactorFriend;
+class FactorCollection;
+
+/** Represents a factor (word, POS, etc).
+ * A Factor has a contiguous identifier and string value.
+ */
+class Factor
+{
+  friend std::ostream& operator<<(std::ostream&, const Factor&);
+
+  // only these classes are allowed to instantiate this class
+  friend class FactorCollection;
+  friend struct FactorFriend;
+
+  // FactorCollection writes here.
+  // This is mutable so the pointer can be changed to pool-backed memory.
+  mutable StringPiece m_string;
+  size_t			m_id;
+
+  //! protected constructor. only friend class, FactorCollection, is allowed to create Factor objects
+  Factor() {}
+
+  // Needed for STL containers.  They'll delegate through FactorFriend, which is never exposed publicly.
+  Factor(const Factor &factor) : m_string(factor.m_string), m_id(factor.m_id) {}
+
+  // Not implemented.  Shouldn't be called.
+  Factor &operator=(const Factor &factor);
+
+public:
+  //! original string representation of the factor
+  StringPiece GetString() const {
+    return m_string;
+  }
+  //! contiguous ID
+  inline size_t GetId() const {
+    return m_id;
+  }
+
+  /** transitive comparison between 2 factors.
+  *	-1 = less than
+  *	+1 = more than
+  *	0	= same
+  */
+  inline int Compare(const Factor &compare) const {
+    if (this < &compare)
+      return -1;
+    if (this > &compare)
+      return 1;
+    return 0;
+  }
+  //! transitive comparison used for adding objects into FactorCollection
+  inline bool operator<(const Factor &compare) const {
+    return this < &compare;
+  }
+
+  // quick equality comparison. Not used
+  inline bool operator==(const Factor &compare) const {
+    return this == &compare;
+  }
+
+  TO_STRING();
+
+};
+
+size_t hash_value(const Factor &f);
+
+}
+

mosesdecoder/moses/FactorCollection.cpp

@@ -0,0 +1,105 @@
+// $Id$
+
+/***********************************************************************
+Moses - factored phrase-based language decoder
+Copyright (C) 2006 University of Edinburgh
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Lesser General Public
+License as published by the Free Software Foundation; either
+version 2.1 of the License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public
+License along with this library; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+***********************************************************************/
+
+#include <boost/version.hpp>
+#ifdef WITH_THREADS
+#include <boost/thread/locks.hpp>
+#endif
+#include <ostream>
+#include <string>
+#include "FactorCollection.h"
+#include "Util.h"
+#include "util/pool.hh"
+
+using namespace std;
+
+namespace Moses
+{
+FactorCollection FactorCollection::s_instance;
+
+const Factor *FactorCollection::AddFactor(const StringPiece &factorString, bool isNonTerminal)
+{
+  FactorFriend to_ins;
+  to_ins.in.m_string = factorString;
+  to_ins.in.m_id = (isNonTerminal) ? m_factorIdNonTerminal : m_factorId;
+  Set & set = (isNonTerminal) ? m_set : m_setNonTerminal;
+  // If we're threaded, hope a read-only lock is sufficient.
+#ifdef WITH_THREADS
+  {
+    // read=lock scope
+    boost::shared_lock<boost::shared_mutex> read_lock(m_accessLock);
+    Set::const_iterator i = set.find(to_ins);
+    if (i != set.end()) return &i->in;
+  }
+  boost::unique_lock<boost::shared_mutex> lock(m_accessLock);
+#endif // WITH_THREADS
+  std::pair<Set::iterator, bool> ret(set.insert(to_ins));
+  if (ret.second) {
+    ret.first->in.m_string.set(
+      memcpy(m_string_backing.Allocate(factorString.size()), factorString.data(), factorString.size()),
+      factorString.size());
+    if (isNonTerminal) {
+      m_factorIdNonTerminal++;
+      UTIL_THROW_IF2(m_factorIdNonTerminal >= moses_MaxNumNonterminals, "Number of non-terminals exceeds maximum size reserved. Adjust parameter moses_MaxNumNonterminals, then recompile");
+    } else {
+      m_factorId++;
+    }
+  }
+  return &ret.first->in;
+}
+
+const Factor *FactorCollection::GetFactor(const StringPiece &factorString, bool isNonTerminal)
+{
+  FactorFriend to_find;
+  to_find.in.m_string = factorString;
+  to_find.in.m_id = (isNonTerminal) ? m_factorIdNonTerminal : m_factorId;
+  Set & set = (isNonTerminal) ? m_set : m_setNonTerminal;
+  {
+    // read=lock scope
+#ifdef WITH_THREADS
+    boost::shared_lock<boost::shared_mutex> read_lock(m_accessLock);
+#endif // WITH_THREADS
+    Set::const_iterator i = set.find(to_find);
+    if (i != set.end()) return &i->in;
+  }
+  return NULL;
+}
+
+
+FactorCollection::~FactorCollection() {}
+
+TO_STRING_BODY(FactorCollection);
+
+// friend
+ostream& operator<<(ostream& out, const FactorCollection& factorCollection)
+{
+#ifdef WITH_THREADS
+  boost::shared_lock<boost::shared_mutex> lock(factorCollection.m_accessLock);
+#endif
+  for (FactorCollection::Set::const_iterator i = factorCollection.m_set.begin(); i != factorCollection.m_set.end(); ++i) {
+    out << i->in;
+  }
+  return out;
+}
+
+}
+
+

mosesdecoder/moses/FactorCollection.h

@@ -0,0 +1,132 @@
+// $Id$
+
+/***********************************************************************
+Moses - factored phrase-based language decoder
+Copyright (C) 2006 University of Edinburgh
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Lesser General Public
+License as published by the Free Software Foundation; either
+version 2.1 of the License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public
+License along with this library; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+***********************************************************************/
+
+#ifndef moses_FactorCollection_h
+#define moses_FactorCollection_h
+
+// reserve space for non-terminal symbols (ensuring consecutive numbering, and allowing quick lookup by ID)
+#ifndef moses_MaxNumNonterminals
+#define moses_MaxNumNonterminals 10000
+#endif
+
+#ifdef WITH_THREADS
+#include <boost/thread/shared_mutex.hpp>
+#endif
+
+#include "util/murmur_hash.hh"
+#include <boost/unordered_set.hpp>
+
+#include <functional>
+#include <string>
+
+#include "util/string_piece.hh"
+#include "util/pool.hh"
+#include "Factor.h"
+
+class System;
+
+namespace Moses
+{
+
+/** We don't want Factor to be copyable by anybody.  But we also want to store
+ * it in an STL container.  The solution is that Factor's copy constructor is
+ * private and friended to FactorFriend.  The STL containers can delegate
+ * copying, so friending the container isn't sufficient.  STL containers see
+ * FactorFriend's public copy constructor and everybody else sees Factor's
+ * private copy constructor.
+ */
+struct FactorFriend {
+  Factor in;
+};
+
+/** collection of factors
+ *
+ * All Factors in moses are accessed and created by a FactorCollection.
+ * By enforcing this strict creation processes (ie, forbidding factors
+ * from being created on the stack, etc), their memory addresses can
+ * be used as keys to uniquely identify them.
+ * Only 1 FactorCollection object should be created.
+ */
+class FactorCollection
+{
+  friend std::ostream& operator<<(std::ostream&, const FactorCollection&);
+  friend class ::System;
+
+  struct HashFactor : public std::unary_function<const FactorFriend &, std::size_t> {
+    std::size_t operator()(const FactorFriend &factor) const {
+      return util::MurmurHashNative(factor.in.m_string.data(), factor.in.m_string.size());
+    }
+  };
+  struct EqualsFactor : public std::binary_function<const FactorFriend &, const FactorFriend &, bool> {
+    bool operator()(const FactorFriend &left, const FactorFriend &right) const {
+      return left.in.GetString() == right.in.GetString();
+    }
+  };
+  typedef boost::unordered_set<FactorFriend, HashFactor, EqualsFactor> Set;
+  Set m_set;
+  Set m_setNonTerminal;
+
+  util::Pool m_string_backing;
+
+  static FactorCollection s_instance;
+#ifdef WITH_THREADS
+  //reader-writer lock
+  mutable boost::shared_mutex m_accessLock;
+#endif
+
+  size_t m_factorIdNonTerminal; /**< unique, contiguous ids, starting from 0, for each non-terminal factor */
+  size_t m_factorId; /**< unique, contiguous ids, starting from moses_MaxNumNonterminals, for each terminal factor */
+
+  //! constructor. only the 1 static variable can be created
+  FactorCollection()
+    : m_factorIdNonTerminal(0)
+    , m_factorId(moses_MaxNumNonterminals) {
+  }
+
+public:
+  static FactorCollection& Instance() {
+    return s_instance;
+  }
+
+  ~FactorCollection();
+
+  /** returns a factor with the same direction, factorType and factorString.
+  *	If a factor already exist in the collection, return the existing factor, if not create a new 1
+  */
+  const Factor *AddFactor(const StringPiece &factorString, bool isNonTerminal = false);
+
+  size_t GetNumNonTerminals() {
+    return m_factorIdNonTerminal;
+  }
+
+  const Factor *GetFactor(const StringPiece &factorString, bool isNonTerminal = false);
+
+  // TODO: remove calls to this function, replacing them with the simpler AddFactor(factorString)
+  const Factor *AddFactor(FactorDirection /*direction*/, FactorType /*factorType*/, const StringPiece &factorString, bool isNonTerminal = false) {
+    return AddFactor(factorString, isNonTerminal);
+  }
+
+  TO_STRING();
+
+};
+
+}
+#endif

mosesdecoder/moses/FactorTypeSet.cpp

@@ -0,0 +1,73 @@
+// $Id$
+
+/***********************************************************************
+Moses - factored phrase-based language decoder
+Copyright (C) 2006 University of Edinburgh
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Lesser General Public
+License as published by the Free Software Foundation; either
+version 2.1 of the License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public
+License along with this library; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+***********************************************************************/
+
+#include "FactorTypeSet.h"
+
+using namespace std;
+
+namespace Moses
+{
+FactorMask::FactorMask(const vector<FactorType> &factors)
+{
+  vector<FactorType>::const_iterator iter;
+  for (iter = factors.begin() ; iter != factors.end() ; ++iter) {
+    this->set(*iter);
+  }
+}
+
+bool FactorMask::IsUseable(const FactorMask &other) const
+{
+  for (size_t i = 0; i < other.size(); ++i) {
+    if (other[i]) {
+      if (!this->operator[](i) ) {
+        return false;
+      }
+    }
+  }
+
+  return true;
+}
+
+TO_STRING_BODY(FactorMask);
+
+// friend
+std::ostream& operator<<(std::ostream& out, const FactorMask& fm)
+{
+  out << "FactorMask<";
+  bool first = true;
+  for (size_t currFactor = 0 ; currFactor < MAX_NUM_FACTORS ; currFactor++) {
+    if (fm[currFactor]) {
+      if (first) {
+        first = false;
+      } else {
+        out << ",";
+      }
+      out << currFactor;
+    }
+  }
+  out << ">";
+
+  return out;
+}
+
+}
+
+

mosesdecoder/moses/FactorTypeSet.h

@@ -0,0 +1,54 @@
+// $Id$
+
+/***********************************************************************
+Moses - factored phrase-based language decoder
+Copyright (C) 2006 University of Edinburgh
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Lesser General Public
+License as published by the Free Software Foundation; either
+version 2.1 of the License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public
+License along with this library; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+***********************************************************************/
+
+#ifndef moses_FactorTypeSet_h
+#define moses_FactorTypeSet_h
+
+#include <iostream>
+#include <bitset>
+#include <vector>
+#include "TypeDef.h"
+#include "Util.h"
+
+namespace Moses
+{
+
+/** set of unique FactorTypes. Used to store what factor types are used in phrase tables etc
+ */
+class FactorMask : public std::bitset<MAX_NUM_FACTORS>
+{
+  friend std::ostream& operator<<(std::ostream&, const FactorMask&);
+
+public:
+  //! construct object from list of FactorType.
+  explicit FactorMask(const std::vector<FactorType> &factors);
+  //! default constructor
+  inline FactorMask() {}
+  //! copy constructor
+  FactorMask(const std::bitset<MAX_NUM_FACTORS>& rhs) : std::bitset<MAX_NUM_FACTORS>(rhs) { }
+
+  bool IsUseable(const FactorMask &other) const;
+
+  TO_STRING();
+};
+
+}
+#endif

mosesdecoder/moses/FilePtr.h

@@ -0,0 +1,89 @@
+// $Id$
+
+/* ---------------------------------------------------------------- */
+/* Copyright 2005 (c) by RWTH Aachen - Lehrstuhl fuer Informatik VI */
+/* Richard Zens                                                     */
+/* ---------------------------------------------------------------- */
+
+#ifndef moses_FilePtr_h
+#define moses_FilePtr_h
+
+#include "File.h"
+
+namespace Moses
+{
+
+/** smart pointer for on-demand loading from file
+ *  requirement: T has a constructor T(FILE*)
+ */
+template<typename T> class FilePtr
+{
+public:
+  typedef T* Ptr;
+private:
+  FILE* f;
+  OFF_T pos;
+  mutable Ptr t;
+public:
+  FilePtr(FILE* f_=0,OFF_T p=0) : f(f_),pos(p),t(0) {}
+  ~FilePtr() {}
+
+  void set(FILE* f_,OFF_T p) {
+    f=f_;
+    pos=p;
+  }
+  void free() {
+    delete t;
+    t=0;
+  }
+
+  T& operator* () {
+    load();
+    return *t;
+  }
+  Ptr operator->() {
+    load();
+    return t;
+  }
+  operator Ptr () {
+    load();
+    return t;
+  }
+
+  const T& operator* () const {
+    load();
+    return *t;
+  }
+  Ptr operator->() const {
+    load();
+    return t;
+  }
+  operator Ptr  () const {
+    load();
+    return t;
+  }
+
+  // direct access to pointer, use with care!
+  Ptr getPtr() {
+    return t;
+  }
+  Ptr getPtr() const {
+    return t;
+  }
+
+  operator bool() const {
+    return (f && pos!=InvalidOffT);
+  }
+
+  void load() const {
+    if(t) return;
+    if(f && pos!=InvalidOffT) {
+      fSeek(f,pos);
+      t=new T(f);
+    }
+  }
+};
+
+}
+
+#endif

mosesdecoder/moses/HypergraphOutput.h

@@ -0,0 +1,107 @@
+// $Id$
+// vim:tabstop=2
+
+/***********************************************************************
+Moses - factored phrase-based language decoder
+Copyright (C) 2014- University of Edinburgh
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Lesser General Public
+License as published by the Free Software Foundation; either
+version 2.1 of the License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public
+License along with this library; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+***********************************************************************/
+
+#ifndef moses_Hypergraph_Output_h
+#define moses_Hypergraph_Output_h
+
+#include <ostream>
+#include "moses/parameters/AllOptions.h"
+
+/**
+* Manage the output of hypergraphs.
+**/
+
+namespace Moses
+{
+
+class ChartHypothesisCollection;
+
+template<class M>
+class HypergraphOutput
+{
+
+public:
+  /** Initialise output directory and create weights file */
+  HypergraphOutput(size_t precision);
+
+  /** Write this hypergraph to file */
+  void Write(const M& manager) const;
+
+private:
+  size_t m_precision;
+  std::string m_hypergraphDir;
+  std::string m_compression;
+  bool m_appendSuffix;
+};
+
+
+/**
+ * ABC for different types of search graph output for chart Moses.
+**/
+class ChartSearchGraphWriter
+{
+protected:
+  AllOptions::ptr m_options;
+  ChartSearchGraphWriter(AllOptions::ptr const& opts) : m_options(opts) { }
+public:
+  virtual void WriteHeader(size_t winners, size_t losers) const = 0;
+  virtual void WriteHypos(const ChartHypothesisCollection& hypos,
+                          const std::map<unsigned, bool> &reachable) const = 0;
+
+};
+
+/** "Moses" format (osg style) */
+class ChartSearchGraphWriterMoses : public virtual ChartSearchGraphWriter
+{
+public:
+  ChartSearchGraphWriterMoses(AllOptions::ptr const& opts,
+                              std::ostream* out, size_t lineNumber)
+    : ChartSearchGraphWriter(opts), m_out(out), m_lineNumber(lineNumber) {}
+  virtual void WriteHeader(size_t, size_t) const {
+    /* do nothing */
+  }
+  virtual void WriteHypos(const ChartHypothesisCollection& hypos,
+                          const std::map<unsigned, bool> &reachable) const;
+
+private:
+  std::ostream* m_out;
+  size_t m_lineNumber;
+};
+
+/** Modified version of Kenneth's lazy hypergraph format */
+class ChartSearchGraphWriterHypergraph : public virtual ChartSearchGraphWriter
+{
+public:
+  ChartSearchGraphWriterHypergraph(AllOptions::ptr const& opts, std::ostream* out)
+    : ChartSearchGraphWriter(opts), m_out(out), m_nodeId(0) { }
+  virtual void WriteHeader(size_t winners, size_t losers) const;
+  virtual void WriteHypos(const ChartHypothesisCollection& hypos,
+                          const std::map<unsigned, bool> &reachable) const;
+
+private:
+  std::ostream* m_out;
+  mutable size_t m_nodeId;
+  mutable std::map<size_t,size_t> m_hypoIdToNodeId;
+};
+
+}
+#endif

mosesdecoder/moses/HypothesisStack.cpp

@@ -0,0 +1,30 @@
+
+#include "HypothesisStack.h"
+
+namespace Moses
+{
+HypothesisStack::~HypothesisStack()
+{
+  // delete all hypos
+  while (m_hypos.begin() != m_hypos.end()) {
+    Remove(m_hypos.begin());
+  }
+}
+
+/** Remove hypothesis pointed to by iterator but don't delete the object. */
+void HypothesisStack::Detach(const HypothesisStack::iterator &iter)
+{
+  m_hypos.erase(iter);
+}
+
+
+void HypothesisStack::Remove(const HypothesisStack::iterator &iter)
+{
+  Hypothesis *h = *iter;
+  Detach(iter);
+  delete h;
+}
+
+
+}
+

mosesdecoder/moses/HypothesisStackNormal.cpp

@@ -0,0 +1,294 @@
+// $Id$
+
+/***********************************************************************
+Moses - factored phrase-based language decoder
+Copyright (C) 2006 University of Edinburgh
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Lesser General Public
+License as published by the Free Software Foundation; either
+version 2.1 of the License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public
+License along with this library; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+***********************************************************************/
+
+#include <algorithm>
+#include <set>
+#include <queue>
+#include "HypothesisStackNormal.h"
+#include "TypeDef.h"
+#include "Util.h"
+#include "Manager.h"
+#include "util/exception.hh"
+
+using namespace std;
+
+namespace Moses
+{
+HypothesisStackNormal::HypothesisStackNormal(Manager& manager) :
+  HypothesisStack(manager)
+{
+  m_nBestIsEnabled = manager.options()->nbest.enabled;
+  m_bestScore = -std::numeric_limits<float>::infinity();
+  m_worstScore = -std::numeric_limits<float>::infinity();
+}
+
+/** remove all hypotheses from the collection */
+void HypothesisStackNormal::RemoveAll()
+{
+  while (m_hypos.begin() != m_hypos.end()) {
+    Remove(m_hypos.begin());
+  }
+}
+
+pair<HypothesisStackNormal::iterator, bool> HypothesisStackNormal::Add(Hypothesis *hypo)
+{
+  std::pair<iterator, bool> ret = m_hypos.insert(hypo);
+  if (ret.second) {
+    // equiv hypo doesn't exists
+    VERBOSE(3,"added hyp to stack");
+
+    // Update best score, if this hypothesis is new best
+    if (hypo->GetFutureScore() > m_bestScore) {
+      VERBOSE(3,", best on stack");
+      m_bestScore = hypo->GetFutureScore();
+      // this may also affect the worst score
+      if ( m_bestScore + m_beamWidth > m_worstScore )
+        m_worstScore = m_bestScore + m_beamWidth;
+    }
+    // update best/worst score for stack diversity 1
+    if ( m_minHypoStackDiversity == 1 &&
+         hypo->GetFutureScore() > GetWorstScoreForBitmap( hypo->GetWordsBitmap() ) ) {
+      SetWorstScoreForBitmap( hypo->GetWordsBitmap().GetID(), hypo->GetFutureScore() );
+    }
+
+    VERBOSE(3,", now size " << m_hypos.size());
+
+    // prune only if stack is twice as big as needed (lazy pruning)
+    size_t toleratedSize = 2*m_maxHypoStackSize-1;
+    // add in room for stack diversity
+    if (m_minHypoStackDiversity) {
+      // so what happens if maxdistortion is negative?
+      toleratedSize += m_minHypoStackDiversity
+                       << m_manager.options()->reordering.max_distortion;
+    }
+
+    if (m_hypos.size() > toleratedSize) {
+      PruneToSize(m_maxHypoStackSize);
+    } else {
+      VERBOSE(3,std::endl);
+    }
+  }
+
+  return ret;
+}
+
+bool HypothesisStackNormal::AddPrune(Hypothesis *hypo)
+{
+  if (hypo->GetFutureScore() == - std::numeric_limits<float>::infinity()) {
+    m_manager.GetSentenceStats().AddDiscarded();
+    VERBOSE(3,"discarded, constraint" << std::endl);
+    delete hypo;
+    return false;
+  }
+
+  // too bad for stack. don't bother adding hypo into collection
+  if (m_manager.options()->search.disable_discarding == false
+      && hypo->GetFutureScore() < m_worstScore
+      && ! ( m_minHypoStackDiversity > 0
+             && hypo->GetFutureScore() >= GetWorstScoreForBitmap( hypo->GetWordsBitmap() ) ) ) {
+    m_manager.GetSentenceStats().AddDiscarded();
+    VERBOSE(3,"discarded, too bad for stack" << std::endl);
+    delete hypo;
+    return false;
+  }
+
+  // over threshold, try to add to collection
+  std::pair<iterator, bool> addRet = Add(hypo);
+  if (addRet.second) {
+    // nothing found. add to collection
+    return true;
+  }
+
+  // equiv hypo exists, recombine with other hypo
+  iterator &iterExisting = addRet.first;
+  Hypothesis *hypoExisting = *iterExisting;
+  assert(iterExisting != m_hypos.end());
+
+  m_manager.GetSentenceStats().AddRecombination(*hypo, **iterExisting);
+
+  // found existing hypo with same target ending.
+  // keep the best 1
+  if (hypo->GetFutureScore() > hypoExisting->GetFutureScore()) {
+    // incoming hypo is better than the one we have
+    VERBOSE(3,"better than matching hyp " << hypoExisting->GetId() << ", recombining, ");
+    if (m_nBestIsEnabled) {
+      hypo->AddArc(hypoExisting);
+      Detach(iterExisting);
+    } else {
+      Remove(iterExisting);
+    }
+
+    bool added = Add(hypo).second;
+    if (!added) {
+      iterExisting = m_hypos.find(hypo);
+      UTIL_THROW2("Offending hypo = " << **iterExisting);
+    }
+    return false;
+  } else {
+    // already storing the best hypo. discard current hypo
+    VERBOSE(3,"worse than matching hyp " << hypoExisting->GetId() << ", recombining" << std::endl)
+    if (m_nBestIsEnabled) {
+      hypoExisting->AddArc(hypo);
+    } else {
+      delete hypo;
+    }
+    return false;
+  }
+}
+
+void HypothesisStackNormal::PruneToSize(size_t newSize)
+{
+  if ( newSize == 0) return; // no limit
+  if ( size() <= newSize ) return; // ok, if not over the limit
+
+  // we need to store a temporary list of hypotheses
+  vector< Hypothesis* > hypos = GetSortedListNOTCONST();
+  bool* included = (bool*) malloc(sizeof(bool) * hypos.size());
+  for(size_t i=0; i<hypos.size(); i++) included[i] = false;
+
+  // clear out original set
+  for( iterator iter = m_hypos.begin(); iter != m_hypos.end(); ) {
+    iterator removeHyp = iter++;
+    Detach(removeHyp);
+  }
+
+  // add best hyps for each coverage according to minStackDiversity
+  if ( m_minHypoStackDiversity > 0 ) {
+    map< WordsBitmapID, size_t > diversityCount;
+    for(size_t i=0; i<hypos.size(); i++) {
+      Hypothesis *hyp = hypos[i];
+      WordsBitmapID coverage = hyp->GetWordsBitmap().GetID();;
+      if (diversityCount.find( coverage ) == diversityCount.end())
+        diversityCount[ coverage ] = 0;
+
+      if (diversityCount[ coverage ] < m_minHypoStackDiversity) {
+        m_hypos.insert( hyp );
+        included[i] = true;
+        diversityCount[ coverage ]++;
+        if (diversityCount[ coverage ] == m_minHypoStackDiversity)
+          SetWorstScoreForBitmap( coverage, hyp->GetFutureScore());
+      }
+    }
+  }
+
+  // only add more if stack not full after satisfying minStackDiversity
+  if ( size() < newSize ) {
+
+    // add best remaining hypotheses
+    for(size_t i=0; i<hypos.size()
+        && size() < newSize
+        && hypos[i]->GetFutureScore() > m_bestScore+m_beamWidth; i++) {
+      if (! included[i]) {
+        m_hypos.insert( hypos[i] );
+        included[i] = true;
+        if (size() == newSize)
+          m_worstScore = hypos[i]->GetFutureScore();
+      }
+    }
+  }
+
+  // delete hypotheses that have not been included
+  for(size_t i=0; i<hypos.size(); i++) {
+    if (! included[i]) {
+      delete hypos[i];
+      m_manager.GetSentenceStats().AddPruning();
+    }
+  }
+  free(included);
+
+  // some reporting....
+  VERBOSE(3,", pruned to size " << size() << endl);
+  IFVERBOSE(3) {
+    TRACE_ERR("stack now contains: ");
+    for(iterator iter = m_hypos.begin(); iter != m_hypos.end(); iter++) {
+      Hypothesis *hypo = *iter;
+      TRACE_ERR( hypo->GetId() << " (" << hypo->GetFutureScore() << ") ");
+    }
+    TRACE_ERR( endl);
+  }
+}
+
+const Hypothesis *HypothesisStackNormal::GetBestHypothesis() const
+{
+  if (!m_hypos.empty()) {
+    const_iterator iter = m_hypos.begin();
+    Hypothesis *bestHypo = *iter;
+    while (++iter != m_hypos.end()) {
+      Hypothesis *hypo = *iter;
+      if (hypo->GetFutureScore() > bestHypo->GetFutureScore())
+        bestHypo = hypo;
+    }
+    return bestHypo;
+  }
+  return NULL;
+}
+
+vector<const Hypothesis*> HypothesisStackNormal::GetSortedList() const
+{
+  vector<const Hypothesis*> ret;
+  ret.reserve(m_hypos.size());
+  std::copy(m_hypos.begin(), m_hypos.end(), std::inserter(ret, ret.end()));
+  sort(ret.begin(), ret.end(), CompareHypothesisTotalScore());
+
+  return ret;
+}
+
+vector<Hypothesis*> HypothesisStackNormal::GetSortedListNOTCONST()
+{
+  vector<Hypothesis*> ret;
+  ret.reserve(m_hypos.size());
+  std::copy(m_hypos.begin(), m_hypos.end(), std::inserter(ret, ret.end()));
+  sort(ret.begin(), ret.end(), CompareHypothesisTotalScore());
+
+  return ret;
+}
+
+void HypothesisStackNormal::CleanupArcList()
+{
+  // only necessary if n-best calculations are enabled
+  if (!m_nBestIsEnabled) return;
+
+  iterator iter;
+  for (iter = m_hypos.begin() ; iter != m_hypos.end() ; ++iter) {
+    Hypothesis *mainHypo = *iter;
+    mainHypo->CleanupArcList(this->m_manager.options()->nbest.nbest_size, this->m_manager.options()->NBestDistinct());
+  }
+}
+
+TO_STRING_BODY(HypothesisStackNormal);
+
+
+// friend
+std::ostream& operator<<(std::ostream& out, const HypothesisStackNormal& hypoColl)
+{
+  HypothesisStackNormal::const_iterator iter;
+
+  for (iter = hypoColl.begin() ; iter != hypoColl.end() ; ++iter) {
+    const Hypothesis &hypo = **iter;
+    out << hypo << endl;
+
+  }
+  return out;
+}
+
+
+}
+

mosesdecoder/moses/IOWrapper.cpp

@@ -0,0 +1,272 @@
+// $Id$
+
+/***********************************************************************
+Moses - factored phrase-based language decoder
+Copyright (c) 2006 University of Edinburgh
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+ * Redistributions of source code must retain the above copyright notice,
+			this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above copyright notice,
+			this list of conditions and the following disclaimer in the documentation
+			and/or other materials provided with the distribution.
+ * Neither the name of the University of Edinburgh nor the names of its contributors
+			may be used to endorse or promote products derived from this software
+			without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS
+BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.
+ ***********************************************************************/
+
+#include <iostream>
+#include <stack>
+#include <boost/algorithm/string.hpp>
+#include <boost/foreach.hpp>
+
+#include "moses/Syntax/KBestExtractor.h"
+#include "moses/Syntax/PVertex.h"
+#include "moses/Syntax/S2T/DerivationWriter.h"
+
+#include "moses/Hypothesis.h"
+#include "moses/TrellisPathList.h"
+#include "moses/StaticData.h"
+#include "moses/InputFileStream.h"
+#include "moses/FF/StatefulFeatureFunction.h"
+#include "moses/TreeInput.h"
+#include "moses/ForestInput.h"
+#include "moses/ConfusionNet.h"
+#include "moses/WordLattice.h"
+#include "moses/ChartManager.h"
+
+#include "IOWrapper.h"
+
+#include <boost/filesystem.hpp>
+#include <boost/iostreams/device/file.hpp>
+#include <boost/iostreams/filter/bzip2.hpp>
+#include <boost/iostreams/filter/gzip.hpp>
+#include <boost/iostreams/filtering_stream.hpp>
+
+using namespace std;
+
+namespace Moses
+{
+
+IOWrapper::IOWrapper(AllOptions const& opts)
+  : m_options(new AllOptions(opts))
+  , m_nBestStream(NULL)
+  , m_surpressSingleBestOutput(false)
+  , m_look_ahead(0)
+  , m_look_back(0)
+  , m_buffered_ahead(0)
+  , spe_src(NULL)
+  , spe_trg(NULL)
+  , spe_aln(NULL)
+{
+  const StaticData &staticData = StaticData::Instance();
+  Parameter const& P = staticData.GetParameter();
+
+  // context buffering for context-sensitive decoding
+  m_look_ahead = m_options->context.look_ahead;
+  m_look_back  = m_options->context.look_back;
+  m_inputType  = m_options->input.input_type;
+
+  UTIL_THROW_IF2((m_look_ahead || m_look_back) && m_inputType != SentenceInput,
+                 "Context-sensitive decoding currently works only with sentence input.");
+
+  m_currentLine = m_options->output.start_translation_id;
+  m_inputFactorOrder = &m_options->input.factor_order;
+
+  size_t nBestSize = m_options->nbest.nbest_size;
+  string nBestFilePath = m_options->nbest.output_file_path;
+
+  staticData.GetParameter().SetParameter<string>(m_inputFilePath, "input-file", "");
+  if (m_inputFilePath.empty()) {
+    m_inputFile = NULL;
+    m_inputStream = &cin;
+  } else {
+    VERBOSE(2,"IO from File" << endl);
+    m_inputFile = new InputFileStream(m_inputFilePath);
+    m_inputStream = m_inputFile;
+  }
+
+  if (nBestSize > 0) {
+    m_nBestOutputCollector.reset(new Moses::OutputCollector(nBestFilePath));
+    if (m_nBestOutputCollector->OutputIsCout()) {
+      m_surpressSingleBestOutput = true;
+    }
+  }
+
+  std::string path;
+  P.SetParameter<std::string>(path, "output-search-graph-extended", "");
+  if (!path.size()) P.SetParameter<std::string>(path, "output-search-graph", "");
+  if (path.size()) m_searchGraphOutputCollector.reset(new OutputCollector(path));
+
+  P.SetParameter<std::string>(path, "output-unknowns", "");
+  if (path.size()) m_unknownsCollector.reset(new OutputCollector(path));
+
+  P.SetParameter<std::string>(path, "alignment-output-file", "");
+  if (path.size()) m_alignmentInfoCollector.reset(new OutputCollector(path));
+
+  P.SetParameter<string>(path, "translation-details", "");
+  if (path.size()) m_detailedTranslationCollector.reset(new OutputCollector(path));
+
+  P.SetParameter<string>(path, "tree-translation-details", "");
+  if (path.size()) m_detailTreeFragmentsOutputCollector.reset(new OutputCollector(path));
+
+  P.SetParameter<string>(path, "output-word-graph", "");
+  if (path.size()) m_wordGraphCollector.reset(new OutputCollector(path));
+
+  size_t latticeSamplesSize = staticData.options()->output.lattice_sample_size;
+  string latticeSamplesFile = staticData.options()->output.lattice_sample_filepath;
+  if (latticeSamplesSize) {
+    m_latticeSamplesCollector.reset(new OutputCollector(latticeSamplesFile));
+    if (m_latticeSamplesCollector->OutputIsCout()) {
+      m_surpressSingleBestOutput = true;
+    }
+  }
+
+  if (!m_surpressSingleBestOutput) {
+    m_singleBestOutputCollector.reset(new Moses::OutputCollector(&std::cout));
+  }
+
+  // setup file pattern for hypergraph output
+  char const* key = "output-search-graph-hypergraph";
+  PARAM_VEC const* p = staticData.GetParameter().GetParam(key);
+  std::string& fmt = m_hypergraph_output_filepattern;
+  // first, determine the output directory
+  if (p && p->size() > 2) fmt = p->at(2);
+  else if (nBestFilePath.size() && nBestFilePath != "-" &&
+           ! boost::starts_with(nBestFilePath, "/dev/stdout")) {
+    fmt = boost::filesystem::path(nBestFilePath).parent_path().string();
+    if (fmt.empty()) fmt = ".";
+  } else fmt = boost::filesystem::current_path().string() + "/hypergraph";
+  if (*fmt.rbegin() != '/') fmt += "/";
+  std::string extension = (p && p->size() > 1 ? p->at(1) : std::string("txt"));
+  UTIL_THROW_IF2(extension != "txt" && extension != "gz" && extension != "bz2",
+                 "Unknown compression type '" << extension
+                 << "' for hypergraph output!");
+  fmt += string("%d.") + extension;
+
+  // input streams for simulated post-editing
+  if (staticData.GetParameter().GetParam("spe-src")) {
+    spe_src = new ifstream(staticData.GetParameter().GetParam("spe-src")->at(0).c_str());
+    spe_trg = new ifstream(staticData.GetParameter().GetParam("spe-trg")->at(0).c_str());
+    spe_aln = new ifstream(staticData.GetParameter().GetParam("spe-aln")->at(0).c_str());
+  }
+}
+
+IOWrapper::~IOWrapper()
+{
+  if (m_inputFile != NULL)
+    delete m_inputFile;
+  // if (m_nBestStream != NULL && !m_surpressSingleBestOutput) {
+  // outputting n-best to file, rather than stdout. need to close file and delete obj
+  // delete m_nBestStream;
+  // }
+
+  // delete m_detailedTranslationReportingStream;
+  // delete m_alignmentInfoStream;
+  // delete m_unknownsStream;
+  // delete m_outputSearchGraphStream;
+  // delete m_outputWordGraphStream;
+  // delete m_latticeSamplesStream;
+}
+
+// InputType*
+// IOWrapper::
+// GetInput(InputType* inputType)
+// {
+//   if(inputType->Read(*m_inputStream, *m_inputFactorOrder)) {
+//     return inputType;
+//   } else {
+//     delete inputType;
+//     return NULL;
+//   }
+// }
+
+boost::shared_ptr<InputType>
+IOWrapper::
+GetBufferedInput()
+{
+  switch(m_inputType) {
+  case SentenceInput:
+    return BufferInput<Sentence>();
+  case ConfusionNetworkInput:
+    return BufferInput<ConfusionNet>();
+  case WordLatticeInput:
+    return BufferInput<WordLattice>();
+  case TreeInputType:
+    return BufferInput<TreeInput>();
+  case TabbedSentenceInput:
+    return BufferInput<TabbedSentence>();
+  case ForestInputType:
+    return BufferInput<ForestInput>();
+  default:
+    TRACE_ERR("Unknown input type: " << m_inputType << "\n");
+    return boost::shared_ptr<InputType>();
+  }
+
+}
+
+boost::shared_ptr<InputType>
+IOWrapper::
+ReadInput(boost::shared_ptr<std::vector<std::string> >* cw)
+{
+#ifdef WITH_THREADS
+  boost::lock_guard<boost::mutex> lock(m_lock);
+#endif
+  boost::shared_ptr<InputType> source = GetBufferedInput();
+  if (source) {
+    source->SetTranslationId(m_currentLine++);
+
+    // when using a sliding context window, remove obsolete past input from buffer:
+    if (m_past_input.size() && m_look_back != std::numeric_limits<size_t>::max()) {
+      list<boost::shared_ptr<InputType> >::iterator m = m_past_input.end();
+      for (size_t cnt = 0; cnt < m_look_back && --m != m_past_input.begin();)
+        cnt += (*m)->GetSize();
+      while (m_past_input.begin() != m) m_past_input.pop_front();
+    }
+
+    if (m_look_back)
+      m_past_input.push_back(source);
+  }
+  if (cw) *cw = GetCurrentContextWindow();
+  return source;
+}
+
+boost::shared_ptr<std::vector<std::string> >
+IOWrapper::
+GetCurrentContextWindow() const
+{
+  boost::shared_ptr<std::vector<string> > context(new std::vector<string>);
+  BOOST_FOREACH(boost::shared_ptr<InputType> const& i, m_past_input)
+  context->push_back(i->ToString());
+  BOOST_FOREACH(boost::shared_ptr<InputType> const& i, m_future_input)
+  context->push_back(i->ToString());
+  return context;
+}
+
+
+
+std::string
+IOWrapper::
+GetHypergraphOutputFileName(size_t const id) const
+{
+  return str(boost::format(m_hypergraph_output_filepattern) % id);
+}
+
+
+} // namespace
+

mosesdecoder/moses/InputFileStream.h

@@ -0,0 +1,48 @@
+// $Id$
+
+/***********************************************************************
+Moses - factored phrase-based language decoder
+Copyright (C) 2006 University of Edinburgh
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Lesser General Public
+License as published by the Free Software Foundation; either
+version 2.1 of the License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public
+License along with this library; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+***********************************************************************/
+
+#ifndef moses_InputFileStream_h
+#define moses_InputFileStream_h
+
+#include <cstdlib>
+#include <fstream>
+#include <string>
+
+namespace Moses
+{
+
+/** Used in place of std::istream, can read zipped files if it ends in .gz
+*/
+class InputFileStream : public std::istream
+{
+protected:
+  std::streambuf *m_streambuf;
+public:
+
+  explicit InputFileStream(const std::string &filePath);
+  ~InputFileStream();
+
+  void Close();
+};
+
+}
+
+#endif

mosesdecoder/moses/LatticeMBR.cpp

@@ -0,0 +1,680 @@
+/*
+ *  LatticeMBR.cpp
+ *  moses-cmd
+ *
+ *  Created by Abhishek Arun on 26/01/2010.
+ *  Copyright 2010 __MyCompanyName__. All rights reserved.
+ *
+ */
+
+#include "LatticeMBR.h"
+#include "moses/StaticData.h"
+#include <algorithm>
+#include <set>
+
+using namespace std;
+
+namespace Moses
+{
+
+size_t bleu_order = 4;
+float UNKNGRAMLOGPROB = -20;
+void GetOutputWords(const TrellisPath &path, vector <Word> &translation)
+{
+  const std::vector<const Hypothesis *> &edges = path.GetEdges();
+
+  // print the surface factor of the translation
+  for (int currEdge = (int)edges.size() - 1 ; currEdge >= 0 ; currEdge--) {
+    const Hypothesis &edge = *edges[currEdge];
+    const Phrase &phrase = edge.GetCurrTargetPhrase();
+    size_t size = phrase.GetSize();
+    for (size_t pos = 0 ; pos < size ; pos++) {
+      translation.push_back(phrase.GetWord(pos));
+    }
+  }
+}
+
+
+void extract_ngrams(const vector<Word >& sentence, map < Phrase, int >  & allngrams)
+{
+  for (int k = 0; k < (int)bleu_order; k++) {
+    for(int i =0; i < max((int)sentence.size()-k,0); i++) {
+      Phrase ngram( k+1);
+      for ( int j = i; j<= i+k; j++) {
+        ngram.AddWord(sentence[j]);
+      }
+      ++allngrams[ngram];
+    }
+  }
+}
+
+
+
+void NgramScores::addScore(const Hypothesis* node, const Phrase& ngram, float score)
+{
+  set<Phrase>::const_iterator ngramIter = m_ngrams.find(ngram);
+  if (ngramIter == m_ngrams.end()) {
+    ngramIter = m_ngrams.insert(ngram).first;
+  }
+  map<const Phrase*,float>& ngramScores = m_scores[node];
+  map<const Phrase*,float>::iterator scoreIter = ngramScores.find(&(*ngramIter));
+  if (scoreIter == ngramScores.end()) {
+    ngramScores[&(*ngramIter)] = score;
+  } else {
+    ngramScores[&(*ngramIter)] = log_sum(score,scoreIter->second);
+  }
+}
+
+NgramScores::NodeScoreIterator NgramScores::nodeBegin(const Hypothesis* node)
+{
+  return m_scores[node].begin();
+}
+
+
+NgramScores::NodeScoreIterator NgramScores::nodeEnd(const Hypothesis* node)
+{
+  return m_scores[node].end();
+}
+
+LatticeMBRSolution::LatticeMBRSolution(const TrellisPath& path, bool isMap) :
+  m_score(0.0f)
+{
+  const std::vector<const Hypothesis *> &edges = path.GetEdges();
+
+  for (int currEdge = (int)edges.size() - 1 ; currEdge >= 0 ; currEdge--) {
+    const Hypothesis &edge = *edges[currEdge];
+    const Phrase &phrase = edge.GetCurrTargetPhrase();
+    size_t size = phrase.GetSize();
+    for (size_t pos = 0 ; pos < size ; pos++) {
+      m_words.push_back(phrase.GetWord(pos));
+    }
+  }
+  if (isMap) {
+    m_mapScore = path.GetFutureScore();
+  } else {
+    m_mapScore = 0;
+  }
+}
+
+
+void LatticeMBRSolution::CalcScore(map<Phrase, float>& finalNgramScores, const vector<float>& thetas, float mapWeight)
+{
+  m_ngramScores.assign(thetas.size()-1, -10000);
+
+  map < Phrase, int > counts;
+  extract_ngrams(m_words,counts);
+
+  //Now score this translation
+  m_score = thetas[0] * m_words.size();
+
+  //Calculate the ngramScores, working in log space at first
+  for (map < Phrase, int >::iterator ngrams = counts.begin(); ngrams != counts.end(); ++ngrams) {
+    float ngramPosterior = UNKNGRAMLOGPROB;
+    map<Phrase,float>::const_iterator ngramPosteriorIt = finalNgramScores.find(ngrams->first);
+    if (ngramPosteriorIt != finalNgramScores.end()) {
+      ngramPosterior = ngramPosteriorIt->second;
+    }
+    size_t ngramSize = ngrams->first.GetSize();
+    m_ngramScores[ngramSize-1] = log_sum(log((float)ngrams->second) + ngramPosterior,m_ngramScores[ngramSize-1]);
+  }
+
+  //convert from log to probability and create weighted sum
+  for (size_t i = 0; i < m_ngramScores.size(); ++i) {
+    m_ngramScores[i] = exp(m_ngramScores[i]);
+    m_score += thetas[i+1] * m_ngramScores[i];
+  }
+
+
+  //The map score
+  m_score += m_mapScore*mapWeight;
+}
+
+
+void pruneLatticeFB(Lattice & connectedHyp, map < const Hypothesis*, set <const Hypothesis* > > & outgoingHyps, map<const Hypothesis*, vector<Edge> >& incomingEdges,
+                    const vector< float> & estimatedScores, const Hypothesis* bestHypo, size_t edgeDensity, float scale)
+{
+
+  //Need hyp 0 in connectedHyp - Find empty hypothesis
+  VERBOSE(2,"Pruning lattice to edge density " << edgeDensity << endl);
+  const Hypothesis* emptyHyp = connectedHyp.at(0);
+  while (emptyHyp->GetId() != 0) {
+    emptyHyp = emptyHyp->GetPrevHypo();
+  }
+  connectedHyp.push_back(emptyHyp); //Add it to list of hyps
+
+  //Need hyp 0's outgoing Hyps
+  for (size_t i = 0; i < connectedHyp.size(); ++i) {
+    if (connectedHyp[i]->GetId() > 0 && connectedHyp[i]->GetPrevHypo()->GetId() == 0)
+      outgoingHyps[emptyHyp].insert(connectedHyp[i]);
+  }
+
+  //sort hyps based on estimated scores - do so by copying to multimap
+  multimap<float, const Hypothesis*> sortHypsByVal;
+  for (size_t i =0; i < estimatedScores.size(); ++i) {
+    sortHypsByVal.insert(make_pair(estimatedScores[i], connectedHyp[i]));
+  }
+
+  multimap<float, const Hypothesis*>::const_iterator it = --sortHypsByVal.end();
+  float bestScore = it->first;
+  //store best score as score of hyp 0
+  sortHypsByVal.insert(make_pair(bestScore, emptyHyp));
+
+
+  IFVERBOSE(3) {
+    for (multimap<float, const Hypothesis*>::const_iterator it = --sortHypsByVal.end(); it != --sortHypsByVal.begin(); --it) {
+      const Hypothesis* currHyp =  it->second;
+      cerr << "Hyp " << currHyp->GetId() << ", estimated score: " << it->first << endl;
+    }
+  }
+
+
+  set <const Hypothesis*> survivingHyps; //store hyps that make the cut in this
+
+  VERBOSE(2, "BEST HYPO TARGET LENGTH : " << bestHypo->GetSize() << endl)
+  size_t numEdgesTotal = edgeDensity * bestHypo->GetSize(); //as per Shankar, aim for (density * target length of MAP solution) arcs
+  size_t numEdgesCreated = 0;
+  VERBOSE(2, "Target edge count: " << numEdgesTotal << endl);
+
+  float prevScore = -999999;
+
+  //now iterate over multimap
+  for (multimap<float, const Hypothesis*>::const_iterator it = --sortHypsByVal.end(); it != --sortHypsByVal.begin(); --it) {
+    float currEstimatedScore = it->first;
+    const Hypothesis* currHyp =  it->second;
+
+    if (numEdgesCreated >= numEdgesTotal && prevScore > currEstimatedScore) //if this hyp has equal estimated score to previous, include its edges too
+      break;
+
+    prevScore = currEstimatedScore;
+    VERBOSE(3, "Num edges created : "<< numEdgesCreated << ", numEdges wanted " << numEdgesTotal << endl)
+    VERBOSE(3, "Considering hyp " << currHyp->GetId() << ", estimated score: " << it->first << endl)
+
+    survivingHyps.insert(currHyp); //CurrHyp made the cut
+
+    // is its best predecessor already included ?
+    if (survivingHyps.find(currHyp->GetPrevHypo()) != survivingHyps.end()) { //yes, then add an edge
+      vector <Edge>& edges = incomingEdges[currHyp];
+      Edge winningEdge(currHyp->GetPrevHypo(),currHyp,scale*(currHyp->GetScore() - currHyp->GetPrevHypo()->GetScore()),currHyp->GetCurrTargetPhrase());
+      edges.push_back(winningEdge);
+      ++numEdgesCreated;
+    }
+
+    //let's try the arcs too
+    const ArcList *arcList = currHyp->GetArcList();
+    if (arcList != NULL) {
+      ArcList::const_iterator iterArcList;
+      for (iterArcList = arcList->begin() ; iterArcList != arcList->end() ; ++iterArcList) {
+        const Hypothesis *loserHypo = *iterArcList;
+        const Hypothesis* loserPrevHypo = loserHypo->GetPrevHypo();
+        if (survivingHyps.find(loserPrevHypo) != survivingHyps.end()) { //found it, add edge
+          double arcScore = loserHypo->GetScore() - loserPrevHypo->GetScore();
+          Edge losingEdge(loserPrevHypo, currHyp, arcScore*scale, loserHypo->GetCurrTargetPhrase());
+          vector <Edge>& edges = incomingEdges[currHyp];
+          edges.push_back(losingEdge);
+          ++numEdgesCreated;
+        }
+      }
+    }
+
+    //Now if a successor node has already been visited, add an edge connecting the two
+    map < const Hypothesis*, set < const Hypothesis* > >::const_iterator outgoingIt = outgoingHyps.find(currHyp);
+
+    if (outgoingIt != outgoingHyps.end()) {//currHyp does have successors
+      const set<const Hypothesis*> & outHyps = outgoingIt->second; //the successors
+      for (set<const Hypothesis*>::const_iterator outHypIts = outHyps.begin(); outHypIts != outHyps.end(); ++outHypIts) {
+        const Hypothesis* succHyp = *outHypIts;
+
+        if (survivingHyps.find(succHyp) == survivingHyps.end()) //Have we encountered the successor yet?
+          continue; //No, move on to next
+
+        //Curr Hyp can be : a) the best predecessor  of succ b) or an arc attached to succ
+        if (succHyp->GetPrevHypo() == currHyp) { //best predecessor
+          vector <Edge>& succEdges = incomingEdges[succHyp];
+          Edge succWinningEdge(currHyp, succHyp, scale*(succHyp->GetScore() - currHyp->GetScore()), succHyp->GetCurrTargetPhrase());
+          succEdges.push_back(succWinningEdge);
+          survivingHyps.insert(succHyp);
+          ++numEdgesCreated;
+        }
+
+        //now, let's find an arc
+        const ArcList *arcList = succHyp->GetArcList();
+        if (arcList != NULL) {
+          ArcList::const_iterator iterArcList;
+          //QUESTION: What happens if there's more than one loserPrevHypo?
+          for (iterArcList = arcList->begin() ; iterArcList != arcList->end() ; ++iterArcList) {
+            const Hypothesis *loserHypo = *iterArcList;
+            const Hypothesis* loserPrevHypo = loserHypo->GetPrevHypo();
+            if (loserPrevHypo == currHyp) { //found it
+              vector <Edge>& succEdges = incomingEdges[succHyp];
+              double arcScore = loserHypo->GetScore() - currHyp->GetScore();
+              Edge losingEdge(currHyp, succHyp,scale* arcScore, loserHypo->GetCurrTargetPhrase());
+              succEdges.push_back(losingEdge);
+              ++numEdgesCreated;
+            }
+          }
+        }
+      }
+    }
+  }
+
+  connectedHyp.clear();
+  for (set <const Hypothesis*>::iterator it =  survivingHyps.begin(); it != survivingHyps.end(); ++it) {
+    connectedHyp.push_back(*it);
+  }
+
+  VERBOSE(2, "Done! Num edges created : "<< numEdgesCreated << ", numEdges wanted " << numEdgesTotal << endl)
+
+  IFVERBOSE(3) {
+    cerr << "Surviving hyps: " ;
+    for (set <const Hypothesis*>::iterator it =  survivingHyps.begin(); it != survivingHyps.end(); ++it) {
+      cerr << (*it)->GetId() << " ";
+    }
+    cerr << endl;
+  }
+
+
+}
+
+void calcNgramExpectations(Lattice & connectedHyp, map<const Hypothesis*, vector<Edge> >& incomingEdges,
+                           map<Phrase, float>& finalNgramScores, bool posteriors)
+{
+
+  sort(connectedHyp.begin(),connectedHyp.end(),ascendingCoverageCmp); //sort by increasing source word cov
+
+  /*cerr << "Lattice:" << endl;
+  for (Lattice::const_iterator i = connectedHyp.begin(); i != connectedHyp.end(); ++i) {
+      const Hypothesis* h = *i;
+      cerr << *h << endl;
+      const vector<Edge>& edges = incomingEdges[h];
+      for (size_t e = 0; e < edges.size(); ++e) {
+          cerr << edges[e];
+      }
+  }*/
+
+  map<const Hypothesis*, float> forwardScore;
+  forwardScore[connectedHyp[0]] = 0.0f; //forward score of hyp 0 is 1 (or 0 in logprob space)
+  set< const Hypothesis *> finalHyps; //store completed hyps
+
+  NgramScores ngramScores;//ngram scores for each hyp
+
+  for (size_t i = 1; i < connectedHyp.size(); ++i) {
+    const Hypothesis* currHyp = connectedHyp[i];
+    if (currHyp->GetWordsBitmap().IsComplete()) {
+      finalHyps.insert(currHyp);
+    }
+
+    VERBOSE(3, "Processing hyp: " << currHyp->GetId() << ", num words cov= " << currHyp->GetWordsBitmap().GetNumWordsCovered() <<  endl)
+
+    vector <Edge> & edges = incomingEdges[currHyp];
+    for (size_t e = 0; e < edges.size(); ++e) {
+      const Edge& edge = edges[e];
+      if (forwardScore.find(currHyp) == forwardScore.end()) {
+        forwardScore[currHyp] = forwardScore[edge.GetTailNode()] + edge.GetScore();
+        VERBOSE(3, "Fwd score["<<currHyp->GetId()<<"] = fwdScore["<<edge.GetTailNode()->GetId() << "] + edge Score: " << edge.GetScore() << endl)
+      } else {
+        forwardScore[currHyp] = log_sum(forwardScore[currHyp], forwardScore[edge.GetTailNode()] + edge.GetScore());
+        VERBOSE(3, "Fwd score["<<currHyp->GetId()<<"] += fwdScore["<<edge.GetTailNode()->GetId() << "] + edge Score: " << edge.GetScore() << endl)
+      }
+    }
+
+    //Process ngrams now
+    for (size_t j =0 ; j < edges.size(); ++j) {
+      Edge& edge = edges[j];
+      const NgramHistory & incomingPhrases = edge.GetNgrams(incomingEdges);
+
+      //let's first score ngrams introduced by this edge
+      for (NgramHistory::const_iterator it = incomingPhrases.begin(); it != incomingPhrases.end(); ++it) {
+        const Phrase& ngram = it->first;
+        const PathCounts& pathCounts = it->second;
+        VERBOSE(4, "Calculating score for: " << it->first << endl)
+
+        for (PathCounts::const_iterator pathCountIt = pathCounts.begin(); pathCountIt != pathCounts.end(); ++pathCountIt) {
+          //Score of an n-gram is forward score of head node of leftmost edge + all edge scores
+          const Path&  path = pathCountIt->first;
+          //cerr << "path count for " << ngram << " is " << pathCountIt->second << endl;
+          float score = forwardScore[path[0]->GetTailNode()];
+          for (size_t i = 0; i < path.size(); ++i) {
+            score += path[i]->GetScore();
+          }
+          //if we're doing expectations, then the number of times the ngram
+          //appears on the path is relevant.
+          size_t count = posteriors ? 1 : pathCountIt->second;
+          for (size_t k = 0; k < count; ++k) {
+            ngramScores.addScore(currHyp,ngram,score);
+          }
+        }
+      }
+
+      //Now score ngrams that are just being propagated from the history
+      for (NgramScores::NodeScoreIterator it = ngramScores.nodeBegin(edge.GetTailNode());
+           it != ngramScores.nodeEnd(edge.GetTailNode()); ++it) {
+        const Phrase & currNgram = *(it->first);
+        float currNgramScore = it->second;
+        VERBOSE(4, "Calculating score for: " << currNgram << endl)
+
+        // For posteriors, don't double count ngrams
+        if (!posteriors || incomingPhrases.find(currNgram) == incomingPhrases.end()) {
+          float score = edge.GetScore() + currNgramScore;
+          ngramScores.addScore(currHyp,currNgram,score);
+        }
+      }
+
+    }
+  }
+
+  float Z = 9999999; //the total score of the lattice
+
+  //Done - Print out ngram posteriors for final hyps
+  for (set< const Hypothesis *>::iterator finalHyp = finalHyps.begin(); finalHyp != finalHyps.end(); ++finalHyp) {
+    const Hypothesis* hyp = *finalHyp;
+
+    for (NgramScores::NodeScoreIterator it = ngramScores.nodeBegin(hyp); it != ngramScores.nodeEnd(hyp); ++it) {
+      const Phrase& ngram = *(it->first);
+      if (finalNgramScores.find(ngram) == finalNgramScores.end()) {
+        finalNgramScores[ngram] = it->second;
+      } else {
+        finalNgramScores[ngram] = log_sum(it->second,  finalNgramScores[ngram]);
+      }
+    }
+
+    if (Z == 9999999) {
+      Z = forwardScore[hyp];
+    } else {
+      Z = log_sum(Z, forwardScore[hyp]);
+    }
+  }
+
+  //Z *= scale;  //scale the score
+
+  for (map<Phrase, float>::iterator finalScoresIt = finalNgramScores.begin();  finalScoresIt != finalNgramScores.end(); ++finalScoresIt) {
+    finalScoresIt->second =  finalScoresIt->second - Z;
+    IFVERBOSE(2) {
+      VERBOSE(2,finalScoresIt->first << " [" << finalScoresIt->second << "]" << endl);
+    }
+  }
+
+}
+
+const NgramHistory& Edge::GetNgrams(map<const Hypothesis*, vector<Edge> > & incomingEdges)
+{
+
+  if (m_ngrams.size() > 0)
+    return m_ngrams;
+
+  const Phrase& currPhrase = GetWords();
+  //Extract the n-grams local to this edge
+  for (size_t start = 0; start < currPhrase.GetSize(); ++start) {
+    for (size_t end = start; end < start + bleu_order; ++end) {
+      if (end < currPhrase.GetSize()) {
+        Phrase edgeNgram(end-start+1);
+        for (size_t index = start; index <= end; ++index) {
+          edgeNgram.AddWord(currPhrase.GetWord(index));
+        }
+        //cout << "Inserting Phrase : " << edgeNgram << endl;
+        vector<const Edge*> edgeHistory;
+        edgeHistory.push_back(this);
+        storeNgramHistory(edgeNgram, edgeHistory);
+      } else {
+        break;
+      }
+    }
+  }
+
+  map<const Hypothesis*, vector<Edge> >::iterator it = incomingEdges.find(m_tailNode);
+  if (it != incomingEdges.end()) { //node has incoming edges
+    vector<Edge> & inEdges = it->second;
+
+    for (vector<Edge>::iterator edge = inEdges.begin(); edge != inEdges.end(); ++edge) {//add the ngrams straddling prev and curr edge
+      const NgramHistory & edgeIncomingNgrams = edge->GetNgrams(incomingEdges);
+      for (NgramHistory::const_iterator edgeInNgramHist = edgeIncomingNgrams.begin(); edgeInNgramHist != edgeIncomingNgrams.end(); ++edgeInNgramHist) {
+        const Phrase& edgeIncomingNgram = edgeInNgramHist->first;
+        const PathCounts &  edgeIncomingNgramPaths = edgeInNgramHist->second;
+        size_t back = min(edgeIncomingNgram.GetSize(), edge->GetWordsSize());
+        const Phrase&  edgeWords = edge->GetWords();
+        IFVERBOSE(3) {
+          cerr << "Edge: "<< *edge <<endl;
+          cerr << "edgeWords: " << edgeWords << endl;
+          cerr << "edgeInNgram: " << edgeIncomingNgram << endl;
+        }
+
+        Phrase edgeSuffix(ARRAY_SIZE_INCR);
+        Phrase ngramSuffix(ARRAY_SIZE_INCR);
+        GetPhraseSuffix(edgeWords,back,edgeSuffix);
+        GetPhraseSuffix(edgeIncomingNgram,back,ngramSuffix);
+
+        if (ngramSuffix == edgeSuffix) { //we've got the suffix of previous edge
+          size_t  edgeInNgramSize =  edgeIncomingNgram.GetSize();
+
+          for (size_t i = 0; i < GetWordsSize() && i + edgeInNgramSize < bleu_order ; ++i) {
+            Phrase newNgram(edgeIncomingNgram);
+            for (size_t j = 0; j <= i ; ++j) {
+              newNgram.AddWord(GetWords().GetWord(j));
+            }
+            VERBOSE(3, "Inserting New Phrase : " << newNgram << endl)
+
+            for (PathCounts::const_iterator pathIt = edgeIncomingNgramPaths.begin(); pathIt !=  edgeIncomingNgramPaths.end(); ++pathIt) {
+              Path newNgramPath = pathIt->first;
+              newNgramPath.push_back(this);
+              storeNgramHistory(newNgram, newNgramPath, pathIt->second);
+            }
+          }
+        }
+      }
+    }
+  }
+  return m_ngrams;
+}
+
+//Add the last lastN words of origPhrase to targetPhrase
+void Edge::GetPhraseSuffix(const Phrase&  origPhrase, size_t lastN, Phrase& targetPhrase) const
+{
+  size_t origSize = origPhrase.GetSize();
+  size_t startIndex = origSize - lastN;
+  for (size_t index = startIndex; index < origPhrase.GetSize(); ++index) {
+    targetPhrase.AddWord(origPhrase.GetWord(index));
+  }
+}
+
+bool Edge::operator< (const Edge& compare ) const
+{
+  if (m_headNode->GetId() < compare.m_headNode->GetId())
+    return true;
+  if (compare.m_headNode->GetId() < m_headNode->GetId())
+    return false;
+  if (m_tailNode->GetId() < compare.m_tailNode->GetId())
+    return true;
+  if (compare.m_tailNode->GetId() < m_tailNode->GetId())
+    return false;
+  return GetScore() <  compare.GetScore();
+}
+
+ostream& operator<< (ostream& out, const Edge& edge)
+{
+  out << "Head: " << edge.m_headNode->GetId()
+      << ", Tail: " << edge.m_tailNode->GetId()
+      << ", Score: " << edge.m_score
+      << ", Phrase: " << edge.m_targetPhrase << endl;
+  return out;
+}
+
+bool ascendingCoverageCmp(const Hypothesis* a, const Hypothesis* b)
+{
+  return (a->GetWordsBitmap().GetNumWordsCovered()
+          <
+          b->GetWordsBitmap().GetNumWordsCovered());
+}
+
+void getLatticeMBRNBest(const Manager& manager, const TrellisPathList& nBestList,
+                        vector<LatticeMBRSolution>& solutions, size_t n)
+{
+  std::map < int, bool > connected;
+  std::vector< const Hypothesis *> connectedList;
+  map<Phrase, float> ngramPosteriors;
+  std::map < const Hypothesis*, set <const Hypothesis*> > outgoingHyps;
+  map<const Hypothesis*, vector<Edge> > incomingEdges;
+  vector< float> estimatedScores;
+  manager.GetForwardBackwardSearchGraph(&connected, &connectedList,
+                                        &outgoingHyps, &estimatedScores);
+  LMBR_Options const& lmbr = manager.options()->lmbr;
+  MBR_Options  const& mbr  = manager.options()->mbr;
+  pruneLatticeFB(connectedList, outgoingHyps, incomingEdges, estimatedScores,
+                 manager.GetBestHypothesis(), lmbr.pruning_factor, mbr.scale);
+  calcNgramExpectations(connectedList, incomingEdges, ngramPosteriors,true);
+
+  vector<float> mbrThetas = lmbr.theta;
+  float p = lmbr.precision;
+  float r = lmbr.ratio;
+  float mapWeight = lmbr.map_weight;
+  if (mbrThetas.size() == 0) {
+    // thetas were not specified on the command line, so use p and r instead
+    mbrThetas.push_back(-1); //Theta 0
+    mbrThetas.push_back(1/(bleu_order*p));
+    for (size_t i = 2; i <= bleu_order; ++i) {
+      mbrThetas.push_back(mbrThetas[i-1] / r);
+    }
+  }
+  IFVERBOSE(2) {
+    VERBOSE(2,"Thetas: ");
+    for (size_t i = 0; i < mbrThetas.size(); ++i) {
+      VERBOSE(2,mbrThetas[i] << " ");
+    }
+    VERBOSE(2,endl);
+  }
+  TrellisPathList::const_iterator iter;
+  size_t ctr = 0;
+  LatticeMBRSolutionComparator comparator;
+  for (iter = nBestList.begin() ; iter != nBestList.end() ; ++iter, ++ctr) {
+    const TrellisPath &path = **iter;
+    solutions.push_back(LatticeMBRSolution(path,iter==nBestList.begin()));
+    solutions.back().CalcScore(ngramPosteriors, mbrThetas, mapWeight);
+    sort(solutions.begin(), solutions.end(), comparator);
+    while (solutions.size() > n) {
+      solutions.pop_back();
+    }
+  }
+  VERBOSE(2,"LMBR Score: " << solutions[0].GetScore() << endl);
+}
+
+vector<Word> doLatticeMBR(const Manager& manager, const TrellisPathList& nBestList)
+{
+
+  vector<LatticeMBRSolution> solutions;
+  getLatticeMBRNBest(manager, nBestList, solutions,1);
+  return solutions.at(0).GetWords();
+}
+
+const TrellisPath doConsensusDecoding(const Manager& manager, const TrellisPathList& nBestList)
+{
+  static const int BLEU_ORDER = 4;
+  static const float SMOOTH = 1;
+
+  //calculate the ngram expectations
+  const StaticData& staticData = StaticData::Instance();
+  std::map < int, bool > connected;
+  std::vector< const Hypothesis *> connectedList;
+  map<Phrase, float> ngramExpectations;
+  std::map < const Hypothesis*, set <const Hypothesis*> > outgoingHyps;
+  map<const Hypothesis*, vector<Edge> > incomingEdges;
+  vector< float> estimatedScores;
+  manager.GetForwardBackwardSearchGraph(&connected, &connectedList, &outgoingHyps, &estimatedScores);
+  LMBR_Options const& lmbr = manager.options()->lmbr;
+  MBR_Options  const&  mbr = manager.options()->mbr;
+  pruneLatticeFB(connectedList, outgoingHyps, incomingEdges, estimatedScores,
+                 manager.GetBestHypothesis(), lmbr.pruning_factor, mbr.scale);
+  calcNgramExpectations(connectedList, incomingEdges, ngramExpectations,false);
+
+  //expected length is sum of expected unigram counts
+  //cerr << "Thread " << pthread_self() <<  " Ngram expectations size: " << ngramExpectations.size() << endl;
+  float ref_length = 0.0f;
+  for (map<Phrase,float>::const_iterator ref_iter = ngramExpectations.begin();
+       ref_iter != ngramExpectations.end(); ++ref_iter) {
+    //cerr << "Ngram: " << ref_iter->first << " score: " <<
+    //    ref_iter->second << endl;
+    if (ref_iter->first.GetSize() == 1) {
+      ref_length += exp(ref_iter->second);
+      //    cerr << "Expected for " << ref_iter->first << " is " << exp(ref_iter->second) << endl;
+    }
+  }
+
+  VERBOSE(2,"REF Length: " << ref_length << endl);
+
+  //use the ngram expectations to rescore the nbest list.
+  TrellisPathList::const_iterator iter;
+  TrellisPathList::const_iterator best = nBestList.end();
+  float bestScore = -100000;
+  //cerr << "nbest list size: " << nBestList.GetSize() << endl;
+  for (iter = nBestList.begin() ; iter != nBestList.end() ; ++iter) {
+    const TrellisPath &path = **iter;
+    vector<Word> words;
+    map<Phrase,int> ngrams;
+    GetOutputWords(path,words);
+    /*for (size_t i = 0; i < words.size(); ++i) {
+        cerr << words[i].GetFactor(0)->GetString() << " ";
+    }
+    cerr << endl;
+    */
+    extract_ngrams(words,ngrams);
+
+    vector<float> comps(2*BLEU_ORDER+1);
+    float logbleu = 0.0;
+    float brevity = 0.0;
+    int hyp_length = words.size();
+    for (int i = 0; i < BLEU_ORDER; ++i) {
+      comps[2*i] = 0.0;
+      comps[2*i+1] = max(hyp_length-i,0);
+    }
+
+    for (map<Phrase,int>::const_iterator hyp_iter = ngrams.begin();
+         hyp_iter != ngrams.end(); ++hyp_iter) {
+      map<Phrase,float>::const_iterator ref_iter = ngramExpectations.find(hyp_iter->first);
+      if (ref_iter != ngramExpectations.end()) {
+        comps[2*(hyp_iter->first.GetSize()-1)] += min(exp(ref_iter->second), (float)(hyp_iter->second));
+      }
+
+    }
+    comps[comps.size()-1] = ref_length;
+    /*for (size_t i = 0; i < comps.size(); ++i) {
+        cerr << comps[i] << " ";
+    }
+    cerr << endl;
+    */
+
+    float score = 0.0f;
+    if (comps[0] != 0) {
+      for (int i=0; i<BLEU_ORDER; i++) {
+        if ( i > 0 ) {
+          logbleu += log((float)comps[2*i]+SMOOTH)-log((float)comps[2*i+1]+SMOOTH);
+        } else {
+          logbleu += log((float)comps[2*i])-log((float)comps[2*i+1]);
+        }
+      }
+      logbleu /= BLEU_ORDER;
+      brevity = 1.0-(float)comps[comps.size()-1]/comps[1]; // comps[comps_n-1] is the ref length, comps[1] is the test length
+      if (brevity < 0.0) {
+        logbleu += brevity;
+      }
+      score =  exp(logbleu);
+    }
+
+    //cerr << "score: " << score << " bestScore: " << bestScore <<  endl;
+    if (score > bestScore) {
+      bestScore = score;
+      best = iter;
+      VERBOSE(2,"NEW BEST: " << score << endl);
+      //for (size_t i = 0; i < comps.size(); ++i) {
+      //    cerr << comps[i] << " ";
+      //}
+      //cerr << endl;
+    }
+  }
+
+  assert (best != nBestList.end());
+  return **best;
+  //vector<Word> bestWords;
+  //GetOutputWords(**best,bestWords);
+  //return bestWords;
+}
+
+}
+
+

mosesdecoder/moses/NonTerminal.cpp

@@ -0,0 +1,21 @@
+
+#include "NonTerminal.h"
+
+using namespace std;
+
+namespace Moses
+{
+std::ostream& operator<<(std::ostream &out, const NonTerminalSet &obj)
+{
+  NonTerminalSet::const_iterator iter;
+  for (iter = obj.begin(); iter != obj.end(); ++iter) {
+    const Word &word = *iter;
+    out << word << " ";
+  }
+
+
+  return out;
+}
+
+
+}

mosesdecoder/moses/PartialTranslOptColl.cpp

@@ -0,0 +1,112 @@
+// $Id$
+
+/***********************************************************************
+Moses - factored phrase-based language decoder
+Copyright (C) 2006 University of Edinburgh
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Lesser General Public
+License as published by the Free Software Foundation; either
+version 2.1 of the License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public
+License along with this library; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+***********************************************************************/
+
+#include "PartialTranslOptColl.h"
+#include <algorithm>
+#include <iostream>
+
+using namespace std;
+
+namespace Moses
+{
+/** constructor, intializes counters and thresholds */
+PartialTranslOptColl::PartialTranslOptColl(size_t const maxSize)
+{
+  m_bestScore = -std::numeric_limits<float>::infinity();
+  m_worstScore = -std::numeric_limits<float>::infinity();
+  m_maxSize = maxSize; // StaticData::Instance().GetMaxNoPartTransOpt();
+  m_totalPruned = 0;
+}
+
+
+/** add a partial translation option to the collection (without pruning) */
+void PartialTranslOptColl::AddNoPrune(TranslationOption *partialTranslOpt)
+{
+  if (partialTranslOpt->GetFutureScore() >= m_worstScore) {
+    m_list.push_back(partialTranslOpt);
+    if (partialTranslOpt->GetFutureScore() > m_bestScore)
+      m_bestScore = partialTranslOpt->GetFutureScore();
+  } else {
+    m_totalPruned++;
+    delete partialTranslOpt;
+  }
+}
+
+/** add a partial translation option to the collection, prune if necessary.
+ * This is done similar to the Prune() in TranslationOptionCollection */
+
+void PartialTranslOptColl::Add(TranslationOption *partialTranslOpt)
+{
+  // add
+  AddNoPrune(partialTranslOpt );
+
+  // done if not too large (lazy pruning, only if twice as large as max)
+  if ( m_list.size() > 2 * m_maxSize ) {
+    Prune();
+  }
+}
+
+
+/** helper, used by pruning */
+bool ComparePartialTranslationOption(const TranslationOption *a, const TranslationOption *b)
+{
+  return a->GetFutureScore() > b->GetFutureScore();
+}
+
+/** pruning, remove partial translation options, if list too big */
+void PartialTranslOptColl::Prune()
+{
+  // done if not too big
+  if ( m_list.size() <= m_maxSize ) {
+    return;
+  }
+
+  //	TRACE_ERR( "pruning partial translation options from size " << m_list.size() << std::endl);
+
+  // find nth element
+  NTH_ELEMENT4(m_list.begin(),
+               m_list.begin() + m_maxSize,
+               m_list.end(),
+               ComparePartialTranslationOption);
+
+  m_worstScore = m_list[ m_maxSize-1 ]->GetFutureScore();
+  // delete the rest
+  for (size_t i = m_maxSize ; i < m_list.size() ; ++i) {
+    delete m_list[i];
+    m_totalPruned++;
+  }
+  m_list.resize(m_maxSize);
+  //	TRACE_ERR( "pruned to size " << m_list.size() << ", total pruned: " << m_totalPruned << std::endl);
+}
+
+// friend
+ostream& operator<<(ostream& out, const PartialTranslOptColl& possibleTranslation)
+{
+  for (size_t i = 0; i < possibleTranslation.m_list.size(); ++i) {
+    const TranslationOption &transOpt = *possibleTranslation.m_list[i];
+    out << transOpt << endl;
+  }
+  return out;
+}
+
+}
+
+

mosesdecoder/moses/RuleCubeQueue.cpp

@@ -0,0 +1,69 @@
+// $Id$
+// vim:tabstop=2
+/***********************************************************************
+ Moses - factored phrase-based language decoder
+ Copyright (C) 2010 Hieu Hoang
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ ***********************************************************************/
+
+#include "RuleCubeQueue.h"
+
+#include "RuleCubeItem.h"
+#include "StaticData.h"
+#include "ChartManager.h"
+
+namespace Moses
+{
+
+RuleCubeQueue::~RuleCubeQueue()
+{
+  while (!m_queue.empty()) {
+    RuleCube *cube = m_queue.top();
+    m_queue.pop();
+    delete cube;
+  }
+}
+
+void RuleCubeQueue::Add(RuleCube *ruleCube)
+{
+  m_queue.push(ruleCube);
+}
+
+ChartHypothesis *RuleCubeQueue::Pop()
+{
+  // pop the most promising rule cube
+  RuleCube *cube = m_queue.top();
+  m_queue.pop();
+
+  // pop the most promising item from the cube and get the corresponding
+  // hypothesis
+  RuleCubeItem *item = cube->Pop(m_manager);
+  if (m_manager.options()->cube.lazy_scoring) {
+    item->CreateHypothesis(cube->GetTranslationOption(), m_manager);
+  }
+  ChartHypothesis *hypo = item->ReleaseHypothesis();
+
+  // if the cube contains more items then push it back onto the queue
+  if (!cube->IsEmpty()) {
+    m_queue.push(cube);
+  } else {
+    delete cube;
+  }
+
+  return hypo;
+}
+
+}

mosesdecoder/moses/RuleCubeQueue.h

@@ -0,0 +1,66 @@
+// $Id$
+// vim:tabstop=2
+/***********************************************************************
+ Moses - factored phrase-based language decoder
+ Copyright (C) 2010 Hieu Hoang
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ ***********************************************************************/
+
+#pragma once
+
+#include "RuleCube.h"
+
+#include <queue>
+#include <vector>
+
+namespace Moses
+{
+
+class ChartManager;
+
+/** Define an ordering between RuleCube based on their best item scores.  This
+ * is used to order items in the priority queue.
+ */
+class RuleCubeOrderer
+{
+public:
+  bool operator()(const RuleCube *p, const RuleCube *q) const {
+    return p->GetTopScore() < q->GetTopScore();
+  }
+};
+
+/** @todo how is this used */
+class RuleCubeQueue
+{
+public:
+  RuleCubeQueue(ChartManager &manager) : m_manager(manager) {}
+  ~RuleCubeQueue();
+
+  void Add(RuleCube *);
+  ChartHypothesis *Pop();
+  bool IsEmpty() const {
+    return m_queue.empty();
+  }
+
+private:
+  typedef std::priority_queue<RuleCube*, std::vector<RuleCube*>,
+          RuleCubeOrderer > Queue;
+
+  Queue m_queue;
+  ChartManager &m_manager;
+};
+
+}

mosesdecoder/moses/SquareMatrix.h

@@ -0,0 +1,86 @@
+// $Id$
+
+/***********************************************************************
+Moses - factored phrase-based language decoder
+Copyright (C) 2006 University of Edinburgh
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Lesser General Public
+License as published by the Free Software Foundation; either
+version 2.1 of the License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public
+License along with this library; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+***********************************************************************/
+
+#ifndef moses_SquareMatrix_h
+#define moses_SquareMatrix_h
+
+#include <iostream>
+#include "TypeDef.h"
+#include "Util.h"
+#include "Bitmap.h"
+
+namespace Moses
+{
+
+//! A square array of floats to store future costs in the phrase-based decoder
+class SquareMatrix
+{
+  friend std::ostream& operator<<(std::ostream &out, const SquareMatrix &matrix);
+protected:
+  const size_t m_size; /**< length of the square (sentence length) */
+  float *m_array; /**< two-dimensional array to store floats */
+
+  SquareMatrix(); // not implemented
+  SquareMatrix(const SquareMatrix &copy); // not implemented
+
+public:
+  SquareMatrix(size_t size)
+    :m_size(size) {
+    m_array = (float*) malloc(sizeof(float) * size * size);
+  }
+  ~SquareMatrix() {
+    free(m_array);
+  }
+
+  // set upper triangle
+  void InitTriangle(float val);
+
+  /** Returns length of the square: typically the sentence length */
+  inline size_t GetSize() const {
+    return m_size;
+  }
+  /** Get a future cost score for a span */
+  inline float GetScore(size_t startPos, size_t endPos) const {
+    return m_array[startPos * m_size + endPos];
+  }
+  /** Set a future cost score for a span */
+  inline void SetScore(size_t startPos, size_t endPos, float value) {
+    m_array[startPos * m_size + endPos] = value;
+  }
+  float CalcEstimatedScore( Bitmap const& ) const;
+  float CalcEstimatedScore( Bitmap const&, size_t startPos, size_t endPos ) const;
+
+  TO_STRING();
+};
+
+inline std::ostream& operator<<(std::ostream &out, const SquareMatrix &matrix)
+{
+  for (size_t endPos = 0 ; endPos < matrix.GetSize() ; endPos++) {
+    for (size_t startPos = 0 ; startPos < matrix.GetSize() ; startPos++)
+      out << matrix.GetScore(startPos, endPos) << " ";
+    out << std::endl;
+  }
+
+  return out;
+}
+
+}
+#endif

mosesdecoder/moses/StackVec.h

@@ -0,0 +1,34 @@
+/***********************************************************************
+ Moses - statistical machine translation system
+ Copyright (C) 2006-2012 University of Edinburgh
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ ***********************************************************************/
+
+#pragma once
+
+#include <vector>
+
+namespace Moses
+{
+
+class ChartCellLabel;
+
+class StackVec : public std::vector<const ChartCellLabel*>
+{
+
+};
+
+}

mosesdecoder/moses/SyntacticLanguageModelFiles.h

@@ -0,0 +1,103 @@
+//
+
+#ifndef moses_SyntacticLanguageModelFiles_h
+#define moses_SyntacticLanguageModelFiles_h
+
+#include "nl-iomacros.h"
+#include "nl-string.h"
+
+namespace Moses
+{
+
+template <class MH, class MO>
+class SyntacticLanguageModelFiles
+{
+
+public:
+
+  SyntacticLanguageModelFiles(const std::vector<std::string>& filePaths);
+  ~SyntacticLanguageModelFiles();
+
+  MH* getHiddenModel();
+  MO* getObservedModel();
+
+private:
+  MH* hiddenModel;
+  MO* observedModel;
+
+};
+
+
+template <class MH, class MO>
+SyntacticLanguageModelFiles<MH,MO>::SyntacticLanguageModelFiles(const std::vector<std::string>& filePaths)
+{
+
+  this->hiddenModel = new MH();
+  this->observedModel = new MO();
+
+  //// I. LOAD MODELS...
+  std::cerr << "Reading syntactic language model files...\n";
+  // For each model file...
+  for ( int a=0, n=filePaths.size(); a<n; a++ ) {                                           // read models
+    FILE* pf = fopen(filePaths[a].c_str(),"r"); // Read model file
+    if(!pf) {
+      std::cerr << "Error loading model file " << filePaths[a] << std::endl;
+      return;
+    }
+    std::cerr << "Loading model \'" << filePaths[a] << "\'...\n";
+    int c=' ';
+    int i=0;
+    int line=1;
+    String sBuff(1000);                          // Lookahead/ctrs/buffers
+    CONSUME_ALL ( pf, c, WHITESPACE(c), line);                                   // Get to first record
+    while ( c!=-1 && c!='\0' && c!='\5' ) {                                      // For each record
+      CONSUME_STR ( pf, c, (c!='\n' && c!='\0' && c!='\5'), sBuff, i, line );    //   Consume line
+      StringInput si(sBuff.c_array());
+      if ( !( sBuff[0]=='#'                                                   //   Accept comments/fields
+              ||  si>>*(this->hiddenModel)>>"\0"!=NULL
+              ||  si>>*(this->observedModel)>>"\0"!=NULL
+            ))
+        std::cerr<<"\nERROR: can't parse \'"<<sBuff<<"\' in line "<<line<<"\n\n";
+      CONSUME_ALL ( pf, c, WHITESPACE(c), line);                                 //   Consume whitespace
+      if ( line%100000==0 ) std::cerr<<"  "<<line<<" lines read...\n";                //   Progress for big models
+    }
+    std::cerr << "Model \'" << filePaths[a] << "\' loaded.\n";
+  }
+
+  std::cerr << "...reading syntactic language model files completed\n";
+
+
+}
+
+
+template <class MH, class MO>
+SyntacticLanguageModelFiles<MH,MO>::~SyntacticLanguageModelFiles()
+{
+
+  VERBOSE(3,"Destructing syntactic language model files" << std::endl);
+  delete hiddenModel;
+  delete observedModel;
+
+}
+
+
+template <class MH, class MO>
+MH* SyntacticLanguageModelFiles<MH,MO>::getHiddenModel()
+{
+
+  return this->hiddenModel;
+
+}
+
+template <class MH, class MO>
+MO* SyntacticLanguageModelFiles<MH,MO>::getObservedModel()
+{
+
+  return this->observedModel;
+
+}
+
+
+}
+
+#endif

mosesdecoder/moses/SyntacticLanguageModelState.h

@@ -0,0 +1,311 @@
+//
+
+#ifndef moses_SyntacticLanguageModelState_h
+#define moses_SyntacticLanguageModelState_h
+
+#include "nl-iomacros.h"
+#include "nl-cpt.h"
+#include "nl-hmm.h"
+
+#include "SyntacticLanguageModelFiles.h"
+#include "moses/FF/FFState.h"
+#include <string>
+
+namespace Moses
+{
+
+template <class MY, class MX, class YS=typename MY::RandVarType, class B=NullBackDat<typename MY::RandVarType> >
+class SyntacticLanguageModelState : public FFState
+{
+public:
+
+  // Initialize an empty LM state
+  SyntacticLanguageModelState( SyntacticLanguageModelFiles<MY,MX>* modelData, int beamSize );
+
+  // Get the next LM state from an existing LM state and the next word
+  SyntacticLanguageModelState( const SyntacticLanguageModelState* prev, std::string word );
+
+
+  ~SyntacticLanguageModelState() {
+    VERBOSE(3,"Destructing SyntacticLanguageModelState" << std::endl);
+    delete randomVariableStore;
+  }
+
+  virtual int Compare(const FFState& other) const;
+
+  // Get the LM score from this LM state
+  double getScore() const;
+
+  double getProb() const;
+
+private:
+
+  void setScore(double score);
+  void printRV();
+
+  SafeArray1D<Id<int>,pair<YS,LogProb> >* randomVariableStore;
+  double prob;
+  double score;
+  int beamSize;
+  SyntacticLanguageModelFiles<MY,MX>* modelData;
+  bool sentenceStart;
+};
+
+
+////////////////////////////////////////////////////////////////////////////////
+
+
+template <class MY, class MX, class YS, class B>
+void SyntacticLanguageModelState<MY,MX,YS,B>::printRV()
+{
+
+  cerr << "*********** BEGIN printRV() ******************" << endl;
+  int size=randomVariableStore->getSize();
+  cerr << "randomVariableStore->getSize() == " << size << endl;
+
+  for (int depth=0; depth<size; depth+=1) {
+
+
+    const pair<YS,LogProb> *data = &(randomVariableStore->get(depth));
+    std::cerr << "randomVariableStore[" << depth << "]\t" << data->first << "\tprob = " << data->second.toProb() << "\tlogProb = " << double(data->second.toInt())/100 << std::endl;
+
+  }
+  cerr << "*********** END printRV() ******************" << endl;
+
+}
+
+// Initialize an empty LM state from grammar files
+//
+//    nArgs is the number of model files
+//    argv is the list of model file names
+//
+template <class MY, class MX, class YS, class B>
+SyntacticLanguageModelState<MY,MX,YS,B>::SyntacticLanguageModelState( SyntacticLanguageModelFiles<MY,MX>* modelData, int beamSize )
+{
+
+  this->randomVariableStore = new SafeArray1D<Id<int>,pair<YS,LogProb> >();
+  this->modelData = modelData;
+  this->beamSize = beamSize;
+
+  // Initialize an empty random variable value
+  YS xBEG;
+  StringInput(String(BEG_STATE).c_array())>>xBEG>>"\0";
+  cerr<<xBEG<<"\n";
+
+  //  cout << "Examining RV store just before RV init" << endl;
+  //printRV();
+
+  // Initialize the random variable store
+  this->randomVariableStore->init(1,pair<YS,LogProb>(xBEG,0));
+
+  this->sentenceStart = true;
+
+  IFVERBOSE(3) {
+    VERBOSE(3,"Examining RV store just after RV init" << endl);
+    printRV();
+  }
+
+  // Get score of final frame in HHMM
+  LogProb l(1.0);
+  //score = l.toDouble();
+  setScore(l.toDouble());
+  //  MY::F_ROOT_OBS = true;
+// this->modelData->getHiddenModel()->setRootObs(true);
+
+
+}
+
+
+template <class MY, class MX, class YS, class B>
+int SyntacticLanguageModelState<MY,MX,YS,B>::Compare(const FFState& other) const
+{
+  /*
+  const SyntacticLanguageModelState<MY,MX,YS,B>& o =
+    static_cast<const SyntacticLanguageModelState<MY,MX,YS,B>&>(other);
+
+  if (o.score > score) return 1;
+  else if (o.score < score) return -1;
+  else return 0;
+  */
+  return 0;
+}
+
+
+template <class MY, class MX, class YS, class B>
+SyntacticLanguageModelState<MY,MX,YS,B>::SyntacticLanguageModelState( const SyntacticLanguageModelState* prev, std::string word )
+{
+
+  // Initialize member variables
+  this->randomVariableStore = new SafeArray1D<Id<int>,pair<YS,LogProb> >();
+  this->modelData = prev->modelData;
+  this->beamSize = prev->beamSize;
+  this->randomVariableStore->init(this->beamSize);
+  this->sentenceStart=false;
+
+  YS ysEND;
+  StringInput(String(END_STATE).c_array())>>ysEND>>"\0";
+
+  // Get HHMM model files
+  MY& mH = *(modelData->getHiddenModel());
+  MX& mO = *(modelData->getObservedModel());
+
+  // Initialize HHMM
+  HMM<MY,MX,YS,B> hmm(mH,mO);
+  int MAX_WORDS  = 2;
+  hmm.init(MAX_WORDS,this->beamSize,prev->randomVariableStore);
+  typename MX::RandVarType x(word.c_str());
+  //  cout << "Examining HHMM just after hmm.init" << endl;
+  //  hmm.debugPrint();
+
+
+  /*  cerr << "*********** BEGIN writeCurr() ******************" << endl;
+  hmm.writeCurr(cout,0);
+  hmm.writeCurr(cout,1);
+  cerr << "*********** END writeCurr() ******************" << endl;
+  */
+  /*
+    {
+
+    int wnum=1;
+    list<TrellNode<YS,B> > lys = hmm.getMLSnodes(ysEND);                                           // get mls list
+        for ( typename list<TrellNode<YS,B> >::iterator i=lys.begin(); i!=lys.end(); i++, wnum++ ) {   // for each frame
+          cout << "HYPOTH " << wnum
+               << " " << i->getBackData()
+               << " " << x
+               << " " << i->getId()
+               << " (" << i->getLogProb() << ")"
+               << endl;                                                                             //   print RV val
+        }
+    }
+    */
+
+
+  /*
+  cerr << "Writing hmm.writeCurr" << endl;
+  hmm.writeCurr(cerr,0);
+  hmm.writeCurr(cerr,1);
+  cerr << "...done writing hmm.writeCurr" << endl;
+  */
+  hmm.getCurrSum();
+
+
+
+  // Initialize observed variable
+  //  typename MX::RandVarType ov;
+  //  ov.set(word.c_str(),mO);
+  //  MY::WORD = ov.getW();
+  //bool endOfSentence = prev->sentenceStart;//true;
+
+  //  std::cerr << "About to give HHMM a word of input:\t" << word << std::endl;
+
+  hmm.updateRanked(x, prev->sentenceStart);
+
+  //  cout << "Examining HHMM just after hmm.updateRanked(" << x << "," << prev->sentenceStart << ")" << endl;
+  //  hmm.debugPrint();
+  /*
+    cerr << "*********** BEGIN writeCurr() ******************" << endl;
+    hmm.writeCurr(cout,0);
+    hmm.writeCurr(cout,1);
+    cerr << "*********** END writeCurr() ******************" << endl;
+    */
+  /*
+  {
+
+    int wnum=1;
+    list<TrellNode<YS,B> > lys = hmm.getMLSnodes(ysEND);                                           // get mls list
+        for ( typename list<TrellNode<YS,B> >::iterator i=lys.begin(); i!=lys.end(); i++, wnum++ ) {   // for each frame
+          cout << "HYPOTH " << wnum
+               << " " << i->getBackData()
+               << " " << x
+               << " " << i->getId()
+               << " (" << i->getLogProb() << ")"
+               << endl;                                                                             //   print RV val
+        }
+    }
+    */
+//  X ov(word.c_str());
+  //mH.setWord(ov);
+  // MY::WORD = ov;//ov.getW();
+
+  // Update HHMM based on observed variable
+  //hmm.updateRanked(ov);
+  //mH.setRootObs(true);
+  //MY::F_ROOT_OBS = false;
+
+  // Get the current score
+  double currSum = hmm.getCurrSum();
+  //VERBOSE(3,"Setting score using currSum for " << scientific << x << " = " << currSum << endl);
+  setScore(currSum);
+  //  cout << "Examining RV store just before RV init via gatherElementsInBeam" << endl;
+  //  printRV();
+
+  // Get new hidden random variable store from HHMM
+  hmm.gatherElementsInBeam(randomVariableStore);
+  //  cout << "Examining RV store just after RV init via gatherElementsInBeam" << endl;
+  //  printRV();
+  /*
+  cerr << "Writing hmm.writeCurr..." << endl;
+  hmm.writeCurr(cerr,0);
+  hmm.writeCurr(cerr,1);
+  cerr << "...done writing hmm.writeCurr" << endl;
+  */
+}
+
+
+template <class MY, class MX, class YS, class B>
+double SyntacticLanguageModelState<MY,MX,YS,B>::getProb() const
+{
+
+  return prob;
+}
+
+template <class MY, class MX, class YS, class B>
+double SyntacticLanguageModelState<MY,MX,YS,B>::getScore() const
+{
+
+  return score;
+}
+
+
+template <class MY, class MX, class YS, class B>
+void SyntacticLanguageModelState<MY,MX,YS,B>::setScore(double score)
+{
+
+
+
+
+  this->prob = score;
+
+  // We want values to range from -100 to 0
+  //
+  // If the minimum positive value for a double is min=4.94065645841246544e-324
+  //    then to scale, we want a logarithmic base such that log_b(min)=-100
+  //
+  // -100 = log(min) / log(b)
+  //
+  // log(b) = log(min) / -100
+  //
+  // b = exp( log(min) / -100 )
+  //
+  // b = 7.44440071921381
+
+  // Check for score==0 to avoid causing -infinity with log(score)
+  if (score==0) {
+    this->score = -100;
+  } else {
+    double x = log(score) / 7.44440071921381;
+    if ( x >= -100) {
+      this->score = x;
+    } else {
+      this->score = -100;
+    }
+  }
+
+  VERBOSE(3,"\tSyntacticLanguageModelState has score=" << this->score << endl);
+
+}
+
+
+}
+
+#endif

mosesdecoder/moses/Syntax/F2S/DerivationWriter.h

@@ -0,0 +1,36 @@
+#pragma once
+
+#include <ostream>
+
+#include "moses/Syntax/KBestExtractor.h"
+#include "moses/Word.h"
+
+namespace Moses
+{
+namespace Syntax
+{
+struct SHyperedge;
+
+namespace F2S
+{
+
+// Writes a string representation of a derivation to a std::ostream.  This is
+// used by the -translation-details / -T option.
+// TODO Merge this with S2T::DerivationWriter.
+class DerivationWriter
+{
+public:
+  // 1-best version.
+  static void Write(const SHyperedge&, std::size_t, std::ostream &);
+
+  // k-best version.
+  static void Write(const KBestExtractor::Derivation &, std::size_t,
+                    std::ostream &);
+private:
+  static void WriteLine(const SHyperedge &, std::size_t, std::ostream &);
+  static void WriteSymbol(const Word &, std::ostream &);
+};
+
+}  // namespace F2S
+}  // namespace Syntax
+}  // namespace Moses

mosesdecoder/moses/Syntax/F2S/Forest.h

@@ -0,0 +1,51 @@
+#pragma once
+
+#include "vector"
+
+#include "moses/Syntax/PVertex.h"
+
+namespace Moses
+{
+namespace Syntax
+{
+namespace F2S
+{
+
+class Forest
+{
+public:
+  struct Vertex;
+
+  struct Hyperedge {
+    Vertex *head;
+    std::vector<Vertex *> tail;
+    float weight;
+  };
+
+  struct Vertex {
+    Vertex(const PVertex &v) : pvertex(v) {}
+    ~Vertex();  // Deletes incoming hyperedges.
+    PVertex pvertex;
+    std::vector<Hyperedge *> incoming;
+  };
+
+  // Constructor.
+  Forest() {}
+
+  // Destructor (deletes vertices).
+  ~Forest();
+
+  // Delete all vertices.
+  void Clear();
+
+  std::vector<Vertex *> vertices;
+
+private:
+  // Copying is not allowed.
+  Forest(const Forest &);
+  Forest &operator=(const Forest &);
+};
+
+}  // namespace F2S
+}  // namespace Syntax
+}  // namespace Moses

mosesdecoder/moses/Syntax/F2S/GlueRuleSynthesizer.cpp

@@ -0,0 +1,85 @@
+#include "GlueRuleSynthesizer.h"
+
+#include <sstream>
+
+#include "moses/FF/UnknownWordPenaltyProducer.h"
+#include "util/string_stream.hh"
+#include "moses/parameters/AllOptions.h"
+namespace Moses
+{
+namespace Syntax
+{
+namespace F2S
+{
+
+GlueRuleSynthesizer::
+GlueRuleSynthesizer(Moses::AllOptions const& opts, HyperTree &trie)
+  : m_input_default_nonterminal(opts.syntax.input_default_non_terminal)
+  , m_output_default_nonterminal(opts.syntax.output_default_non_terminal)
+  , m_hyperTree(trie)
+{
+  Word *lhs = NULL;
+  m_dummySourcePhrase.CreateFromString(Input, opts.input.factor_order, "hello", &lhs);
+  delete lhs;
+}
+
+void GlueRuleSynthesizer::SynthesizeRule(const Forest::Hyperedge &e)
+{
+  HyperPath source;
+  SynthesizeHyperPath(e, source);
+  TargetPhrase *tp = SynthesizeTargetPhrase(e);
+  TargetPhraseCollection::shared_ptr tpc
+  = GetOrCreateTargetPhraseCollection(m_hyperTree, source);
+  tpc->Add(tp);
+}
+
+void GlueRuleSynthesizer::SynthesizeHyperPath(const Forest::Hyperedge &e,
+    HyperPath &path)
+{
+  path.nodeSeqs.clear();
+  path.nodeSeqs.resize(2);
+  path.nodeSeqs[0].push_back(e.head->pvertex.symbol[0]->GetId());
+  for (std::vector<Forest::Vertex*>::const_iterator p = e.tail.begin();
+       p != e.tail.end(); ++p) {
+    const Forest::Vertex &child = **p;
+    path.nodeSeqs[1].push_back(child.pvertex.symbol[0]->GetId());
+  }
+}
+
+TargetPhrase*
+GlueRuleSynthesizer::
+SynthesizeTargetPhrase(const Forest::Hyperedge &e)
+{
+  const UnknownWordPenaltyProducer &unknownWordPenaltyProducer =
+    UnknownWordPenaltyProducer::Instance();
+
+  TargetPhrase *targetPhrase = new TargetPhrase();
+
+  util::StringStream alignmentSS;
+  for (std::size_t i = 0; i < e.tail.size(); ++i) {
+    const Word &symbol = e.tail[i]->pvertex.symbol;
+    if (symbol.IsNonTerminal()) {
+      targetPhrase->AddWord(m_output_default_nonterminal);
+    } else {
+      // TODO Check this
+      Word &targetWord = targetPhrase->AddWord();
+      targetWord.CreateUnknownWord(symbol);
+    }
+    alignmentSS << i << "-" << i << " ";
+  }
+
+  // Assign the lowest possible score so that glue rules are only used when
+  // absolutely required.
+  float score = LOWEST_SCORE;
+  targetPhrase->GetScoreBreakdown().Assign(&unknownWordPenaltyProducer, score);
+  targetPhrase->EvaluateInIsolation(m_dummySourcePhrase);
+  Word *targetLhs = new Word(m_output_default_nonterminal);
+  targetPhrase->SetTargetLHS(targetLhs);
+  targetPhrase->SetAlignmentInfo(alignmentSS.str());
+
+  return targetPhrase;
+}
+
+}  // F2S
+}  // Syntax
+}  // Moses

mosesdecoder/moses/Syntax/F2S/GlueRuleSynthesizer.h

@@ -0,0 +1,40 @@
+#pragma once
+
+#include "moses/Phrase.h"
+#include "moses/TargetPhrase.h"
+
+#include "HyperTree.h"
+#include "HyperTreeCreator.h"
+#include "Forest.h"
+
+namespace Moses
+{
+class AllOptions;
+namespace Syntax
+{
+namespace F2S
+{
+
+class GlueRuleSynthesizer : public HyperTreeCreator
+{
+  Word m_input_default_nonterminal;
+  Word m_output_default_nonterminal;
+public:
+  GlueRuleSynthesizer(Moses::AllOptions const& opts, HyperTree &);
+
+  // Synthesize the minimal, monotone rule that can be applied to the given
+  // hyperedge and add it to the rule trie.
+  void SynthesizeRule(const Forest::Hyperedge &);
+
+private:
+  void SynthesizeHyperPath(const Forest::Hyperedge &, HyperPath &);
+
+  TargetPhrase *SynthesizeTargetPhrase(const Forest::Hyperedge &);
+
+  HyperTree &m_hyperTree;
+  Phrase m_dummySourcePhrase;
+};
+
+}  // F2S
+}  // Syntax
+}  // Moses

mosesdecoder/moses/Syntax/F2S/HyperPath.cpp

@@ -0,0 +1,20 @@
+#include "HyperPath.h"
+
+#include <limits>
+
+namespace Moses
+{
+namespace Syntax
+{
+namespace F2S
+{
+
+const std::size_t HyperPath::kEpsilon =
+  std::numeric_limits<std::size_t>::max()-1;
+
+const std::size_t HyperPath::kComma =
+  std::numeric_limits<std::size_t>::max()-2;
+
+}  // namespace F2S
+}  // namespace Syntax
+}  // namespace Moses

mosesdecoder/moses/Syntax/F2S/HyperPath.h

@@ -0,0 +1,34 @@
+#pragma once
+
+#include <vector>
+
+#include "moses/Factor.h"
+
+namespace Moses
+{
+namespace Syntax
+{
+namespace F2S
+{
+
+// A HyperPath for representing the source-side tree fragment of a
+// tree-to-string rule.  See this paper:
+//
+//  Hui Zhang, Min Zhang, Haizhou Li, and Chew Lim Tan
+//  "Fast Translation Rule Matching for Syntax-based Statistical Machine
+//   Translation"
+//  In proceedings of EMNLP 2009
+//
+struct HyperPath {
+public:
+  typedef std::vector<std::size_t> NodeSeq;
+
+  static const std::size_t kEpsilon;
+  static const std::size_t kComma;
+
+  std::vector<NodeSeq> nodeSeqs;
+};
+
+}  // namespace F2S
+}  // namespace Syntax
+}  // namespace Moses

mosesdecoder/moses/Syntax/F2S/HyperPathLoader.h

@@ -0,0 +1,66 @@
+#pragma once
+
+#include <stack>
+#include <vector>
+
+#include "util/string_piece.hh"
+
+#include "moses/FactorCollection.h"
+#include "moses/TypeDef.h"
+
+#include "HyperPath.h"
+#include "TreeFragmentTokenizer.h"
+
+namespace Moses
+{
+namespace Syntax
+{
+namespace F2S
+{
+
+// Parses a string representation of a tree fragment, adding the terminals
+// and non-terminals to FactorCollection::Instance() and building a
+// HyperPath object.
+//
+// This class is designed to be used during rule table loading.  Since every
+// rule has a tree fragment on the source-side, Load() may be called millions
+// of times.  The algorithm therefore sacrifices readability for speed and
+// shoehorns everything into two passes over the input token sequence.
+//
+class HyperPathLoader
+{
+public:
+  void Load(const StringPiece &, HyperPath &);
+
+private:
+  struct NodeTuple {
+    int index;          // Preorder index of the node.
+    int parent;         // Preorder index of the node's parent.
+    int depth;          // Depth of the node.
+    std::size_t symbol; // Either the factor ID of a tree terminal/non-terminal
+    // or for virtual nodes, HyperPath::kEpsilon.
+  };
+
+  // Determine the height of the current tree fragment (stored in m_tokenSeq).
+  int DetermineHeight() const;
+
+  // Generate the preorder sequence of NodeTuples for the current tree fragment,
+  // including virtual nodes.
+  void GenerateNodeTupleSeq(int height);
+
+  const Factor *AddTerminalFactor(const StringPiece &s) {
+    return FactorCollection::Instance().AddFactor(s, false);
+  }
+
+  const Factor *AddNonTerminalFactor(const StringPiece &s) {
+    return FactorCollection::Instance().AddFactor(s, true);
+  }
+
+  std::vector<TreeFragmentToken> m_tokenSeq;
+  std::vector<NodeTuple> m_nodeTupleSeq;
+  std::stack<int> m_parentStack;
+};
+
+}  // namespace F2S
+}  // namespace Syntax
+}  // namespace Moses

mosesdecoder/moses/Syntax/F2S/HyperTree.cpp

@@ -0,0 +1,70 @@
+#include "HyperTree.h"
+
+namespace Moses
+{
+namespace Syntax
+{
+namespace F2S
+{
+
+void HyperTree::Node::Prune(std::size_t tableLimit)
+{
+  // Recusively prune child nodes.
+  for (Map::iterator p = m_map.begin(); p != m_map.end(); ++p) {
+    p->second.Prune(tableLimit);
+  }
+  // Prune TargetPhraseCollection at this node.
+  m_targetPhraseCollection->Prune(true, tableLimit);
+}
+
+void HyperTree::Node::Sort(std::size_t tableLimit)
+{
+  // Recusively sort child nodes.
+  for (Map::iterator p = m_map.begin(); p != m_map.end(); ++p) {
+    p->second.Sort(tableLimit);
+  }
+  // Sort TargetPhraseCollection at this node.
+  m_targetPhraseCollection->Sort(true, tableLimit);
+}
+
+HyperTree::Node *HyperTree::Node::GetOrCreateChild(
+  const HyperPath::NodeSeq &nodeSeq)
+{
+  return &m_map[nodeSeq];
+}
+
+const HyperTree::Node *HyperTree::Node::GetChild(
+  const HyperPath::NodeSeq &nodeSeq) const
+{
+  Map::const_iterator p = m_map.find(nodeSeq);
+  return (p == m_map.end()) ? NULL : &p->second;
+}
+
+TargetPhraseCollection::shared_ptr HyperTree::GetOrCreateTargetPhraseCollection(
+  const HyperPath &hyperPath)
+{
+  Node &node = GetOrCreateNode(hyperPath);
+  return node.GetTargetPhraseCollection();
+}
+
+HyperTree::Node &HyperTree::GetOrCreateNode(const HyperPath &hyperPath)
+{
+  const std::size_t height = hyperPath.nodeSeqs.size();
+  Node *node = &m_root;
+  for (std::size_t i = 0; i < height; ++i) {
+    const HyperPath::NodeSeq &nodeSeq = hyperPath.nodeSeqs[i];
+    node = node->GetOrCreateChild(nodeSeq);
+  }
+  return *node;
+}
+
+void HyperTree::SortAndPrune(std::size_t tableLimit)
+{
+  if (tableLimit) {
+    m_root.Sort(tableLimit);
+  }
+}
+
+}  // namespace F2S
+}  // namespace Syntax
+}  // namespace Moses

mosesdecoder/moses/Syntax/F2S/HyperTree.h

@@ -0,0 +1,92 @@
+#pragma once
+
+#include <map>
+#include <vector>
+
+#include <boost/unordered_map.hpp>
+
+#include "moses/Syntax/RuleTable.h"
+#include "moses/TargetPhraseCollection.h"
+
+#include "HyperPath.h"
+
+namespace Moses
+{
+namespace Syntax
+{
+namespace F2S
+{
+
+// A HyperTree for representing a tree-to-string rule table.  See this paper:
+//
+//  Hui Zhang, Min Zhang, Haizhou Li, and Chew Lim Tan
+//  "Fast Translation Rule Matching for Syntax-based Statistical Machine
+//   Translation"
+//  In proceedings of EMNLP 2009
+//
+class HyperTree : public RuleTable
+{
+public:
+  class Node
+  {
+  public:
+    typedef boost::unordered_map<HyperPath::NodeSeq, Node> Map;
+
+    bool IsLeaf() const {
+      return m_map.empty();
+    }
+
+    bool HasRules() const {
+      return !m_targetPhraseCollection->IsEmpty();
+    }
+
+    void Prune(std::size_t tableLimit);
+    void Sort(std::size_t tableLimit);
+
+    Node *GetOrCreateChild(const HyperPath::NodeSeq &);
+
+    const Node *GetChild(const HyperPath::NodeSeq &) const;
+
+    TargetPhraseCollection::shared_ptr
+    GetTargetPhraseCollection() const {
+      return m_targetPhraseCollection;
+    }
+
+    TargetPhraseCollection::shared_ptr
+    GetTargetPhraseCollection() {
+      return m_targetPhraseCollection;
+    }
+
+    const Map &GetMap() const {
+      return m_map;
+    }
+
+    Node() : m_targetPhraseCollection(new TargetPhraseCollection) { }
+
+  private:
+    Map m_map;
+    TargetPhraseCollection::shared_ptr m_targetPhraseCollection;
+  };
+
+  HyperTree(const RuleTableFF *ff) : RuleTable(ff) { }
+
+  const Node &GetRootNode() const {
+    return m_root;
+  }
+
+private:
+  friend class HyperTreeCreator;
+
+  TargetPhraseCollection::shared_ptr
+  GetOrCreateTargetPhraseCollection(const HyperPath &);
+
+  Node &GetOrCreateNode(const HyperPath &);
+
+  void SortAndPrune(std::size_t);
+
+  Node m_root;
+};
+
+}  // namespace F2S
+}  // namespace Syntax
+}  // namespace Moses

mosesdecoder/moses/Syntax/F2S/HyperTreeLoader.cpp

@@ -0,0 +1,169 @@
+#include "HyperTreeLoader.h"
+
+#include <sys/stat.h>
+
+#include <cmath>
+#include <cstdlib>
+#include <fstream>
+#include <string>
+#include <iterator>
+#include <algorithm>
+#include <iostream>
+
+#include "moses/FactorCollection.h"
+#include "moses/Word.h"
+#include "moses/Util.h"
+#include "moses/Timer.h"
+#include "moses/InputFileStream.h"
+#include "moses/StaticData.h"
+#include "moses/Range.h"
+#include "moses/ChartTranslationOptionList.h"
+#include "moses/FactorCollection.h"
+#include "moses/Syntax/RuleTableFF.h"
+#include "moses/parameters/AllOptions.h"
+#include "util/file_piece.hh"
+#include "util/string_piece.hh"
+#include "util/tokenize_piece.hh"
+#include "util/double-conversion/double-conversion.h"
+#include "util/exception.hh"
+
+#include "HyperPath.h"
+#include "HyperPathLoader.h"
+#include "HyperTree.h"
+
+namespace Moses
+{
+
+namespace Syntax
+{
+namespace F2S
+{
+
+bool HyperTreeLoader::Load(AllOptions const& opts,
+                           const std::vector<FactorType> &input,
+                           const std::vector<FactorType> &output,
+                           const std::string &inFile,
+                           const RuleTableFF &ff,
+                           HyperTree &trie,
+                           boost::unordered_set<std::size_t> &sourceTermSet)
+{
+  PrintUserTime(std::string("Start loading HyperTree"));
+
+  sourceTermSet.clear();
+
+  std::size_t count = 0;
+
+  std::ostream *progress = NULL;
+  IFVERBOSE(1) progress = &std::cerr;
+  util::FilePiece in(inFile.c_str(), progress);
+
+  // reused variables
+  std::vector<float> scoreVector;
+  StringPiece line;
+
+  double_conversion::StringToDoubleConverter converter(double_conversion::StringToDoubleConverter::NO_FLAGS, NAN, NAN, "inf", "nan");
+
+  HyperPathLoader hyperPathLoader;
+
+  Phrase dummySourcePhrase;
+  {
+    Word *lhs = NULL;
+    dummySourcePhrase.CreateFromString(Input, input, "hello", &lhs);
+    delete lhs;
+  }
+
+  while(true) {
+    try {
+      line = in.ReadLine();
+    } catch (const util::EndOfFileException &e) {
+      break;
+    }
+
+    util::TokenIter<util::MultiCharacter> pipes(line, "|||");
+    StringPiece sourceString(*pipes);
+    StringPiece targetString(*++pipes);
+    StringPiece scoreString(*++pipes);
+
+    StringPiece alignString;
+    if (++pipes) {
+      StringPiece temp(*pipes);
+      alignString = temp;
+    }
+
+    ++pipes;  // counts
+
+    scoreVector.clear();
+    for (util::TokenIter<util::AnyCharacter, true> s(scoreString, " \t"); s; ++s) {
+      int processed;
+      float score = converter.StringToFloat(s->data(), s->length(), &processed);
+      UTIL_THROW_IF2(std::isnan(score), "Bad score " << *s << " on line " << count);
+      scoreVector.push_back(FloorScore(TransformScore(score)));
+    }
+    const std::size_t numScoreComponents = ff.GetNumScoreComponents();
+    if (scoreVector.size() != numScoreComponents) {
+      UTIL_THROW2("Size of scoreVector != number (" << scoreVector.size() << "!="
+                  << numScoreComponents << ") of score components on line " << count);
+    }
+
+    // Source-side
+    HyperPath sourceFragment;
+    hyperPathLoader.Load(sourceString, sourceFragment);
+    ExtractSourceTerminalSetFromHyperPath(sourceFragment, sourceTermSet);
+
+    // Target-side
+    TargetPhrase *targetPhrase = new TargetPhrase(&ff);
+    Word *targetLHS = NULL;
+    targetPhrase->CreateFromString(Output, output, targetString, &targetLHS);
+    targetPhrase->SetTargetLHS(targetLHS);
+    targetPhrase->SetAlignmentInfo(alignString);
+
+    if (++pipes) {
+      StringPiece sparseString(*pipes);
+      targetPhrase->SetSparseScore(&ff, sparseString);
+    }
+
+    if (++pipes) {
+      StringPiece propertiesString(*pipes);
+      targetPhrase->SetProperties(propertiesString);
+    }
+
+    targetPhrase->GetScoreBreakdown().Assign(&ff, scoreVector);
+    targetPhrase->EvaluateInIsolation(dummySourcePhrase,
+                                      ff.GetFeaturesToApply());
+
+    // Add rule to trie.
+    TargetPhraseCollection::shared_ptr phraseColl
+    = GetOrCreateTargetPhraseCollection(trie, sourceFragment);
+    phraseColl->Add(targetPhrase);
+
+    count++;
+  }
+
+  // sort and prune each target phrase collection
+  if (ff.GetTableLimit()) {
+    SortAndPrune(trie, ff.GetTableLimit());
+  }
+
+  return true;
+}
+
+void HyperTreeLoader::ExtractSourceTerminalSetFromHyperPath(
+  const HyperPath &hp, boost::unordered_set<std::size_t> &sourceTerminalSet)
+{
+  for (std::vector<HyperPath::NodeSeq>::const_iterator p = hp.nodeSeqs.begin();
+       p != hp.nodeSeqs.end(); ++p) {
+    for (std::vector<std::size_t>::const_iterator q = p->begin();
+         q != p->end(); ++q) {
+      const std::size_t factorId = *q;
+      if (factorId >= moses_MaxNumNonterminals &&
+          factorId != HyperPath::kComma &&
+          factorId != HyperPath::kEpsilon) {
+        sourceTerminalSet.insert(factorId);
+      }
+    }
+  }
+}
+
+}  // namespace F2S
+}  // namespace Syntax
+}  // namespace Moses

mosesdecoder/moses/Syntax/F2S/HyperTreeLoader.h

@@ -0,0 +1,41 @@
+#pragma once
+
+#include <istream>
+#include <vector>
+
+#include <boost/unordered_set.hpp>
+
+#include "moses/TypeDef.h"
+#include "moses/Syntax/RuleTableFF.h"
+
+#include "HyperPath.h"
+#include "HyperTree.h"
+#include "HyperTreeCreator.h"
+
+namespace Moses
+{
+class AllOptions;
+namespace Syntax
+{
+namespace F2S
+{
+
+class HyperTreeLoader : public HyperTreeCreator
+{
+public:
+  bool Load(AllOptions const& opts,
+            const std::vector<FactorType> &input,
+            const std::vector<FactorType> &output,
+            const std::string &inFile,
+            const RuleTableFF &,
+            HyperTree &,
+            boost::unordered_set<std::size_t> &);
+
+private:
+  void ExtractSourceTerminalSetFromHyperPath(
+    const HyperPath &, boost::unordered_set<std::size_t> &);
+};
+
+}  // namespace F2S
+}  // namespace Syntax
+}  // namespace Moses

mosesdecoder/moses/Syntax/F2S/PHyperedgeToSHyperedgeBundle.h

@@ -0,0 +1,34 @@
+#pragma once
+
+#include "moses/Syntax/PHyperedge.h"
+#include "moses/Syntax/PVertex.h"
+#include "moses/Syntax/SHyperedgeBundle.h"
+
+#include "PVertexToStackMap.h"
+
+namespace Moses
+{
+namespace Syntax
+{
+namespace F2S
+{
+
+// Given a PHyperedge object and SStackSet produces a SHyperedgeBundle object.
+inline void PHyperedgeToSHyperedgeBundle(const PHyperedge &hyperedge,
+    const PVertexToStackMap &stackMap,
+    SHyperedgeBundle &bundle)
+{
+  bundle.inputWeight = hyperedge.label.inputWeight;
+  bundle.translations = hyperedge.label.translations;
+  bundle.stacks.clear();
+  for (std::vector<PVertex*>::const_iterator p = hyperedge.tail.begin();
+       p != hyperedge.tail.end(); ++p) {
+    PVertexToStackMap::const_iterator q = stackMap.find(*p);
+    const SVertexStack &stack = q->second;
+    bundle.stacks.push_back(&stack);
+  }
+}
+
+}  // F2S
+}  // Syntax
+}  // Moses

mosesdecoder/moses/Syntax/F2S/PVertexToStackMap.h

@@ -0,0 +1,20 @@
+#pragma once
+
+#include <boost/unordered_map.hpp>
+
+#include "moses/Syntax/PVertex.h"
+#include "moses/Syntax/SVertexStack.h"
+
+
+namespace Moses
+{
+namespace Syntax
+{
+namespace F2S
+{
+
+typedef boost::unordered_map<const PVertex *, SVertexStack> PVertexToStackMap;
+
+}  // namespace F2S
+}  // namespace Syntax
+}  // namespace Moses

mosesdecoder/moses/Syntax/F2S/RuleMatcherCallback.h

@@ -0,0 +1,51 @@
+#pragma once
+
+#include "moses/Syntax/BoundedPriorityContainer.h"
+#include "moses/Syntax/PHyperedge.h"
+#include "moses/Syntax/PVertex.h"
+#include "moses/Syntax/SHyperedgeBundle.h"
+#include "moses/Syntax/SHyperedgeBundleScorer.h"
+
+#include "PHyperedgeToSHyperedgeBundle.h"
+#include "PVertexToStackMap.h"
+
+namespace Moses
+{
+namespace Syntax
+{
+namespace F2S
+{
+
+class RuleMatcherCallback
+{
+private:
+  typedef BoundedPriorityContainer<SHyperedgeBundle> Container;
+
+public:
+  RuleMatcherCallback(const PVertexToStackMap &stackMap, std::size_t ruleLimit)
+    : m_stackMap(stackMap)
+    , m_container(ruleLimit) {}
+
+  void operator()(const PHyperedge &hyperedge) {
+    PHyperedgeToSHyperedgeBundle(hyperedge, m_stackMap, m_tmpBundle);
+    float score = SHyperedgeBundleScorer::Score(m_tmpBundle);
+    m_container.SwapIn(m_tmpBundle, score);
+  }
+
+  void ClearContainer() {
+    m_container.LazyClear();
+  }
+
+  const Container &GetContainer() {
+    return m_container;
+  }
+
+private:
+  const PVertexToStackMap &m_stackMap;
+  SHyperedgeBundle m_tmpBundle;
+  BoundedPriorityContainer<SHyperedgeBundle> m_container;
+};
+
+}  // F2S
+}  // Syntax
+}  // Moses

mosesdecoder/moses/Syntax/F2S/RuleMatcherHyperTree-inl.h

@@ -0,0 +1,203 @@
+#pragma once
+
+namespace Moses
+{
+namespace Syntax
+{
+namespace F2S
+{
+
+template<typename Callback>
+RuleMatcherHyperTree<Callback>::RuleMatcherHyperTree(const HyperTree &ruleTrie)
+  : m_ruleTrie(ruleTrie)
+{
+}
+
+template<typename Callback>
+void RuleMatcherHyperTree<Callback>::EnumerateHyperedges(
+  const Forest::Vertex &v, Callback &callback)
+{
+  const HyperTree::Node &root = m_ruleTrie.GetRootNode();
+  HyperPath::NodeSeq nodeSeq(1, v.pvertex.symbol[0]->GetId());
+  const HyperTree::Node *child = root.GetChild(nodeSeq);
+  if (!child) {
+    return;
+  }
+
+  m_hyperedge.head = const_cast<PVertex*>(&v.pvertex);
+
+  // Initialize the queue.
+  MatchItem item;
+  item.annotatedFNS.fns = FNS(1, &v);
+  item.trieNode = child;
+  m_queue.push(item);
+
+  while (!m_queue.empty()) {
+    MatchItem item = m_queue.front();
+    m_queue.pop();
+    if (item.trieNode->HasRules()) {
+      const FNS &fns = item.annotatedFNS.fns;
+      // Set the output hyperedge's tail.
+      m_hyperedge.tail.clear();
+      for (FNS::const_iterator p = fns.begin(); p != fns.end(); ++p) {
+        const Forest::Vertex *v = *p;
+        m_hyperedge.tail.push_back(const_cast<PVertex *>(&(v->pvertex)));
+      }
+      // Set the output hyperedge label's input weight.
+      m_hyperedge.label.inputWeight = 0.0f;
+      for (std::vector<const Forest::Hyperedge *>::const_iterator
+           p = item.annotatedFNS.fragment.begin();
+           p != item.annotatedFNS.fragment.end(); ++p) {
+        m_hyperedge.label.inputWeight += (*p)->weight;
+      }
+      // Set the output hyperedge label's translation set pointer.
+      m_hyperedge.label.translations
+      = item.trieNode->GetTargetPhraseCollection();
+      // Pass the output hyperedge to the callback.
+      callback(m_hyperedge);
+    }
+    PropagateNextLexel(item);
+  }
+}
+
+template<typename Callback>
+void RuleMatcherHyperTree<Callback>::PropagateNextLexel(const MatchItem &item)
+{
+  std::vector<AnnotatedFNS> tfns;
+  std::vector<AnnotatedFNS> rfns;
+  std::vector<AnnotatedFNS> rfns2;
+
+  const HyperTree::Node &trieNode = *(item.trieNode);
+  const HyperTree::Node::Map &map = trieNode.GetMap();
+
+  for (HyperTree::Node::Map::const_iterator p = map.begin();
+       p != map.end(); ++p) {
+    const HyperPath::NodeSeq &edgeLabel = p->first;
+    const HyperTree::Node &child = p->second;
+
+    const int numSubSeqs = CountCommas(edgeLabel) + 1;
+
+    std::size_t pos = 0;
+    for (int i = 0; i < numSubSeqs; ++i) {
+      const FNS &fns = item.annotatedFNS.fns;
+      tfns.clear();
+      if (edgeLabel[pos] == HyperPath::kEpsilon) {
+        AnnotatedFNS x;
+        x.fns = FNS(1, fns[i]);
+        tfns.push_back(x);
+        pos += 2;
+      } else {
+        const int subSeqLength = SubSeqLength(edgeLabel, pos);
+        const std::vector<Forest::Hyperedge*> &incoming = fns[i]->incoming;
+        for (std::vector<Forest::Hyperedge *>::const_iterator q =
+               incoming.begin(); q != incoming.end(); ++q) {
+          const Forest::Hyperedge &edge = **q;
+          if (MatchChildren(edge.tail, edgeLabel, pos, subSeqLength)) {
+            tfns.resize(tfns.size()+1);
+            tfns.back().fns.assign(edge.tail.begin(), edge.tail.end());
+            tfns.back().fragment.push_back(&edge);
+          }
+        }
+        pos += subSeqLength + 1;
+      }
+      if (tfns.empty()) {
+        rfns.clear();
+        break;
+      } else if (i == 0) {
+        rfns.swap(tfns);
+      } else {
+        CartesianProduct(rfns, tfns, rfns2);
+        rfns.swap(rfns2);
+      }
+    }
+
+    for (typename std::vector<AnnotatedFNS>::const_iterator q = rfns.begin();
+         q != rfns.end(); ++q) {
+      MatchItem newItem;
+      newItem.annotatedFNS.fns = q->fns;
+      newItem.annotatedFNS.fragment = item.annotatedFNS.fragment;
+      newItem.annotatedFNS.fragment.insert(newItem.annotatedFNS.fragment.end(),
+                                           q->fragment.begin(),
+                                           q->fragment.end());
+      newItem.trieNode = &child;
+      m_queue.push(newItem);
+    }
+  }
+}
+
+template<typename Callback>
+void RuleMatcherHyperTree<Callback>::CartesianProduct(
+  const std::vector<AnnotatedFNS> &x,
+  const std::vector<AnnotatedFNS> &y,
+  std::vector<AnnotatedFNS> &z)
+{
+  z.clear();
+  z.reserve(x.size() * y.size());
+  for (typename std::vector<AnnotatedFNS>::const_iterator p = x.begin();
+       p != x.end(); ++p) {
+    const AnnotatedFNS &a = *p;
+    for (typename std::vector<AnnotatedFNS>::const_iterator q = y.begin();
+         q != y.end(); ++q) {
+      const AnnotatedFNS &b = *q;
+      // Create a new AnnotatedFNS.
+      z.resize(z.size()+1);
+      AnnotatedFNS &c = z.back();
+      // Combine frontier node sequences from a and b.
+      c.fns.reserve(a.fns.size() + b.fns.size());
+      c.fns.assign(a.fns.begin(), a.fns.end());
+      c.fns.insert(c.fns.end(), b.fns.begin(), b.fns.end());
+      // Combine tree fragments from a and b.
+      c.fragment.reserve(a.fragment.size() + b.fragment.size());
+      c.fragment.assign(a.fragment.begin(), a.fragment.end());
+      c.fragment.insert(c.fragment.end(), b.fragment.begin(), b.fragment.end());
+    }
+  }
+}
+
+template<typename Callback>
+bool RuleMatcherHyperTree<Callback>::MatchChildren(
+  const std::vector<Forest::Vertex *> &children,
+  const HyperPath::NodeSeq &edgeLabel,
+  std::size_t pos,
+  std::size_t subSeqSize)
+{
+  if (children.size() != subSeqSize) {
+    return false;
+  }
+  for (size_t i = 0; i < subSeqSize; ++i) {
+    if (edgeLabel[pos+i] != children[i]->pvertex.symbol[0]->GetId()) {
+      return false;
+    }
+  }
+  return true;
+}
+
+template<typename Callback>
+int RuleMatcherHyperTree<Callback>::CountCommas(const HyperPath::NodeSeq &seq)
+{
+  int count = 0;
+  for (std::vector<std::size_t>::const_iterator p = seq.begin();
+       p != seq.end(); ++p) {
+    if (*p == HyperPath::kComma) {
+      ++count;
+    }
+  }
+  return count;
+}
+
+template<typename Callback>
+int RuleMatcherHyperTree<Callback>::SubSeqLength(const HyperPath::NodeSeq &seq,
+    int pos)
+{
+  int length = 0;
+  HyperPath::NodeSeq::size_type curpos = pos;
+  while (curpos != seq.size() && seq[curpos] != HyperPath::kComma) {
+    ++curpos;
+    ++length;
+  }
+  return length;
+}
+
+}  // namespace F2S
+}  // namespace Syntax
+}  // namespace Moses

mosesdecoder/moses/Syntax/F2S/RuleMatcherHyperTree.h

@@ -0,0 +1,78 @@
+#pragma once
+
+#include "moses/Syntax/PHyperedge.h"
+
+#include "Forest.h"
+#include "HyperTree.h"
+#include "RuleMatcher.h"
+
+namespace Moses
+{
+namespace Syntax
+{
+namespace F2S
+{
+
+// Rule matcher based on the algorithm from this paper:
+//
+//  Hui Zhang, Min Zhang, Haizhou Li, and Chew Lim Tan
+//  "Fast Translation Rule Matching for Syntax-based Statistical Machine
+//   Translation"
+//  In proceedings of EMNLP 2009
+//
+template<typename Callback>
+class RuleMatcherHyperTree : public RuleMatcher<Callback>
+{
+public:
+  RuleMatcherHyperTree(const HyperTree &);
+
+  ~RuleMatcherHyperTree() {}
+
+  void EnumerateHyperedges(const Forest::Vertex &, Callback &);
+
+private:
+  // Frontier node sequence.
+  typedef std::vector<const Forest::Vertex *> FNS;
+
+  // An AnnotatedFNS is a FNS annotated with the set of forest hyperedges that
+  // constitute the tree fragment from which it was derived.
+  struct AnnotatedFNS {
+    FNS fns;
+    std::vector<const Forest::Hyperedge *> fragment;
+  };
+
+  // A MatchItem is like the FP structure in Zhang et al. (2009), but it also
+  // records the set of forest hyperedges that constitute the matched tree
+  // fragment.
+  struct MatchItem {
+    AnnotatedFNS annotatedFNS;
+    const HyperTree::Node *trieNode;
+  };
+
+  // Implements the Cartsian product operation from line 16 of Algorithm 4
+  // (Zhang et al., 2009), which in this implementation also involves
+  // combining the fragment information associated with the FNS objects.
+  void CartesianProduct(const std::vector<AnnotatedFNS> &,
+                        const std::vector<AnnotatedFNS> &,
+                        std::vector<AnnotatedFNS> &);
+
+  int CountCommas(const HyperPath::NodeSeq &);
+
+  bool MatchChildren(const std::vector<Forest::Vertex *> &,
+                     const HyperPath::NodeSeq &, std::size_t, std::size_t);
+
+  void PropagateNextLexel(const MatchItem &);
+
+  int SubSeqLength(const HyperPath::NodeSeq &, int);
+
+  const HyperTree &m_ruleTrie;
+  PHyperedge m_hyperedge;
+  std::queue<MatchItem> m_queue;  // Called "SFP" in Zhang et al. (2009)
+};
+
+}  // namespace F2S
+}  // namespace Syntax
+}  // namespace Moses
+
+// Implementation
+#include "RuleMatcherHyperTree-inl.h"

mosesdecoder/moses/Syntax/F2S/TopologicalSorter.cpp

@@ -0,0 +1,55 @@
+#include "TopologicalSorter.h"
+
+namespace Moses
+{
+namespace Syntax
+{
+namespace F2S
+{
+
+void TopologicalSorter::Sort(const Forest &forest,
+                             std::vector<const Forest::Vertex *> &permutation)
+{
+  permutation.clear();
+  BuildPredSets(forest);
+  m_visited.clear();
+  for (std::vector<Forest::Vertex *>::const_iterator
+       p = forest.vertices.begin(); p != forest.vertices.end(); ++p) {
+    if (m_visited.find(*p) == m_visited.end()) {
+      Visit(**p, permutation);
+    }
+  }
+}
+
+void TopologicalSorter::BuildPredSets(const Forest &forest)
+{
+  m_predSets.clear();
+  for (std::vector<Forest::Vertex *>::const_iterator
+       p = forest.vertices.begin(); p != forest.vertices.end(); ++p) {
+    const Forest::Vertex *head = *p;
+    for (std::vector<Forest::Hyperedge *>::const_iterator
+         q = head->incoming.begin(); q != head->incoming.end(); ++q) {
+      for (std::vector<Forest::Vertex *>::const_iterator
+           r = (*q)->tail.begin(); r != (*q)->tail.end(); ++r) {
+        m_predSets[head].insert(*r);
+      }
+    }
+  }
+}
+
+void TopologicalSorter::Visit(const Forest::Vertex &v,
+                              std::vector<const Forest::Vertex *> &permutation)
+{
+  m_visited.insert(&v);
+  const VertexSet &predSet = m_predSets[&v];
+  for (VertexSet::const_iterator p = predSet.begin(); p != predSet.end(); ++p) {
+    if (m_visited.find(*p) == m_visited.end()) {
+      Visit(**p, permutation);
+    }
+  }
+  permutation.push_back(&v);
+}
+
+}  // namespace F2S
+}  // namespace Syntax
+}  // namespace Moses

mosesdecoder/moses/Syntax/F2S/TopologicalSorter.h

@@ -0,0 +1,34 @@
+#pragma once
+
+#include <vector>
+
+#include <boost/unordered_map.hpp>
+#include <boost/unordered_set.hpp>
+
+#include "Forest.h"
+
+namespace Moses
+{
+namespace Syntax
+{
+namespace F2S
+{
+
+class TopologicalSorter
+{
+public:
+  void Sort(const Forest &, std::vector<const Forest::Vertex *> &);
+
+private:
+  typedef boost::unordered_set<const Forest::Vertex *> VertexSet;
+
+  void BuildPredSets(const Forest &);
+  void Visit(const Forest::Vertex &, std::vector<const Forest::Vertex *> &);
+
+  boost::unordered_set<const Forest::Vertex *> m_visited;
+  boost::unordered_map<const Forest::Vertex *, VertexSet> m_predSets;
+};
+
+}  // namespace F2S
+}  // namespace Syntax
+}  // namespace Moses

mosesdecoder/moses/Syntax/F2S/TreeFragmentTokenizer.cpp

@@ -0,0 +1,100 @@
+#include "TreeFragmentTokenizer.h"
+
+#include <cctype>
+
+namespace Moses
+{
+namespace Syntax
+{
+namespace F2S
+{
+
+TreeFragmentToken::TreeFragmentToken(TreeFragmentTokenType t,
+                                     StringPiece v, std::size_t p)
+  : type(t)
+  , value(v)
+  , pos(p)
+{
+}
+
+TreeFragmentTokenizer::TreeFragmentTokenizer()
+  : value_(TreeFragmentToken_EOS, "", -1)
+{
+}
+
+TreeFragmentTokenizer::TreeFragmentTokenizer(const StringPiece &s)
+  : str_(s)
+  , value_(TreeFragmentToken_EOS, "", -1)
+  , iter_(s.begin())
+  , end_(s.end())
+  , pos_(0)
+{
+  ++(*this);
+}
+
+TreeFragmentTokenizer &TreeFragmentTokenizer::operator++()
+{
+  while (iter_ != end_ && (*iter_ == ' ' || *iter_ == '\t')) {
+    ++iter_;
+    ++pos_;
+  }
+
+  if (iter_ == end_) {
+    value_ = TreeFragmentToken(TreeFragmentToken_EOS, "", pos_);
+    return *this;
+  }
+
+  if (*iter_ == '[') {
+    value_ = TreeFragmentToken(TreeFragmentToken_LSB, "[", pos_);
+    ++iter_;
+    ++pos_;
+  } else if (*iter_ == ']') {
+    value_ = TreeFragmentToken(TreeFragmentToken_RSB, "]", pos_);
+    ++iter_;
+    ++pos_;
+  } else {
+    std::size_t start = pos_;
+    while (true) {
+      ++iter_;
+      ++pos_;
+      if (iter_ == end_ || *iter_ == ' ' || *iter_ == '\t') {
+        break;
+      }
+      if (*iter_ == '[' || *iter_ == ']') {
+        break;
+      }
+    }
+    StringPiece word = str_.substr(start, pos_-start);
+    value_ = TreeFragmentToken(TreeFragmentToken_WORD, word, start);
+  }
+
+  return *this;
+}
+
+TreeFragmentTokenizer TreeFragmentTokenizer::operator++(int)
+{
+  TreeFragmentTokenizer tmp(*this);
+  ++*this;
+  return tmp;
+}
+
+bool operator==(const TreeFragmentTokenizer &lhs,
+                const TreeFragmentTokenizer &rhs)
+{
+  if (lhs.value_.type == TreeFragmentToken_EOS ||
+      rhs.value_.type == TreeFragmentToken_EOS) {
+    return lhs.value_.type == TreeFragmentToken_EOS &&
+           rhs.value_.type == TreeFragmentToken_EOS;
+  }
+  return lhs.iter_ == rhs.iter_;
+}
+
+bool operator!=(const TreeFragmentTokenizer &lhs,
+                const TreeFragmentTokenizer &rhs)
+{
+  return !(lhs == rhs);
+}
+
+}  // namespace F2S
+}  // namespace Syntax
+}  // namespace Moses

mosesdecoder/moses/Syntax/Manager.h

@@ -0,0 +1,70 @@
+#pragma once
+
+#include <boost/unordered_set.hpp>
+#include "moses/InputType.h"
+#include "moses/BaseManager.h"
+
+#include "KBestExtractor.h"
+
+namespace Moses
+{
+namespace Syntax
+{
+
+// Common base class for Moses::Syntax managers.
+class Manager : public BaseManager
+{
+public:
+  Manager(ttasksptr const& ttask);
+
+  // Virtual functions from Moses::BaseManager that are implemented the same
+  // way for all Syntax managers.
+  void OutputBest(OutputCollector *collector) const;
+  void OutputNBest(OutputCollector *collector) const;
+  void OutputUnknowns(OutputCollector *collector) const;
+
+  // Virtual functions from Moses::BaseManager that are no-ops for all Syntax
+  // managers.
+  void OutputAlignment(OutputCollector *collector) const {}
+  void OutputDetailedTreeFragmentsTranslationReport(
+    OutputCollector *collector) const {}
+  void OutputLatticeSamples(OutputCollector *collector) const {}
+  void OutputSearchGraph(OutputCollector *collector) const {}
+  // void OutputSearchGraphHypergraph() const {}
+
+  void
+  OutputSearchGraphAsHypergraph
+  ( std::string const& fname, size_t const precision ) const
+  { }
+
+  void OutputSearchGraphSLF() const {}
+  void OutputWordGraph(OutputCollector *collector) const {}
+  void OutputDetailedTranslationReport(OutputCollector *collector) const {}
+
+  void CalcDecoderStatistics() const {}
+
+  // Syntax-specific virtual functions that derived classes must implement.
+  virtual void ExtractKBest(
+    std::size_t k,
+    std::vector<boost::shared_ptr<KBestExtractor::Derivation> > &kBestList,
+    bool onlyDistinct=false) const = 0;
+  virtual const SHyperedge *GetBestSHyperedge() const = 0;
+
+protected:
+  boost::unordered_set<Word> m_oovs;
+
+private:
+  // Syntax-specific helper functions used to implement OutputNBest.
+  void OutputNBestList(OutputCollector *collector,
+                       const KBestExtractor::KBestVec &nBestList,
+                       long translationId) const;
+
+  std::size_t OutputAlignmentNBest(Alignments &retAlign,
+                                   const KBestExtractor::Derivation &d,
+                                   std::size_t startTarget) const;
+
+  std::size_t CalcSourceSize(const KBestExtractor::Derivation &d) const;
+};
+
+}  // Syntax
+}  // Moses

mosesdecoder/moses/Syntax/PVertex.h

@@ -0,0 +1,25 @@
+#pragma once
+
+#include "moses/Word.h"
+#include "moses/Range.h"
+
+namespace Moses
+{
+namespace Syntax
+{
+
+struct PVertex {
+public:
+  PVertex(const Range &wr, const Word &w) : span(wr), symbol(w) {}
+
+  Range span;
+  Word symbol;
+};
+
+inline bool operator==(const PVertex &v, const PVertex &w)
+{
+  return v.span == w.span && v.symbol == w.symbol;
+}
+
+}  // Syntax
+}  // Moses

mosesdecoder/moses/Syntax/RuleTable.h

@@ -0,0 +1,24 @@
+#pragma once
+
+namespace Moses
+{
+namespace Syntax
+{
+
+class RuleTableFF;
+
+// Base class for any data structure representing a synchronous
+// grammar, like a trie (for S2T) or a DFA (for T2S).
+class RuleTable
+{
+public:
+  RuleTable(const RuleTableFF *ff) : m_ff(ff) {}
+
+  virtual ~RuleTable() {}
+
+protected:
+  const RuleTableFF *m_ff;
+};
+
+}  // Syntax
+}  // Moses