Upload 235 files

.gitattributes

@@ -126,3 +126,21 @@ mosesdecoder/phrase-extract/syntax-common/bin/gcc-9/release/link-static/threadin
 mosesdecoder/phrase-extract/syntax-common/bin/gcc-9/release/link-static/threading-multi/tree_test filter=lfs diff=lfs merge=lfs -text
 mosesdecoder/probingpt/bin/gcc-9/release/link-static/threading-multi/CreateProbingPT filter=lfs diff=lfs merge=lfs -text
 mosesdecoder/probingpt/bin/gcc-9/release/link-static/threading-multi/libprobingpt.a filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/util/bin/file_piece_test.test/gcc-9/release/link-static/threading-multi/file_piece_test filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/util/bin/gcc-9/release/link-static/threading-multi/bit_packing_test filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/util/bin/gcc-9/release/link-static/threading-multi/integer_to_string_test filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/util/bin/gcc-9/release/link-static/threading-multi/joint_sort_test filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/util/bin/gcc-9/release/link-static/threading-multi/multi_intersection_test filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/util/bin/gcc-9/release/link-static/threading-multi/pcqueue_test filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/util/bin/gcc-9/release/link-static/threading-multi/probing_hash_table_test filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/util/bin/gcc-9/release/link-static/threading-multi/random_test filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/util/bin/gcc-9/release/link-static/threading-multi/sized_iterator_test filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/util/bin/gcc-9/release/link-static/threading-multi/sorted_uniform_test filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/util/bin/gcc-9/release/link-static/threading-multi/string_stream_test filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/util/bin/gcc-9/release/link-static/threading-multi/tempfile_test filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/util/bin/gcc-9/release/link-static/threading-multi/tokenize_piece_test filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/util/bin/gcc-9/release/link-static/threading-multi/tokenize_test filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/util/stream/bin/gcc-9/release/link-static/threading-multi/io_test filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/util/stream/bin/gcc-9/release/link-static/threading-multi/rewindable_stream_test filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/util/stream/bin/gcc-9/release/link-static/threading-multi/sort_test filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/util/stream/bin/gcc-9/release/link-static/threading-multi/stream_test filter=lfs diff=lfs merge=lfs -text

mosesdecoder/search/Jamfile

@@ -0,0 +1 @@
+fakelib search : edge_generator.cc nbest.cc rule.cc vertex.cc ../lm//kenlm ../util//kenutil /top//boost_system : : : <include>.. ;

mosesdecoder/search/applied.hh

@@ -0,0 +1,87 @@
+#ifndef SEARCH_APPLIED__
+#define SEARCH_APPLIED__
+
+#include "search/edge.hh"
+#include "search/header.hh"
+#include "util/pool.hh"
+
+#include <cmath>
+
+namespace search {
+
+// A full hypothesis: a score, arity of the rule, a pointer to the decoder's rule (Note), and pointers to non-terminals that were substituted.
+template <class Below> class GenericApplied : public Header {
+  public:
+    GenericApplied() {}
+
+    GenericApplied(void *location, PartialEdge partial)
+      : Header(location) {
+      memcpy(Base(), partial.Base(), kHeaderSize);
+      Below *child_out = Children();
+      const PartialVertex *part = partial.NT();
+      const PartialVertex *const part_end_loop = part + partial.GetArity();
+      for (; part != part_end_loop; ++part, ++child_out)
+        *child_out = Below(part->End());
+    }
+
+    GenericApplied(void *location, Score score, Arity arity, Note note, Moses::Range range) : Header(location, arity) {
+      SetScore(score);
+      SetNote(note);
+      SetRange(range);
+    }
+
+    explicit GenericApplied(History from) : Header(from) {}
+
+
+    // These are arrays of length GetArity().
+    Below *Children() {
+      return reinterpret_cast<Below*>(After());
+    }
+    const Below *Children() const {
+      return reinterpret_cast<const Below*>(After());
+    }
+
+    static std::size_t Size(Arity arity) {
+      return kHeaderSize + arity * sizeof(const Below);
+    }
+};
+
+// Applied rule that references itself.
+class Applied : public GenericApplied<Applied> {
+  private:
+    typedef GenericApplied<Applied> P;
+
+  public:
+    Applied() {}
+    Applied(void *location, PartialEdge partial) : P(location, partial) {}
+    Applied(History from) : P(from) {}
+};
+
+// How to build single-best hypotheses.
+class SingleBest {
+  public:
+    typedef PartialEdge Combine;
+
+    void Add(PartialEdge &existing, PartialEdge add) const {
+      if (!existing.Valid() || existing.GetScore() < add.GetScore())
+        existing = add;
+    }
+
+    NBestComplete Complete(PartialEdge partial) {
+      if (!partial.Valid())
+        return NBestComplete(NULL, lm::ngram::ChartState(), -INFINITY);
+      void *place_final = pool_.Allocate(Applied::Size(partial.GetArity()));
+      Applied(place_final, partial);
+      return NBestComplete(
+          place_final,
+          partial.CompletedState(),
+          partial.GetScore());
+    }
+
+  private:
+    util::Pool pool_;
+};
+
+} // namespace search
+
+#endif // SEARCH_APPLIED__

mosesdecoder/search/config.hh

@@ -0,0 +1,38 @@
+#ifndef SEARCH_CONFIG__
+#define SEARCH_CONFIG__
+
+#include "search/types.hh"
+
+namespace search {
+
+struct NBestConfig {
+  explicit NBestConfig(unsigned int in_size) {
+    keep = in_size;
+    size = in_size;
+  }
+
+  unsigned int keep, size;
+};
+
+class Config {
+  public:
+    Config(Score lm_weight, unsigned int pop_limit, const NBestConfig &nbest) :
+      lm_weight_(lm_weight), pop_limit_(pop_limit), nbest_(nbest) {}
+
+    Score LMWeight() const { return lm_weight_; }
+
+    unsigned int PopLimit() const { return pop_limit_; }
+
+    const NBestConfig &GetNBest() const { return nbest_; }
+
+  private:
+    Score lm_weight_;
+
+    unsigned int pop_limit_;
+
+    NBestConfig nbest_;
+};
+
+} // namespace search
+
+#endif // SEARCH_CONFIG__

mosesdecoder/search/context.hh

@@ -0,0 +1,49 @@
+#ifndef SEARCH_CONTEXT__
+#define SEARCH_CONTEXT__
+
+#include "search/config.hh"
+#include "search/vertex.hh"
+
+#include <boost/pool/object_pool.hpp>
+
+namespace search {
+
+class ContextBase {
+  public:
+    explicit ContextBase(const Config &config) : config_(config) {}
+
+    VertexNode *NewVertexNode() {
+      VertexNode *ret = vertex_node_pool_.construct();
+      assert(ret);
+      return ret;
+    }
+
+    void DeleteVertexNode(VertexNode *node) {
+      vertex_node_pool_.destroy(node);
+    }
+
+    unsigned int PopLimit() const { return config_.PopLimit(); }
+
+    Score LMWeight() const { return config_.LMWeight(); }
+
+    const Config &GetConfig() const { return config_; }
+
+  private:
+    boost::object_pool<VertexNode> vertex_node_pool_;
+
+    Config config_;
+};
+
+template <class Model> class Context : public ContextBase {
+  public:
+    Context(const Config &config, const Model &model) : ContextBase(config), model_(model) {}
+
+    const Model &LanguageModel() const { return model_; }
+
+  private:
+    const Model &model_;
+};
+
+} // namespace search
+
+#endif // SEARCH_CONTEXT__

mosesdecoder/search/edge.hh

@@ -0,0 +1,54 @@
+#ifndef SEARCH_EDGE__
+#define SEARCH_EDGE__
+
+#include "lm/state.hh"
+#include "search/header.hh"
+#include "search/types.hh"
+#include "search/vertex.hh"
+#include "util/pool.hh"
+
+#include <functional>
+
+#include <stdint.h>
+
+namespace search {
+
+// Copyable, but the copy will be shallow.
+class PartialEdge : public Header {
+  public:
+    // Allow default construction for STL.
+    PartialEdge() {}
+
+    PartialEdge(util::Pool &pool, Arity arity)
+      : Header(pool.Allocate(Size(arity, arity + 1)), arity) {}
+
+    PartialEdge(util::Pool &pool, Arity arity, Arity chart_states)
+      : Header(pool.Allocate(Size(arity, chart_states)), arity) {}
+
+    // Non-terminals
+    const PartialVertex *NT() const {
+      return reinterpret_cast<const PartialVertex*>(After());
+    }
+    PartialVertex *NT() {
+      return reinterpret_cast<PartialVertex*>(After());
+    }
+
+    const lm::ngram::ChartState &CompletedState() const {
+      return *Between();
+    }
+    const lm::ngram::ChartState *Between() const {
+      return reinterpret_cast<const lm::ngram::ChartState*>(After() + GetArity() * sizeof(PartialVertex));
+    }
+    lm::ngram::ChartState *Between() {
+      return reinterpret_cast<lm::ngram::ChartState*>(After() + GetArity() * sizeof(PartialVertex));
+    }
+
+  private:
+    static std::size_t Size(Arity arity, Arity chart_states) {
+      return kHeaderSize + arity * sizeof(PartialVertex) + chart_states * sizeof(lm::ngram::ChartState);
+    }
+};
+
+
+} // namespace search
+#endif // SEARCH_EDGE__

mosesdecoder/search/edge_generator.cc

@@ -0,0 +1,116 @@
+#include "search/edge_generator.hh"
+
+#include "lm/left.hh"
+#include "lm/model.hh"
+#include "lm/partial.hh"
+#include "search/context.hh"
+#include "search/vertex.hh"
+
+#include <numeric>
+
+namespace search {
+
+namespace {
+
+template <class Model> void FastScore(const Context<Model> &context, Arity victim, Arity before_idx, Arity incomplete, const PartialVertex &previous_vertex, PartialEdge update) {
+  lm::ngram::ChartState *between = update.Between();
+  lm::ngram::ChartState *before = &between[before_idx], *after = &between[before_idx + 1];
+
+  float adjustment = 0.0;
+  const lm::ngram::ChartState &previous_reveal = previous_vertex.State();
+  const PartialVertex &update_nt = update.NT()[victim];
+  const lm::ngram::ChartState &update_reveal = update_nt.State();
+  if ((update_reveal.left.length > previous_reveal.left.length) || (update_reveal.left.full && !previous_reveal.left.full)) {
+    adjustment += lm::ngram::RevealAfter(context.LanguageModel(), before->left, before->right, update_reveal.left, previous_reveal.left.length);
+  }
+  if ((update_reveal.right.length > previous_reveal.right.length) || (update_nt.RightFull() && !previous_vertex.RightFull())) {
+    adjustment += lm::ngram::RevealBefore(context.LanguageModel(), update_reveal.right, previous_reveal.right.length, update_nt.RightFull(), after->left, after->right);
+  }
+  if (update_nt.Complete()) {
+    if (update_reveal.left.full) {
+      before->left.full = true;
+    } else {
+      assert(update_reveal.left.length == update_reveal.right.length);
+      adjustment += lm::ngram::Subsume(context.LanguageModel(), before->left, before->right, after->left, after->right, update_reveal.left.length);
+    }
+    before->right = after->right;
+    // Shift the others shifted one down, covering after.
+    for (lm::ngram::ChartState *cover = after; cover < between + incomplete; ++cover) {
+      *cover = *(cover + 1);
+    }
+  }
+  update.SetScore(update.GetScore() + adjustment * context.LMWeight());
+}
+
+} // namespace
+
+template <class Model> PartialEdge EdgeGenerator::Pop(Context<Model> &context) {
+  assert(!generate_.empty());
+  PartialEdge top = generate_.top();
+  generate_.pop();
+  PartialVertex *const top_nt = top.NT();
+  const Arity arity = top.GetArity();
+
+  Arity victim = 0;
+  Arity victim_completed;
+  Arity incomplete;
+  unsigned char lowest_niceness = 255;
+  // Select victim or return if complete.
+  {
+    Arity completed = 0;
+    for (Arity i = 0; i != arity; ++i) {
+      if (top_nt[i].Complete()) {
+        ++completed;
+      } else if (top_nt[i].Niceness() < lowest_niceness) {
+        lowest_niceness = top_nt[i].Niceness();
+        victim = i;
+        victim_completed = completed;
+      }
+    }
+    if (lowest_niceness == 255) {
+      return top;
+    }
+    incomplete = arity - completed;
+  }
+
+  PartialVertex old_value(top_nt[victim]);
+  PartialVertex alternate_changed;
+  if (top_nt[victim].Split(alternate_changed)) {
+    PartialEdge alternate(partial_edge_pool_, arity, incomplete + 1);
+    alternate.SetScore(top.GetScore() + alternate_changed.Bound() - old_value.Bound());
+
+    alternate.SetNote(top.GetNote());
+    alternate.SetRange(top.GetRange());
+
+    PartialVertex *alternate_nt = alternate.NT();
+    for (Arity i = 0; i < victim; ++i) alternate_nt[i] = top_nt[i];
+    alternate_nt[victim] = alternate_changed;
+    for (Arity i = victim + 1; i < arity; ++i) alternate_nt[i] = top_nt[i];
+
+    memcpy(alternate.Between(), top.Between(), sizeof(lm::ngram::ChartState) * (incomplete + 1));
+
+    // TODO: dedupe?
+    generate_.push(alternate);
+  }
+
+#ifndef NDEBUG
+  Score before = top.GetScore();
+#endif
+  // top is now the continuation.
+  FastScore(context, victim, victim - victim_completed, incomplete, old_value, top);
+  // TODO: dedupe?
+  generate_.push(top);
+  assert(lowest_niceness != 254 || top.GetScore() == before);
+
+  // Invalid indicates no new hypothesis generated.
+  return PartialEdge();
+}
+
+template PartialEdge EdgeGenerator::Pop(Context<lm::ngram::RestProbingModel> &context);
+template PartialEdge EdgeGenerator::Pop(Context<lm::ngram::ProbingModel> &context);
+template PartialEdge EdgeGenerator::Pop(Context<lm::ngram::TrieModel> &context);
+template PartialEdge EdgeGenerator::Pop(Context<lm::ngram::QuantTrieModel> &context);
+template PartialEdge EdgeGenerator::Pop(Context<lm::ngram::ArrayTrieModel> &context);
+template PartialEdge EdgeGenerator::Pop(Context<lm::ngram::QuantArrayTrieModel> &context);
+
+} // namespace search

mosesdecoder/search/edge_generator.hh

@@ -0,0 +1,56 @@
+#ifndef SEARCH_EDGE_GENERATOR__
+#define SEARCH_EDGE_GENERATOR__
+
+#include "search/edge.hh"
+#include "search/types.hh"
+
+#include <queue>
+
+namespace lm {
+namespace ngram {
+struct ChartState;
+} // namespace ngram
+} // namespace lm
+
+namespace search {
+
+template <class Model> class Context;
+
+class EdgeGenerator {
+  public:
+    EdgeGenerator() {}
+
+    PartialEdge AllocateEdge(Arity arity) {
+      return PartialEdge(partial_edge_pool_, arity);
+    }
+
+    void AddEdge(PartialEdge edge) {
+      generate_.push(edge);
+    }
+
+    bool Empty() const { return generate_.empty(); }
+
+    // Pop.  If there's a complete hypothesis, return it.  Otherwise return an invalid PartialEdge.
+    template <class Model> PartialEdge Pop(Context<Model> &context);
+
+    template <class Model, class Output> void Search(Context<Model> &context, Output &output) {
+      unsigned to_pop = context.PopLimit();
+      while (to_pop > 0 && !generate_.empty()) {
+        PartialEdge got(Pop(context));
+        if (got.Valid()) {
+          output.NewHypothesis(got);
+          --to_pop;
+        }
+      }
+      output.FinishedSearch();
+    }
+
+  private:
+    util::Pool partial_edge_pool_;
+
+    typedef std::priority_queue<PartialEdge> Generate;
+    Generate generate_;
+};
+
+} // namespace search
+#endif // SEARCH_EDGE_GENERATOR__

mosesdecoder/search/header.hh

@@ -0,0 +1,72 @@
+#ifndef SEARCH_HEADER__
+#define SEARCH_HEADER__
+
+// Header consisting of Score, Arity, Note and Range
+
+#include "search/types.hh"
+#include "moses/Range.h"
+
+#include <stdint.h>
+
+namespace search {
+
+// Copying is shallow.
+class Header {
+  public:
+    bool Valid() const { return base_; }
+
+    Score GetScore() const {
+      return *reinterpret_cast<const float*>(base_);
+    }
+    void SetScore(Score to) {
+      *reinterpret_cast<float*>(base_) = to;
+    }
+    bool operator<(const Header &other) const {
+      return GetScore() < other.GetScore();
+    }
+    bool operator>(const Header &other) const {
+      return GetScore() > other.GetScore();
+    }
+
+    Arity GetArity() const {
+      return *reinterpret_cast<const Arity*>(base_ + sizeof(Score));
+    }
+
+    Note GetNote() const {
+      return *reinterpret_cast<const Note*>(base_ + sizeof(Score) + sizeof(Arity));
+    }
+    void SetNote(Note to) {
+      *reinterpret_cast<Note*>(base_ + sizeof(Score) + sizeof(Arity)) = to;
+    }
+
+    Moses::Range GetRange() const {
+      return *reinterpret_cast<const Moses::Range*>(base_ + sizeof(Score) + sizeof(Arity) + sizeof(Note));
+    }
+    void SetRange(Moses::Range to) {
+      *reinterpret_cast<Moses::Range*>(base_ + sizeof(Score) + sizeof(Arity) + sizeof(Note)) = to;
+    }
+
+    uint8_t *Base() { return base_; }
+    const uint8_t *Base() const { return base_; }
+
+  protected:
+    Header() : base_(NULL) {}
+
+    explicit Header(void *base) : base_(static_cast<uint8_t*>(base)) {}
+
+    Header(void *base, Arity arity) : base_(static_cast<uint8_t*>(base)) {
+      *reinterpret_cast<Arity*>(base_ + sizeof(Score)) = arity;
+    }
+
+    static const std::size_t kHeaderSize = sizeof(Score) + sizeof(Arity) + sizeof(Note) + sizeof(Moses::Range);
+
+    uint8_t *After() { return base_ + kHeaderSize; }
+    const uint8_t *After() const { return base_ + kHeaderSize; }
+
+  private:
+    uint8_t *base_;
+};
+
+} // namespace search
+
+#endif // SEARCH_HEADER__

mosesdecoder/search/nbest.cc

@@ -0,0 +1,106 @@
+#include "search/nbest.hh"
+
+#include "util/pool.hh"
+#include "moses/Util.h"
+
+#include <algorithm>
+#include <functional>
+#include <queue>
+#include <cassert>
+#include <cmath>
+
+namespace search {
+
+NBestList::NBestList(std::vector<PartialEdge> &partials, util::Pool &entry_pool, std::size_t keep) {
+  assert(!partials.empty());
+  std::vector<PartialEdge>::iterator end;
+  if (partials.size() > keep) {
+    end = partials.begin() + keep;
+    NTH_ELEMENT4(partials.begin(), end, partials.end(), std::greater<PartialEdge>());
+  } else {
+    end = partials.end();
+  }
+  for (std::vector<PartialEdge>::const_iterator i(partials.begin()); i != end; ++i) {
+    queue_.push(QueueEntry(entry_pool.Allocate(QueueEntry::Size(i->GetArity())), *i));
+  }
+}
+
+Score NBestList::TopAfterConstructor() const {
+  assert(revealed_.empty());
+  return queue_.top().GetScore();
+}
+
+const std::vector<Applied> &NBestList::Extract(util::Pool &pool, std::size_t n) {
+  while (revealed_.size() < n && !queue_.empty()) {
+    MoveTop(pool);
+  }
+  return revealed_;
+}
+
+Score NBestList::Visit(util::Pool &pool, std::size_t index) {
+  if (index + 1 < revealed_.size())
+    return revealed_[index + 1].GetScore() - revealed_[index].GetScore();
+  if (queue_.empty())
+    return -INFINITY;
+  if (index + 1 == revealed_.size())
+    return queue_.top().GetScore() - revealed_[index].GetScore();
+  assert(index == revealed_.size());
+
+  MoveTop(pool);
+
+  if (queue_.empty()) return -INFINITY;
+  return queue_.top().GetScore() - revealed_[index].GetScore();
+}
+
+Applied NBestList::Get(util::Pool &pool, std::size_t index) {
+  assert(index <= revealed_.size());
+  if (index == revealed_.size()) MoveTop(pool);
+  return revealed_[index];
+}
+
+void NBestList::MoveTop(util::Pool &pool) {
+  assert(!queue_.empty());
+  QueueEntry entry(queue_.top());
+  queue_.pop();
+  RevealedRef *const children_begin = entry.Children();
+  RevealedRef *const children_end = children_begin + entry.GetArity();
+  Score basis = entry.GetScore();
+  for (RevealedRef *child = children_begin; child != children_end; ++child) {
+    Score change = child->in_->Visit(pool, child->index_);
+    if (change != -INFINITY) {
+      assert(change < 0.001);
+      QueueEntry new_entry(pool.Allocate(QueueEntry::Size(entry.GetArity())), basis + change, entry.GetArity(), entry.GetNote(), entry.GetRange());
+      std::copy(children_begin, child, new_entry.Children());
+      RevealedRef *update = new_entry.Children() + (child - children_begin);
+      update->in_ = child->in_;
+      update->index_ = child->index_ + 1;
+      std::copy(child + 1, children_end, update + 1);
+      queue_.push(new_entry);
+    }
+    // Gesmundo, A. and Henderson, J. Faster Cube Pruning, IWSLT 2010.
+    if (child->index_) break;
+  }
+
+  // Convert QueueEntry to Applied.  This leaves some unused memory.
+  void *overwrite = entry.Children();
+  for (unsigned int i = 0; i < entry.GetArity(); ++i) {
+    RevealedRef from(*(static_cast<const RevealedRef*>(overwrite) + i));
+    *(static_cast<Applied*>(overwrite) + i) = from.in_->Get(pool, from.index_);
+  }
+  revealed_.push_back(Applied(entry.Base()));
+}
+
+NBestComplete NBest::Complete(std::vector<PartialEdge> &partials) {
+  assert(!partials.empty());
+  NBestList *list = list_pool_.construct(partials, entry_pool_, config_.keep);
+  return NBestComplete(
+      list,
+      partials.front().CompletedState(), // All partials have the same state
+      list->TopAfterConstructor());
+}
+
+const std::vector<Applied> &NBest::Extract(History history) {
+  return static_cast<NBestList*>(history)->Extract(entry_pool_, config_.size);
+}
+
+} // namespace search

mosesdecoder/search/nbest.hh

@@ -0,0 +1,80 @@
+#ifndef SEARCH_NBEST__
+#define SEARCH_NBEST__
+
+#include "search/applied.hh"
+#include "search/config.hh"
+#include "search/edge.hh"
+
+#include <boost/pool/object_pool.hpp>
+
+#include <cstddef>
+#include <queue>
+#include <vector>
+#include <cassert>
+
+namespace search {
+
+class NBestList;
+
+class NBestList {
+  private:
+    class RevealedRef {
+      public:
+        explicit RevealedRef(History history)
+          : in_(static_cast<NBestList*>(history)), index_(0) {}
+
+      private:
+        friend class NBestList;
+
+        NBestList *in_;
+        std::size_t index_;
+    };
+
+    typedef GenericApplied<RevealedRef> QueueEntry;
+
+  public:
+    NBestList(std::vector<PartialEdge> &existing, util::Pool &entry_pool, std::size_t keep);
+
+    Score TopAfterConstructor() const;
+
+    const std::vector<Applied> &Extract(util::Pool &pool, std::size_t n);
+
+  private:
+    Score Visit(util::Pool &pool, std::size_t index);
+
+    Applied Get(util::Pool &pool, std::size_t index);
+
+    void MoveTop(util::Pool &pool);
+
+    typedef std::vector<Applied> Revealed;
+    Revealed revealed_;
+
+    typedef std::priority_queue<QueueEntry> Queue;
+    Queue queue_;
+};
+
+class NBest {
+  public:
+    typedef std::vector<PartialEdge> Combine;
+
+    explicit NBest(const NBestConfig &config) : config_(config) {}
+
+    void Add(std::vector<PartialEdge> &existing, PartialEdge addition) const {
+      existing.push_back(addition);
+    }
+
+    NBestComplete Complete(std::vector<PartialEdge> &partials);
+
+    const std::vector<Applied> &Extract(History root);
+
+  private:
+    const NBestConfig config_;
+
+    boost::object_pool<NBestList> list_pool_;
+
+    util::Pool entry_pool_;
+};
+
+} // namespace search
+
+#endif // SEARCH_NBEST__

mosesdecoder/search/rule.cc

@@ -0,0 +1,43 @@
+#include "search/rule.hh"
+
+#include "lm/model.hh"
+#include "search/context.hh"
+
+#include <ostream>
+
+#include <cmath>
+
+namespace search {
+
+template <class Model> ScoreRuleRet ScoreRule(const Model &model, const std::vector<lm::WordIndex> &words, lm::ngram::ChartState *writing) {
+  ScoreRuleRet ret;
+  ret.prob = 0.0;
+  ret.oov = 0;
+  const lm::WordIndex oov = model.GetVocabulary().NotFound(), bos = model.GetVocabulary().BeginSentence();
+  lm::ngram::RuleScore<Model> scorer(model, *(writing++));
+  std::vector<lm::WordIndex>::const_iterator word = words.begin();
+  if (word != words.end() && *word == bos) {
+    scorer.BeginSentence();
+    ++word;
+  }
+  for (; word != words.end(); ++word) {
+    if (*word == kNonTerminal) {
+      ret.prob += scorer.Finish();
+      scorer.Reset(*(writing++));
+    } else {
+      if (*word == oov) ++ret.oov;
+      scorer.Terminal(*word);
+    }
+  }
+  ret.prob += scorer.Finish();
+  return ret;
+}
+
+template ScoreRuleRet ScoreRule(const lm::ngram::RestProbingModel &model, const std::vector<lm::WordIndex> &words, lm::ngram::ChartState *writing);
+template ScoreRuleRet ScoreRule(const lm::ngram::ProbingModel &model, const std::vector<lm::WordIndex> &words, lm::ngram::ChartState *writing);
+template ScoreRuleRet ScoreRule(const lm::ngram::TrieModel &model, const std::vector<lm::WordIndex> &words, lm::ngram::ChartState *writing);
+template ScoreRuleRet ScoreRule(const lm::ngram::QuantTrieModel &model, const std::vector<lm::WordIndex> &words, lm::ngram::ChartState *writing);
+template ScoreRuleRet ScoreRule(const lm::ngram::ArrayTrieModel &model, const std::vector<lm::WordIndex> &words, lm::ngram::ChartState *writing);
+template ScoreRuleRet ScoreRule(const lm::ngram::QuantArrayTrieModel &model, const std::vector<lm::WordIndex> &words, lm::ngram::ChartState *writing);
+
+} // namespace search

mosesdecoder/search/rule.hh

@@ -0,0 +1,25 @@
+#ifndef SEARCH_RULE__
+#define SEARCH_RULE__
+
+#include "lm/left.hh"
+#include "lm/word_index.hh"
+#include "search/types.hh"
+
+#include <vector>
+
+namespace search {
+
+const lm::WordIndex kNonTerminal = lm::kMaxWordIndex;
+
+struct ScoreRuleRet {
+  Score prob;
+  unsigned int oov;
+};
+
+// Pass <s> and </s> normally.
+// Indicate non-terminals with kNonTerminal.
+template <class Model> ScoreRuleRet ScoreRule(const Model &model, const std::vector<lm::WordIndex> &words, lm::ngram::ChartState *state_out);
+
+} // namespace search
+
+#endif // SEARCH_RULE__

mosesdecoder/search/types.hh

@@ -0,0 +1,31 @@
+#ifndef SEARCH_TYPES__
+#define SEARCH_TYPES__
+
+#include <stdint.h>
+
+namespace lm { namespace ngram { struct ChartState; } }
+
+namespace search {
+
+typedef float Score;
+
+typedef uint32_t Arity;
+
+union Note {
+  const void *vp;
+};
+
+typedef void *History;
+
+struct NBestComplete {
+  NBestComplete(History in_history, const lm::ngram::ChartState &in_state, Score in_score)
+    : history(in_history), state(&in_state), score(in_score) {}
+
+  History history;
+  const lm::ngram::ChartState *state;
+  Score score;
+};
+
+} // namespace search
+
+#endif // SEARCH_TYPES__

mosesdecoder/search/vertex.cc

@@ -0,0 +1,204 @@
+#include "search/vertex.hh"
+
+#include "search/context.hh"
+
+#include <boost/unordered_map.hpp>
+
+#include <algorithm>
+#include <functional>
+#include <cassert>
+
+namespace search {
+
+namespace {
+
+const uint64_t kCompleteAdd = static_cast<uint64_t>(-1);
+
+class DivideLeft {
+  public:
+    explicit DivideLeft(unsigned char index)
+      : index_(index) {}
+
+    uint64_t operator()(const lm::ngram::ChartState &state) const {
+      return (index_ < state.left.length) ?
+        state.left.pointers[index_] :
+        (kCompleteAdd - state.left.full);
+    }
+
+  private:
+    unsigned char index_;
+};
+
+class DivideRight {
+  public:
+    explicit DivideRight(unsigned char index)
+      : index_(index) {}
+
+    uint64_t operator()(const lm::ngram::ChartState &state) const {
+      return (index_ < state.right.length) ?
+        static_cast<uint64_t>(state.right.words[index_]) :
+        (kCompleteAdd - state.left.full);
+    }
+
+  private:
+    unsigned char index_;
+};
+
+template <class Divider> void Split(const Divider &divider, const std::vector<HypoState> &hypos, std::vector<VertexNode> &extend) {
+  // Map from divider to index in extend.
+  typedef boost::unordered_map<uint64_t, std::size_t> Lookup;
+  Lookup lookup;
+  for (std::vector<HypoState>::const_iterator i = hypos.begin(); i != hypos.end(); ++i) {
+    uint64_t key = divider(i->state);
+    std::pair<Lookup::iterator, bool> res(lookup.insert(std::make_pair(key, extend.size())));
+    if (res.second) {
+      extend.resize(extend.size() + 1);
+      extend.back().AppendHypothesis(*i);
+    } else {
+      extend[res.first->second].AppendHypothesis(*i);
+    }
+  }
+  //assert((extend.size() != 1) || (hypos.size() == 1));
+}
+
+lm::WordIndex Identify(const lm::ngram::Right &right, unsigned char index) {
+  return right.words[index];
+}
+
+uint64_t Identify(const lm::ngram::Left &left, unsigned char index) {
+  return left.pointers[index];
+}
+
+template <class Side> class DetermineSame {
+  public:
+    DetermineSame(const Side &side, unsigned char guaranteed)
+      : side_(side), guaranteed_(guaranteed), shared_(side.length), complete_(true) {}
+
+    void Consider(const Side &other) {
+      if (shared_ != other.length) {
+        complete_ = false;
+        if (shared_ > other.length)
+          shared_ = other.length;
+      }
+      for (unsigned char i = guaranteed_; i < shared_; ++i) {
+        if (Identify(side_, i) != Identify(other, i)) {
+          shared_ = i;
+          complete_ = false;
+          return;
+        }
+      }
+    }
+
+    unsigned char Shared() const { return shared_; }
+
+    bool Complete() const { return complete_; }
+
+  private:
+    const Side &side_;
+    unsigned char guaranteed_, shared_;
+    bool complete_;
+};
+
+// Custom enum to save memory: valid values of policy_.
+// Alternate and there is still alternation to do.
+const unsigned char kPolicyAlternate = 0;
+// Branch based on left state only, because right ran out or this is a left tree.
+const unsigned char kPolicyOneLeft = 1;
+// Branch based on right state only.
+const unsigned char kPolicyOneRight = 2;
+// Reveal everything in the next branch.  Used to terminate the left/right policies.
+//    static const unsigned char kPolicyEverything = 3;
+
+} // namespace
+
+namespace {
+struct GreaterByScore : public std::binary_function<const HypoState &, const HypoState &, bool> {
+  bool operator()(const HypoState &first, const HypoState &second) const {
+    return first.score > second.score;
+  }
+};
+} // namespace
+
+void VertexNode::FinishRoot() {
+  std::sort(hypos_.begin(), hypos_.end(), GreaterByScore());
+  extend_.clear();
+  // HACK: extend to one hypo so that root can be blank.
+  state_.left.full = false;
+  state_.left.length = 0;
+  state_.right.length = 0;
+  right_full_ = false;
+  niceness_ = 0;
+  policy_ = kPolicyAlternate;
+  if (hypos_.size() == 1) {
+    extend_.resize(1);
+    extend_.front().AppendHypothesis(hypos_.front());
+    extend_.front().FinishedAppending(0, 0);
+  }
+  if (hypos_.empty()) {
+    bound_ = -INFINITY;
+  } else {
+    bound_ = hypos_.front().score;
+  }
+}
+
+void VertexNode::FinishedAppending(const unsigned char common_left, const unsigned char common_right) {
+  assert(!hypos_.empty());
+  assert(extend_.empty());
+  bound_ = hypos_.front().score;
+  state_ = hypos_.front().state;
+  bool all_full = state_.left.full;
+  bool all_non_full = !state_.left.full;
+  DetermineSame<lm::ngram::Left> left(state_.left, common_left);
+  DetermineSame<lm::ngram::Right> right(state_.right, common_right);
+  for (std::vector<HypoState>::const_iterator i = hypos_.begin() + 1; i != hypos_.end(); ++i) {
+    all_full &= i->state.left.full;
+    all_non_full &= !i->state.left.full;
+    left.Consider(i->state.left);
+    right.Consider(i->state.right);
+  }
+  state_.left.full = all_full && left.Complete();
+  right_full_ = all_full && right.Complete();
+  state_.left.length = left.Shared();
+  state_.right.length = right.Shared();
+
+  if (!all_full && !all_non_full) {
+    policy_ = kPolicyAlternate;
+  } else if (left.Complete()) {
+    policy_ = kPolicyOneRight;
+  } else if (right.Complete()) {
+    policy_ = kPolicyOneLeft;
+  } else {
+    policy_ = kPolicyAlternate;
+  }
+  niceness_ = state_.left.length + state_.right.length;
+}
+
+void VertexNode::BuildExtend() {
+  // Already built.
+  if (!extend_.empty()) return;
+  // Nothing to build since this is a leaf.
+  if (hypos_.size() <= 1) return;
+  bool left_branch = true;
+  switch (policy_) {
+    case kPolicyAlternate:
+      left_branch = (state_.left.length <= state_.right.length);
+      break;
+    case kPolicyOneLeft:
+      left_branch = true;
+      break;
+    case kPolicyOneRight:
+      left_branch = false;
+      break;
+  }
+  if (left_branch) {
+    Split(DivideLeft(state_.left.length), hypos_, extend_);
+  } else {
+    Split(DivideRight(state_.right.length), hypos_, extend_);
+  }
+  for (std::vector<VertexNode>::iterator i = extend_.begin(); i != extend_.end(); ++i) {
+    // TODO: provide more here for branching?
+    i->FinishedAppending(state_.left.length, state_.right.length);
+  }
+}
+
+} // namespace search

mosesdecoder/search/vertex.hh

@@ -0,0 +1,196 @@
+#ifndef SEARCH_VERTEX__
+#define SEARCH_VERTEX__
+
+#include "lm/left.hh"
+#include "search/types.hh"
+
+#include <boost/unordered_set.hpp>
+
+#include <queue>
+#include <vector>
+#include <cmath>
+#include <stdint.h>
+
+namespace search {
+
+class ContextBase;
+
+struct HypoState {
+  History history;
+  lm::ngram::ChartState state;
+  Score score;
+};
+
+class VertexNode {
+  public:
+    VertexNode() {}
+
+    void InitRoot() { hypos_.clear(); }
+
+    /* The steps of building a VertexNode:
+     * 1. Default construct.
+     * 2. AppendHypothesis at least once, possibly multiple times.
+     * 3. FinishAppending with the number of words on left and right guaranteed
+     * to be common.
+     * 4. If !Complete(), call BuildExtend to construct the extensions
+     */
+    // Must default construct, call AppendHypothesis 1 or more times then do FinishedAppending.
+    void AppendHypothesis(const NBestComplete &best) {
+      assert(hypos_.empty() || !(hypos_.front().state == *best.state));
+      HypoState hypo;
+      hypo.history = best.history;
+      hypo.state = *best.state;
+      hypo.score = best.score;
+      hypos_.push_back(hypo);
+    }
+    void AppendHypothesis(const HypoState &hypo) {
+      hypos_.push_back(hypo);
+    }
+
+    // Sort hypotheses for the root.
+    void FinishRoot();
+
+    void FinishedAppending(const unsigned char common_left, const unsigned char common_right);
+
+    void BuildExtend();
+
+    // Should only happen to a root node when the entire vertex is empty.
+    bool Empty() const {
+      return hypos_.empty() && extend_.empty();
+    }
+
+    bool Complete() const {
+      // HACK: prevent root from being complete.  TODO: allow root to be complete.
+      return hypos_.size() == 1 && extend_.empty();
+    }
+
+    const lm::ngram::ChartState &State() const { return state_; }
+    bool RightFull() const { return right_full_; }
+
+    // Priority relative to other non-terminals.  0 is highest.
+    unsigned char Niceness() const { return niceness_; }
+
+    Score Bound() const {
+      return bound_;
+    }
+
+    // Will be invalid unless this is a leaf.
+    const History End() const {
+      assert(hypos_.size() == 1);
+      return hypos_.front().history;
+    }
+
+    VertexNode &operator[](size_t index) {
+      assert(!extend_.empty());
+      return extend_[index];
+    }
+
+    size_t Size() const {
+      return extend_.size();
+    }
+
+  private:
+    // Hypotheses to be split.
+    std::vector<HypoState> hypos_;
+
+    std::vector<VertexNode> extend_;
+
+    lm::ngram::ChartState state_;
+    bool right_full_;
+
+    unsigned char niceness_;
+
+    unsigned char policy_;
+
+    Score bound_;
+};
+
+class PartialVertex {
+  public:
+    PartialVertex() {}
+
+    explicit PartialVertex(VertexNode &back) : back_(&back), index_(0) {}
+
+    bool Empty() const { return back_->Empty(); }
+
+    bool Complete() const { return back_->Complete(); }
+
+    const lm::ngram::ChartState &State() const { return back_->State(); }
+    bool RightFull() const { return back_->RightFull(); }
+
+    Score Bound() const { return index_ ? (*back_)[index_].Bound() : back_->Bound(); }
+
+    unsigned char Niceness() const { return back_->Niceness(); }
+
+    // Split into continuation and alternative, rendering this the continuation.
+    bool Split(PartialVertex &alternative) {
+      assert(!Complete());
+      back_->BuildExtend();
+      bool ret;
+      if (index_ + 1 < back_->Size()) {
+        alternative.index_ = index_ + 1;
+        alternative.back_ = back_;
+        ret = true;
+      } else {
+        ret = false;
+      }
+      back_ = &((*back_)[index_]);
+      index_ = 0;
+      return ret;
+    }
+
+    const History End() const {
+      return back_->End();
+    }
+
+  private:
+    VertexNode *back_;
+    unsigned int index_;
+};
+
+template <class Output> class VertexGenerator;
+
+class Vertex {
+  public:
+    Vertex() {}
+
+    //PartialVertex RootFirst() const { return PartialVertex(right_); }
+    PartialVertex RootAlternate() { return PartialVertex(root_); }
+    //PartialVertex RootLast() const { return PartialVertex(left_); }
+
+    bool Empty() const {
+      return root_.Empty();
+    }
+
+    Score Bound() const {
+      return root_.Bound();
+    }
+
+    const History BestChild() {
+      // left_ and right_ are not set at the root.
+      PartialVertex top(RootAlternate());
+      if (top.Empty()) {
+        return History();
+      } else {
+        PartialVertex continuation;
+        while (!top.Complete()) {
+          top.Split(continuation);
+        }
+        return top.End();
+      }
+    }
+
+  private:
+    template <class Output> friend class VertexGenerator;
+    template <class Output> friend class RootVertexGenerator;
+    VertexNode root_;
+
+    // These will not be set for the root vertex.
+    // Branches only on left state.
+    //VertexNode left_;
+    // Branches only on right state.
+    //VertexNode right_;
+};
+
+} // namespace search
+#endif // SEARCH_VERTEX__

mosesdecoder/search/vertex_generator.hh

@@ -0,0 +1,75 @@
+#ifndef SEARCH_VERTEX_GENERATOR__
+#define SEARCH_VERTEX_GENERATOR__
+
+#include "search/edge.hh"
+#include "search/types.hh"
+#include "search/vertex.hh"
+
+namespace lm {
+namespace ngram {
+struct ChartState;
+} // namespace ngram
+} // namespace lm
+
+namespace search {
+
+class ContextBase;
+
+// Output makes the single-best or n-best list.
+template <class Output> class VertexGenerator {
+  public:
+    VertexGenerator(ContextBase &context, Vertex &gen, Output &nbest) : context_(context), gen_(gen), nbest_(nbest) {}
+
+    void NewHypothesis(PartialEdge partial) {
+      nbest_.Add(existing_[hash_value(partial.CompletedState())], partial);
+    }
+
+    void FinishedSearch() {
+      gen_.root_.InitRoot();
+      for (typename Existing::iterator i(existing_.begin()); i != existing_.end(); ++i) {
+        gen_.root_.AppendHypothesis(nbest_.Complete(i->second));
+      }
+      existing_.clear();
+      gen_.root_.FinishRoot();
+    }
+
+    Vertex &Generating() { return gen_; }
+
+  private:
+    ContextBase &context_;
+
+    Vertex &gen_;
+
+    typedef boost::unordered_map<uint64_t, typename Output::Combine> Existing;
+    Existing existing_;
+
+    Output &nbest_;
+};
+
+// Special case for root vertex: everything should come together into the root
+// node.  In theory, this should happen naturally due to state collapsing with
+// <s> and </s>.  If that's the case, VertexGenerator is fine, though it will
+// make one connection.
+template <class Output> class RootVertexGenerator {
+  public:
+    RootVertexGenerator(Vertex &gen, Output &out) : gen_(gen), out_(out) {}
+
+    void NewHypothesis(PartialEdge partial) {
+      out_.Add(combine_, partial);
+    }
+
+    void FinishedSearch() {
+      gen_.root_.InitRoot();
+      gen_.root_.AppendHypothesis(out_.Complete(combine_));
+      gen_.root_.FinishRoot();
+    }
+
+  private:
+    Vertex &gen_;
+
+    typename Output::Combine combine_;
+    Output &out_;
+};
+
+} // namespace search
+#endif // SEARCH_VERTEX_GENERATOR__

mosesdecoder/symal/Jamfile

@@ -0,0 +1,2 @@
+exe symal : symal.cpp cmd.c ;
+

mosesdecoder/symal/cmd.c

@@ -0,0 +1,642 @@
+
+// $Id$
+
+#include	<stdarg.h>
+#include	<stdio.h>
+#include	<stdlib.h>
+#include	<ctype.h>
+#include	<string.h>
+
+#include	"cmd.h"
+
+#ifdef WIN32
+#		define popen	_popen
+#		define pclose _pclose
+#endif
+
+typedef struct {
+  enum CommandType Type;
+  const char *Name,
+        *ArgStr;
+  void	*Val;
+  const void *p;
+} Cmd_T;
+
+static const Enum_T	BoolEnum[] = {
+  {	"FALSE",	0	},
+  {	"TRUE",		1	},
+  {	0,		0	}
+};
+
+#ifdef	NEEDSTRDUP
+char	*strdup();
+#endif
+
+#define	FALSE	0
+#define	TRUE	1
+
+#define	LINSIZ		10240
+#define	MAXPARAM	256
+
+static Cmd_T	cmds[MAXPARAM+1];
+static const char *SepString = " \t\n";
+
+/// Return cmd->p, as an int.
+static int get_p_int(const Cmd_T *cmd)
+{
+  return *(const int *)cmd->p;
+}
+
+/// Return cmd->p, as a pointer to a null-terminated array of Enum_T.
+static const Enum_T *get_p_enums(const Cmd_T *cmd)
+{
+  return (const Enum_T *)cmd->p;
+}
+
+/// Return cmd->p, as a pointer to a string.
+static const char *get_p_char(const Cmd_T *cmd)
+{
+  return (const char *)cmd->p;
+}
+
+/// Return cmd->p, as a pointer to an array of two ints.
+static const int *get_p_range(const Cmd_T *cmd)
+{
+  return (const int *)cmd->p;
+}
+
+/// Return cmd->Val, as a pointer to int.
+static int *get_val_int_ptr(const Cmd_T *cmd)
+{
+  return (int *)cmd->Val;
+}
+
+/// Return the int at which cmd->Val points.
+static int get_val_int(const Cmd_T *cmd)
+{
+  return *get_val_int_ptr(cmd);
+}
+
+/// Update the int at which cmd->Val points.
+static void update_val_int(const Cmd_T *cmd, int value)
+{
+  *get_val_int_ptr(cmd) = value;
+}
+
+/// Return cmd->Val, as a pointer to double.
+static double *get_val_double_ptr(const Cmd_T *cmd)
+{
+  return (double *)cmd->Val;
+}
+
+/// Return the double at which cmd->Val points.
+static double get_val_double(const Cmd_T *cmd)
+{
+  return *get_val_double_ptr(cmd);
+}
+
+/// Return cmd->Val as a pointer to a string pointer.
+static const char **get_val_char_ptr(const Cmd_T *cmd)
+{
+  return (const char **)cmd->Val;
+}
+
+/// Return the string pointer at which cmd->Val points.
+static const char *get_val_char(const Cmd_T *cmd)
+{
+  return *get_val_char_ptr(cmd);
+}
+
+/// Update the string pointer at which cmd->Val points.
+static void update_val_char(const Cmd_T *cmd, const char *s)
+{
+  *get_val_char_ptr(cmd) = s;
+}
+
+int DeclareParams(const char *ParName, ...)
+{
+  va_list		args;
+  static int	ParamN = 0;
+
+  va_start(args, ParName);
+  for(; ParName;) {
+    int c,
+        j = 0;
+    if(ParamN==MAXPARAM) {
+      fprintf(stderr, "Too many parameters !!\n");
+      break;
+    }
+    for(c=1; j<ParamN&&(c=strcmp(cmds[j].Name,ParName))<0; j++)
+      ;
+    if(!c) {
+      fprintf(stderr,
+              "Warning: parameter \"%s\" declared twice.\n",
+              ParName);
+    }
+    for(c=ParamN; c>j; c--) {
+      cmds[c] = cmds[c-1];
+    }
+    cmds[j].Name = ParName;
+    cmds[j].Type = va_arg(args, enum CommandType);
+    cmds[j].Val = va_arg(args, void *);
+    switch(cmds[j].Type) {
+    case CMDENUMTYPE:	/* get the pointer to Enum_T struct  */
+      cmds[j].p = va_arg(args, void *);
+      break;
+    case CMDSUBRANGETYPE: {	/* get the two extremes		     */
+      int *subrange = calloc(2, sizeof(int));
+      cmds[j].p = subrange;
+      subrange[0] = va_arg(args, int);
+      subrange[1] = va_arg(args, int);
+    }
+    break;
+    case CMDGTETYPE:	/* get lower or upper bound	     */
+    case CMDLTETYPE: {
+      int *value = calloc(1, sizeof(int));
+      cmds[j].p = value;
+      value[0] = va_arg(args, int);
+    }
+    break;
+    case CMDSTRARRAYTYPE: {	/* get the separators string	     */
+      const char *s = va_arg(args, const char *);
+      cmds[j].p = (s ? strdup(s) : NULL);
+    }
+    break;
+    case CMDBOOLTYPE:
+      cmds[j].Type = CMDENUMTYPE;
+      cmds[j].p = BoolEnum;
+      break;
+    case CMDDOUBLETYPE:	/* nothing else is needed	     */
+    case CMDINTTYPE:
+    case CMDSTRINGTYPE:
+      break;
+    default:
+      fprintf(stderr, "%s: %s %d %s \"%s\"\n",
+              "DeclareParam()", "Unknown Type",
+              cmds[j].Type, "for parameter", cmds[j].Name);
+      exit(1);
+    }
+    ParamN++;
+    ParName = va_arg(args, const char *);
+  }
+  cmds[ParamN].Name = NULL;
+  va_end(args);
+  return 0;
+}
+
+static char *GetLine(FILE *fp, int n, char *Line)
+{
+  int	offs=0;
+
+  for(;;) {
+    int j, l;
+    if(!fgets(Line+offs, n-offs, fp)) {
+      return NULL;
+    }
+    if(Line[offs]=='#') continue;
+    l = strlen(Line+offs)-1;
+    Line[offs+l] = 0;
+    for(j=offs; Line[j] && isspace(Line[j]); j++, l--)
+      ;
+    if(l<1) continue;
+    if(j > offs) {
+      char	*s = Line+offs,
+             *q = Line+j;
+
+      while((*s++=*q++))
+        ;
+    }
+    if(Line[offs+l-1]=='\\') {
+      offs += l;
+      Line[offs-1] = ' ';
+    } else {
+      break;
+    }
+  }
+  return Line;
+}
+
+static void EnumError(const Cmd_T *cmd, const char *s)
+{
+  const Enum_T	*en;
+
+  fprintf(stderr,
+          "Invalid value \"%s\" for parameter \"%s\"\n", s, cmd->Name);
+  fprintf(stderr, "Valid values are:\n");
+  for(en=get_p_enums(cmd); en->Name; en++) {
+    if(*en->Name) {
+      fprintf(stderr, "    %s\n", en->Name);
+    }
+  }
+  fprintf(stderr, "\n");
+  exit(1);
+}
+
+static void GteError(const Cmd_T *cmd, int n)
+{
+  fprintf(stderr,
+          "Value %d out of range for parameter \"%s\"\n", n, cmd->Name);
+  fprintf(stderr, "Valid values must be greater than or equal to  %d\n",
+          get_p_int(cmd));
+  exit(1);
+}
+
+static void LteError(const Cmd_T *cmd, int n)
+{
+  fprintf(stderr,
+          "Value %d out of range for parameter \"%s\"\n", n, cmd->Name);
+  fprintf(stderr, "Valid values must be less than or equal to  %d\n",
+          get_p_int(cmd));
+  exit(1);
+}
+
+static void SubrangeError(const Cmd_T *cmd, int n)
+{
+  const int *subrange = get_p_range(cmd);
+  fprintf(stderr,
+          "Value %d out of range for parameter \"%s\"\n", n, cmd->Name);
+  fprintf(stderr, "Valid values range from %d to %d\n",
+          subrange[0], subrange[1]);
+  exit(1);
+}
+
+static void SetEnum(Cmd_T *cmd, const char *s)
+{
+  const Enum_T	*en;
+
+  for(en=get_p_enums(cmd); en->Name; en++) {
+    if(*en->Name && !strcmp(s, en->Name)) {
+      update_val_int(cmd, en->Idx);
+      return;
+    }
+  }
+  EnumError(cmd, s);
+}
+
+static void SetSubrange(Cmd_T *cmd, const char *s)
+{
+  int	n;
+  const int *subrange = get_p_range(cmd);
+
+  if(sscanf(s, "%d", &n)!=1) {
+    fprintf(stderr,
+            "Integer value required for parameter \"%s\"\n",
+            cmd->Name);
+    exit(1);
+  }
+  if(n < subrange[0] || n > subrange[1]) {
+    SubrangeError(cmd, n);
+  }
+  update_val_int(cmd, n);
+}
+
+static void SetGte(Cmd_T *cmd, const char *s)
+{
+  int	n;
+
+  if(sscanf(s, "%d", &n)!=1) {
+    fprintf(stderr,
+            "Integer value required for parameter \"%s\"\n",
+            cmd->Name);
+    exit(1);
+  }
+  if(n<get_p_int(cmd)) {
+    GteError(cmd, n);
+  }
+  update_val_int(cmd, n);
+}
+
+static char **str2array(const char *s, const char *sep)
+{
+  const char *p;
+  char **a;
+  int	n = 0,
+      l;
+
+  if(!sep) sep = SepString;
+  p = s += strspn(s, sep);
+  while(*p) {
+    p += strcspn(p, sep);
+    p += strspn(p, sep);
+    ++n;
+  }
+  a = calloc(n+1, sizeof(char *));
+  p = s;
+  n = 0;
+  while(*p) {
+    l = strcspn(p, sep);
+    a[n] = malloc(l+1);
+    memcpy(a[n], p, l);
+    a[n][l] = 0;
+    ++n;
+    p += l;
+    p += strspn(p, sep);
+  }
+  return a;
+}
+
+static void SetStrArray(Cmd_T *cmd, const char *s)
+{
+  *(char***)cmd->Val = str2array(s, get_p_char(cmd));
+}
+
+static void SetLte(Cmd_T *cmd, const char *s)
+{
+  int	n;
+
+  if(sscanf(s, "%d", &n)!=1) {
+    fprintf(stderr,
+            "Integer value required for parameter \"%s\"\n",
+            cmd->Name);
+    exit(1);
+  }
+  if(n > get_p_int(cmd)) {
+    LteError(cmd, n);
+  }
+  update_val_int(cmd, n);
+}
+
+static void SetParam(Cmd_T *cmd, const char *s)
+{
+  if(!*s && cmd->Type != CMDSTRINGTYPE) {
+    fprintf(stderr,
+            "WARNING: No value specified for parameter \"%s\"\n",
+            cmd->Name);
+    return;
+  }
+  switch(cmd->Type) {
+  case CMDDOUBLETYPE:
+    if(sscanf(s, "%lf", get_val_double_ptr(cmd))!=1) {
+      fprintf(stderr,
+              "Float value required for parameter \"%s\"\n",
+              cmd->Name);
+      exit(1);
+    }
+    break;
+  case CMDENUMTYPE:
+    SetEnum(cmd, s);
+    break;
+  case CMDINTTYPE:
+    if(sscanf(s, "%d", get_val_int_ptr(cmd))!=1) {
+      fprintf(stderr,
+              "Integer value required for parameter \"%s\"\n",
+              cmd->Name);
+      exit(1);
+    }
+    break;
+  case CMDSTRINGTYPE:
+    update_val_char(cmd,
+                    (strcmp(s, "<NULL>") && strcmp(s, "NULL"))
+                    ? strdup(s)
+                    : 0);
+    break;
+  case CMDSTRARRAYTYPE:
+    SetStrArray(cmd, s);
+    break;
+  case CMDGTETYPE:
+    SetGte(cmd, s);
+    break;
+  case CMDLTETYPE:
+    SetLte(cmd, s);
+    break;
+  case CMDSUBRANGETYPE:
+    SetSubrange(cmd, s);
+    break;
+  default:
+    fprintf(stderr, "%s: %s %d %s \"%s\"\n",
+            "SetParam",
+            "Unknown Type",
+            cmd->Type,
+            "for parameter",
+            cmd->Name);
+    exit(1);
+  }
+  cmd->ArgStr = strdup(s);
+}
+
+static int Scan(const char *ProgName, Cmd_T *cmds, char *Line)
+{
+  char	*q,
+        *p;
+  int	i,
+      hl,
+      HasToMatch = FALSE,
+      c0,
+      c;
+
+  p = Line+strspn(Line, SepString);
+  hl = strcspn(p, SepString);
+  if(!hl) {
+    return 0;
+  }
+  q = strchr(p, '/');
+  if(q && q-p<hl) {
+    *q = 0;
+    if(strcmp(p, ProgName)) {
+      *q = '/';
+      return 0;
+    }
+    *q = '/';
+    HasToMatch=TRUE;
+    p = q+1;
+  }
+  hl = strcspn(p, SepString);
+  if(!hl) {
+    return 0;
+  }
+  c0 = p[hl];
+  p[hl] = 0;
+  for(i=0, c=1; cmds[i].Name&&(c=strcmp(cmds[i].Name, p))<0; i++)
+    ;
+  p[hl] = c0;
+
+  if (c)
+    return HasToMatch && c;
+
+  SetParam(cmds+i, p+hl+strspn(p+hl, SepString));
+  return 0;
+}
+
+static void PrintEnum(const Cmd_T *cmd, int ValFlag, FILE *fp)
+{
+  const Enum_T	*en;
+
+  fprintf(fp, "%s", cmd->Name);
+  if(ValFlag) {
+    for(en=get_p_enums(cmd); en->Name; en++) {
+      if(*en->Name && en->Idx==get_val_int(cmd)) {
+        fprintf(fp, ": %s", en->Name);
+      }
+    }
+  }
+  fprintf(fp, "\n");
+}
+
+static void PrintStrArray(const Cmd_T *cmd, int ValFlag, FILE *fp)
+{
+  char	*indent,
+        **s = *(char***)cmd->Val;
+  int	l = 4+strlen(cmd->Name);
+
+  fprintf(fp, "%s", cmd->Name);
+  indent = malloc(l+2);
+  memset(indent, ' ', l+1);
+  indent[l+1] = 0;
+  if(ValFlag) {
+    fprintf(fp, ": %s", s ? (*s ? *s++ : "NULL") : "");
+    if(s) while(*s) {
+        fprintf(fp, "\n%s %s", indent, *s++);
+      }
+  }
+  free(indent);
+  fprintf(fp, "\n");
+}
+
+static void PrintParam(const Cmd_T *cmd, int ValFlag, FILE *fp)
+{
+  fprintf(fp, "%4s", "");
+  switch(cmd->Type) {
+  case CMDDOUBLETYPE:
+    fprintf(fp, "%s", cmd->Name);
+    if(ValFlag) fprintf(fp, ": %22.15e", get_val_double(cmd));
+    fprintf(fp, "\n");
+    break;
+  case CMDENUMTYPE:
+    PrintEnum(cmd, ValFlag, fp);
+    break;
+  case CMDINTTYPE:
+  case CMDSUBRANGETYPE:
+  case CMDGTETYPE:
+  case CMDLTETYPE:
+    fprintf(fp, "%s", cmd->Name);
+    if(ValFlag) fprintf(fp, ": %d", get_val_int(cmd));
+    fprintf(fp, "\n");
+    break;
+  case CMDSTRINGTYPE:
+    fprintf(fp, "%s", cmd->Name);
+    if(ValFlag) {
+      const char *value = get_val_char(cmd);
+      if(value) {
+        fprintf(fp, ": \"%s\"", value);
+      } else {
+        fprintf(fp, ": %s", "NULL");
+      }
+    }
+    fprintf(fp, "\n");
+    break;
+  case CMDSTRARRAYTYPE:
+    PrintStrArray(cmd, ValFlag, fp);
+    break;
+  default:
+    fprintf(stderr, "%s: %s %d %s \"%s\"\n",
+            "PrintParam",
+            "Unknown Type",
+            cmd->Type,
+            "for parameter",
+            cmd->Name);
+    exit(1);
+  }
+}
+
+static void PrintParams(int ValFlag, FILE *fp)
+{
+  int	i;
+
+  fflush(fp);
+  if(ValFlag) {
+    fprintf(fp, "Parameters Values:\n");
+  } else {
+    fprintf(fp, "Parameters:\n");
+  }
+  for(i=0; cmds[i].Name; i++) PrintParam(cmds+i, ValFlag, fp);
+  fprintf(fp, "\n");
+  fflush(fp);
+}
+
+static void CmdError(const char *opt)
+{
+  fprintf(stderr, "Invalid option \"%s\"\n", opt);
+  fprintf(stderr, "This program expectes the following parameters:\n");
+  PrintParams(FALSE, stderr);
+  exit(0);
+}
+
+int GetParams(int *n, char ***a, const char *CmdFileName)
+{
+  char	*Line,
+        *ProgName;
+  int	argc = *n;
+  char	**argv = *a,
+          *s;
+  FILE	*fp;
+  int	IsPipe;
+
+#ifdef	MSDOS
+#define	PATHSEP '\\'
+  char	*dot = NULL;
+#else
+#define	PATHSEP '/'
+#endif
+
+  if(!(Line=malloc(LINSIZ))) {
+    fprintf(stderr, "GetParams(): Unable to alloc %d bytes\n",
+            LINSIZ);
+    exit(1);
+  }
+  if((ProgName=strrchr(*argv, PATHSEP))) {
+    ++ProgName;
+  } else {
+    ProgName = *argv;
+  }
+#ifdef	MSDOS
+  if(dot=strchr(ProgName, '.')) *dot = 0;
+#endif
+  --argc;
+  ++argv;
+  for(;;) {
+    if(argc && argv[0][0]=='-' && argv[0][1]=='=') {
+      CmdFileName = argv[0]+2;
+      ++argv;
+      --argc;
+    }
+    if(!CmdFileName) {
+      break;
+    }
+    IsPipe = !strncmp(CmdFileName, "@@", 2);
+    fp = IsPipe
+         ? popen(CmdFileName+2, "r")
+         : strcmp(CmdFileName, "-")
+         ? fopen(CmdFileName, "r")
+         : stdin;
+    if(!fp) {
+      fprintf(stderr, "Unable to open command file %s\n",
+              CmdFileName);
+      exit(1);
+    }
+    while(GetLine(fp, LINSIZ, Line) && strcmp(Line, "\\End")) {
+      if(Scan(ProgName, cmds, Line)) {
+        CmdError(Line);
+      }
+    }
+    if(fp!=stdin) {
+      if(IsPipe) pclose(fp);
+      else fclose(fp);
+    }
+    CmdFileName = NULL;
+  }
+  while(argc && **argv=='-' && (s=strchr(*argv, '='))) {
+    *s = ' ';
+    sprintf(Line, "%s/%s", ProgName, *argv+1);
+    *s = '=';
+    if(Scan(ProgName, cmds, Line)) CmdError(*argv);
+    --argc;
+    ++argv;
+  }
+  *n = argc;
+  *a = argv;
+#ifdef MSDOS
+  if(dot) *dot = '.';
+#endif
+  free(Line);
+  return 0;
+}

mosesdecoder/symal/cmd.h

@@ -0,0 +1,35 @@
+
+// $Id$
+
+#if !defined(CMD_H)
+
+#define	CMD_H
+
+enum CommandType {
+  CMDDOUBLETYPE = 1,
+  CMDENUMTYPE,
+  CMDINTTYPE,
+  CMDSTRINGTYPE,
+  CMDSUBRANGETYPE,
+  CMDGTETYPE,
+  CMDLTETYPE,
+  CMDSTRARRAYTYPE,
+  CMDBOOLTYPE
+};
+
+typedef struct {
+  const char	*Name;
+  int	Idx;
+} Enum_T;
+
+#ifdef  __cplusplus
+extern "C" {
+#endif
+
+  int DeclareParams(const char *, ...);
+  int GetParams(int *n, char ***a, const char *CmdFileName);
+
+#ifdef  __cplusplus
+}
+#endif
+#endif

mosesdecoder/symal/symal.cpp

@@ -0,0 +1,518 @@
+// $Id$
+
+#include <cassert>
+#include <iomanip>
+#include <iostream>
+#include <fstream>
+#include <sstream>
+#include <stdexcept>
+#include <string>
+#include <list>
+#include <vector>
+#include <set>
+#include <algorithm>
+#include <cstring>
+#include "cmd.h"
+
+using namespace std;
+
+const int MAX_WORD = 10000;  // maximum lengthsource/target strings
+const int MAX_M = 400;       // maximum length of source strings
+const int MAX_N = 400;       // maximum length of target strings
+
+enum Alignment {
+  UNION = 1,
+  INTERSECT,
+  GROW,
+  SRCTOTGT,
+  TGTTOSRC,
+};
+
+const Enum_T END_ENUM = {0, 0};
+
+namespace
+{
+Enum_T AlignEnum [] = {
+  {    "union",                        UNION },
+  {    "u",                            UNION },
+  {    "intersect",                    INTERSECT},
+  {    "i",                            INTERSECT},
+  {    "grow",                         GROW },
+  {    "g",                            GROW },
+  {    "srctotgt",                     SRCTOTGT },
+  {    "s2t",                          SRCTOTGT },
+  {    "tgttosrc",                     TGTTOSRC },
+  {    "t2s",                          TGTTOSRC },
+  END_ENUM
+};
+
+Enum_T BoolEnum [] = {
+  {    "true",        true },
+  {    "yes",         true },
+  {    "y",           true },
+  {    "false",       false },
+  {    "no",          false },
+  {    "n",           false },
+  END_ENUM
+};
+
+// global variables and constants
+
+int* fa; //counters of covered foreign positions
+int* ea; //counters of covered english positions
+int** A; //alignment matrix with information symmetric/direct/inverse alignments
+
+int verbose=0;
+
+//read an alignment pair from the input stream.
+
+int lc = 0;
+
+int getals(istream& inp,int& m, int *a,int& n, int *b)
+{
+  char w[MAX_WORD], dummy[10];
+  int i,j,freq;
+  if (inp >> freq) {
+    ++lc;
+    //target sentence
+    inp >> n;
+    assert(n<MAX_N);
+    for (i=1; i<=n; i++) {
+      inp >> setw(MAX_WORD) >> w;
+      if (strlen(w)>=MAX_WORD-1) {
+        cerr << lc << ": target len=" << strlen(w) << " is not less than MAX_WORD-1="
+             << MAX_WORD-1 << endl;
+        assert(strlen(w)<MAX_WORD-1);
+      }
+    }
+
+    inp >> dummy; //# separator
+    // inverse alignment
+    for (i=1; i<=n; i++) inp >> b[i];
+
+    //source sentence
+    inp >> m;
+    assert(m<MAX_M);
+    for (j=1; j<=m; j++) {
+      inp >> setw(MAX_WORD) >> w;
+      if (strlen(w)>=MAX_WORD-1) {
+        cerr << lc << ": source len=" << strlen(w) << " is not less than MAX_WORD-1="
+             << MAX_WORD-1 << endl;
+        assert(strlen(w)<MAX_WORD-1);
+      }
+    }
+
+    inp >> dummy; //# separator
+
+    // direct alignment
+    for (j=1; j<=m; j++) {
+      inp >> a[j];
+      assert(0<=a[j] && a[j]<=n);
+    }
+
+    //check inverse alignemnt
+    for (i=1; i<=n; i++)
+      assert(0<=b[i] && b[i]<=m);
+
+    return 1;
+
+  } else
+    return 0;
+}
+
+
+//compute union alignment
+int prunionalignment(ostream& out,int m,int *a,int n,int* b)
+{
+
+  ostringstream sout;
+
+  for (int j=1; j<=m; j++)
+    if (a[j])
+      sout << j-1 << "-" << a[j]-1 << " ";
+
+  for (int i=1; i<=n; i++)
+    if (b[i] && a[b[i]]!=i)
+      sout << b[i]-1 <<  "-" << i-1 << " ";
+
+  //fix the last " "
+  string str = sout.str();
+  if (str.length() == 0)
+    str = "\n";
+  else
+    str.replace(str.length()-1,1,"\n");
+
+  out << str;
+  out.flush();
+
+  return 1;
+}
+
+
+//Compute intersection alignment
+
+int printersect(ostream& out,int m,int *a,int n,int* b)
+{
+
+  ostringstream sout;
+
+  for (int j=1; j<=m; j++)
+    if (a[j] && b[a[j]]==j)
+      sout << j-1 << "-" << a[j]-1 << " ";
+
+  //fix the last " "
+  string str = sout.str();
+  if (str.length() == 0)
+    str = "\n";
+  else
+    str.replace(str.length()-1,1,"\n");
+
+  out << str;
+  out.flush();
+
+  return 1;
+}
+
+//Compute target-to-source alignment
+
+int printtgttosrc(ostream& out,int m,int *a,int n,int* b)
+{
+
+  ostringstream sout;
+
+  for (int i=1; i<=n; i++)
+    if (b[i])
+      sout << b[i]-1 << "-" << i-1 << " ";
+
+  //fix the last " "
+  string str = sout.str();
+  if (str.length() == 0)
+    str = "\n";
+  else
+    str.replace(str.length()-1,1,"\n");
+
+  out << str;
+  out.flush();
+
+  return 1;
+}
+
+//Compute source-to-target alignment
+
+int printsrctotgt(ostream& out,int m,int *a,int n,int* b)
+{
+
+  ostringstream sout;
+
+  for (int j=1; j<=m; j++)
+    if (a[j])
+      sout << j-1 << "-" << a[j]-1 << " ";
+
+  //fix the last " "
+  string str = sout.str();
+  if (str.length() == 0)
+    str = "\n";
+  else
+    str.replace(str.length()-1,1,"\n");
+
+  out << str;
+  out.flush();
+
+  return 1;
+}
+
+//Compute Grow Diagonal Alignment
+//Nice property: you will never introduce more points
+//than the unionalignment alignemt. Hence, you will always be able
+//to represent the grow alignment as the unionalignment of a
+//directed and inverted alignment
+
+int printgrow(ostream& out,int m,int *a,int n,int* b, bool diagonal=false,bool isfinal=false,bool bothuncovered=false)
+{
+
+  ostringstream sout;
+
+  vector <pair <int,int> > neighbors; //neighbors
+
+  pair <int,int> entry;
+
+  neighbors.push_back(make_pair(-1,-0));
+  neighbors.push_back(make_pair(0,-1));
+  neighbors.push_back(make_pair(1,0));
+  neighbors.push_back(make_pair(0,1));
+
+
+  if (diagonal) {
+    neighbors.push_back(make_pair(-1,-1));
+    neighbors.push_back(make_pair(-1,1));
+    neighbors.push_back(make_pair(1,-1));
+    neighbors.push_back(make_pair(1,1));
+  }
+
+
+  int i,j;
+  size_t o;
+
+
+  //covered foreign and english positions
+
+  memset(fa,0,(m+1)*sizeof(int));
+  memset(ea,0,(n+1)*sizeof(int));
+
+  //matrix to quickly check if one point is in the symmetric
+  //alignment (value=2), direct alignment (=1) and inverse alignment
+
+  for (int i=1; i<=n; i++) memset(A[i],0,(m+1)*sizeof(int));
+
+  set <pair <int,int> > currentpoints; //symmetric alignment
+  set <pair <int,int> > unionalignment; //union alignment
+
+  pair <int,int> point; //variable to store points
+  set<pair <int,int> >::const_iterator k; //iterator over sets
+
+  //fill in the alignments
+  for (j=1; j<=m; j++) {
+    if (a[j]) {
+      unionalignment.insert(make_pair(a[j],j));
+      if (b[a[j]]==j) {
+        fa[j]=1;
+        ea[a[j]]=1;
+        A[a[j]][j]=2;
+        currentpoints.insert(make_pair(a[j],j));
+      } else
+        A[a[j]][j]=-1;
+    }
+  }
+
+  for (i=1; i<=n; i++)
+    if (b[i] && a[b[i]]!=i) { //not intersection
+      unionalignment.insert(make_pair(i,b[i]));
+      A[i][b[i]]=1;
+    }
+
+
+  int added=1;
+
+  while (added) {
+    added=0;
+    ///scan the current alignment
+    for (k=currentpoints.begin(); k!=currentpoints.end(); k++) {
+      //cout << "{"<< (k->second)-1 << "-" << (k->first)-1 << "}";
+      for (o=0; o<neighbors.size(); o++) {
+        //cout << "go over check all neighbors\n";
+        point.first=k->first+neighbors[o].first;
+        point.second=k->second+neighbors[o].second;
+        //cout << point.second-1 << " " << point.first-1 << "\n";
+        //check if neighbor is inside 'matrix'
+        if (point.first>0 && point.first <=n && point.second>0 && point.second<=m)
+          //check if neighbor is in the unionalignment alignment
+          if (b[point.first]==point.second || a[point.second]==point.first) {
+            //cout << "In unionalignment ";cout.flush();
+            //check if it connects at least one uncovered word
+            if (!(ea[point.first] && fa[point.second])) {
+              //insert point in currentpoints!
+              currentpoints.insert(point);
+              A[point.first][point.second]=2;
+              ea[point.first]=1;
+              fa[point.second]=1;
+              added=1;
+              //cout << "added grow: " << point.second-1 << "-" << point.first-1 << "\n";cout.flush();
+            }
+          }
+      }
+    }
+  }
+
+  if (isfinal) {
+    for (k=unionalignment.begin(); k!=unionalignment.end(); k++)
+      if (A[k->first][k->second]==1) {
+        point.first=k->first;
+        point.second=k->second;
+        //one of the two words is not covered yet
+        //cout << "{" << point.second-1 << "-" << point.first-1 << "} ";
+        if ((bothuncovered &&  !ea[point.first] && !fa[point.second]) ||
+            (!bothuncovered && !(ea[point.first] && fa[point.second]))) {
+          //add it!
+          currentpoints.insert(point);
+          A[point.first][point.second]=2;
+          //keep track of new covered positions
+          ea[point.first]=1;
+          fa[point.second]=1;
+
+          //added=1;
+          //cout << "added final: " << point.second-1 << "-" << point.first-1 << "\n";
+        }
+      }
+
+    for (k=unionalignment.begin(); k!=unionalignment.end(); k++)
+      if (A[k->first][k->second]==-1) {
+        point.first=k->first;
+        point.second=k->second;
+        //one of the two words is not covered yet
+        //cout << "{" << point.second-1 << "-" << point.first-1 << "} ";
+        if ((bothuncovered &&  !ea[point.first] && !fa[point.second]) ||
+            (!bothuncovered && !(ea[point.first] && fa[point.second]))) {
+          //add it!
+          currentpoints.insert(point);
+          A[point.first][point.second]=2;
+          //keep track of new covered positions
+          ea[point.first]=1;
+          fa[point.second]=1;
+
+          //added=1;
+          //cout << "added final: " << point.second-1 << "-" << point.first-1 << "\n";
+        }
+      }
+  }
+
+
+  for (k=currentpoints.begin(); k!=currentpoints.end(); k++)
+    sout << k->second-1 << "-" << k->first-1 << " ";
+
+
+  //fix the last " "
+  string str = sout.str();
+  if (str.length() == 0)
+    str = "\n";
+  else
+    str.replace(str.length()-1,1,"\n");
+
+  out << str;
+  out.flush();
+  return 1;
+
+  return 1;
+}
+
+} // namespace
+
+
+//Main file here
+
+
+int main(int argc, char** argv)
+{
+
+  int alignment=0;
+  char* input= NULL;
+  char* output= NULL;
+  int diagonal=false;
+  int isfinal=false;
+  int bothuncovered=false;
+
+
+  DeclareParams("a", CMDENUMTYPE,  &alignment, AlignEnum,
+                "alignment", CMDENUMTYPE,  &alignment, AlignEnum,
+                "d", CMDENUMTYPE,  &diagonal, BoolEnum,
+                "diagonal", CMDENUMTYPE,  &diagonal, BoolEnum,
+                "f", CMDENUMTYPE,  &isfinal, BoolEnum,
+                "final", CMDENUMTYPE,  &isfinal, BoolEnum,
+                "b", CMDENUMTYPE,  &bothuncovered, BoolEnum,
+                "both", CMDENUMTYPE,  &bothuncovered, BoolEnum,
+                "i", CMDSTRINGTYPE, &input,
+                "o", CMDSTRINGTYPE, &output,
+                "v", CMDENUMTYPE,  &verbose, BoolEnum,
+                "verbose", CMDENUMTYPE,  &verbose, BoolEnum,
+
+                NULL);
+
+  GetParams(&argc, &argv, NULL);
+
+  if (alignment==0) {
+    cerr << "usage: symal [-i=<inputfile>] [-o=<outputfile>] -a=[u|i|g] -d=[yes|no] -b=[yes|no] -f=[yes|no] \n"
+         << "Input file or std must be in .bal format (see script giza2bal.pl).\n";
+
+    exit(1);
+  }
+
+  istream *inp = &std::cin;
+  ostream *out = &std::cout;
+
+  try {
+    if (input) {
+      fstream *fin = new fstream(input,ios::in);
+      if (!fin->is_open()) throw runtime_error("cannot open " + string(input));
+      inp = fin;
+    }
+
+    if (output) {
+      fstream *fout = new fstream(output,ios::out);
+      if (!fout->is_open()) throw runtime_error("cannot open " + string(output));
+      out = fout;
+    }
+
+    int a[MAX_M],b[MAX_N],m,n;
+    fa=new int[MAX_M+1];
+    ea=new int[MAX_N+1];
+
+
+    int sents = 0;
+    A=new int *[MAX_N+1];
+    for (int i=1; i<=MAX_N; i++) A[i]=new int[MAX_M+1];
+
+    switch (alignment) {
+    case UNION:
+      cerr << "symal: computing union alignment\n";
+      while(getals(*inp,m,a,n,b)) {
+        prunionalignment(*out,m,a,n,b);
+        sents++;
+      }
+      cerr << "Sents: " << sents << endl;
+      break;
+    case INTERSECT:
+      cerr << "symal: computing intersect alignment\n";
+      while(getals(*inp,m,a,n,b)) {
+        printersect(*out,m,a,n,b);
+        sents++;
+      }
+      cerr << "Sents: " << sents << endl;
+      break;
+    case GROW:
+      cerr << "symal: computing grow alignment: diagonal ("
+           << diagonal << ") final ("<< isfinal << ")"
+           <<  "both-uncovered (" << bothuncovered <<")\n";
+
+      while(getals(*inp,m,a,n,b))
+        printgrow(*out,m,a,n,b,diagonal,isfinal,bothuncovered);
+
+      break;
+    case TGTTOSRC:
+      cerr << "symal: computing target-to-source alignment\n";
+
+      while(getals(*inp,m,a,n,b)) {
+        printtgttosrc(*out,m,a,n,b);
+        sents++;
+      }
+      cerr << "Sents: " << sents << endl;
+      break;
+    case SRCTOTGT:
+      cerr << "symal: computing source-to-target alignment\n";
+
+      while(getals(*inp,m,a,n,b)) {
+        printsrctotgt(*out,m,a,n,b);
+        sents++;
+      }
+      cerr << "Sents: " << sents << endl;
+      break;
+    default:
+      throw runtime_error("Unknown alignment");
+    }
+
+    delete [] fa;
+    delete [] ea;
+    for (int i=1; i<=MAX_N; i++) delete [] A[i];
+    delete [] A;
+
+    if (inp != &std::cin) {
+      delete inp;
+    }
+    if (out != &std::cout) {
+      delete inp;
+    }
+  } catch (const std::exception &e) {
+    cerr << e.what() << std::endl;
+    exit(1);
+  }
+
+  exit(0);
+}

mosesdecoder/symal/symal.vcproj

@@ -0,0 +1,204 @@
+<?xml version="1.0" encoding="Windows-1252"?>
+<VisualStudioProject
+	ProjectType="Visual C++"
+	Version="9.00"
+	Name="symal"
+	ProjectGUID="{6716FB26-8298-47A3-A915-958AF0AC80F8}"
+	RootNamespace="symal"
+	Keyword="Win32Proj"
+	TargetFrameworkVersion="131072"
+	>
+	<Platforms>
+		<Platform
+			Name="Win32"
+		/>
+	</Platforms>
+	<ToolFiles>
+	</ToolFiles>
+	<Configurations>
+		<Configuration
+			Name="Debug|Win32"
+			OutputDirectory="$(SolutionDir)$(ConfigurationName)"
+			IntermediateDirectory="$(ConfigurationName)"
+			ConfigurationType="1"
+			CharacterSet="1"
+			>
+			<Tool
+				Name="VCPreBuildEventTool"
+			/>
+			<Tool
+				Name="VCCustomBuildTool"
+			/>
+			<Tool
+				Name="VCXMLDataGeneratorTool"
+			/>
+			<Tool
+				Name="VCWebServiceProxyGeneratorTool"
+			/>
+			<Tool
+				Name="VCMIDLTool"
+			/>
+			<Tool
+				Name="VCCLCompilerTool"
+				Optimization="0"
+				PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_DEPRECATE;__STDC__"
+				MinimalRebuild="true"
+				BasicRuntimeChecks="3"
+				RuntimeLibrary="3"
+				UsePrecompiledHeader="0"
+				WarningLevel="3"
+				Detect64BitPortabilityProblems="true"
+				DebugInformationFormat="4"
+			/>
+			<Tool
+				Name="VCManagedResourceCompilerTool"
+			/>
+			<Tool
+				Name="VCResourceCompilerTool"
+			/>
+			<Tool
+				Name="VCPreLinkEventTool"
+			/>
+			<Tool
+				Name="VCLinkerTool"
+				LinkIncremental="2"
+				GenerateDebugInformation="true"
+				SubSystem="1"
+				RandomizedBaseAddress="1"
+				DataExecutionPrevention="0"
+				TargetMachine="1"
+			/>
+			<Tool
+				Name="VCALinkTool"
+			/>
+			<Tool
+				Name="VCManifestTool"
+			/>
+			<Tool
+				Name="VCXDCMakeTool"
+			/>
+			<Tool
+				Name="VCBscMakeTool"
+			/>
+			<Tool
+				Name="VCFxCopTool"
+			/>
+			<Tool
+				Name="VCAppVerifierTool"
+			/>
+			<Tool
+				Name="VCPostBuildEventTool"
+			/>
+		</Configuration>
+		<Configuration
+			Name="Release|Win32"
+			OutputDirectory="$(SolutionDir)$(ConfigurationName)"
+			IntermediateDirectory="$(ConfigurationName)"
+			ConfigurationType="1"
+			CharacterSet="1"
+			WholeProgramOptimization="1"
+			>
+			<Tool
+				Name="VCPreBuildEventTool"
+			/>
+			<Tool
+				Name="VCCustomBuildTool"
+			/>
+			<Tool
+				Name="VCXMLDataGeneratorTool"
+			/>
+			<Tool
+				Name="VCWebServiceProxyGeneratorTool"
+			/>
+			<Tool
+				Name="VCMIDLTool"
+			/>
+			<Tool
+				Name="VCCLCompilerTool"
+				PreprocessorDefinitions="WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_DEPRECATE;__STDC__"
+				RuntimeLibrary="2"
+				UsePrecompiledHeader="0"
+				WarningLevel="3"
+				Detect64BitPortabilityProblems="true"
+				DebugInformationFormat="3"
+			/>
+			<Tool
+				Name="VCManagedResourceCompilerTool"
+			/>
+			<Tool
+				Name="VCResourceCompilerTool"
+			/>
+			<Tool
+				Name="VCPreLinkEventTool"
+			/>
+			<Tool
+				Name="VCLinkerTool"
+				LinkIncremental="1"
+				GenerateDebugInformation="true"
+				SubSystem="1"
+				OptimizeReferences="2"
+				EnableCOMDATFolding="2"
+				RandomizedBaseAddress="1"
+				DataExecutionPrevention="0"
+				TargetMachine="1"
+			/>
+			<Tool
+				Name="VCALinkTool"
+			/>
+			<Tool
+				Name="VCManifestTool"
+			/>
+			<Tool
+				Name="VCXDCMakeTool"
+			/>
+			<Tool
+				Name="VCBscMakeTool"
+			/>
+			<Tool
+				Name="VCFxCopTool"
+			/>
+			<Tool
+				Name="VCAppVerifierTool"
+			/>
+			<Tool
+				Name="VCPostBuildEventTool"
+			/>
+		</Configuration>
+	</Configurations>
+	<References>
+	</References>
+	<Files>
+		<Filter
+			Name="Source Files"
+			Filter="cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx"
+			UniqueIdentifier="{4FC737F1-C7A5-4376-A066-2A32D752A2FF}"
+			>
+			<File
+				RelativePath=".\cmd.c"
+				>
+			</File>
+			<File
+				RelativePath=".\symal.cpp"
+				>
+			</File>
+		</Filter>
+		<Filter
+			Name="Header Files"
+			Filter="h;hpp;hxx;hm;inl;inc;xsd"
+			UniqueIdentifier="{93995380-89BD-4b04-88EB-625FBE52EBFB}"
+			>
+			<File
+				RelativePath=".\cmd.h"
+				>
+			</File>
+		</Filter>
+		<Filter
+			Name="Resource Files"
+			Filter="rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav"
+			UniqueIdentifier="{67DA6AB6-F800-4c08-8B7A-83BB121AAD01}"
+			>
+		</Filter>
+	</Files>
+	<Globals>
+	</Globals>
+</VisualStudioProject>

mosesdecoder/util/CMakeLists.txt

@@ -0,0 +1,81 @@
+cmake_minimum_required(VERSION 2.8.8)
+#
+# The KenLM cmake files make use of add_library(... OBJECTS ...)
+# 
+# This syntax allows grouping of source files when compiling
+# (effectively creating "fake" libraries based on source subdirs).
+# 
+# This syntax was only added in cmake version 2.8.8
+#
+# see http://www.cmake.org/Wiki/CMake/Tutorials/Object_Library
+
+
+# This CMake file was created by Lane Schwartz <dowobeha@gmail.com>
+
+
+# Explicitly list the source files for this subdirectory
+#
+# If you add any source files to this subdirectory
+#    that should be included in the kenlm library,
+#        (this excludes any unit test files)
+#    you should add them to the following list:
+#
+# Because we do not set PARENT_SCOPE in the following definition,
+#    CMake files in the parent directory won't be able to access this variable.
+#
+set(KENLM_UTIL_SOURCE 
+		bit_packing.cc 
+		ersatz_progress.cc 
+		exception.cc 
+		file.cc 
+		file_piece.cc 
+		float_to_string.cc
+		integer_to_string.cc
+		mmap.cc 
+		murmur_hash.cc 
+		parallel_read.cc
+		pool.cc 
+		read_compressed.cc 
+		scoped.cc 
+		string_piece.cc 
+		usage.cc
+	)
+
+# This directory has children that need to be processed
+add_subdirectory(double-conversion)
+add_subdirectory(stream)
+
+
+# Group these objects together for later use. 
+#
+# Given add_library(foo OBJECT ${my_foo_sources}),
+# refer to these objects as $<TARGET_OBJECTS:foo>
+#
+add_library(kenlm_util OBJECT ${KENLM_UTIL_DOUBLECONVERSION_SOURCE} ${KENLM_UTIL_STREAM_SOURCE} ${KENLM_UTIL_SOURCE})
+
+
+
+# Only compile and run unit tests if tests should be run
+if(BUILD_TESTING)
+
+  # Explicitly list the Boost test files to be compiled
+  set(KENLM_BOOST_TESTS_LIST
+    bit_packing_test
+    joint_sort_test
+    multi_intersection_test
+    probing_hash_table_test
+    read_compressed_test
+    sorted_uniform_test
+    tokenize_piece_test
+  )
+
+  AddTests(TESTS ${KENLM_BOOST_TESTS_LIST}
+           DEPENDS $<TARGET_OBJECTS:kenlm_util>
+           LIBRARIES ${Boost_LIBRARIES} pthread)
+
+  # file_piece_test requires an extra command line parameter
+  KenLMAddTest(TEST file_piece_test
+               DEPENDS $<TARGET_OBJECTS:kenlm_util>
+               LIBRARIES ${Boost_LIBRARIES} pthread
+               TEST_ARGS ${CMAKE_CURRENT_SOURCE_DIR}/file_piece.cc)
+endif()

mosesdecoder/util/Jamfile

@@ -0,0 +1,41 @@
+local compressed_flags = <include>.. <define>HAVE_ZLIB ;
+local compressed_deps = /top//z ;
+if [ test_library "bz2" ] && [ test_header "bzlib.h" ] {
+  external-lib bz2 ;
+  compressed_flags += <define>HAVE_BZLIB ;
+  compressed_deps += bz2 ;
+}
+if [ test_library "lzma" ] && [ test_header "lzma.h" ] {
+  external-lib lzma ;
+  compressed_flags += <define>HAVE_XZLIB ;
+  compressed_deps += lzma ;
+}
+
+#rt is needed for clock_gettime on linux.  But it's already included with threading=multi
+lib rt ;
+
+obj read_compressed.o : read_compressed.cc : $(compressed_flags) ;
+alias read_compressed : read_compressed.o $(compressed_deps) ;
+obj read_compressed_test.o : read_compressed_test.cc /top//boost_unit_test_framework : $(compressed_flags) ;
+obj file_piece_test.o : file_piece_test.cc /top//boost_unit_test_framework : $(compressed_flags) ;
+
+fakelib parallel_read : parallel_read.cc : <threading>multi:<source>/top//boost_thread <threading>multi:<define>WITH_THREADS : : <include>.. ;
+
+fakelib kenutil : [ glob *.cc : parallel_read.cc read_compressed.cc *_main.cc *_test.cc ] read_compressed parallel_read double-conversion//double-conversion : <include>.. <os>LINUX,<threading>single:<source>rt : : <include>.. ;
+
+exe cat_compressed : cat_compressed_main.cc kenutil ;
+
+#Does not install this
+exe probing_hash_table_benchmark : probing_hash_table_benchmark_main.cc kenutil ;
+
+alias programs : cat_compressed ;
+
+import testing ;
+
+run file_piece_test.o kenutil /top//boost_unit_test_framework : : file_piece.cc ;
+for local t in [ glob *_test.cc : file_piece_test.cc read_compressed_test.cc ] {
+    local name = [ MATCH "(.*)\.cc" : $(t) ] ;
+    unit-test $(name) : $(t) kenutil /top//boost_unit_test_framework /top//boost_filesystem /top//boost_system ;
+}
+
+build-project stream ;

mosesdecoder/util/bin/file_piece_test.test/gcc-9/release/link-static/threading-multi/file_piece_test

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ab8564bc029fa8f47779ac76015e129d019fb75c9c8bbda96ccb54e419e67a5c
+size 1747304

mosesdecoder/util/bin/file_piece_test.test/gcc-9/release/link-static/threading-multi/file_piece_test.output

@@ -0,0 +1,8 @@
+Boost.Test WARNING: token "util/file_piece.cc" does not correspond to the Boost.Test argument 
+                    and should be placed after all Boost.Test arguments and the -- separator.
+                    For example: file_piece_test --random -- util/file_piece.cc
+Running 6 test cases...
+
+*** No errors detected
+
+EXIT STATUS: 0

mosesdecoder/util/bin/file_piece_test.test/gcc-9/release/link-static/threading-multi/file_piece_test.run

@@ -0,0 +1,8 @@
+Boost.Test WARNING: token "util/file_piece.cc" does not correspond to the Boost.Test argument 
+                    and should be placed after all Boost.Test arguments and the -- separator.
+                    For example: file_piece_test --random -- util/file_piece.cc
+Running 6 test cases...
+
+*** No errors detected
+
+EXIT STATUS: 0

mosesdecoder/util/bin/file_piece_test.test/gcc-9/release/link-static/threading-multi/file_piece_test.test

@@ -0,0 +1 @@
+passed

mosesdecoder/util/bin/gcc-9/release/link-static/threading-multi/bit_packing_test

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:488bd05ca0bdcd9baf07b6c4618502d8b9cdf5b1ef97a55149be622a2e33cc60
+size 1701840

mosesdecoder/util/bin/gcc-9/release/link-static/threading-multi/integer_to_string_test

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:47f73d0d532194f98ca0b5de04960b79173ae4cb6291106f75af83f478b2ee94
+size 1715968