Upload folder using huggingface_hub

.gitattributes

@@ -36,3 +36,166 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 HiSd/data.blm.sd filter=lfs diff=lfs merge=lfs -text
 HiSd/phrase-table.minphr filter=lfs diff=lfs merge=lfs -text
 HiSd/reordering-table.minlexr filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/bin/build_binary filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/bin/consolidate filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/bin/consolidate-direct filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/bin/consolidate-reverse filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/bin/CreateOnDiskPt filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/bin/CreateProbingPT filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/bin/dump_counts filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/bin/evaluator filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/bin/extract filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/bin/extract-ghkm filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/bin/extract-lex filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/bin/extract-mixed-syntax filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/bin/extract-rules filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/bin/extractor filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/bin/filter filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/bin/filter-rule-table filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/bin/fragment filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/bin/gcc-9/debug/empty_test_static filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/bin/hgdecode filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/bin/kbmira filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/bin/kenlm_benchmark filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/bin/lexical-reordering-score filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/bin/lmbrgrid filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/bin/lmplz filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/bin/merge-sorted filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/bin/mert filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/bin/moses filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/bin/moses2 filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/bin/mosesserver filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/bin/postprocess-egret-forests filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/bin/prepare-expected-bleu-training filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/bin/pro filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/bin/processLexicalTable filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/bin/processLexicalTableMin filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/bin/processPhraseTableMin filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/bin/prunePhraseTable filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/bin/query filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/bin/queryLexicalTable filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/bin/queryPhraseTableMin filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/bin/relax-parse filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/bin/score filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/bin/score-stsg filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/bin/sentence-bleu-nbest filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/bin/statistics filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/bin/train-expected-bleu filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/bin/vwtrainer filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/cmph-2.0.2/lib/libcmph.a filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/cmph-2.0.2/src/.libs/libcmph.a filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/contrib/expected-bleu-training/bin/gcc-9/release/link-static/threading-multi/prepare-expected-bleu-training filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/contrib/expected-bleu-training/bin/gcc-9/release/link-static/threading-multi/train-expected-bleu filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/contrib/promix/test_data/esen.ep.model.filtered/phrase-table.0-0.1.1 filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/contrib/promix/test_data/esen.ep.model.filtered/phrase-table.0-0.1.1.binphr.tgtdata.wa filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/contrib/server/bin/gcc-9/release/link-static/threading-multi/mosesserver filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/lib/libmert_lib.a filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/lib/libmoses.a filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/lib/libmoses2_lib.a filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/lib/libmoses2decoder.a filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/lib/libprobingpt.a filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/lib/libsyntax_common.a filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/lm/bin/gcc-9/release/link-static/threading-multi/build_binary filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/lm/bin/gcc-9/release/link-static/threading-multi/fragment filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/lm/bin/gcc-9/release/link-static/threading-multi/kenlm_benchmark filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/lm/bin/gcc-9/release/link-static/threading-multi/query filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/lm/bin/left_test.test/gcc-9/release/link-static/threading-multi/left_test filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/lm/bin/model_test.test/gcc-9/release/link-static/threading-multi/model_test filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/lm/bin/model_test.test/gcc-9/release/link-static/threading-multi/model_test.o filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/lm/bin/partial_test.test/gcc-9/release/link-static/threading-multi/partial_test filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/lm/builder/bin/gcc-9/release/link-static/threading-multi/dump_counts filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/lm/builder/bin/gcc-9/release/link-static/threading-multi/lmplz filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/lm/filter/bin/gcc-9/release/link-static/threading-multi/filter filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/mert/bin/gcc-9/release/link-static/threading-multi/bleu_scorer_test filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/mert/bin/gcc-9/release/link-static/threading-multi/data_test filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/mert/bin/gcc-9/release/link-static/threading-multi/evaluator filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/mert/bin/gcc-9/release/link-static/threading-multi/extractor filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/mert/bin/gcc-9/release/link-static/threading-multi/feature_data_test filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/mert/bin/gcc-9/release/link-static/threading-multi/forest_rescore_test filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/mert/bin/gcc-9/release/link-static/threading-multi/hgdecode filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/mert/bin/gcc-9/release/link-static/threading-multi/hypergraph_test filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/mert/bin/gcc-9/release/link-static/threading-multi/kbmira filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/mert/bin/gcc-9/release/link-static/threading-multi/libmert_lib.a filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/mert/bin/gcc-9/release/link-static/threading-multi/mert filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/mert/bin/gcc-9/release/link-static/threading-multi/mira_feature_vector_test filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/mert/bin/gcc-9/release/link-static/threading-multi/ngram_test filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/mert/bin/gcc-9/release/link-static/threading-multi/optimizer_factory_test filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/mert/bin/gcc-9/release/link-static/threading-multi/point_test filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/mert/bin/gcc-9/release/link-static/threading-multi/pro filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/mert/bin/gcc-9/release/link-static/threading-multi/reference_test filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/mert/bin/gcc-9/release/link-static/threading-multi/sentence-bleu-nbest filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/mert/bin/gcc-9/release/link-static/threading-multi/singleton_test filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/mert/bin/gcc-9/release/link-static/threading-multi/TER/tools.o filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/mert/bin/gcc-9/release/link-static/threading-multi/timer_test filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/mert/bin/gcc-9/release/link-static/threading-multi/util_test filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/mert/bin/gcc-9/release/link-static/threading-multi/vocabulary_test filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/mert/evaluator filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/mert/extractor filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/mert/hgdecode filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/mert/kbmira filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/mert/mert filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/mert/pro filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/mert/sentence-bleu-nbest filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/misc/bin/gcc-9/release/link-static/threading-multi/merge-sorted filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/misc/bin/gcc-9/release/link-static/threading-multi/processLexicalTable filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/misc/bin/gcc-9/release/link-static/threading-multi/processLexicalTableMin filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/misc/bin/gcc-9/release/link-static/threading-multi/processPhraseTableMin filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/misc/bin/gcc-9/release/link-static/threading-multi/prunePhraseTable filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/misc/bin/gcc-9/release/link-static/threading-multi/queryLexicalTable filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/misc/bin/gcc-9/release/link-static/threading-multi/queryPhraseTableMin filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/moses/bin/gcc-9/release/link-static/threading-multi/libmoses.a filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/moses/bin/gcc-9/release/link-static/threading-multi/moses_test filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/moses/LM/bin/BackwardTest.test/gcc-9/release/link-static/threading-multi/BackwardTest filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/moses/TranslationModel/UG/util/ibm1-align filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/moses-cmd/bin/gcc-9/release/link-static/threading-multi/lmbrgrid filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/moses-cmd/bin/gcc-9/release/link-static/threading-multi/moses filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/moses-cmd/bin/gcc-9/release/link-static/threading-multi/vwtrainer filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/moses2/bin/gcc-9/release/link-static/threading-multi/libmoses2_lib.a filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/moses2/bin/gcc-9/release/link-static/threading-multi/libmoses2decoder.a filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/moses2/bin/gcc-9/release/link-static/threading-multi/moses2 filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/OnDiskPt/bin/gcc-9/release/link-static/threading-multi/CreateOnDiskPt filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/phrase-extract/bin/gcc-9/release/link-static/threading-multi/consolidate filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/phrase-extract/bin/gcc-9/release/link-static/threading-multi/consolidate-direct filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/phrase-extract/bin/gcc-9/release/link-static/threading-multi/consolidate-reverse filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/phrase-extract/bin/gcc-9/release/link-static/threading-multi/extract filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/phrase-extract/bin/gcc-9/release/link-static/threading-multi/extract-lex filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/phrase-extract/bin/gcc-9/release/link-static/threading-multi/extract-rules filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/phrase-extract/bin/gcc-9/release/link-static/threading-multi/relax-parse filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/phrase-extract/bin/gcc-9/release/link-static/threading-multi/score filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/phrase-extract/bin/gcc-9/release/link-static/threading-multi/statistics filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/phrase-extract/bin/ScoreFeatureTest.test/gcc-9/release/link-static/threading-multi/ScoreFeatureTest filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/phrase-extract/extract-ghkm/bin/gcc-9/release/link-static/threading-multi/extract-ghkm filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/phrase-extract/extract-mixed-syntax/bin/gcc-9/release/link-static/threading-multi/extract-mixed-syntax filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/phrase-extract/filter-rule-table/bin/gcc-9/release/link-static/threading-multi/filter-rule-table filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/phrase-extract/lexical-reordering/bin/gcc-9/release/link-static/threading-multi/lexical-reordering-score filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/phrase-extract/postprocess-egret-forests/bin/gcc-9/release/link-static/threading-multi/postprocess-egret-forests filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/phrase-extract/score-stsg/bin/gcc-9/release/link-static/threading-multi/score-stsg filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/phrase-extract/syntax-common/bin/gcc-9/release/link-static/threading-multi/libsyntax_common.a filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/phrase-extract/syntax-common/bin/gcc-9/release/link-static/threading-multi/tree_fragment_tokenizer_test filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/phrase-extract/syntax-common/bin/gcc-9/release/link-static/threading-multi/tree_test filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/probingpt/bin/gcc-9/release/link-static/threading-multi/CreateProbingPT filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/probingpt/bin/gcc-9/release/link-static/threading-multi/libprobingpt.a filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/util/bin/file_piece_test.test/gcc-9/release/link-static/threading-multi/file_piece_test filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/util/bin/gcc-9/release/link-static/threading-multi/bit_packing_test filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/util/bin/gcc-9/release/link-static/threading-multi/integer_to_string_test filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/util/bin/gcc-9/release/link-static/threading-multi/joint_sort_test filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/util/bin/gcc-9/release/link-static/threading-multi/multi_intersection_test filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/util/bin/gcc-9/release/link-static/threading-multi/pcqueue_test filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/util/bin/gcc-9/release/link-static/threading-multi/probing_hash_table_test filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/util/bin/gcc-9/release/link-static/threading-multi/random_test filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/util/bin/gcc-9/release/link-static/threading-multi/sized_iterator_test filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/util/bin/gcc-9/release/link-static/threading-multi/sorted_uniform_test filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/util/bin/gcc-9/release/link-static/threading-multi/string_stream_test filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/util/bin/gcc-9/release/link-static/threading-multi/tempfile_test filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/util/bin/gcc-9/release/link-static/threading-multi/tokenize_piece_test filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/util/bin/gcc-9/release/link-static/threading-multi/tokenize_test filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/util/stream/bin/gcc-9/release/link-static/threading-multi/io_test filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/util/stream/bin/gcc-9/release/link-static/threading-multi/rewindable_stream_test filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/util/stream/bin/gcc-9/release/link-static/threading-multi/sort_test filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/util/stream/bin/gcc-9/release/link-static/threading-multi/stream_test filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/xmlrpc-c-1.51.06/lib/abyss/src/libxmlrpc_abyss.a filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/xmlrpc-c-1.51.06/lib/libxmlrpc++.a filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/xmlrpc-c-1.51.06/lib/libxmlrpc.a filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/xmlrpc-c-1.51.06/lib/libxmlrpc_abyss.a filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/xmlrpc-c-1.51.06/src/cpp/libxmlrpc++.a filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/xmlrpc-c-1.51.06/src/libxmlrpc.a filter=lfs diff=lfs merge=lfs -text

mosesdecoder/.beautify-ignore

@@ -0,0 +1,38 @@
+# Files and directories that beautify.py should not clean up.
+#
+# This file is not as advanced as, say, .gitignore.  It only supports files
+# and directory paths relative to the project root, one per line, no globs,
+# no quotes.
+#
+# Leading and trailing whitespace is stripped from filenames, but internal
+# whitespace is preserved.
+#
+# Lines starting with a hash mark, such as this one, are comments.  The hash
+# mark must be the first character on the line.  Blank lines are ignored.
+#
+# The .beautify-ignore file must be encoded in UTF-8.
+
+boost
+contrib
+irstlm
+jam-files
+lm
+mingw/MosesGUI/icons_rc.py
+mingw/MosesGUI/Ui_credits.py
+mingw/MosesGUI/Ui_mainWindow.py
+moses/TranslationModel/UG
+moses/server
+moses/parameters
+moses/thread_safe_container.h
+phrase-extract/pcfg-common
+phrase-extract/syntax-common
+randlm
+# Filename suffixes in here are language codes, so e.g. ".pl" means
+# Polish, not Perl.
+scripts/share/nonbreaking_prefixes
+search
+srilm
+util
+xmlrpc-c
+.git
+util/ug_cache_with_timeout.h

mosesdecoder/.gitignore

@@ -0,0 +1,90 @@
+tools
+*.d
+*.pyc
+*.lo
+*.o
+*.so
+*.lo
+*.o
+*.la
+*.a
+*.swp
+*.save
+*.cmd
+*~
+*.gch
+dist*
+jam-files/bjam
+jam-files/engine/bootstrap
+jam-files/engine/bin.*
+lm/build_binary
+lm/query
+mert/evaluator
+mert/extractor
+mert/hgdecode
+mert/mert
+mert/megam_i686.opt
+mert/pro
+mert/kbmira
+misc/processLexicalTable
+misc/processPhraseTable
+misc/queryLexicalTable
+mira/mira
+mira/Makefile
+mira/Makefile.in
+misc/queryPhraseTable
+moses-chart-cmd/src/moses_chart
+moses-cmd/src/checkplf
+moses-cmd/src/lmbrgrid
+moses-cmd/src/moses
+regression-testing/moses-reg-test-data-*
+regression-testing/tests/mert.extractor-bin/FEATSTAT*
+regression-testing/tests/mert.extractor-bin/SCORESTAT*
+scripts/ems/biconcor/biconcor
+scripts/release-exclude
+scripts/training/cmert-0.5/mert
+scripts/training/compact-rule-table/tools/compactify
+scripts/training/eppex/counter
+scripts/training/eppex/eppex
+scripts/training/lexical-reordering/score
+scripts/training/memscore/memscore
+scripts/training/mbr/mbr
+scripts/training/phrase-extract/consolidate
+scripts/training/phrase-extract/consolidate-direct
+scripts/training/phrase-extract/consolidate-reverse
+scripts/training/phrase-extract/extract
+scripts/training/phrase-extract/extract-ghkm/tools/extract-ghkm
+scripts/training/phrase-extract/extract-lex
+scripts/training/phrase-extract/extract-rules
+scripts/training/phrase-extract/relax-parse
+scripts/training/phrase-extract/score
+scripts/training/phrase-extract/statistics
+scripts/training/symal/symal
+dist
+bin
+previous.sh
+contrib/other-builds/*.xcodeproj/project.xcworkspace/
+contrib/other-builds/*.xcodeproj/xcuserdata/
+*/*.xcodeproj/project.xcworkspace
+*/*.xcodeproj/xcuserdata
+
+mert/sentence-bleu
+mert/sentence-bleu-nbest
+._*
+.DS_Store
+*.pbxuser
+*.mode1v3
+
+*.exe
+build/
+nbproject/
+
+mingw/MosesGUI/MosesGUI.e4p
+mingw/MosesGUI/_eric4project/
+
+contrib/m4m/merge-sorted
+mert/hgdecode
+.bash_history*
+doxygen.conf
+doxy
+opt

mosesdecoder/.gitmodules

@@ -0,0 +1,9 @@
+[submodule "contrib/arrow-pipelines/python/pcl"]
+	path = contrib/arrow-pipelines/python/pcl
+	url = https://github.com/ianj-als/pcl.git
+[submodule "contrib/omtc/omtc"]
+	path = contrib/omtc/omtc
+	url = https://github.com/ianj-als/omtc.git
+[submodule "regtest"]
+	path = regtest
+	url = https://github.com/moses-smt/moses-regression-tests

mosesdecoder/.travis.yml

@@ -0,0 +1,24 @@
+sudo: false
+dist: trusty
+language: c
+compiler: gcc
+env:
+  matrix: 
+addons:
+  apt:
+    sources:
+      - ubuntu-toolchain-r-test
+    packages:
+      - subversion
+      - automake
+      - libtool
+      - zlib1g-dev
+      - libbz2-dev
+      - liblzma-dev
+      - libboost-all-dev
+      - libgoogle-perftools-dev
+      - libxmlrpc-c++.*-dev
+      - cmake
+      - csh
+script:
+- ./bjam -j4

mosesdecoder/COPYING

@@ -0,0 +1,460 @@
+
+                  GNU LESSER GENERAL PUBLIC LICENSE
+                       Version 2.1, February 1999
+
+ Copyright (C) 1991, 1999 Free Software Foundation, Inc.
+	51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+[This is the first released version of the Lesser GPL.  It also counts
+ as the successor of the GNU Library Public License, version 2, hence
+ the version number 2.1.]
+
+                            Preamble
+
+  The licenses for most software are designed to take away your
+freedom to share and change it.  By contrast, the GNU General Public
+Licenses are intended to guarantee your freedom to share and change
+free software--to make sure the software is free for all its users.
+
+  This license, the Lesser General Public License, applies to some
+specially designated software packages--typically libraries--of the
+Free Software Foundation and other authors who decide to use it.  You
+can use it too, but we suggest you first think carefully about whether
+this license or the ordinary General Public License is the better
+strategy to use in any particular case, based on the explanations
+below.
+
+  When we speak of free software, we are referring to freedom of use,
+not price.  Our General Public Licenses are designed to make sure that
+you have the freedom to distribute copies of free software (and charge
+for this service if you wish); that you receive source code or can get
+it if you want it; that you can change the software and use pieces of
+it in new free programs; and that you are informed that you can do
+these things.
+
+  To protect your rights, we need to make restrictions that forbid
+distributors to deny you these rights or to ask you to surrender these
+rights.  These restrictions translate to certain responsibilities for
+you if you distribute copies of the library or if you modify it.
+
+  For example, if you distribute copies of the library, whether gratis
+or for a fee, you must give the recipients all the rights that we gave
+you.  You must make sure that they, too, receive or can get the source
+code.  If you link other code with the library, you must provide
+complete object files to the recipients, so that they can relink them
+with the library after making changes to the library and recompiling
+it.  And you must show them these terms so they know their rights.
+
+  We protect your rights with a two-step method: (1) we copyright the
+library, and (2) we offer you this license, which gives you legal
+permission to copy, distribute and/or modify the library.
+
+  To protect each distributor, we want to make it very clear that
+there is no warranty for the free library.  Also, if the library is
+modified by someone else and passed on, the recipients should know
+that what they have is not the original version, so that the original
+author's reputation will not be affected by problems that might be
+introduced by others.
+
+  Finally, software patents pose a constant threat to the existence of
+any free program.  We wish to make sure that a company cannot
+effectively restrict the users of a free program by obtaining a
+restrictive license from a patent holder.  Therefore, we insist that
+any patent license obtained for a version of the library must be
+consistent with the full freedom of use specified in this license.
+
+  Most GNU software, including some libraries, is covered by the
+ordinary GNU General Public License.  This license, the GNU Lesser
+General Public License, applies to certain designated libraries, and
+is quite different from the ordinary General Public License.  We use
+this license for certain libraries in order to permit linking those
+libraries into non-free programs.
+
+  When a program is linked with a library, whether statically or using
+a shared library, the combination of the two is legally speaking a
+combined work, a derivative of the original library.  The ordinary
+General Public License therefore permits such linking only if the
+entire combination fits its criteria of freedom.  The Lesser General
+Public License permits more lax criteria for linking other code with
+the library.
+
+  We call this license the "Lesser" General Public License because it
+does Less to protect the user's freedom than the ordinary General
+Public License.  It also provides other free software developers Less
+of an advantage over competing non-free programs.  These disadvantages
+are the reason we use the ordinary General Public License for many
+libraries.  However, the Lesser license provides advantages in certain
+special circumstances.
+
+  For example, on rare occasions, there may be a special need to
+encourage the widest possible use of a certain library, so that it
+becomes a de-facto standard.  To achieve this, non-free programs must
+be allowed to use the library.  A more frequent case is that a free
+library does the same job as widely used non-free libraries.  In this
+case, there is little to gain by limiting the free library to free
+software only, so we use the Lesser General Public License.
+
+  In other cases, permission to use a particular library in non-free
+programs enables a greater number of people to use a large body of
+free software.  For example, permission to use the GNU C Library in
+non-free programs enables many more people to use the whole GNU
+operating system, as well as its variant, the GNU/Linux operating
+system.
+
+  Although the Lesser General Public License is Less protective of the
+users' freedom, it does ensure that the user of a program that is
+linked with the Library has the freedom and the wherewithal to run
+that program using a modified version of the Library.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.  Pay close attention to the difference between a
+"work based on the library" and a "work that uses the library".  The
+former contains code derived from the library, whereas the latter must
+be combined with the library in order to run.
+
+                  GNU LESSER GENERAL PUBLIC LICENSE
+   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+  0. This License Agreement applies to any software library or other
+program which contains a notice placed by the copyright holder or
+other authorized party saying it may be distributed under the terms of
+this Lesser General Public License (also called "this License").
+Each licensee is addressed as "you".
+
+  A "library" means a collection of software functions and/or data
+prepared so as to be conveniently linked with application programs
+(which use some of those functions and data) to form executables.
+
+  The "Library", below, refers to any such software library or work
+which has been distributed under these terms.  A "work based on the
+Library" means either the Library or any derivative work under
+copyright law: that is to say, a work containing the Library or a
+portion of it, either verbatim or with modifications and/or translated
+straightforwardly into another language.  (Hereinafter, translation is
+included without limitation in the term "modification".)
+
+  "Source code" for a work means the preferred form of the work for
+making modifications to it.  For a library, complete source code means
+all the source code for all modules it contains, plus any associated
+interface definition files, plus the scripts used to control
+compilation and installation of the library.
+
+  Activities other than copying, distribution and modification are not
+covered by this License; they are outside its scope.  The act of
+running a program using the Library is not restricted, and output from
+such a program is covered only if its contents constitute a work based
+on the Library (independent of the use of the Library in a tool for
+writing it).  Whether that is true depends on what the Library does
+and what the program that uses the Library does.
+
+  1. You may copy and distribute verbatim copies of the Library's
+complete source code as you receive it, in any medium, provided that
+you conspicuously and appropriately publish on each copy an
+appropriate copyright notice and disclaimer of warranty; keep intact
+all the notices that refer to this License and to the absence of any
+warranty; and distribute a copy of this License along with the
+Library.
+
+  You may charge a fee for the physical act of transferring a copy,
+and you may at your option offer warranty protection in exchange for a
+fee.
+
+  2. You may modify your copy or copies of the Library or any portion
+of it, thus forming a work based on the Library, and copy and
+distribute such modifications or work under the terms of Section 1
+above, provided that you also meet all of these conditions:
+
+    a) The modified work must itself be a software library.
+
+    b) You must cause the files modified to carry prominent notices
+    stating that you changed the files and the date of any change.
+
+    c) You must cause the whole of the work to be licensed at no
+    charge to all third parties under the terms of this License.
+
+    d) If a facility in the modified Library refers to a function or a
+    table of data to be supplied by an application program that uses
+    the facility, other than as an argument passed when the facility
+    is invoked, then you must make a good faith effort to ensure that,
+    in the event an application does not supply such function or
+    table, the facility still operates, and performs whatever part of
+    its purpose remains meaningful.
+
+    (For example, a function in a library to compute square roots has
+    a purpose that is entirely well-defined independent of the
+    application.  Therefore, Subsection 2d requires that any
+    application-supplied function or table used by this function must
+    be optional: if the application does not supply it, the square
+    root function must still compute square roots.)
+
+These requirements apply to the modified work as a whole.  If
+identifiable sections of that work are not derived from the Library,
+and can be reasonably considered independent and separate works in
+themselves, then this License, and its terms, do not apply to those
+sections when you distribute them as separate works.  But when you
+distribute the same sections as part of a whole which is a work based
+on the Library, the distribution of the whole must be on the terms of
+this License, whose permissions for other licensees extend to the
+entire whole, and thus to each and every part regardless of who wrote
+it.
+
+Thus, it is not the intent of this section to claim rights or contest
+your rights to work written entirely by you; rather, the intent is to
+exercise the right to control the distribution of derivative or
+collective works based on the Library.
+
+In addition, mere aggregation of another work not based on the Library
+with the Library (or with a work based on the Library) on a volume of
+a storage or distribution medium does not bring the other work under
+the scope of this License.
+
+  3. You may opt to apply the terms of the ordinary GNU General Public
+License instead of this License to a given copy of the Library.  To do
+this, you must alter all the notices that refer to this License, so
+that they refer to the ordinary GNU General Public License, version 2,
+instead of to this License.  (If a newer version than version 2 of the
+ordinary GNU General Public License has appeared, then you can specify
+that version instead if you wish.)  Do not make any other change in
+these notices.
+
+  Once this change is made in a given copy, it is irreversible for
+that copy, so the ordinary GNU General Public License applies to all
+subsequent copies and derivative works made from that copy.
+
+  This option is useful when you wish to copy part of the code of
+the Library into a program that is not a library.
+
+  4. You may copy and distribute the Library (or a portion or
+derivative of it, under Section 2) in object code or executable form
+under the terms of Sections 1 and 2 above provided that you accompany
+it with the complete corresponding machine-readable source code, which
+must be distributed under the terms of Sections 1 and 2 above on a
+medium customarily used for software interchange.
+
+  If distribution of object code is made by offering access to copy
+from a designated place, then offering equivalent access to copy the
+source code from the same place satisfies the requirement to
+distribute the source code, even though third parties are not
+compelled to copy the source along with the object code.
+
+  5. A program that contains no derivative of any portion of the
+Library, but is designed to work with the Library by being compiled or
+linked with it, is called a "work that uses the Library".  Such a
+work, in isolation, is not a derivative work of the Library, and
+therefore falls outside the scope of this License.
+
+  However, linking a "work that uses the Library" with the Library
+creates an executable that is a derivative of the Library (because it
+contains portions of the Library), rather than a "work that uses the
+library".  The executable is therefore covered by this License.
+Section 6 states terms for distribution of such executables.
+
+  When a "work that uses the Library" uses material from a header file
+that is part of the Library, the object code for the work may be a
+derivative work of the Library even though the source code is not.
+Whether this is true is especially significant if the work can be
+linked without the Library, or if the work is itself a library.  The
+threshold for this to be true is not precisely defined by law.
+
+  If such an object file uses only numerical parameters, data
+structure layouts and accessors, and small macros and small inline
+functions (ten lines or less in length), then the use of the object
+file is unrestricted, regardless of whether it is legally a derivative
+work.  (Executables containing this object code plus portions of the
+Library will still fall under Section 6.)
+
+  Otherwise, if the work is a derivative of the Library, you may
+distribute the object code for the work under the terms of Section 6.
+Any executables containing that work also fall under Section 6,
+whether or not they are linked directly with the Library itself.
+
+  6. As an exception to the Sections above, you may also combine or
+link a "work that uses the Library" with the Library to produce a
+work containing portions of the Library, and distribute that work
+under terms of your choice, provided that the terms permit
+modification of the work for the customer's own use and reverse
+engineering for debugging such modifications.
+
+  You must give prominent notice with each copy of the work that the
+Library is used in it and that the Library and its use are covered by
+this License.  You must supply a copy of this License.  If the work
+during execution displays copyright notices, you must include the
+copyright notice for the Library among them, as well as a reference
+directing the user to the copy of this License.  Also, you must do one
+of these things:
+
+    a) Accompany the work with the complete corresponding
+    machine-readable source code for the Library including whatever
+    changes were used in the work (which must be distributed under
+    Sections 1 and 2 above); and, if the work is an executable linked
+    with the Library, with the complete machine-readable "work that
+    uses the Library", as object code and/or source code, so that the
+    user can modify the Library and then relink to produce a modified
+    executable containing the modified Library.  (It is understood
+    that the user who changes the contents of definitions files in the
+    Library will not necessarily be able to recompile the application
+    to use the modified definitions.)
+
+    b) Use a suitable shared library mechanism for linking with the
+    Library.  A suitable mechanism is one that (1) uses at run time a
+    copy of the library already present on the user's computer system,
+    rather than copying library functions into the executable, and (2)
+    will operate properly with a modified version of the library, if
+    the user installs one, as long as the modified version is
+    interface-compatible with the version that the work was made with.
+
+    c) Accompany the work with a written offer, valid for at least
+    three years, to give the same user the materials specified in
+    Subsection 6a, above, for a charge no more than the cost of
+    performing this distribution.
+
+    d) If distribution of the work is made by offering access to copy
+    from a designated place, offer equivalent access to copy the above
+    specified materials from the same place.
+
+    e) Verify that the user has already received a copy of these
+    materials or that you have already sent this user a copy.
+
+  For an executable, the required form of the "work that uses the
+Library" must include any data and utility programs needed for
+reproducing the executable from it.  However, as a special exception,
+the materials to be distributed need not include anything that is
+normally distributed (in either source or binary form) with the major
+components (compiler, kernel, and so on) of the operating system on
+which the executable runs, unless that component itself accompanies
+the executable.
+
+  It may happen that this requirement contradicts the license
+restrictions of other proprietary libraries that do not normally
+accompany the operating system.  Such a contradiction means you cannot
+use both them and the Library together in an executable that you
+distribute.
+
+  7. You may place library facilities that are a work based on the
+Library side-by-side in a single library together with other library
+facilities not covered by this License, and distribute such a combined
+library, provided that the separate distribution of the work based on
+the Library and of the other library facilities is otherwise
+permitted, and provided that you do these two things:
+
+    a) Accompany the combined library with a copy of the same work
+    based on the Library, uncombined with any other library
+    facilities.  This must be distributed under the terms of the
+    Sections above.
+
+    b) Give prominent notice with the combined library of the fact
+    that part of it is a work based on the Library, and explaining
+    where to find the accompanying uncombined form of the same work.
+
+  8. You may not copy, modify, sublicense, link with, or distribute
+the Library except as expressly provided under this License.  Any
+attempt otherwise to copy, modify, sublicense, link with, or
+distribute the Library is void, and will automatically terminate your
+rights under this License.  However, parties who have received copies,
+or rights, from you under this License will not have their licenses
+terminated so long as such parties remain in full compliance.
+
+  9. You are not required to accept this License, since you have not
+signed it.  However, nothing else grants you permission to modify or
+distribute the Library or its derivative works.  These actions are
+prohibited by law if you do not accept this License.  Therefore, by
+modifying or distributing the Library (or any work based on the
+Library), you indicate your acceptance of this License to do so, and
+all its terms and conditions for copying, distributing or modifying
+the Library or works based on it.
+
+  10. Each time you redistribute the Library (or any work based on the
+Library), the recipient automatically receives a license from the
+original licensor to copy, distribute, link with or modify the Library
+subject to these terms and conditions.  You may not impose any further
+restrictions on the recipients' exercise of the rights granted herein.
+You are not responsible for enforcing compliance by third parties with
+this License.
+
+  11. If, as a consequence of a court judgment or allegation of patent
+infringement or for any other reason (not limited to patent issues),
+conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot
+distribute so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you
+may not distribute the Library at all.  For example, if a patent
+license would not permit royalty-free redistribution of the Library by
+all those who receive copies directly or indirectly through you, then
+the only way you could satisfy both it and this License would be to
+refrain entirely from distribution of the Library.
+
+If any portion of this section is held invalid or unenforceable under
+any particular circumstance, the balance of the section is intended to
+apply, and the section as a whole is intended to apply in other
+circumstances.
+
+It is not the purpose of this section to induce you to infringe any
+patents or other property right claims or to contest validity of any
+such claims; this section has the sole purpose of protecting the
+integrity of the free software distribution system which is
+implemented by public license practices.  Many people have made
+generous contributions to the wide range of software distributed
+through that system in reliance on consistent application of that
+system; it is up to the author/donor to decide if he or she is willing
+to distribute software through any other system and a licensee cannot
+impose that choice.
+
+This section is intended to make thoroughly clear what is believed to
+be a consequence of the rest of this License.
+
+  12. If the distribution and/or use of the Library is restricted in
+certain countries either by patents or by copyrighted interfaces, the
+original copyright holder who places the Library under this License
+may add an explicit geographical distribution limitation excluding those
+countries, so that distribution is permitted only in or among
+countries not thus excluded.  In such case, this License incorporates
+the limitation as if written in the body of this License.
+
+  13. The Free Software Foundation may publish revised and/or new
+versions of the Lesser General Public License from time to time.
+Such new versions will be similar in spirit to the present version,
+but may differ in detail to address new problems or concerns.
+
+Each version is given a distinguishing version number.  If the Library
+specifies a version number of this License which applies to it and
+"any later version", you have the option of following the terms and
+conditions either of that version or of any later version published by
+the Free Software Foundation.  If the Library does not specify a
+license version number, you may choose any version ever published by
+the Free Software Foundation.
+
+  14. If you wish to incorporate parts of the Library into other free
+programs whose distribution conditions are incompatible with these,
+write to the author to ask for permission.  For software which is
+copyrighted by the Free Software Foundation, write to the Free
+Software Foundation; we sometimes make exceptions for this.  Our
+decision will be guided by the two goals of preserving the free status
+of all derivatives of our free software and of promoting the sharing
+and reuse of software generally.
+
+                            NO WARRANTY
+
+  15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO
+WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW.
+EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR
+OTHER PARTIES PROVIDE THE LIBRARY "AS IS" WITHOUT WARRANTY OF ANY
+KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE.  THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE
+LIBRARY IS WITH YOU.  SHOULD THE LIBRARY PROVE DEFECTIVE, YOU ASSUME
+THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
+
+  16. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN
+WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY
+AND/OR REDISTRIBUTE THE LIBRARY AS PERMITTED ABOVE, BE LIABLE TO YOU
+FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR
+CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE
+LIBRARY (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING
+RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A
+FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF
+SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH
+DAMAGES.
+

mosesdecoder/Jamroot

@@ -0,0 +1,345 @@
+#BUILDING MOSES
+
+#PACKAGES
+#Language models (optional):
+#--with-irstlm=/path/to/irstlm 
+#--with-srilm=/path/to/srilm See moses/LM/Jamfile for more options.
+#--with-maxent-srilm=true (requires a maxent-enabled version of SRILM to be specified via --with-srilm)
+#--with-nplm=/path/to/nplm
+#--with-randlm=/path/to/randlm
+#KenLM is always compiled.  
+#
+#--with-boost=/path/to/boost
+#If Boost is in a non-standard location, specify it here.  This directory is
+#expected to contain include and lib or lib64.  
+#
+#--with-xmlrpc-c=/path/to/xmlrpc-c for libxmlrpc-c (used by server)
+#Note that, like language models, this is the --prefix where the library was
+#installed, not some executable within the library.  
+#
+#--no-xmlrpc-c
+# Don't use xmlrpc-c library, even if it exists. Don't build moses server
+#
+#Compact phrase table and compact lexical reordering table
+#--with-cmph=/path/to/cmph
+#
+#Thread-caching malloc (if present, used for multi-threaded builds by default)
+#--without-tcmalloc does not compile with tcmalloc even if present
+#--full-tcmalloc links against the full version (useful for memory profiling)
+#
+#REGRESSION TESTING
+#--with-regtest=/path/to/moses-reg-test-data
+#
+#INSTALLATION
+#--prefix=/path/to/prefix sets the install prefix [default is source root].
+#--bindir=/path/to/prefix/bin sets the bin directory [PREFIX/bin]
+#--libdir=/path/to/prefix/lib sets the lib directory [PREFIX/lib]
+#--includedir=/path/to/prefix/include installs headers.  
+#  Does not install if missing.  No argument defaults to PREFIX/include .
+#--install-scripts=/path/to/scripts copies scripts into a directory.
+#  Does not install if missing.  No argument defaults to PREFIX/scripts .
+#--git appends the git revision to the prefix directory.
+#
+#
+#BUILD OPTIONS
+# By default, the build is multi-threaded, optimized, and statically linked.  
+# Pass these to change the build:
+#
+# threading=single|multi         controls threading (default multi)
+#
+# variant=release|debug|profile  builds optimized (default), for debug, or for
+#                                profiling
+#
+# link=static|shared             controls preferred linking (default static)
+# --static                       forces static linking (the default will fall
+#                                back to shared)
+#
+# debug-symbols=on|off           include or exclude (default) debugging 
+#                                information also known as -g
+# --notrace                      compiles without TRACE macros
+#
+# --enable-boost-pool            uses Boost pools for the memory SCFG tabgle
+#
+# --enable-mpi                   switch on mpi
+# --without-libsegfault          does not link with libSegFault
+#
+# --max-kenlm-order              maximum ngram order that kenlm can process (default 6)
+#
+# --max-factors                  maximum number of factors (default 4)
+#
+# --unlabelled-source            ignore source labels (redundant in hiero or string-to-tree system)
+#                                for better performance
+#CONTROLLING THE BUILD
+#-a to build from scratch
+#-j$NCPUS to compile in parallel
+#--clean to clean
+#--debug-build to build with Og. Only available with gcc 4.8+
+
+import os ;
+import option ;
+import modules ;
+import path ;
+path-constant TOP : . ;
+
+include $(TOP)/jam-files/sanity.jam ;
+
+home = [ os.environ "HOME" ] ;
+if [ path.exists $(home)/moses-environment.jam ] 
+{ 
+  # for those of use who don't like typing in command line bjam options all day long
+  include $(home)/moses-environment.jam ; 
+}
+include $(TOP)/jam-files/check-environment.jam ; # get resource locations 
+                                                 # from environment variables
+include $(TOP)/jam-files/xmlrpc-c.jam ; # xmlrpc-c stuff for the server
+# include $(TOP)/jam-files/curlpp.jam ;   # curlpp stuff for bias lookup (MMT only)
+
+# exit "done" : 0 ;
+
+max-order = [ option.get "max-kenlm-order" : 6 : 6 ] ;
+if ! [ option.get "max-kenlm-order" ] 
+{
+  # some classes in Moses pull in header files from KenLM, so this needs to be 
+  # defined here, not in moses/lm/Jamfile 
+  option.set "max-kenlm-order" : 6 ;
+  requirements += <define>KENLM_MAX_ORDER=$(max-order) ;
+}
+# exit "all done" : 0 ; 
+
+boost 104400 ;
+external-lib z ;
+
+#lib dl : : <runtime-link>static:<link>static <runtime-link>shared:<link>shared ;
+#requirements += <library>dl ;
+requirements += <cxxflags>-std=c++0x ;
+
+# Allow moses to report the git commit hash of the version used for compilation
+moses_githash = [ _shell "git describe --dirty" ] ;
+requirements += <define>MOSES_VERSION_ID=\\\"$(moses_githash)\\\" ;
+
+if ! [ option.get "without-tcmalloc" : : "yes" ] && [ test_library "tcmalloc_minimal" ] {
+  if [ option.get "full-tcmalloc" : : "yes" ] {
+    external-lib unwind ;
+    external-lib tcmalloc_and_profiler : : unwind ;
+    requirements += <library>tcmalloc_and_profiler <library>unwind <cflags>-fno-omit-frame-pointer <cxxflags>-fno-omit-frame-pointer ;
+  } else {
+    external-lib tcmalloc_minimal ;
+    requirements += <threading>multi:<library>tcmalloc_minimal ;
+  }
+} else {
+  echo "Tip: install tcmalloc for faster threading.  See BUILD-INSTRUCTIONS.txt for more information." ;
+}
+
+if [ option.get "filter-warnings" : : "yes" ] {
+  # given the low coding standards in Moses, we may want to filter out
+  # warnings about poor coding practice that no-one is ever going to fix 
+  # anyway ...
+  requirements += <cxxflags>-Wno-deprecated ;
+  requirements += <cxxflags>-Wno-reorder ;
+  requirements += <cxxflags>-Wno-sign-compare ;
+  requirements += <cxxflags>-Wno-unused-but-set-variable ;
+  requirements += <cxxflags>-Wno-unused-result ;
+  requirements += <cxxflags>-Wno-unused-variable ;
+  requirements += <cxxflags>-Wno-comment ;
+  requirements += <cxxflags>-Wno-strict-aliasing ;
+  requirements += <cxxflags>-Wno-overloaded-virtual ;
+}
+
+if [ option.get "debug-build" : : "yes" ] {
+   requirements += <cxxflags>-Og ;
+   echo "Building with -Og to enable easier profiling and debugging. Only available on gcc 4.8+." ;
+}
+
+if [ option.get "with-address-sanitizer" : : "yes" ] {
+  requirements += <cxxflags>-fsanitize=address ;
+  requirements += <cxxflags>-fno-omit-frame-pointer ;
+  requirements += <linkflags>-fsanitize=address ;
+  echo "Building with AddressSanitizer to enable debugging of memory errors. Only available on gcc 4.8+." ;
+}
+
+if [ option.get "enable-mpi" : : "yes" ] {
+  import mpi ;
+  using mpi ;
+  external-lib boost_mpi ;
+  external-lib boost_serialization ;
+  requirements += <define>MPI_ENABLE ;
+  requirements += <library>mpi ;
+  requirements += <library>boost_mpi ;
+  requirements += <library>boost_serialization ;
+}
+
+mmt = [ option.get "mmt" ] ; 
+if $(mmt) {
+  requirements += <define>MMT ;
+  requirements += <include>$(mmt) ; 
+  mmt_githash = [ _shell "cd $(mmt) && git describe --dirty" ] ;
+  requirements += <define>MMT_VERSION_ID=\\\"$(mmt_githash)\\\" ;
+}
+
+requirements += [ option.get "notrace" : <define>TRACE_ENABLE=1 ] ;
+requirements += [ option.get "enable-boost-pool" : : <define>USE_BOOST_POOL ] ;
+requirements += [ option.get "with-mm" : : <define>PT_UG ] ;
+requirements += [ option.get "with-mm" : : <define>MAX_NUM_FACTORS=4 ] ;
+requirements += [ option.get "unlabelled-source" : : <define>UNLABELLED_SOURCE ] ;
+
+if [ option.get "with-oxlm" ] {
+  external-lib gomp ;
+  requirements += <library>boost_serialization ;
+  requirements += <library>gomp ;
+}
+
+if [ option.get "with-cmph" : : "yes" ] {
+  requirements += <define>HAVE_CMPH ;
+}
+
+if [ option.get "with-icu" : : "yes" ]
+{
+  external-lib icuuc ;
+  external-lib icuio ;
+  external-lib icui18n ;
+  requirements += <library>icuuc/<link>shared ;
+  requirements += <library>icuio/<link>shared ;
+  requirements += <library>icui18n/<link>shared ;
+  requirements += <cxxflags>-fPIC ;
+  requirements += <address-model>64 ;
+#  requirements += <runtime-link>shared ;
+}
+
+# for probing pt
+external-lib boost_serialization ;
+requirements += <library>boost_serialization/<runtime-link>static ;
+
+if [ option.get "with-vw" ] {
+  requirements += <define>HAVE_VW ;
+}
+
+project : default-build
+  <threading>multi
+  <warnings>on
+  <debug-symbols>off
+  <variant>release
+  <link>static
+  ;
+
+#Apparently OS X likes to link against iconv for fgetsUTF8.
+lib iconv ;
+requirements += <os>MACOSX:<library>iconv ;
+
+project : requirements 
+  <threading>multi:<define>WITH_THREADS
+  <threading>multi:<library>boost_thread
+  <library>boost_system
+  <library>boost_program_options
+  <define>_FILE_OFFSET_BITS=64 <define>_LARGE_FILES
+  $(requirements)
+  <include>.
+  ;
+
+
+#Add directories here if you want their incidental targets too (i.e. tests).
+build-projects lm util phrase-extract phrase-extract/syntax-common search moses moses/LM mert moses-cmd scripts regression-testing ;
+# contrib/mira
+
+if [ option.get "with-mm-extras" : : "yes" ]
+{
+  alias mm-extras :
+  moses/TranslationModel/UG//bitext-find 
+  moses/TranslationModel/UG//ptable-describe-features 
+  moses/TranslationModel/UG//count-ptable-features 
+  moses/TranslationModel/UG//ptable-sigtest-filter 
+  moses/TranslationModel/UG//ptable-lookup 
+  moses/TranslationModel/UG//ptable-lookup-corpus 
+  moses/TranslationModel/UG//check-coverage 
+  moses/TranslationModel/UG/mm//mtt-demo1 
+  moses/TranslationModel/UG/mm//mtt-dump 
+  moses/TranslationModel/UG/mm//mam2symal 
+  moses/TranslationModel/UG/mm//mam_verify 
+  moses/TranslationModel/UG/mm//mmlex-lookup 
+  moses/TranslationModel/UG/mm//mtt-count-words 
+  moses/TranslationModel/UG/mm//calc-coverage 
+  moses/TranslationModel/UG//try-align 
+  ;
+} 
+else
+{
+  alias mm-extras ;
+}
+
+if [ option.get "with-mm" : : "yes" ]
+{
+ alias mm :  
+  moses/TranslationModel/UG/mm//mtt-build 
+  moses/TranslationModel/UG/mm//symal2mam 
+  moses/TranslationModel/UG/mm//mmlex-build 
+  ;
+}
+else
+{
+ alias mm ; 
+}
+
+if [ option.get "with-rephraser" : : "yes" ]
+{
+  alias rephraser :
+  contrib/rephraser//paraphrase
+  ;
+}
+else
+{
+  alias rephraser ;
+}
+
+alias programs : 
+lm//programs 
+moses-cmd//programs 
+OnDiskPt//CreateOnDiskPt 
+OnDiskPt//queryOnDiskPt 
+mert//programs 
+misc//programs 
+symal 
+phrase-extract 
+phrase-extract//lexical-reordering 
+phrase-extract//extract-ghkm 
+phrase-extract//pcfg-extract 
+phrase-extract//pcfg-score 
+phrase-extract//extract-mixed-syntax 
+phrase-extract//score-stsg
+phrase-extract//filter-rule-table
+phrase-extract//postprocess-egret-forests
+biconcor 
+# contrib/mira//mira 
+contrib/server//mosesserver 
+mm
+mm-extras
+rephraser
+contrib/c++tokenizer//tokenizer
+contrib/expected-bleu-training//train-expected-bleu
+contrib/expected-bleu-training//prepare-expected-bleu-training
+
+probingpt//programs
+moses2//programs
+;
+
+
+install-bin-libs programs ;
+install-headers headers-base : [ path.glob-tree biconcor contrib lm mert misc moses-cmd OnDiskPt phrase-extract symal util : *.hh *.h ] : . ;
+install-headers headers-moses : moses//headers-to-install : moses ;
+
+alias install : prefix-bin prefix-lib headers-base headers-moses ;
+
+if ! [ option.get "includedir" : : $(prefix)/include ] {
+  explicit install headers-base headers-moses ;
+}
+
+if [ path.exists $(TOP)/dist ] && $(prefix) != dist {
+  echo "You have a $(TOP)/dist directory, but the build system now places files directly in the root i.e. $(TOP)/bin ." ;
+  echo "To disable this message, delete $(TOP)/dist ." ;
+  echo ;
+}
+
+#local temp = [ _shell "bash source ./s.sh" ] ;
+local temp = [ _shell "mkdir -p $(PREFIX)/bin" ] ;
+local temp = [ _shell "rm -f $(PREFIX)/bin/moses_chart" ] ;
+local temp = [ _shell "cd $(PREFIX)/bin && ln -sf moses moses_chart" ] ;
+local temp = [ _shell "cd $(PREFIX)/bin && ln -sf CreateProbingPT CreateProbingPT2" ] ;
+

mosesdecoder/OnDiskPt/Jamfile

@@ -0,0 +1,5 @@
+fakelib OnDiskPt : OnDiskWrapper.cpp SourcePhrase.cpp TargetPhrase.cpp Word.cpp Phrase.cpp PhraseNode.cpp TargetPhraseCollection.cpp Vocab.cpp OnDiskQuery.cpp ../moses//headers ;
+
+exe CreateOnDiskPt : Main.cpp ..//boost_filesystem ../moses//moses OnDiskPt ;
+exe queryOnDiskPt : queryOnDiskPt.cpp ..//boost_filesystem ../moses//moses OnDiskPt ;
+

mosesdecoder/OnDiskPt/Main.cpp

@@ -0,0 +1,273 @@
+// $Id$
+/***********************************************************************
+ Moses - factored phrase-based, hierarchical and syntactic language decoder
+ Copyright (C) 2009 Hieu Hoang
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ ***********************************************************************/
+
+#include <algorithm>
+#include <iostream>
+#include <string>
+#include <vector>
+#include <iterator>
+#include <cassert>
+#include "moses/InputFileStream.h"
+#include "moses/Timer.h"
+#include "moses/Util.h"
+#include "OnDiskWrapper.h"
+#include "SourcePhrase.h"
+#include "TargetPhrase.h"
+#include "TargetPhraseCollection.h"
+#include "Word.h"
+#include "Vocab.h"
+#include "Main.h"
+
+using namespace std;
+using namespace OnDiskPt;
+
+int main (int argc, char * const argv[])
+{
+  // insert code here...
+  Moses::ResetUserTime();
+  Moses::PrintUserTime("Starting");
+
+  if (argc != 8) {
+    std::cerr << "Usage: " << argv[0] << " numSourceFactors numTargetFactors numScores tableLimit sortScoreIndex inputPath outputPath" << std::endl;
+    return 1;
+  }
+
+  int numSourceFactors	= Moses::Scan<int>(argv[1])
+                          , numTargetFactors	= Moses::Scan<int>(argv[2])
+                              , numScores				= Moses::Scan<int>(argv[3])
+                                  , tableLimit				= Moses::Scan<int>(argv[4]);
+  TargetPhraseCollection::s_sortScoreInd			= Moses::Scan<int>(argv[5]);
+  assert(TargetPhraseCollection::s_sortScoreInd < numScores);
+
+  const string filePath 	= argv[6]
+                            ,destPath	= argv[7];
+
+  Moses::InputFileStream inStream(filePath);
+
+  OnDiskWrapper onDiskWrapper;
+  onDiskWrapper.BeginSave(destPath, numSourceFactors, numTargetFactors, numScores);
+
+  PhraseNode &rootNode = onDiskWrapper.GetRootSourceNode();
+  size_t lineNum = 0;
+  string line;
+
+  while(getline(inStream, line)) {
+    lineNum++;
+    if (lineNum%1000 == 0) cerr << "." << flush;
+    if (lineNum%10000 == 0) cerr << ":" << flush;
+    if (lineNum%100000 == 0) cerr << lineNum << flush;
+    //cerr << lineNum << " " << line << endl;
+
+    std::vector<float> misc(1);
+    SourcePhrase sourcePhrase;
+    TargetPhrase *targetPhrase = new TargetPhrase(numScores);
+    OnDiskPt::PhrasePtr spShort = Tokenize(sourcePhrase, *targetPhrase, line, onDiskWrapper, numScores, misc);
+    assert(misc.size() == onDiskWrapper.GetNumCounts());
+
+    rootNode.AddTargetPhrase(sourcePhrase, targetPhrase, onDiskWrapper, tableLimit, misc, spShort);
+  }
+
+  rootNode.Save(onDiskWrapper, 0, tableLimit);
+  onDiskWrapper.EndSave();
+
+  Moses::PrintUserTime("Finished");
+
+  //pause();
+  return 0;
+
+} // main()
+
+bool Flush(const OnDiskPt::SourcePhrase *prevSourcePhrase, const OnDiskPt::SourcePhrase *currSourcePhrase)
+{
+  if (prevSourcePhrase == NULL)
+    return false;
+
+  assert(currSourcePhrase);
+  bool ret = (*currSourcePhrase > *prevSourcePhrase);
+  //cerr << *prevSourcePhrase << endl << *currSourcePhrase << " " << ret << endl << endl;
+
+  return ret;
+}
+
+OnDiskPt::PhrasePtr Tokenize(SourcePhrase &sourcePhrase, TargetPhrase &targetPhrase, const std::string &lineStr, OnDiskWrapper &onDiskWrapper, int numScores, vector<float> &misc)
+{
+  char line[lineStr.size() + 1];
+  strcpy(line, lineStr.c_str());
+
+  stringstream sparseFeatures, property;
+
+  size_t scoreInd = 0;
+
+  // MAIN LOOP
+  size_t stage = 0;
+  /*	0 = source phrase
+   1 = target phrase
+   2 = scores
+   3 = align
+   4 = count
+   7 = properties
+   */
+  char *tok = strtok (line," ");
+  OnDiskPt::PhrasePtr out(new Phrase());
+  while (tok != NULL) {
+    if (0 == strcmp(tok, "|||")) {
+      ++stage;
+    } else {
+      switch (stage) {
+      case 0: {
+        WordPtr w = Tokenize(sourcePhrase, tok, true, true, onDiskWrapper, 1);
+        if (w != NULL)
+          out->AddWord(w);
+
+        break;
+      }
+      case 1: {
+        Tokenize(targetPhrase, tok, false, true, onDiskWrapper, 0);
+        break;
+      }
+      case 2: {
+        float score = Moses::Scan<float>(tok);
+        targetPhrase.SetScore(score, scoreInd);
+        ++scoreInd;
+        break;
+      }
+      case 3: {
+        //targetPhrase.Create1AlignFromString(tok);
+        targetPhrase.CreateAlignFromString(tok);
+        break;
+      }
+      case 4: {
+        // store only the 3rd one (rule count)
+        float val = Moses::Scan<float>(tok);
+        misc[0] = val;
+        break;
+      }
+      case 5: {
+        // sparse features
+        sparseFeatures << tok << " ";
+        break;
+      }
+      case 6: {
+        property << tok << " ";
+        break;
+      }
+      default:
+        cerr << "ERROR in line " << line << endl;
+        assert(false);
+        break;
+      }
+    }
+
+    tok = strtok (NULL, " ");
+  } // while (tok != NULL)
+
+  assert(scoreInd == numScores);
+  targetPhrase.SetSparseFeatures(Moses::Trim(sparseFeatures.str()));
+  targetPhrase.SetProperty(Moses::Trim(property.str()));
+  targetPhrase.SortAlign();
+  return out;
+} // Tokenize()
+
+OnDiskPt::WordPtr Tokenize(OnDiskPt::Phrase &phrase
+                           , const std::string &token, bool addSourceNonTerm, bool addTargetNonTerm
+                           , OnDiskPt::OnDiskWrapper &onDiskWrapper, int retSourceTarget)
+{
+  // retSourceTarget: 0 = don't return anything. 1 = source, 2 = target
+
+  bool nonTerm = false;
+  size_t tokSize = token.size();
+  int comStr =token.compare(0, 1, "[");
+
+  if (comStr == 0) {
+    comStr = token.compare(tokSize - 1, 1, "]");
+    nonTerm = comStr == 0;
+  }
+
+  OnDiskPt::WordPtr out;
+  if (nonTerm) {
+    // non-term
+    size_t splitPos		= token.find_first_of("[", 2);
+    string wordStr	= token.substr(0, splitPos);
+
+    if (splitPos == string::npos) {
+      // lhs - only 1 word
+      WordPtr word(new Word());
+      word->CreateFromString(wordStr, onDiskWrapper.GetVocab());
+      phrase.AddWord(word);
+    } else {
+      // source & target non-terms
+      if (addSourceNonTerm) {
+        WordPtr word(new Word());
+        word->CreateFromString(wordStr, onDiskWrapper.GetVocab());
+        phrase.AddWord(word);
+
+        if (retSourceTarget == 1) {
+          out = word;
+        }
+      }
+
+      wordStr = token.substr(splitPos, tokSize - splitPos);
+      if (addTargetNonTerm) {
+        WordPtr word(new Word());
+        word->CreateFromString(wordStr, onDiskWrapper.GetVocab());
+        phrase.AddWord(word);
+
+        if (retSourceTarget == 2) {
+          out = word;
+        }
+      }
+
+    }
+  } else {
+    // term
+    WordPtr word(new Word());
+    word->CreateFromString(token, onDiskWrapper.GetVocab());
+    phrase.AddWord(word);
+    out = word;
+  }
+
+  return out;
+}
+
+void InsertTargetNonTerminals(std::vector<std::string> &sourceToks, const std::vector<std::string> &targetToks, const ::AlignType &alignments)
+{
+  for (int ind = alignments.size() - 1; ind >= 0; --ind) {
+    const ::AlignPair &alignPair = alignments[ind];
+    size_t sourcePos = alignPair.first
+                       ,targetPos = alignPair.second;
+
+    const string &target = targetToks[targetPos];
+    sourceToks.insert(sourceToks.begin() + sourcePos + 1, target);
+
+  }
+}
+
+class AlignOrderer
+{
+public:
+  bool operator()(const ::AlignPair &a, const ::AlignPair &b) const {
+    return a.first < b.first;
+  }
+};
+
+void SortAlign(::AlignType &alignments)
+{
+  std::sort(alignments.begin(), alignments.end(), AlignOrderer());
+}

mosesdecoder/OnDiskPt/Main.h

@@ -0,0 +1,39 @@
+#pragma once
+// $Id$
+/***********************************************************************
+ Moses - factored phrase-based, hierarchical and syntactic language decoder
+ Copyright (C) 2009 Hieu Hoang
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ ***********************************************************************/
+#include <string>
+#include "SourcePhrase.h"
+#include "TargetPhrase.h"
+
+typedef std::pair<size_t, size_t>  AlignPair;
+typedef std::vector<AlignPair> AlignType;
+
+OnDiskPt::WordPtr Tokenize(OnDiskPt::Phrase &phrase
+                           , const std::string &token, bool addSourceNonTerm, bool addTargetNonTerm
+                           , OnDiskPt::OnDiskWrapper &onDiskWrapper, int retSourceTarget);
+OnDiskPt::PhrasePtr Tokenize(OnDiskPt::SourcePhrase &sourcePhrase, OnDiskPt::TargetPhrase &targetPhrase
+                             , const std::string &lineStr, OnDiskPt::OnDiskWrapper &onDiskWrapper
+                             , int numScores
+                             , std::vector<float> &misc);
+
+void InsertTargetNonTerminals(std::vector<std::string> &sourceToks, const std::vector<std::string> &targetToks, const AlignType &alignments);
+void SortAlign(AlignType &alignments);
+bool Flush(const OnDiskPt::SourcePhrase *prevSource, const OnDiskPt::SourcePhrase *currSource);
+

mosesdecoder/OnDiskPt/OnDiskQuery.cpp

@@ -0,0 +1,83 @@
+#include "OnDiskQuery.h"
+
+namespace OnDiskPt
+{
+
+void OnDiskQuery::Tokenize(Phrase &phrase,
+                           const std::string &token,
+                           bool addSourceNonTerm,
+                           bool addTargetNonTerm)
+{
+  bool nonTerm = false;
+  size_t tokSize = token.size();
+  int comStr =token.compare(0, 1, "[");
+
+  if (comStr == 0) {
+    comStr = token.compare(tokSize - 1, 1, "]");
+    nonTerm = comStr == 0;
+  }
+
+  if (nonTerm) {
+    // non-term
+    size_t splitPos = token.find_first_of("[", 2);
+    std::string wordStr = token.substr(0, splitPos);
+
+    if (splitPos == std::string::npos) {
+      // lhs - only 1 word
+      WordPtr word (new Word());
+      word->CreateFromString(wordStr, m_wrapper.GetVocab());
+      phrase.AddWord(word);
+    } else {
+      // source & target non-terms
+      if (addSourceNonTerm) {
+        WordPtr word( new Word());
+        word->CreateFromString(wordStr, m_wrapper.GetVocab());
+        phrase.AddWord(word);
+      }
+
+      wordStr = token.substr(splitPos, tokSize - splitPos);
+      if (addTargetNonTerm) {
+        WordPtr word(new Word());
+        word->CreateFromString(wordStr, m_wrapper.GetVocab());
+        phrase.AddWord(word);
+      }
+
+    }
+  } else {
+    // term
+    WordPtr word(new Word());
+    word->CreateFromString(token, m_wrapper.GetVocab());
+    phrase.AddWord(word);
+  }
+}
+
+SourcePhrase OnDiskQuery::Tokenize(const std::vector<std::string>& tokens)
+{
+  SourcePhrase sourcePhrase;
+  if (tokens.size() > 0) {
+    std::vector<std::string>::const_iterator token = tokens.begin();
+    for (; token + 1 != tokens.end(); ++token) {
+      Tokenize(sourcePhrase, *token, true, true);
+    }
+    // last position. LHS non-term
+    Tokenize(sourcePhrase, *token, false, true);
+  }
+  return sourcePhrase;
+}
+
+const PhraseNode* OnDiskQuery::Query(const SourcePhrase& sourcePhrase)
+{
+  const PhraseNode *node = &m_wrapper.GetRootSourceNode();
+  assert(node);
+
+  for (size_t pos = 0; pos < sourcePhrase.GetSize(); ++pos) {
+    const Word &word = sourcePhrase.GetWord(pos);
+    node = node->GetChild(word, m_wrapper);
+    if (node == NULL) {
+      break;
+    }
+  }
+  return node;
+}
+
+}

mosesdecoder/OnDiskPt/OnDiskQuery.h

@@ -0,0 +1,39 @@
+#pragma once
+#include <string>
+#include <vector>
+#include "OnDiskWrapper.h"
+#include "Phrase.h"
+#include "SourcePhrase.h"
+#include "Word.h"
+#include "PhraseNode.h"
+
+
+namespace OnDiskPt
+{
+
+class OnDiskQuery
+{
+private:
+  OnDiskWrapper &m_wrapper;
+
+public:
+
+  OnDiskQuery(OnDiskWrapper &wrapper):m_wrapper(wrapper) {}
+
+  void Tokenize(Phrase &phrase,
+                const std::string &token,
+                bool addSourceNonTerm,
+                bool addTargetNonTerm);
+
+  SourcePhrase Tokenize(const std::vector<std::string>& tokens);
+
+  const PhraseNode *Query(const SourcePhrase& sourcePhrase);
+
+  inline const PhraseNode *Query(const std::vector<std::string>& tokens) {
+    return Query(Tokenize(tokens));
+  }
+
+};
+
+
+}

mosesdecoder/OnDiskPt/OnDiskWrapper.cpp

@@ -0,0 +1,223 @@
+// $Id$
+/***********************************************************************
+ Moses - factored phrase-based, hierarchical and syntactic language decoder
+ Copyright (C) 2009 Hieu Hoang
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ ***********************************************************************/
+#ifdef WIN32
+#include <direct.h>
+#endif
+#include <sys/stat.h>
+#include <string>
+#include "OnDiskWrapper.h"
+#include "moses/Util.h"
+#include "util/exception.hh"
+#include "util/string_stream.hh"
+
+using namespace std;
+
+namespace OnDiskPt
+{
+
+int OnDiskWrapper::VERSION_NUM = 7;
+
+OnDiskWrapper::OnDiskWrapper()
+{
+}
+
+OnDiskWrapper::~OnDiskWrapper()
+{
+  delete m_rootSourceNode;
+}
+
+void OnDiskWrapper::BeginLoad(const std::string &filePath)
+{
+  if (!OpenForLoad(filePath)) {
+    UTIL_THROW(util::FileOpenException, "Couldn't open for loading: " << filePath);
+  }
+
+  if (!m_vocab.Load(*this))
+    UTIL_THROW(util::FileOpenException, "Couldn't load vocab");
+
+  uint64_t rootFilePos = GetMisc("RootNodeOffset");
+  m_rootSourceNode = new PhraseNode(rootFilePos, *this);
+}
+
+bool OnDiskWrapper::OpenForLoad(const std::string &filePath)
+{
+  m_fileSource.open((filePath + "/Source.dat").c_str(), ios::in | ios::binary);
+  UTIL_THROW_IF(!m_fileSource.is_open(),
+                util::FileOpenException,
+                "Couldn't open file " << filePath << "/Source.dat");
+
+  m_fileTargetInd.open((filePath + "/TargetInd.dat").c_str(), ios::in | ios::binary);
+  UTIL_THROW_IF(!m_fileTargetInd.is_open(),
+                util::FileOpenException,
+                "Couldn't open file " << filePath << "/TargetInd.dat");
+
+  m_fileTargetColl.open((filePath + "/TargetColl.dat").c_str(), ios::in | ios::binary);
+  UTIL_THROW_IF(!m_fileTargetColl.is_open(),
+                util::FileOpenException,
+                "Couldn't open file " << filePath << "/TargetColl.dat");
+
+  m_fileVocab.open((filePath + "/Vocab.dat").c_str(), ios::in);
+  UTIL_THROW_IF(!m_fileVocab.is_open(),
+                util::FileOpenException,
+                "Couldn't open file " << filePath << "/Vocab.dat");
+
+  m_fileMisc.open((filePath + "/Misc.dat").c_str(), ios::in);
+  UTIL_THROW_IF(!m_fileMisc.is_open(),
+                util::FileOpenException,
+                "Couldn't open file " << filePath << "/Misc.dat");
+
+  // set up root node
+  LoadMisc();
+  m_numSourceFactors = GetMisc("NumSourceFactors");
+  m_numTargetFactors = GetMisc("NumTargetFactors");
+  m_numScores = GetMisc("NumScores");
+
+  return true;
+}
+
+bool OnDiskWrapper::LoadMisc()
+{
+  char line[100000];
+
+  while(m_fileMisc.getline(line, 100000)) {
+    vector<string> tokens;
+    Moses::Tokenize(tokens, line);
+    UTIL_THROW_IF2(tokens.size() != 2, "Except key value. Found " << line);
+
+
+    const string &key = tokens[0];
+    m_miscInfo[key] =  Moses::Scan<uint64_t>(tokens[1]);
+  }
+
+  return true;
+}
+
+void OnDiskWrapper::BeginSave(const std::string &filePath
+                              , int numSourceFactors, int	numTargetFactors, int numScores)
+{
+  m_numSourceFactors = numSourceFactors;
+  m_numTargetFactors = numTargetFactors;
+  m_numScores = numScores;
+  m_filePath = filePath;
+
+#ifdef WIN32
+  mkdir(filePath.c_str());
+#else
+  mkdir(filePath.c_str(), 0777);
+#endif
+
+  m_fileSource.open((filePath + "/Source.dat").c_str(), ios::out | ios::in | ios::binary | ios::ate | ios::trunc);
+  UTIL_THROW_IF(!m_fileSource.is_open(),
+                util::FileOpenException,
+                "Couldn't open file " << filePath << "/Source.dat");
+
+  m_fileTargetInd.open((filePath + "/TargetInd.dat").c_str(), ios::out | ios::binary | ios::ate | ios::trunc);
+  UTIL_THROW_IF(!m_fileTargetInd.is_open(),
+                util::FileOpenException,
+                "Couldn't open file " << filePath << "/TargetInd.dat");
+
+  m_fileTargetColl.open((filePath + "/TargetColl.dat").c_str(), ios::out | ios::binary | ios::ate | ios::trunc);
+  UTIL_THROW_IF(!m_fileTargetColl.is_open(),
+                util::FileOpenException,
+                "Couldn't open file " << filePath << "/TargetColl.dat");
+
+  m_fileVocab.open((filePath + "/Vocab.dat").c_str(), ios::out | ios::ate | ios::trunc);
+  UTIL_THROW_IF(!m_fileVocab.is_open(),
+                util::FileOpenException,
+                "Couldn't open file " << filePath << "/Vocab.dat");
+
+  m_fileMisc.open((filePath + "/Misc.dat").c_str(), ios::out | ios::ate | ios::trunc);
+  UTIL_THROW_IF(!m_fileMisc.is_open(),
+                util::FileOpenException,
+                "Couldn't open file " << filePath << "/Misc.dat");
+
+  // offset by 1. 0 offset is reserved
+  char c = 0xff;
+  m_fileSource.write(&c, 1);
+  UTIL_THROW_IF2(1 != m_fileSource.tellp(),
+                 "Couldn't write to stream m_fileSource");
+
+  m_fileTargetInd.write(&c, 1);
+  UTIL_THROW_IF2(1 != m_fileTargetInd.tellp(),
+                 "Couldn't write to stream m_fileTargetInd");
+
+  m_fileTargetColl.write(&c, 1);
+  UTIL_THROW_IF2(1 != m_fileTargetColl.tellp(),
+                 "Couldn't write to stream m_fileTargetColl");
+
+  // set up root node
+  UTIL_THROW_IF2(GetNumCounts() != 1,
+                 "Not sure what this is...");
+
+  vector<float> counts(GetNumCounts());
+  counts[0] = DEFAULT_COUNT;
+  m_rootSourceNode = new PhraseNode();
+  m_rootSourceNode->AddCounts(counts);
+}
+
+void OnDiskWrapper::EndSave()
+{
+  bool ret = m_rootSourceNode->Saved();
+  UTIL_THROW_IF2(!ret, "Root node not saved");
+
+  GetVocab().Save(*this);
+
+  SaveMisc();
+
+  m_fileMisc.close();
+  m_fileVocab.close();
+  m_fileSource.close();
+  m_fileTarget.close();
+  m_fileTargetInd.close();
+  m_fileTargetColl.close();
+}
+
+void OnDiskWrapper::SaveMisc()
+{
+  m_fileMisc << "Version " << VERSION_NUM << endl;
+  m_fileMisc << "NumSourceFactors " << m_numSourceFactors << endl;
+  m_fileMisc << "NumTargetFactors " << m_numTargetFactors << endl;
+  m_fileMisc << "NumScores " << m_numScores << endl;
+  m_fileMisc << "RootNodeOffset " << m_rootSourceNode->GetFilePos() << endl;
+}
+
+size_t OnDiskWrapper::GetSourceWordSize() const
+{
+  return sizeof(uint64_t) + sizeof(char);
+}
+
+size_t OnDiskWrapper::GetTargetWordSize() const
+{
+  return sizeof(uint64_t) + sizeof(char);
+}
+
+uint64_t OnDiskWrapper::GetMisc(const std::string &key) const
+{
+  std::map<std::string, uint64_t>::const_iterator iter;
+  iter = m_miscInfo.find(key);
+  UTIL_THROW_IF2(iter == m_miscInfo.end()
+                 , "Couldn't find value for key " << key
+                );
+
+  return iter->second;
+}
+
+
+}

mosesdecoder/OnDiskPt/OnDiskWrapper.h

@@ -0,0 +1,111 @@
+#pragma once
+// $Id$
+/***********************************************************************
+ Moses - factored phrase-based, hierarchical and syntactic language decoder
+ Copyright (C) 2009 Hieu Hoang
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ ***********************************************************************/
+#include <string>
+#include <fstream>
+#include "Vocab.h"
+#include "PhraseNode.h"
+
+namespace OnDiskPt
+{
+const float DEFAULT_COUNT = 66666;
+
+/** Global class with misc information need to create and use the on-disk rule table.
+ * 1 object of this class should be instantiated per rule table.
+ * Currently only hierarchical/syntax models use this, but can & should be used with pb models too
+ */
+class OnDiskWrapper
+{
+protected:
+  Vocab m_vocab;
+  std::string m_filePath;
+  int m_numSourceFactors, m_numTargetFactors, m_numScores;
+  std::fstream m_fileMisc, m_fileVocab, m_fileSource, m_fileTarget, m_fileTargetInd, m_fileTargetColl;
+
+  size_t m_defaultNodeSize;
+  PhraseNode *m_rootSourceNode;
+
+  std::map<std::string, uint64_t> m_miscInfo;
+
+  void SaveMisc();
+  bool OpenForLoad(const std::string &filePath);
+  bool LoadMisc();
+
+public:
+  static int VERSION_NUM;
+
+  OnDiskWrapper();
+  ~OnDiskWrapper();
+
+  void BeginLoad(const std::string &filePath);
+
+  void BeginSave(const std::string &filePath
+                 , int numSourceFactors, int	numTargetFactors, int numScores);
+  void EndSave();
+
+  Vocab &GetVocab() {
+    return m_vocab;
+  }
+  const Vocab &GetVocab() const {
+    return m_vocab;
+  }
+
+  size_t GetSourceWordSize() const;
+  size_t GetTargetWordSize() const;
+
+  std::fstream &GetFileSource() {
+    return m_fileSource;
+  }
+  std::fstream &GetFileTargetInd() {
+    return m_fileTargetInd;
+  }
+  std::fstream &GetFileTargetColl() {
+    return m_fileTargetColl;
+  }
+  std::fstream &GetFileVocab() {
+    return m_fileVocab;
+  }
+
+  size_t GetNumSourceFactors() const {
+    return m_numSourceFactors;
+  }
+  size_t GetNumTargetFactors() const {
+    return m_numTargetFactors;
+  }
+
+  size_t GetNumScores() const {
+    return m_numScores;
+  }
+  size_t GetNumCounts() const {
+    return 1;
+  }
+
+  PhraseNode &GetRootSourceNode() {
+    return *m_rootSourceNode;
+  }
+  const PhraseNode &GetRootSourceNode() const {
+    return *m_rootSourceNode;
+  }
+
+  uint64_t GetMisc(const std::string &key) const;
+
+};
+
+}

mosesdecoder/OnDiskPt/Phrase.cpp

@@ -0,0 +1,108 @@
+// $Id$
+/***********************************************************************
+ Moses - factored phrase-based, hierarchical and syntactic language decoder
+ Copyright (C) 2009 Hieu Hoang
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ ***********************************************************************/
+#include <iostream>
+#include "moses/Util.h"
+#include "Phrase.h"
+#include "util/exception.hh"
+
+using namespace std;
+
+namespace OnDiskPt
+{
+
+
+void Phrase::AddWord(WordPtr word)
+{
+  m_words.push_back(word);
+}
+
+void Phrase::AddWord(WordPtr word, size_t pos)
+{
+  UTIL_THROW_IF2(!(pos < m_words.size()),
+                 "Trying to get word " << pos << " when phrase size is " << m_words.size());
+  m_words.insert(m_words.begin() + pos + 1, word);
+}
+
+int Phrase::Compare(const Phrase &compare) const
+{
+  int ret = 0;
+  for (size_t pos = 0; pos < GetSize(); ++pos) {
+    if (pos >= compare.GetSize()) {
+      // we're bigger than the other. Put 1st
+      ret = -1;
+      break;
+    }
+
+    const Word &thisWord = GetWord(pos)
+                           ,&compareWord = compare.GetWord(pos);
+    int wordRet = thisWord.Compare(compareWord);
+    if (wordRet != 0) {
+      ret = wordRet;
+      break;
+    }
+  }
+
+  if (ret == 0) {
+    assert(compare.GetSize() >= GetSize());
+    ret = (compare.GetSize() > GetSize()) ? 1 : 0;
+  }
+  return ret;
+}
+
+//! transitive comparison
+bool Phrase::operator<(const Phrase &compare) const
+{
+  int ret = Compare(compare);
+  return ret < 0;
+}
+
+bool Phrase::operator>(const Phrase &compare) const
+{
+  int ret = Compare(compare);
+  return ret > 0;
+}
+
+bool Phrase::operator==(const Phrase &compare) const
+{
+  int ret = Compare(compare);
+  return ret == 0;
+}
+
+void Phrase::DebugPrint(ostream &out, const Vocab &vocab) const
+{
+  for (size_t pos = 0; pos < GetSize(); ++pos) {
+    const Word &word = GetWord(pos);
+    word.DebugPrint(out, vocab);
+    out << " ";
+  }
+}
+
+std::ostream& operator<<(std::ostream &out, const Phrase &phrase)
+{
+  for (size_t pos = 0; pos < phrase.GetSize(); ++pos) {
+    const Word &word = phrase.GetWord(pos);
+    out << word << " ";
+  }
+
+  return out;
+}
+
+}
+

mosesdecoder/OnDiskPt/Phrase.h

@@ -0,0 +1,66 @@
+#pragma once
+// $Id$
+/***********************************************************************
+ Moses - factored phrase-based, hierarchical and syntactic language decoder
+ Copyright (C) 2009 Hieu Hoang
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ ***********************************************************************/
+#include <vector>
+#include <iostream>
+#include <boost/shared_ptr.hpp>
+#include "Word.h"
+
+namespace OnDiskPt
+{
+class Vocab;
+
+
+/** A contiguous phrase. SourcePhrase & TargetPhrase inherit from this and add the on-disk functionality
+ */
+class Phrase
+{
+  friend std::ostream& operator<<(std::ostream&, const Phrase&);
+
+protected:
+  std::vector<WordPtr>	m_words;
+
+public:
+  Phrase() {
+  }
+
+  virtual ~Phrase() {}
+
+  void AddWord(WordPtr word);
+  void AddWord(WordPtr word, size_t pos);
+
+  const Word &GetWord(size_t pos) const {
+    return *m_words[pos];
+  }
+  size_t GetSize() const {
+    return m_words.size();
+  }
+
+  virtual void DebugPrint(std::ostream &out, const Vocab &vocab) const;
+
+  int Compare(const Phrase &compare) const;
+  bool operator<(const Phrase &compare) const;
+  bool operator>(const Phrase &compare) const;
+  bool operator==(const Phrase &compare) const;
+};
+
+typedef boost::shared_ptr<Phrase> PhrasePtr;
+
+}

mosesdecoder/OnDiskPt/PhraseNode.cpp

@@ -0,0 +1,268 @@
+// $Id$
+/***********************************************************************
+ Moses - factored phrase-based, hierarchical and syntactic language decoder
+ Copyright (C) 2009 Hieu Hoang
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ ***********************************************************************/
+#include "PhraseNode.h"
+#include "OnDiskWrapper.h"
+#include "TargetPhraseCollection.h"
+#include "SourcePhrase.h"
+#include "moses/Util.h"
+#include "util/exception.hh"
+
+using namespace std;
+
+namespace OnDiskPt
+{
+
+size_t PhraseNode::GetNodeSize(size_t numChildren, size_t wordSize, size_t countSize)
+{
+  size_t ret = sizeof(uint64_t) * 2 // num children, value
+               + (wordSize + sizeof(uint64_t)) * numChildren // word + ptr to next source node
+               + sizeof(float) * countSize; // count info
+  return ret;
+}
+
+PhraseNode::PhraseNode()
+  : m_value(0)
+  ,m_currChild(NULL)
+  ,m_saved(false)
+  ,m_memLoad(NULL)
+{
+}
+
+PhraseNode::PhraseNode(uint64_t filePos, OnDiskWrapper &onDiskWrapper)
+  :m_counts(onDiskWrapper.GetNumCounts())
+{
+  // load saved node
+  m_filePos = filePos;
+
+  size_t countSize = onDiskWrapper.GetNumCounts();
+
+  std::fstream &file = onDiskWrapper.GetFileSource();
+  file.seekg(filePos);
+  assert(filePos == (uint64_t)file.tellg());
+
+  file.read((char*) &m_numChildrenLoad, sizeof(uint64_t));
+
+  size_t memAlloc = GetNodeSize(m_numChildrenLoad, onDiskWrapper.GetSourceWordSize(), countSize);
+  m_memLoad = (char*) malloc(memAlloc);
+
+  // go to start of node again
+  file.seekg(filePos);
+  assert(filePos == (uint64_t)file.tellg());
+
+  // read everything into memory
+  file.read(m_memLoad, memAlloc);
+  assert(filePos + memAlloc == (uint64_t)file.tellg());
+
+  // get value
+  m_value = ((uint64_t*)m_memLoad)[1];
+
+  // get counts
+  float *memFloat = (float*) (m_memLoad + sizeof(uint64_t) * 2);
+
+  assert(countSize == 1);
+  m_counts[0] = memFloat[0];
+
+  m_memLoadLast = m_memLoad + memAlloc;
+}
+
+PhraseNode::~PhraseNode()
+{
+  free(m_memLoad);
+}
+
+float PhraseNode::GetCount(size_t ind) const
+{
+  return m_counts[ind];
+}
+
+void PhraseNode::Save(OnDiskWrapper &onDiskWrapper, size_t pos, size_t tableLimit)
+{
+  UTIL_THROW_IF2(m_saved, "Already saved");
+
+  // save this node
+  m_targetPhraseColl.Sort(tableLimit);
+  m_targetPhraseColl.Save(onDiskWrapper);
+  m_value = m_targetPhraseColl.GetFilePos();
+
+  size_t numCounts = onDiskWrapper.GetNumCounts();
+
+  size_t memAlloc = GetNodeSize(GetSize(), onDiskWrapper.GetSourceWordSize(), numCounts);
+  char *mem = (char*) malloc(memAlloc);
+  //memset(mem, 0xfe, memAlloc);
+
+  size_t memUsed = 0;
+  uint64_t *memArray = (uint64_t*) mem;
+  memArray[0] = GetSize(); // num of children
+  memArray[1] = m_value;   // file pos of corresponding target phrases
+  memUsed += 2 * sizeof(uint64_t);
+
+  // count info
+  float *memFloat = (float*) (mem + memUsed);
+  UTIL_THROW_IF2(numCounts != 1, "Can only store 1 phrase count");
+  memFloat[0] = (m_counts.size() == 0) ? DEFAULT_COUNT : m_counts[0]; // if count = 0, put in very large num to make sure its still used. HACK
+  memUsed += sizeof(float) * numCounts;
+
+  // recursively save chm_countsildren
+  ChildColl::iterator iter;
+  for (iter = m_children.begin(); iter != m_children.end(); ++iter) {
+    const Word &childWord = iter->first;
+    PhraseNode &childNode = iter->second;
+
+    // recursive
+    if (!childNode.Saved())
+      childNode.Save(onDiskWrapper, pos + 1, tableLimit);
+
+    char *currMem = mem + memUsed;
+    size_t wordMemUsed = childWord.WriteToMemory(currMem);
+    memUsed += wordMemUsed;
+
+    uint64_t *memArray = (uint64_t*) (mem + memUsed);
+    memArray[0] = childNode.GetFilePos();
+    memUsed += sizeof(uint64_t);
+
+  }
+
+  // save this node
+  //Moses::DebugMem(mem, memAlloc);
+  assert(memUsed == memAlloc);
+
+  std::fstream &file = onDiskWrapper.GetFileSource();
+  m_filePos = file.tellp();
+  file.seekp(0, ios::end);
+  file.write(mem, memUsed);
+
+  uint64_t endPos = file.tellp();
+  assert(m_filePos + memUsed == endPos);
+
+  free(mem);
+
+  m_children.clear();
+  m_saved = true;
+}
+
+void PhraseNode::AddTargetPhrase(const SourcePhrase &sourcePhrase, TargetPhrase *targetPhrase
+                                 , OnDiskWrapper &onDiskWrapper, size_t tableLimit
+                                 , const std::vector<float> &counts, OnDiskPt::PhrasePtr spShort)
+{
+  AddTargetPhrase(0, sourcePhrase, targetPhrase, onDiskWrapper, tableLimit, counts, spShort);
+}
+
+void PhraseNode::AddTargetPhrase(size_t pos, const SourcePhrase &sourcePhrase
+                                 , TargetPhrase *targetPhrase, OnDiskWrapper &onDiskWrapper
+                                 , size_t tableLimit, const std::vector<float> &counts, OnDiskPt::PhrasePtr spShort)
+{
+  size_t phraseSize = sourcePhrase.GetSize();
+  if (pos < phraseSize) {
+    const Word &word = sourcePhrase.GetWord(pos);
+
+    PhraseNode &node = m_children[word];
+    if (m_currChild != &node) {
+      // new node
+      node.SetPos(pos);
+
+      if (m_currChild) {
+        m_currChild->Save(onDiskWrapper, pos, tableLimit);
+      }
+
+      m_currChild = &node;
+    }
+
+    // keep searching for target phrase node..
+    node.AddTargetPhrase(pos + 1, sourcePhrase, targetPhrase, onDiskWrapper, tableLimit, counts, spShort);
+  } else {
+    // drilled down to the right node
+    m_counts = counts;
+    targetPhrase->SetSourcePhrase(spShort);
+    m_targetPhraseColl.AddTargetPhrase(targetPhrase);
+  }
+}
+
+const PhraseNode *PhraseNode::GetChild(const Word &wordSought, OnDiskWrapper &onDiskWrapper) const
+{
+  const PhraseNode *ret = NULL;
+
+  int l = 0;
+  int r = m_numChildrenLoad - 1;
+  int x;
+
+  while (r >= l) {
+    x = (l + r) / 2;
+
+    Word wordFound;
+    uint64_t childFilePos;
+    GetChild(wordFound, childFilePos, x, onDiskWrapper);
+
+    if (wordSought == wordFound) {
+      ret = new PhraseNode(childFilePos, onDiskWrapper);
+      break;
+    }
+    if (wordSought < wordFound)
+      r = x - 1;
+    else
+      l = x + 1;
+  }
+
+  return ret;
+}
+
+void PhraseNode::GetChild(Word &wordFound, uint64_t &childFilePos, size_t ind, OnDiskWrapper &onDiskWrapper) const
+{
+
+  size_t wordSize = onDiskWrapper.GetSourceWordSize();
+  size_t childSize = wordSize + sizeof(uint64_t);
+
+  char *currMem = m_memLoad
+                  + sizeof(uint64_t) * 2 // size & file pos of target phrase coll
+                  + sizeof(float) * onDiskWrapper.GetNumCounts() // count info
+                  + childSize * ind;
+
+  size_t memRead = ReadChild(wordFound, childFilePos, currMem);
+  assert(memRead == childSize);
+}
+
+size_t PhraseNode::ReadChild(Word &wordFound, uint64_t &childFilePos, const char *mem) const
+{
+  size_t memRead = wordFound.ReadFromMemory(mem);
+
+  const char *currMem = mem + memRead;
+  uint64_t *memArray = (uint64_t*) (currMem);
+  childFilePos = memArray[0];
+
+  memRead += sizeof(uint64_t);
+  return memRead;
+}
+
+TargetPhraseCollection::shared_ptr
+PhraseNode::
+GetTargetPhraseCollection(size_t tableLimit, OnDiskWrapper &onDiskWrapper) const
+{
+  TargetPhraseCollection::shared_ptr ret(new TargetPhraseCollection);
+  if (m_value > 0) ret->ReadFromFile(tableLimit, m_value, onDiskWrapper);
+  return ret;
+}
+
+std::ostream& operator<<(std::ostream &out, const PhraseNode &node)
+{
+  out << "node (" << node.GetFilePos() << "," << node.GetValue() << "," << node.m_pos << ")";
+  return out;
+}
+
+}
+

mosesdecoder/OnDiskPt/PhraseNode.h

@@ -0,0 +1,108 @@
+#pragma once
+// $Id$
+/***********************************************************************
+ Moses - factored phrase-based, hierarchical and syntactic language decoder
+ Copyright (C) 2009 Hieu Hoang
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ ***********************************************************************/
+#include <fstream>
+#include <vector>
+#include <map>
+#include "Word.h"
+#include "TargetPhraseCollection.h"
+#include "Phrase.h"
+
+namespace OnDiskPt
+{
+
+class OnDiskWrapper;
+class SourcePhrase;
+
+/** A node in the source tree trie */
+class PhraseNode
+{
+  friend std::ostream& operator<<(std::ostream&, const PhraseNode&);
+protected:
+  uint64_t m_filePos, m_value;
+
+  typedef std::map<Word, PhraseNode> ChildColl;
+  ChildColl m_children;
+  PhraseNode *m_currChild;
+  bool m_saved;
+  size_t m_pos;
+  std::vector<float> m_counts;
+
+  TargetPhraseCollection m_targetPhraseColl;
+
+  char *m_memLoad, *m_memLoadLast;
+  uint64_t m_numChildrenLoad;
+
+  void AddTargetPhrase(size_t pos, const SourcePhrase &sourcePhrase
+                       , TargetPhrase *targetPhrase, OnDiskWrapper &onDiskWrapper
+                       , size_t tableLimit, const std::vector<float> &counts, OnDiskPt::PhrasePtr spShort);
+  size_t ReadChild(Word &wordFound, uint64_t &childFilePos, const char *mem) const;
+  void GetChild(Word &wordFound, uint64_t &childFilePos, size_t ind, OnDiskWrapper &onDiskWrapper) const;
+
+public:
+  static size_t GetNodeSize(size_t numChildren, size_t wordSize, size_t countSize);
+
+  PhraseNode(); // unsaved node
+  PhraseNode(uint64_t filePos, OnDiskWrapper &onDiskWrapper); // load saved node
+  ~PhraseNode();
+
+  void Add(const Word &word, uint64_t nextFilePos, size_t wordSize);
+  void Save(OnDiskWrapper &onDiskWrapper, size_t pos, size_t tableLimit);
+
+  void AddTargetPhrase(const SourcePhrase &sourcePhrase, TargetPhrase *targetPhrase
+                       , OnDiskWrapper &onDiskWrapper, size_t tableLimit
+                       , const std::vector<float> &counts, OnDiskPt::PhrasePtr spShort);
+
+  uint64_t GetFilePos() const {
+    return m_filePos;
+  }
+  uint64_t GetValue() const {
+    return m_value;
+  }
+  void SetValue(uint64_t value) {
+    m_value = value;
+  }
+  size_t GetSize() const {
+    return m_children.size();
+  }
+
+  bool Saved() const {
+    return m_saved;
+  }
+
+  void SetPos(size_t pos) {
+    m_pos = pos;
+  }
+
+  const PhraseNode *GetChild(const Word &wordSought, OnDiskWrapper &onDiskWrapper) const;
+
+  TargetPhraseCollection::shared_ptr
+  GetTargetPhraseCollection(size_t tableLimit,
+                            OnDiskWrapper &onDiskWrapper) const;
+
+  void AddCounts(const std::vector<float> &counts) {
+    m_counts = counts;
+  }
+  float GetCount(size_t ind) const;
+
+};
+
+}
+

mosesdecoder/OnDiskPt/SourcePhrase.cpp

@@ -0,0 +1,27 @@
+// $Id$
+/***********************************************************************
+ Moses - factored phrase-based, hierarchical and syntactic language decoder
+ Copyright (C) 2009 Hieu Hoang
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ ***********************************************************************/
+#include "SourcePhrase.h"
+
+namespace OnDiskPt
+{
+
+}
+
+

mosesdecoder/OnDiskPt/SourcePhrase.h

@@ -0,0 +1,38 @@
+#pragma once
+// $Id$
+/***********************************************************************
+ Moses - factored phrase-based, hierarchical and syntactic language decoder
+ Copyright (C) 2009 Hieu Hoang
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ ***********************************************************************/
+#include <vector>
+#include "Phrase.h"
+#include "Word.h"
+
+namespace OnDiskPt
+{
+
+/** A source phrase. No extension of a norm Phrase class because source phrases are saved as tries.
+ */
+class SourcePhrase: public Phrase
+{
+protected:
+
+public:
+};
+
+
+}

mosesdecoder/OnDiskPt/TargetPhrase.cpp

@@ -0,0 +1,402 @@
+// $Id$
+/***********************************************************************
+ Moses - factored phrase-based, hierarchical and syntactic language decoder
+ Copyright (C) 2009 Hieu Hoang
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ ***********************************************************************/
+
+#include <algorithm>
+#include <iostream>
+#include "moses/Util.h"
+#include "TargetPhrase.h"
+#include "OnDiskWrapper.h"
+#include "util/exception.hh"
+
+#include <boost/algorithm/string.hpp>
+
+using namespace std;
+
+namespace OnDiskPt
+{
+
+TargetPhrase::TargetPhrase(size_t numScores)
+  :m_scores(numScores)
+{
+}
+
+TargetPhrase::TargetPhrase(const 	TargetPhrase &copy)
+  :Phrase(copy)
+  ,m_scores(copy.m_scores)
+{
+
+}
+
+TargetPhrase::~TargetPhrase()
+{
+}
+
+void TargetPhrase::SetLHS(WordPtr lhs)
+{
+  AddWord(lhs);
+}
+
+void TargetPhrase::Create1AlignFromString(const std::string &align1Str)
+{
+  vector<size_t> alignPoints;
+  Moses::Tokenize<size_t>(alignPoints, align1Str, "-");
+  UTIL_THROW_IF2(alignPoints.size() != 2, "Incorrectly formatted word alignment: " << align1Str);
+  m_align.push_back(pair<size_t, size_t>(alignPoints[0], alignPoints[1]) );
+}
+
+void TargetPhrase::CreateAlignFromString(const std::string &alignStr)
+{
+  vector<std::string> alignPairs;
+  boost::split(alignPairs, alignStr, boost::is_any_of("\t "));
+  for (size_t i = 0; i < alignPairs.size(); ++i) {
+    vector<size_t> alignPoints;
+    Moses::Tokenize<size_t>(alignPoints, alignPairs[i], "-");
+    m_align.push_back(pair<size_t, size_t>(alignPoints[0], alignPoints[1]) );
+  }
+}
+
+
+void TargetPhrase::SetScore(float score, size_t ind)
+{
+  assert(ind < m_scores.size());
+  m_scores[ind] = score;
+}
+
+class AlignOrderer
+{
+public:
+  bool operator()(const AlignPair &a, const AlignPair &b) const {
+    return a.first < b.first;
+  }
+};
+
+void TargetPhrase::SortAlign()
+{
+  std::sort(m_align.begin(), m_align.end(), AlignOrderer());
+}
+
+char *TargetPhrase::WriteToMemory(OnDiskWrapper &onDiskWrapper, size_t &memUsed) const
+{
+  size_t phraseSize = GetSize();
+  size_t targetWordSize = onDiskWrapper.GetTargetWordSize();
+
+  const PhrasePtr sp = GetSourcePhrase();
+  size_t spSize = sp->GetSize();
+  size_t sourceWordSize = onDiskWrapper.GetSourceWordSize();
+
+  size_t memNeeded = sizeof(uint64_t)						// num of words
+                     + targetWordSize * phraseSize	// actual words. lhs as last words
+                     + sizeof(uint64_t)					// num source words
+                     + sourceWordSize * spSize;   // actual source words
+
+  memUsed = 0;
+  uint64_t *mem = (uint64_t*) malloc(memNeeded);
+
+  // write size
+  mem[0] = phraseSize;
+  memUsed += sizeof(uint64_t);
+
+  // write each word
+  for (size_t pos = 0; pos < phraseSize; ++pos) {
+    const Word &word = GetWord(pos);
+    char *currPtr = (char*)mem + memUsed;
+    memUsed += word.WriteToMemory((char*) currPtr);
+  }
+
+  // write size of source phrase and all source words
+  char *currPtr = (char*)mem + memUsed;
+  uint64_t *memTmp = (uint64_t*) currPtr;
+  memTmp[0] = spSize;
+  memUsed += sizeof(uint64_t);
+  for (size_t pos = 0; pos < spSize; ++pos) {
+    const Word &word = sp->GetWord(pos);
+    char *currPtr = (char*)mem + memUsed;
+    memUsed += word.WriteToMemory((char*) currPtr);
+  }
+
+  assert(memUsed == memNeeded);
+  return (char *) mem;
+}
+
+void TargetPhrase::Save(OnDiskWrapper &onDiskWrapper)
+{
+  // save in target ind
+  size_t memUsed;
+  char *mem = WriteToMemory(onDiskWrapper, memUsed);
+
+  std::fstream &file = onDiskWrapper.GetFileTargetInd();
+
+  uint64_t startPos = file.tellp();
+
+  file.seekp(0, ios::end);
+  file.write(mem, memUsed);
+
+#ifndef NDEBUG
+  uint64_t endPos = file.tellp();
+  assert(startPos + memUsed == endPos);
+#endif
+
+  m_filePos = startPos;
+  free(mem);
+}
+
+char *TargetPhrase::WriteOtherInfoToMemory(OnDiskWrapper &onDiskWrapper, size_t &memUsed) const
+{
+  // allocate mem
+  size_t numScores = onDiskWrapper.GetNumScores()
+                     ,numAlign = GetAlign().size();
+  size_t sparseFeatureSize = m_sparseFeatures.size();
+  size_t propSize = m_property.size();
+
+  size_t memNeeded = sizeof(uint64_t) // file pos (phrase id)
+                     + sizeof(uint64_t) + 2 * sizeof(uint64_t) * numAlign // align
+                     + sizeof(float) * numScores // scores
+                     + sizeof(uint64_t) + sparseFeatureSize // sparse features string
+                     + sizeof(uint64_t) + propSize; // property string
+
+  char *mem = (char*) malloc(memNeeded);
+  //memset(mem, 0, memNeeded);
+
+  memUsed = 0;
+
+  // phrase id
+  memcpy(mem, &m_filePos, sizeof(uint64_t));
+  memUsed += sizeof(uint64_t);
+
+  // align
+  size_t tmp = WriteAlignToMemory(mem + memUsed);
+  memUsed += tmp;
+
+  // scores
+  memUsed += WriteScoresToMemory(mem + memUsed);
+
+  // sparse features
+  memUsed += WriteStringToMemory(mem + memUsed, m_sparseFeatures);
+
+  // property string
+  memUsed += WriteStringToMemory(mem + memUsed, m_property);
+
+  //DebugMem(mem, memNeeded);
+  assert(memNeeded == memUsed);
+  return mem;
+}
+
+size_t TargetPhrase::WriteStringToMemory(char *mem, const std::string &str) const
+{
+  size_t memUsed = 0;
+  uint64_t *memTmp = (uint64_t*) mem;
+
+  size_t strSize = str.size();
+  memTmp[0] = strSize;
+  memUsed += sizeof(uint64_t);
+
+  const char *charStr = str.c_str();
+  memcpy(mem + memUsed, charStr, strSize);
+  memUsed += strSize;
+
+  return memUsed;
+}
+
+size_t TargetPhrase::WriteAlignToMemory(char *mem) const
+{
+  size_t memUsed = 0;
+
+  // num of alignments
+  uint64_t numAlign = m_align.size();
+  memcpy(mem, &numAlign, sizeof(numAlign));
+  memUsed += sizeof(numAlign);
+
+  // actual alignments
+  AlignType::const_iterator iter;
+  for (iter = m_align.begin(); iter != m_align.end(); ++iter) {
+    const AlignPair &alignPair = *iter;
+
+    memcpy(mem + memUsed, &alignPair.first, sizeof(alignPair.first));
+    memUsed += sizeof(alignPair.first);
+
+    memcpy(mem + memUsed, &alignPair.second, sizeof(alignPair.second));
+    memUsed += sizeof(alignPair.second);
+  }
+
+  return memUsed;
+}
+
+size_t TargetPhrase::WriteScoresToMemory(char *mem) const
+{
+  float *scoreMem = (float*) mem;
+
+  for (size_t ind = 0; ind < m_scores.size(); ++ind)
+    scoreMem[ind] = m_scores[ind];
+
+  size_t memUsed = sizeof(float) * m_scores.size();
+  return memUsed;
+}
+
+uint64_t TargetPhrase::ReadOtherInfoFromFile(uint64_t filePos, std::fstream &fileTPColl)
+{
+  assert(filePos == (uint64_t)fileTPColl.tellg());
+
+  uint64_t memUsed = 0;
+  fileTPColl.read((char*) &m_filePos, sizeof(uint64_t));
+  memUsed += sizeof(uint64_t);
+  assert(m_filePos != 0);
+
+  memUsed += ReadAlignFromFile(fileTPColl);
+  assert((memUsed + filePos) == (uint64_t)fileTPColl.tellg());
+
+  memUsed += ReadScoresFromFile(fileTPColl);
+  assert((memUsed + filePos) == (uint64_t)fileTPColl.tellg());
+
+  // sparse features
+  memUsed += ReadStringFromFile(fileTPColl, m_sparseFeatures);
+
+  // properties
+  memUsed += ReadStringFromFile(fileTPColl, m_property);
+
+  return memUsed;
+}
+
+uint64_t TargetPhrase::ReadStringFromFile(std::fstream &fileTPColl, std::string &outStr)
+{
+  uint64_t bytesRead = 0;
+
+  uint64_t strSize;
+  fileTPColl.read((char*) &strSize, sizeof(uint64_t));
+  bytesRead += sizeof(uint64_t);
+
+  if (strSize) {
+    char *mem = (char*) malloc(strSize + 1);
+    mem[strSize] = '\0';
+    fileTPColl.read(mem, strSize);
+    outStr = string(mem);
+    free(mem);
+
+    bytesRead += strSize;
+  }
+
+  return bytesRead;
+}
+
+uint64_t TargetPhrase::ReadFromFile(std::fstream &fileTP)
+{
+  uint64_t bytesRead = 0;
+
+  fileTP.seekg(m_filePos);
+
+  uint64_t numWords;
+  fileTP.read((char*) &numWords, sizeof(uint64_t));
+  bytesRead += sizeof(uint64_t);
+
+  for (size_t ind = 0; ind < numWords; ++ind) {
+    WordPtr word(new Word());
+    bytesRead += word->ReadFromFile(fileTP);
+    AddWord(word);
+  }
+
+  // read source words
+  uint64_t numSourceWords;
+  fileTP.read((char*) &numSourceWords, sizeof(uint64_t));
+  bytesRead += sizeof(uint64_t);
+
+  PhrasePtr sp(new SourcePhrase());
+  for (size_t ind = 0; ind < numSourceWords; ++ind) {
+    WordPtr word( new Word());
+    bytesRead += word->ReadFromFile(fileTP);
+    sp->AddWord(word);
+  }
+  SetSourcePhrase(sp);
+
+  return bytesRead;
+}
+
+uint64_t TargetPhrase::ReadAlignFromFile(std::fstream &fileTPColl)
+{
+  uint64_t bytesRead = 0;
+
+  uint64_t numAlign;
+  fileTPColl.read((char*) &numAlign, sizeof(uint64_t));
+  bytesRead += sizeof(uint64_t);
+
+  for (size_t ind = 0; ind < numAlign; ++ind) {
+    AlignPair alignPair;
+    fileTPColl.read((char*) &alignPair.first, sizeof(uint64_t));
+    fileTPColl.read((char*) &alignPair.second, sizeof(uint64_t));
+    m_align.push_back(alignPair);
+
+    bytesRead += sizeof(uint64_t) * 2;
+  }
+
+  return bytesRead;
+}
+
+uint64_t TargetPhrase::ReadScoresFromFile(std::fstream &fileTPColl)
+{
+  UTIL_THROW_IF2(m_scores.size() == 0, "Translation rules must must have some scores");
+
+  uint64_t bytesRead = 0;
+
+  for (size_t ind = 0; ind < m_scores.size(); ++ind) {
+    fileTPColl.read((char*) &m_scores[ind], sizeof(float));
+
+    bytesRead += sizeof(float);
+  }
+
+  std::transform(m_scores.begin(),m_scores.end(),m_scores.begin(), Moses::TransformScore);
+  std::transform(m_scores.begin(),m_scores.end(),m_scores.begin(), Moses::FloorScore);
+
+  return bytesRead;
+}
+
+void TargetPhrase::DebugPrint(ostream &out, const Vocab &vocab) const
+{
+  Phrase::DebugPrint(out, vocab);
+
+  for (size_t ind = 0; ind < m_align.size(); ++ind) {
+    const AlignPair &alignPair = m_align[ind];
+    out << alignPair.first << "-" << alignPair.second << " ";
+  }
+  out << ", ";
+
+  for (size_t ind = 0; ind < m_scores.size(); ++ind) {
+    out << m_scores[ind] << " ";
+  }
+
+  return;
+}
+
+std::ostream& operator<<(std::ostream &out, const TargetPhrase &phrase)
+{
+  out << (const Phrase&) phrase << ", " ;
+
+  for (size_t ind = 0; ind < phrase.m_align.size(); ++ind) {
+    const AlignPair &alignPair = phrase.m_align[ind];
+    out << alignPair.first << "-" << alignPair.second << " ";
+  }
+  out << ", ";
+
+  for (size_t ind = 0; ind < phrase.m_scores.size(); ++ind) {
+    out << phrase.m_scores[ind] << " ";
+  }
+
+  return out;
+}
+
+} // namespace
+

mosesdecoder/OnDiskPt/TargetPhrase.h

@@ -0,0 +1,127 @@
+#pragma once
+// $Id$
+/***********************************************************************
+ Moses - factored phrase-based, hierarchical and syntactic language decoder
+ Copyright (C) 2009 Hieu Hoang
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ ***********************************************************************/
+
+#include <fstream>
+#include <string>
+#include <vector>
+#include "Word.h"
+#include "Phrase.h"
+#include "SourcePhrase.h"
+
+namespace Moses
+{
+class PhraseDictionary;
+class TargetPhrase;
+class Phrase;
+}
+
+namespace OnDiskPt
+{
+
+typedef std::pair<uint64_t, uint64_t>  AlignPair;
+typedef std::vector<AlignPair> AlignType;
+
+class Vocab;
+
+/** A target phrase, with the score breakdowns, alignment info and assorted other information it need.
+ *  Readable and writeable to disk
+ */
+class TargetPhrase: public Phrase
+{
+  friend std::ostream& operator<<(std::ostream&, const TargetPhrase&);
+protected:
+  AlignType m_align;
+  PhrasePtr m_sourcePhrase;
+  std::string m_sparseFeatures, m_property;
+
+  std::vector<float> m_scores;
+  uint64_t m_filePos;
+
+  size_t WriteAlignToMemory(char *mem) const;
+  size_t WriteScoresToMemory(char *mem) const;
+  size_t WriteStringToMemory(char *mem, const std::string &str) const;
+
+  uint64_t ReadAlignFromFile(std::fstream &fileTPColl);
+  uint64_t ReadScoresFromFile(std::fstream &fileTPColl);
+  uint64_t ReadStringFromFile(std::fstream &fileTPColl, std::string &outStr);
+
+public:
+  TargetPhrase() {
+  }
+  TargetPhrase(size_t numScores);
+  TargetPhrase(const TargetPhrase &copy);
+  virtual ~TargetPhrase();
+
+  void SetSourcePhrase(PhrasePtr p) {
+    m_sourcePhrase = p;
+  }
+  const PhrasePtr GetSourcePhrase() const {
+    return m_sourcePhrase;
+  }
+  const std::vector<float> &GetScores() const {
+    return m_scores;
+  }
+
+  void SetLHS(WordPtr lhs);
+
+  void Create1AlignFromString(const std::string &align1Str);
+  void CreateAlignFromString(const std::string &align1Str);
+  void SetScore(float score, size_t ind);
+
+  const AlignType &GetAlign() const {
+    return m_align;
+  }
+  void SortAlign();
+
+  char *WriteToMemory(OnDiskWrapper &onDiskWrapper, size_t &memUsed) const;
+  char *WriteOtherInfoToMemory(OnDiskWrapper &onDiskWrapper, size_t &memUsed) const;
+  void Save(OnDiskWrapper &onDiskWrapper);
+
+  uint64_t GetFilePos() const {
+    return m_filePos;
+  }
+  float GetScore(size_t ind) const {
+    return m_scores[ind];
+  }
+
+  uint64_t ReadOtherInfoFromFile(uint64_t filePos, std::fstream &fileTPColl);
+  uint64_t ReadFromFile(std::fstream &fileTP);
+
+  virtual void DebugPrint(std::ostream &out, const Vocab &vocab) const;
+
+  const std::string &GetProperty() const {
+    return m_property;
+  }
+
+  void SetProperty(const std::string &value) {
+    m_property = value;
+  }
+
+  const std::string &GetSparseFeatures() const {
+    return m_sparseFeatures;
+  }
+
+  void SetSparseFeatures(const std::string &value) {
+    m_sparseFeatures = value;
+  }
+};
+
+}

mosesdecoder/OnDiskPt/TargetPhraseCollection.cpp

@@ -0,0 +1,171 @@
+// $Id$
+/***********************************************************************
+ Moses - factored phrase-based, hierarchical and syntactic language decoder
+ Copyright (C) 2009 Hieu Hoang
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ ***********************************************************************/
+
+#include <algorithm>
+#include <iostream>
+#include "moses/Util.h"
+#include "TargetPhraseCollection.h"
+#include "Vocab.h"
+#include "OnDiskWrapper.h"
+
+using namespace std;
+
+namespace OnDiskPt
+{
+
+size_t TargetPhraseCollection::s_sortScoreInd;
+
+TargetPhraseCollection::TargetPhraseCollection()
+  :m_filePos(777)
+{}
+
+TargetPhraseCollection::TargetPhraseCollection(const TargetPhraseCollection &copy)
+  :m_filePos(copy.m_filePos)
+  ,m_debugStr(copy.m_debugStr)
+{
+}
+
+TargetPhraseCollection::~TargetPhraseCollection()
+{
+  Moses::RemoveAllInColl(m_coll);
+}
+
+void TargetPhraseCollection::AddTargetPhrase(TargetPhrase *targetPhrase)
+{
+  m_coll.push_back(targetPhrase);
+}
+
+void TargetPhraseCollection::Sort(size_t tableLimit)
+{
+  std::sort(m_coll.begin(), m_coll.end(), TargetPhraseOrderByScore());
+
+  if (tableLimit && m_coll.size() > tableLimit) {
+    CollType::iterator iter;
+    for (iter = m_coll.begin() + tableLimit ; iter != m_coll.end(); ++iter) {
+      delete *iter;
+    }
+    m_coll.resize(tableLimit);
+  }
+}
+
+void TargetPhraseCollection::Save(OnDiskWrapper &onDiskWrapper)
+{
+  std::fstream &file = onDiskWrapper.GetFileTargetColl();
+
+  size_t memUsed = sizeof(uint64_t);
+  char *mem = (char*) malloc(memUsed);
+
+  // size of coll
+  uint64_t numPhrases = GetSize();
+  ((uint64_t*)mem)[0] = numPhrases;
+
+  // MAIN LOOP
+  CollType::iterator iter;
+  for (iter = m_coll.begin(); iter != m_coll.end(); ++iter) {
+    // save phrase
+    TargetPhrase &targetPhrase = **iter;
+    targetPhrase.Save(onDiskWrapper);
+
+    // save coll
+    size_t memUsedTPOtherInfo;
+    char *memTPOtherInfo = targetPhrase.WriteOtherInfoToMemory(onDiskWrapper, memUsedTPOtherInfo);
+
+    // expand existing mem
+    mem = (char*) realloc(mem, memUsed + memUsedTPOtherInfo);
+    memcpy(mem + memUsed, memTPOtherInfo, memUsedTPOtherInfo);
+    memUsed += memUsedTPOtherInfo;
+
+    free(memTPOtherInfo);
+  }
+
+  // total number of bytes
+  //((uint64_t*)mem)[0] = (uint64_t) memUsed;
+
+  uint64_t startPos = file.tellp();
+  file.seekp(0, ios::end);
+  file.write((char*) mem, memUsed);
+
+  free(mem);
+
+#ifndef NDEBUG
+  uint64_t endPos = file.tellp();
+  assert(startPos + memUsed == endPos);
+#endif
+  m_filePos = startPos;
+
+}
+
+void TargetPhraseCollection::ReadFromFile(size_t tableLimit, uint64_t filePos, OnDiskWrapper &onDiskWrapper)
+{
+  fstream &fileTPColl = onDiskWrapper.GetFileTargetColl();
+  fstream &fileTP = onDiskWrapper.GetFileTargetInd();
+
+  size_t numScores = onDiskWrapper.GetNumScores();
+
+
+  uint64_t numPhrases;
+
+  uint64_t currFilePos = filePos;
+  fileTPColl.seekg(filePos);
+  fileTPColl.read((char*) &numPhrases, sizeof(uint64_t));
+
+  // table limit
+  if (tableLimit) {
+    numPhrases = std::min(numPhrases, (uint64_t) tableLimit);
+  }
+
+  currFilePos += sizeof(uint64_t);
+
+  for (size_t ind = 0; ind < numPhrases; ++ind) {
+    TargetPhrase *tp = new TargetPhrase(numScores);
+
+    uint64_t sizeOtherInfo = tp->ReadOtherInfoFromFile(currFilePos, fileTPColl);
+    tp->ReadFromFile(fileTP);
+
+    currFilePos += sizeOtherInfo;
+
+    m_coll.push_back(tp);
+  }
+}
+
+uint64_t TargetPhraseCollection::GetFilePos() const
+{
+  return m_filePos;
+}
+
+const std::string TargetPhraseCollection::GetDebugStr() const
+{
+  return m_debugStr;
+}
+
+void TargetPhraseCollection::SetDebugStr(const std::string &str)
+{
+  m_debugStr = str;
+}
+
+const TargetPhrase &TargetPhraseCollection::GetTargetPhrase(size_t ind) const
+{
+  assert(ind < GetSize());
+  return *m_coll[ind];
+}
+
+}
+
+

mosesdecoder/OnDiskPt/TargetPhraseCollection.h

@@ -0,0 +1,84 @@
+// $Id$
+/***********************************************************************
+ Moses - factored phrase-based, hierarchical and syntactic language decoder
+ Copyright (C) 2009 Hieu Hoang
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ ***********************************************************************/
+#pragma once
+
+#include "TargetPhrase.h"
+#include "Vocab.h"
+#include <boost/shared_ptr.hpp>
+
+namespace Moses
+{
+class TargetPhraseCollection;
+class PhraseDictionary;
+}
+
+namespace OnDiskPt
+{
+
+/** A vector of target phrases
+ */
+class TargetPhraseCollection
+{
+  class TargetPhraseOrderByScore
+  {
+  public:
+    bool operator()(const TargetPhrase* a, const TargetPhrase *b) const {
+      return a->GetScore(s_sortScoreInd) > b->GetScore(s_sortScoreInd);
+    }
+  };
+
+protected:
+  typedef std::vector<TargetPhrase*> CollType;
+  CollType m_coll;
+  uint64_t m_filePos;
+  std::string m_debugStr;
+
+public:
+  typedef boost::shared_ptr<TargetPhraseCollection const> shared_const_ptr;
+  typedef boost::shared_ptr<TargetPhraseCollection> shared_ptr;
+
+  static size_t s_sortScoreInd;
+
+  TargetPhraseCollection();
+  TargetPhraseCollection(const TargetPhraseCollection &copy);
+
+  ~TargetPhraseCollection();
+  void AddTargetPhrase(TargetPhrase *targetPhrase);
+  void Sort(size_t tableLimit);
+
+  void Save(OnDiskWrapper &onDiskWrapper);
+
+  size_t GetSize() const {
+    return m_coll.size();
+  }
+
+  const TargetPhrase &GetTargetPhrase(size_t ind) const;
+
+  uint64_t GetFilePos() const;
+
+  void ReadFromFile(size_t tableLimit, uint64_t filePos, OnDiskWrapper &onDiskWrapper);
+
+  const std::string GetDebugStr() const;
+  void SetDebugStr(const std::string &str);
+
+};
+
+}
+

mosesdecoder/OnDiskPt/Vocab.cpp

@@ -0,0 +1,101 @@
+// $Id$
+/***********************************************************************
+ Moses - factored phrase-based, hierarchical and syntactic language decoder
+ Copyright (C) 2009 Hieu Hoang
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ ***********************************************************************/
+#include <string>
+#include <fstream>
+#include "OnDiskWrapper.h"
+#include "Vocab.h"
+#include "moses/Util.h"
+#include "util/exception.hh"
+
+using namespace std;
+
+namespace OnDiskPt
+{
+
+bool Vocab::Load(OnDiskWrapper &onDiskWrapper)
+{
+  fstream &file = onDiskWrapper.GetFileVocab();
+
+  string line;
+  while(getline(file, line)) {
+    vector<string> tokens;
+    Moses::Tokenize(tokens, line);
+    UTIL_THROW_IF2(tokens.size() != 2, "Vocab file corrupted");
+    const string &key = tokens[0];
+    m_vocabColl[key] =  Moses::Scan<uint64_t>(tokens[1]);
+  }
+
+  // create lookup
+  // assume contiguous vocab id
+  m_lookup.resize(m_vocabColl.size() + 1);
+  m_nextId = m_lookup.size();
+
+  CollType::const_iterator iter;
+  for (iter = m_vocabColl.begin(); iter != m_vocabColl.end(); ++iter) {
+    uint32_t vocabId = iter->second;
+    const std::string &word = iter->first;
+
+    m_lookup[vocabId] = word;
+  }
+
+  return true;
+}
+
+void Vocab::Save(OnDiskWrapper &onDiskWrapper)
+{
+  fstream &file = onDiskWrapper.GetFileVocab();
+  CollType::const_iterator iterVocab;
+  for (iterVocab = m_vocabColl.begin(); iterVocab != m_vocabColl.end(); ++iterVocab) {
+    const string &word = iterVocab->first;
+    uint32_t vocabId = iterVocab->second;
+
+    file << word << " " << vocabId << endl;
+  }
+}
+
+uint64_t Vocab::AddVocabId(const std::string &str)
+{
+  // find string id
+  CollType::const_iterator iter = m_vocabColl.find(str);
+  if (iter == m_vocabColl.end()) {
+    // add new vocab entry
+    m_vocabColl[str] = m_nextId;
+    return m_nextId++;
+  } else {
+    // return existing entry
+    return iter->second;
+  }
+}
+
+uint64_t Vocab::GetVocabId(const std::string &str, bool &found) const
+{
+  // find string id
+  CollType::const_iterator iter = m_vocabColl.find(str);
+  if (iter == m_vocabColl.end()) {
+    found = false;
+    return 0; //return whatever
+  } else {
+    // return existing entry
+    found = true;
+    return iter->second;
+  }
+}
+
+}

mosesdecoder/OnDiskPt/Vocab.h

@@ -0,0 +1,58 @@
+#pragma once
+// $Id$
+/***********************************************************************
+ Moses - factored phrase-based, hierarchical and syntactic language decoder
+ Copyright (C) 2009 Hieu Hoang
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ ***********************************************************************/
+#include <string>
+#include <map>
+#include "moses/TypeDef.h"
+
+
+namespace OnDiskPt
+{
+
+class OnDiskWrapper;
+
+/* A bidirectional map of string<->contiguous id
+ * No distinction between source and target language
+ */
+class Vocab
+{
+protected:
+  typedef std::map<std::string, uint64_t> CollType;
+  CollType m_vocabColl;
+
+  std::vector<std::string> m_lookup; // opposite of m_vocabColl
+  uint64_t m_nextId; // starts @ 1
+
+public:
+  Vocab()
+    :m_nextId(1) {
+  }
+  uint64_t AddVocabId(const std::string &str);
+  uint64_t GetVocabId(const std::string &str, bool &found) const;
+  const std::string &GetString(uint64_t vocabId) const {
+    return m_lookup[vocabId];
+  }
+
+  bool Load(OnDiskWrapper &onDiskWrapper);
+  void Save(OnDiskWrapper &onDiskWrapper);
+};
+
+}
+

mosesdecoder/OnDiskPt/Word.cpp

@@ -0,0 +1,144 @@
+// $Id$
+/***********************************************************************
+ Moses - factored phrase-based, hierarchical and syntactic language decoder
+ Copyright (C) 2009 Hieu Hoang
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ ***********************************************************************/
+
+#include <boost/algorithm/string/predicate.hpp>
+#include "moses/Util.h"
+#include "Word.h"
+
+#include "util/tokenize_piece.hh"
+#include "util/exception.hh"
+
+using namespace std;
+using namespace boost::algorithm;
+
+namespace OnDiskPt
+{
+
+Word::Word(const Word &copy)
+  :m_isNonTerminal(copy.m_isNonTerminal)
+  ,m_vocabId(copy.m_vocabId)
+{}
+
+Word::~Word()
+{}
+
+void Word::CreateFromString(const std::string &inString, Vocab &vocab)
+{
+  if (starts_with(inString, "[") && ends_with(inString, "]")) {
+    // non-term
+    m_isNonTerminal = true;
+    string str = inString.substr(1, inString.size() - 2);
+    m_vocabId = vocab.AddVocabId(str);
+  } else {
+    m_isNonTerminal = false;
+    m_vocabId = vocab.AddVocabId(inString);
+  }
+
+}
+
+size_t Word::WriteToMemory(char *mem) const
+{
+  uint64_t *vocabMem = (uint64_t*) mem;
+  vocabMem[0] = m_vocabId;
+
+  size_t size = sizeof(uint64_t);
+
+  // is non-term
+  char bNonTerm = (char) m_isNonTerminal;
+  mem[size] = bNonTerm;
+  ++size;
+
+  return size;
+}
+
+size_t Word::ReadFromMemory(const char *mem)
+{
+  uint64_t *vocabMem = (uint64_t*) mem;
+  m_vocabId = vocabMem[0];
+
+  size_t memUsed = sizeof(uint64_t);
+
+  // is non-term
+  char bNonTerm;
+  bNonTerm = mem[memUsed];
+  m_isNonTerminal = (bool) bNonTerm;
+  ++memUsed;
+
+  return memUsed;
+}
+
+size_t Word::ReadFromFile(std::fstream &file)
+{
+  const size_t memAlloc = sizeof(uint64_t) + sizeof(char);
+  char mem[sizeof(uint64_t) + sizeof(char)];
+  file.read(mem, memAlloc);
+
+  size_t memUsed = ReadFromMemory(mem);
+  assert(memAlloc == memUsed);
+
+  return memAlloc;
+}
+
+int Word::Compare(const Word &compare) const
+{
+  int ret;
+
+  if (m_isNonTerminal != compare.m_isNonTerminal)
+    return m_isNonTerminal ?-1 : 1;
+
+  if (m_vocabId < compare.m_vocabId)
+    ret = -1;
+  else if (m_vocabId > compare.m_vocabId)
+    ret = 1;
+  else
+    ret = 0;
+
+  return ret;
+}
+
+bool Word::operator<(const Word &compare) const
+{
+  int ret = Compare(compare);
+  return ret < 0;
+}
+
+bool Word::operator==(const Word &compare) const
+{
+  int ret = Compare(compare);
+  return ret == 0;
+}
+
+void Word::DebugPrint(ostream &out, const Vocab &vocab) const
+{
+  const string &str = vocab.GetString(m_vocabId);
+  out << str;
+}
+
+std::ostream& operator<<(std::ostream &out, const Word &word)
+{
+  out << "(";
+  out << word.m_vocabId;
+
+  out << (word.m_isNonTerminal ? "n" : "t");
+  out << ")";
+
+  return out;
+}
+}

mosesdecoder/OnDiskPt/Word.h

@@ -0,0 +1,91 @@
+#pragma once
+// $Id$
+/***********************************************************************
+ Moses - factored phrase-based, hierarchical and syntactic language decoder
+ Copyright (C) 2009 Hieu Hoang
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ ***********************************************************************/
+#include <string>
+#include <vector>
+#include <iostream>
+#include <fstream>
+#include <boost/shared_ptr.hpp>
+#include "Vocab.h"
+
+namespace Moses
+{
+class Word;
+}
+
+namespace OnDiskPt
+{
+class Vocab;
+
+/* A wrapper around a vocab id, and a boolean indicating whther it is a term or non-term.
+ * Factors can be represented by using a vocab string with | character, eg go|VB
+ */
+class Word
+{
+  friend std::ostream& operator<<(std::ostream&, const Word&);
+
+private:
+  bool m_isNonTerminal;
+  uint64_t m_vocabId;
+
+public:
+  explicit Word() {
+  }
+
+  explicit Word(bool isNonTerminal)
+    :m_isNonTerminal(isNonTerminal)
+    ,m_vocabId(0) {
+  }
+
+  Word(const Word &copy);
+  ~Word();
+
+
+  void CreateFromString(const std::string &inString, Vocab &vocab);
+  bool IsNonTerminal() const {
+    return m_isNonTerminal;
+  }
+
+  size_t WriteToMemory(char *mem) const;
+  size_t ReadFromMemory(const char *mem);
+  size_t ReadFromFile(std::fstream &file);
+
+  uint64_t GetVocabId() const {
+    return m_vocabId;
+  }
+
+  void SetVocabId(uint64_t vocabId) {
+    m_vocabId = vocabId;
+  }
+
+  void DebugPrint(std::ostream &out, const Vocab &vocab) const;
+  inline const std::string &GetString(const Vocab &vocab) const {
+    return vocab.GetString(m_vocabId);
+  }
+
+  int Compare(const Word &compare) const;
+  bool operator<(const Word &compare) const;
+  bool operator==(const Word &compare) const;
+
+};
+
+typedef boost::shared_ptr<Word> WordPtr;
+}
+

mosesdecoder/OnDiskPt/bin/gcc-9/release/link-static/threading-multi/CreateOnDiskPt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:924fb66d9f0e64d799938679376511b174bd77ad4cbe1d218e33a9c3278402a3
+size 9824568

mosesdecoder/OnDiskPt/queryOnDiskPt.cpp

@@ -0,0 +1,86 @@
+// Query binary phrase tables.
+// Christian Hardmeier, 16 May 2010
+
+#include <cstdlib>
+#include <cstring>
+#include <string>
+#include <vector>
+
+#include "moses/Util.h"
+#include "OnDiskWrapper.h"
+#include "SourcePhrase.h"
+#include "OnDiskQuery.h"
+
+using namespace std;
+using namespace OnDiskPt;
+
+void usage();
+
+typedef unsigned int uint;
+
+int main(int argc, char **argv)
+{
+  int tableLimit = 20;
+  std::string ttable = "";
+  // bool useAlignments = false;
+
+  for(int i = 1; i < argc; i++) {
+    if(!strcmp(argv[i], "-tlimit")) {
+      if(i + 1 == argc)
+        usage();
+      tableLimit = atoi(argv[++i]);
+    } else if(!strcmp(argv[i], "-t")) {
+      if(i + 1 == argc)
+        usage();
+      ttable = argv[++i];
+    } else
+      usage();
+  }
+
+  if(ttable == "")
+    usage();
+
+  OnDiskWrapper onDiskWrapper;
+  onDiskWrapper.BeginLoad(ttable);
+  OnDiskQuery onDiskQuery(onDiskWrapper);
+
+  cerr << "Ready..." << endl;
+
+  std::string line;
+  while(getline(std::cin, line)) {
+    std::vector<std::string> tokens;
+    tokens = Moses::Tokenize(line, " ");
+
+    cerr << "line: " << line << endl;
+    const PhraseNode* node = onDiskQuery.Query(tokens);
+
+    if (node) {
+      // source phrase points to a bunch of rules
+      TargetPhraseCollection::shared_ptr coll = node->GetTargetPhraseCollection(tableLimit, onDiskWrapper);
+      string str = coll->GetDebugStr();
+      cout << "Found " << coll->GetSize() << endl;
+
+      for (size_t ind = 0; ind < coll->GetSize(); ++ind) {
+        const TargetPhrase &targetPhrase = coll->GetTargetPhrase(ind);
+        cerr << "  ";
+        targetPhrase.DebugPrint(cerr, onDiskWrapper.GetVocab());
+        cerr << endl;
+      }
+    } else {
+      cout << "Not found" << endl;
+    }
+
+    std::cout << '\n';
+    std::cout.flush();
+  }
+
+  cerr << "Finished." << endl;
+}
+
+void usage()
+{
+  std::cerr << "Usage: queryOnDiskPt [-n <nscores>] [-a] -t <ttable>\n"
+            "-tlimit <table limit>      max number of rules per source phrase (default: 20)\n"
+            "-t <ttable>       phrase table\n";
+  exit(1);
+}

mosesdecoder/README

@@ -0,0 +1,14 @@
+Instructions for building and installing Moses are online:
+
+   http://www.statmt.org/moses/?n=Development.GetStarted
+
+Questions should be directed to the mailing list (don't forget to register before sending emails):
+   http://mailman.mit.edu/mailman/listinfo/moses-support
+
+Some of the code is not originally part of Moses, but is periodically copied
+into the source tree from elsewhere:
+
+ * "bjam-files" is taken from Boost.
+ * "util" and "lm" are taken from KenLM: https://github.com/kpu/kenlm
+
+

mosesdecoder/azure-pipelines.yml

@@ -0,0 +1,100 @@
+# Starter pipeline
+# Start with a minimal pipeline that you can customize to build and deploy your code.
+# Add steps that build, run tests, deploy, and more:
+# https://aka.ms/yaml
+
+trigger:
+- master
+
+pool:
+  #vmImage: 'ubuntu-latest'
+  vmImage: 'ubuntu-16.04'
+
+steps:
+
+- script: |
+    echo Printing some environment information
+    echo HOME: $HOME
+    echo
+    echo UBUNTU VERSION: 
+    cat /etc/lsb-release
+    echo
+    echo CPU INFO
+    cat /proc/cpuinfo
+    echo 
+    echo MEM INFO
+    cat /proc/meminfo
+    echo
+    echo DISK INFO
+    df -h
+    echo 
+    echo PWD: $PWD
+    echo
+    ls
+  displayName: 'Printing some environment information'
+
+
+## Installation commands for Ubuntu 
+- script: |
+    sudo apt-get install  \
+      g++  \
+      git  \
+      subversion \
+      automake \
+      libtool \
+      zlib1g-dev \
+      libicu-dev \
+      libboost-all-dev \
+      libssl-dev \
+      libbz2-dev \
+      liblzma-dev \
+      python-dev \
+      graphviz \
+      imagemagick \
+      make \
+      cmake \
+      libgoogle-perftools-dev \
+      autoconf \
+      doxygen 
+  displayName: 'Install Ubuntu packages'
+
+- script: |   
+    wget "https://sourceforge.net/projects/cmph/files/v2.0.2/cmph-2.0.2.tar.gz/download"
+    mv download  cmph-2.0.2.tar.gz  
+    tar xvzf cmph-2.0.2.tar.gz
+    cd cmph-2.0.2
+    ./configure --prefix=$PWD
+    make
+    make install
+    cd ..
+  displayName: 'Build and Install cmph'    
+
+- script: |   
+    wget "https://sourceforge.net/projects/xmlrpc-c/files/Xmlrpc-c%20Super%20Stable/1.51.06/xmlrpc-c-1.51.06.tgz/download"
+    mv download  xmlrpc-c-1.51.06.tgz
+    tar xvzf xmlrpc-c-1.51.06.tgz
+    cd xmlrpc-c-1.51.06
+    ./configure --prefix=$PWD
+    make
+    make install
+    sudo ldconfig
+    cd ..
+  displayName: 'Build and Install xmlrpc-c'    
+
+- script: |   
+    ./bjam \
+      --with-cmph=$PWD/cmph-2.0.2 \
+      --with-xmlrpc-c=$PWD/xmlrpc-c-1.51.06 \
+      -j2
+  displayName: 'Build Moses'    
+
+# - script: |   
+#     ./bjam \
+#       -j2
+#   displayName: 'Build Moses'  
+
+# - task: ComponentGovernanceComponentDetection@0
+#   inputs:
+#     scanType: 'Register'
+#     verbosity: 'Verbose'
+#     alertWarningLevel: 'High' 

mosesdecoder/biconcor/Alignment.cpp

@@ -0,0 +1,222 @@
+#include "Alignment.h"
+
+#include <fstream>
+#include <string>
+#include <cstdlib>
+#include <cstring>
+
+namespace
+{
+
+const int LINE_MAX_LENGTH = 10000;
+
+} // namespace
+
+using namespace std;
+
+void Alignment::Create(const string& fileName)
+{
+  ifstream textFile;
+  char line[LINE_MAX_LENGTH];
+
+  // count the number of words first;
+  textFile.open(fileName.c_str());
+
+  if (!textFile) {
+    cerr << "No such file or directory: " << fileName << endl;
+    exit(1);
+  }
+
+  istream *fileP = &textFile;
+  m_size = 0;
+  m_sentenceCount = 0;
+  while(!fileP->eof()) {
+    SAFE_GETLINE((*fileP), line, LINE_MAX_LENGTH, '\n');
+    if (fileP->eof()) break;
+    vector<string> alignmentSequence = Tokenize( line );
+    m_size += alignmentSequence.size();
+    m_sentenceCount++;
+  }
+  textFile.close();
+  cerr << m_size << " alignment points" << endl;
+
+  // allocate memory
+  m_array = (int*) calloc( sizeof(int), m_size*2 );
+  m_sentenceEnd = (INDEX*) calloc( sizeof( INDEX ), m_sentenceCount );
+
+  if (m_array == NULL) {
+    cerr << "Error: cannot allocate memory to m_array" << endl;
+    exit(1);
+  }
+
+  if (m_sentenceEnd == NULL) {
+    cerr << "Error: cannot allocate memory to m_sentenceEnd" << endl;
+    exit(1);
+  }
+
+  // fill the array
+  int alignmentPointIndex = 0;
+  int sentenceId = 0;
+
+  textFile.open(fileName.c_str());
+  if (!textFile) {
+    cerr << "Failed to open " << fileName << endl;
+    exit(1);
+  }
+
+  fileP = &textFile;
+  while(!fileP->eof()) {
+    SAFE_GETLINE((*fileP), line, LINE_MAX_LENGTH, '\n');
+    if (fileP->eof()) break;
+    vector<string> alignmentSequence = Tokenize( line );
+    for(size_t i=0; i<alignmentSequence.size(); i++) {
+      int s,t;
+      // cout << "scaning " << alignmentSequence[i].c_str() << endl;
+      if (! sscanf(alignmentSequence[i].c_str(), "%d-%d", &s, &t)) {
+        cerr << "WARNING: " << alignmentSequence[i] << " is a bad alignment point in sentence " << sentenceId << endl;
+      }
+      m_array[alignmentPointIndex++] = (char) s;
+      m_array[alignmentPointIndex++] = (char) t;
+    }
+    m_sentenceEnd[ sentenceId++ ] = alignmentPointIndex - 2;
+  }
+  textFile.close();
+  cerr << "done reading " << (alignmentPointIndex/2) << " alignment points, " << sentenceId << " sentences." << endl;
+}
+
+Alignment::Alignment()
+  : m_array(NULL),
+    m_sentenceEnd(NULL),
+    m_size(0),
+    m_sentenceCount(0) {}
+
+Alignment::~Alignment()
+{
+  if (m_array != NULL) {
+    free(m_array);
+  }
+  if (m_sentenceEnd != NULL) {
+    free(m_sentenceEnd);
+  }
+}
+
+vector<string> Alignment::Tokenize( const char input[] )
+{
+  vector< string > token;
+  bool betweenWords = true;
+  int start=0;
+  int i=0;
+  for(; input[i] != '\0'; i++) {
+    bool isSpace = (input[i] == ' ' || input[i] == '\t');
+
+    if (!isSpace && betweenWords) {
+      start = i;
+      betweenWords = false;
+    } else if (isSpace && !betweenWords) {
+      token.push_back( string( input+start, i-start ) );
+      betweenWords = true;
+    }
+  }
+  if (!betweenWords)
+    token.push_back( string( input+start, i-start ) );
+  return token;
+}
+
+bool Alignment::PhraseAlignment( INDEX sentence, int target_length,
+                                 int source_start, int source_end,
+                                 int &target_start, int &target_end,
+                                 int &pre_null, int &post_null )
+{
+  // get index for first alignment point
+  INDEX sentenceStart = 0;
+  if (sentence > 0) {
+    sentenceStart = m_sentenceEnd[ sentence-1 ] + 2;
+  }
+
+  // get target phrase boundaries
+  target_start = target_length;
+  target_end = 0;
+  for(INDEX ap = sentenceStart; ap <= m_sentenceEnd[ sentence ]; ap += 2 ) {
+    int source = m_array[ ap ];
+    if (source >= source_start && source <= source_end ) {
+      int target =  m_array[ ap+1 ];
+      if (target < target_start) target_start = target;
+      if (target > target_end )  target_end   = target;
+    }
+  }
+  if (target_start == target_length) {
+    return false; // done if no alignment points
+  }
+
+  // check consistency
+  for(INDEX ap = sentenceStart; ap <= m_sentenceEnd[ sentence ]; ap += 2 ) {
+    int target =  m_array[ ap+1 ];
+    if (target >= target_start && target <= target_end ) {
+      int source = m_array[ ap ];
+      if (source < source_start || source > source_end) {
+        return false; // alignment point out of range
+      }
+    }
+  }
+
+  // create array for unaligned words
+  for( int i=0; i<target_length; i++ ) {
+    m_unaligned[i] = true;
+  }
+  for(INDEX ap = sentenceStart; ap <= m_sentenceEnd[ sentence ]; ap += 2 ) {
+    int target =  m_array[ ap+1 ];
+    m_unaligned[ target ] = false;
+  }
+
+  // prior unaligned words
+  pre_null = 0;
+  for(int target = target_start-1; target >= 0 && m_unaligned[ target ]; target--) {
+    pre_null++;
+  }
+
+  // post unaligned words;
+  post_null = 0;
+  for(int target = target_end+1; target < target_length && m_unaligned[ target ]; target++) {
+    post_null++;
+  }
+  return true;
+}
+
+void Alignment::Save(const string& fileName ) const
+{
+  FILE *pFile = fopen ( (fileName + ".align").c_str() , "w" );
+  if (pFile == NULL) {
+    cerr << "Cannot open " << fileName << ".align" << endl;
+    exit(1);
+  }
+
+  fwrite( &m_size, sizeof(INDEX), 1, pFile );
+  fwrite( m_array, sizeof(int), m_size*2, pFile ); // corpus
+
+  fwrite( &m_sentenceCount, sizeof(INDEX), 1, pFile );
+  fwrite( m_sentenceEnd, sizeof(INDEX), m_sentenceCount, pFile); // sentence index
+  fclose( pFile );
+}
+
+void Alignment::Load(const string& fileName )
+{
+  FILE *pFile = fopen ( (fileName + ".align").c_str() , "r" );
+  if (pFile == NULL) {
+    cerr << "no such file or directory: " << fileName << ".align" << endl;
+    exit(1);
+  }
+
+  cerr << "loading from " << fileName << ".align" << endl;
+
+  fread( &m_size, sizeof(INDEX), 1, pFile );
+  cerr << "alignment points in corpus: " << m_size << endl;
+  m_array = (int*) calloc( sizeof(int), m_size*2 );
+  fread( m_array, sizeof(int), m_size*2, pFile ); // corpus
+
+  fread( &m_sentenceCount, sizeof(INDEX), 1, pFile );
+  cerr << "sentences in corpus: " << m_sentenceCount << endl;
+  m_sentenceEnd = (INDEX*) calloc( sizeof(INDEX), m_sentenceCount );
+  fread( m_sentenceEnd, sizeof(INDEX), m_sentenceCount, pFile); // sentence index
+  fclose( pFile );
+  cerr << "done loading\n";
+}

mosesdecoder/biconcor/Alignment.h

@@ -0,0 +1,47 @@
+#pragma once
+
+#include "Vocabulary.h"
+
+class Alignment
+{
+public:
+  typedef unsigned int INDEX;
+
+private:
+  int *m_array;
+  INDEX *m_sentenceEnd;
+  INDEX m_size;
+  INDEX m_sentenceCount;
+  char m_unaligned[ 256 ]; // here for speed (local to PhraseAlignment)
+
+  // No copying allowed.
+  Alignment(const Alignment&);
+  void operator=(const Alignment&);
+
+public:
+  Alignment();
+  ~Alignment();
+
+  void Create(const std::string& fileName );
+  bool PhraseAlignment( INDEX sentence, int target_length,
+                        int source_start, int source_end,
+                        int &target_start, int &target_end,
+                        int &pre_null, int &post_null );
+  void Load(const std::string& fileName );
+  void Save(const std::string& fileName ) const;
+  std::vector<std::string> Tokenize( const char input[] );
+
+  INDEX GetSentenceStart( INDEX sentence ) const {
+    if (sentence == 0) return 0;
+    return m_sentenceEnd[ sentence-1 ] + 2;
+  }
+  INDEX GetNumberOfAlignmentPoints( INDEX sentence ) const {
+    return ( m_sentenceEnd[ sentence ] - GetSentenceStart( sentence ) ) / 2;
+  }
+  int GetSourceWord( INDEX sentence, INDEX alignment_point ) const {
+    return m_array[ GetSentenceStart( sentence ) + alignment_point*2 ];
+  }
+  int GetTargetWord( INDEX sentence, INDEX alignment_point ) const {
+    return m_array[ GetSentenceStart( sentence ) + alignment_point*2 + 1 ];
+  }
+};

mosesdecoder/biconcor/CMakeLists.txt

@@ -0,0 +1,5 @@
+project(biconcor)
+
+FILE(GLOB biconcor_source *.cpp)
+
+add_executable(biconcor ${biconcor_source})

mosesdecoder/biconcor/Jamfile

@@ -0,0 +1,2 @@
+exe biconcor : Vocabulary.cpp SuffixArray.cpp TargetCorpus.cpp Alignment.cpp Mismatch.cpp PhrasePair.cpp PhrasePairCollection.cpp biconcor.cpp base64.cpp ;
+exe phrase-lookup : Vocabulary.cpp SuffixArray.cpp phrase-lookup.cpp ;

mosesdecoder/biconcor/Mismatch.cpp

@@ -0,0 +1,292 @@
+#include "Mismatch.h"
+
+#include <fstream>
+#include <iostream>
+#include <cstring>
+#include <string>
+#include <cstdlib>
+
+#include "SuffixArray.h"
+#include "TargetCorpus.h"
+#include "Alignment.h"
+#include "Vocabulary.h"
+
+using namespace std;
+
+enum {
+  UNANNOTATED = 0,
+  PRE_ALIGNED = 1,
+  POST_ALIGNED = 2,
+  UNALIGNED = 3,
+  MISALIGNED = 4,
+  ALIGNED = 5
+};
+
+Mismatch::Mismatch( SuffixArray *sa, TargetCorpus *tc, Alignment *a, INDEX sentence_id, INDEX position, int source_length, int target_length, int source_start, int source_end )
+  :m_suffixArray(sa)
+  ,m_targetCorpus(tc)
+  ,m_alignment(a)
+  ,m_sentence_id(sentence_id)
+  ,m_source_length(source_length)
+  ,m_target_length(target_length)
+  ,m_source_position(position)
+  ,m_source_start(source_start)
+  ,m_source_end(source_end)
+  ,m_unaligned(true)
+{
+  // initialize unaligned indexes
+  for (int i = 0; i < m_source_length; i++) {
+    m_source_unaligned[i] = true;
+  }
+  for (int i = 0; i < m_target_length; i++) {
+    m_target_unaligned[i] = true;
+  }
+  m_num_alignment_points =
+    m_alignment->GetNumberOfAlignmentPoints( sentence_id );
+  for(INDEX ap=0; ap<m_num_alignment_points; ap++) {
+    m_source_unaligned[ (int)m_alignment->GetSourceWord( sentence_id, ap ) ] = false;
+    m_target_unaligned[ (int)m_alignment->GetTargetWord( sentence_id, ap ) ] = false;
+  }
+  for(int i = source_start; i <= source_end; i++) {
+    if (!m_source_unaligned[ i ]) {
+      m_unaligned = false;
+    }
+  }
+}
+
+Mismatch::~Mismatch () {}
+
+void Mismatch::PrintClippedHTML( ostream* out, int width )
+{
+  int source_annotation[256], target_annotation[256];
+  vector< string > label_class;
+  label_class.push_back( "" );
+  label_class.push_back( "mismatch_pre_aligned" );
+  label_class.push_back( "mismatch_post_aligned" );
+  label_class.push_back( "null_aligned" );
+  label_class.push_back( "mismatch_misaligned" );
+  label_class.push_back( "mismatch_aligned" );
+
+  for(int i=0; i<m_source_length; i++) source_annotation[i] = UNANNOTATED;
+  for(int i=0; i<m_target_length; i++) target_annotation[i] = UNANNOTATED;
+
+  if (m_unaligned) {
+    // find alignment points for prior and next word(s) and
+    // center target phrase around those.
+    bool found_aligned = false;
+    for(int i=1; i<m_source_length && !found_aligned; i++) {
+      if (m_source_start-i >= 0) {
+        int word_id =  m_source_start-i;
+        source_annotation[ word_id ] = UNALIGNED;
+        if (!m_source_unaligned[ word_id ]) {
+          found_aligned = true;
+          LabelSourceMatches( source_annotation, target_annotation, word_id, PRE_ALIGNED );
+        }
+      }
+
+      if (m_source_end+i < m_source_length) {
+        int word_id = m_source_end+i;
+        source_annotation[ word_id ] = UNALIGNED;
+        if (!m_source_unaligned[ word_id ]) {
+          found_aligned = true;
+          LabelSourceMatches( source_annotation, target_annotation, word_id, POST_ALIGNED );
+        }
+      }
+    }
+
+  }
+  // misalignment
+  else {
+    // label aligned output words
+    for(int i=m_source_start; i<=m_source_end; i++)
+      LabelSourceMatches( source_annotation, target_annotation, i, ALIGNED );
+
+    // find first and last
+    int target_start = -1;
+    int target_end = -1;
+    for(int i=0; i<m_target_length; i++)
+      if (target_annotation[i] == ALIGNED) {
+        if (target_start == -1)
+          target_start = i;
+        target_end = i;
+      }
+    // go over all enclosed target words
+    for(int i=target_start; i<=target_end; i++) {
+      // label other target words as unaligned or misaligned
+      if (m_target_unaligned[ i ])
+        target_annotation[ i ] = UNALIGNED;
+      else {
+        if (target_annotation[ i ] != ALIGNED)
+          target_annotation[ i ] = MISALIGNED;
+        // loop over aligned source words
+        for(INDEX ap=0; ap<m_num_alignment_points; ap++) {
+          if (m_alignment->GetTargetWord( m_sentence_id, ap ) == i) {
+            int source_word = m_alignment->GetSourceWord( m_sentence_id, ap );
+            // if not part of the source phrase -> also misaligned
+            if (source_word < m_source_start || source_word > m_source_end)
+              source_annotation[ source_word ] = MISALIGNED;
+          }
+        }
+      }
+    }
+    // closure
+    bool change = true;
+    while(change) {
+      change = false;
+      for(INDEX ap=0; ap<m_num_alignment_points; ap++) {
+        int source_word = m_alignment->GetSourceWord( m_sentence_id, ap );
+        int target_word = m_alignment->GetTargetWord( m_sentence_id, ap );
+        if (source_annotation[source_word] != UNANNOTATED &&
+            target_annotation[target_word] == UNANNOTATED) {
+          target_annotation[target_word] = MISALIGNED;
+          change = true;
+        }
+        if (source_annotation[source_word] == UNANNOTATED &&
+            target_annotation[target_word] != UNANNOTATED) {
+          source_annotation[source_word] = MISALIGNED;
+          change = true;
+        }
+      }
+    }
+  }
+
+  // print source
+  // shorten source context if too long
+  int sentence_start = m_source_position - m_source_start;
+  int context_space = width/2;
+  for(int i=m_source_start; i<=m_source_end; i++)
+    context_space -= m_suffixArray->GetWord( sentence_start + i ).size() + 1;
+  context_space /= 2;
+
+  int remaining = context_space;
+  int start_word = m_source_start;
+  for(; start_word>0 && remaining>0; start_word--)
+    remaining -= m_suffixArray->GetWord( sentence_start + start_word-1 ).size() + 1;
+  if (remaining<0 || start_word == -1) start_word++;
+
+  remaining = context_space;
+  int end_word = m_source_end;
+  for(; end_word<m_source_length && remaining>0; end_word++)
+    remaining -= m_suffixArray->GetWord( sentence_start + end_word ).size() + 1;
+  end_word--;
+
+  // output with markup
+  *out << "<tr><td class=\"pp_source_left\">";
+  char current_label = UNANNOTATED;
+  if (start_word>0) {
+    current_label = source_annotation[start_word-1];
+    *out << "... ";
+  }
+  for(int i=start_word; i<=end_word; i++) {
+    // change to phrase block
+    if (i == m_source_start) {
+      if (current_label != UNANNOTATED && i!=start_word)
+        *out << "</span>";
+      *out << "</td><td class=\"pp_source\">";
+      current_label = UNANNOTATED;
+    }
+
+    // change to labeled word
+    else if (source_annotation[i] != current_label &&
+             source_annotation[i] != ALIGNED) {
+      if (current_label != UNANNOTATED && i!=start_word)
+        *out << "</span>";
+      if (source_annotation[i] != UNANNOTATED)
+        *out << "<span class=\""
+             << label_class[ source_annotation[i] ]
+             << "\">";
+      current_label = source_annotation[i];
+    }
+
+    // output word
+    *out << m_suffixArray->GetWord( sentence_start + i ) << " ";
+
+    // change to right context block
+    if (i == m_source_end) {
+      *out << "</td><td class=\"pp_source_right\">";
+      current_label = UNANNOTATED;
+    }
+  }
+
+  if (current_label != UNANNOTATED && end_word>m_source_end)
+    *out << "</span>";
+  if (end_word<m_source_length-1)
+    *out << "... ";
+
+  // print target
+  // shorten target context if too long
+  int target_start = -1;
+  int target_end=0;
+  for(int i=0; i<m_target_length; i++)
+    if (target_annotation[i] != UNANNOTATED) {
+      if (target_start == -1)
+        target_start = i;
+      target_end = i;
+    }
+
+  context_space = width/2;
+  for(int i=target_start; i<=target_end; i++)
+    context_space -= m_targetCorpus->GetWord( m_sentence_id, i ).size() + 1;
+  while (context_space < 0) { // shorten matched part, if too long
+    context_space +=
+      m_targetCorpus->GetWord( m_sentence_id, target_start ).size() +
+      m_targetCorpus->GetWord( m_sentence_id, target_end ).size() + 2;
+    target_start++;
+    target_end--;
+  }
+  context_space /= 2;
+
+  remaining = context_space;
+  start_word = target_start;
+  for(; start_word>0 && remaining>0; start_word--) {
+    //cerr << "remaining: " << remaining << ", start_word: " << start_word << endl;
+    remaining -= m_targetCorpus->GetWord( m_sentence_id, start_word-1 ).size() + 1;
+  }
+  if (remaining<0 || start_word == -1) start_word++;
+
+  remaining = context_space;
+  end_word = target_end;
+  for(; end_word<m_target_length && remaining>0; end_word++) {
+    //cerr << "remaining: " << remaining << ", end_word: " << end_word << endl;
+    remaining -= m_targetCorpus->GetWord( m_sentence_id, end_word ).size() + 1;
+  }
+  end_word--;
+
+  // output with markup
+  *out << "</td><td class=\"mismatch_target\">";
+  current_label = UNANNOTATED;
+  if (start_word>0) {
+    current_label = target_annotation[start_word-1];
+    *out << "... ";
+  }
+  for(int i=start_word; i<=end_word; i++) {
+    if (target_annotation[i] != current_label) {
+      if (current_label != UNANNOTATED && i!=start_word)
+        *out << "</span>";
+      if (target_annotation[i] != UNANNOTATED)
+        *out << "<span class=\""
+             << label_class[ target_annotation[i] ]
+             << "\">";
+      current_label = target_annotation[i];
+    }
+
+    // output word
+    *out << m_targetCorpus->GetWord( m_sentence_id, i ) << " ";
+  }
+
+  if (current_label != UNANNOTATED && end_word>target_end)
+    *out << "</span>";
+  if (end_word<m_target_length-1)
+    *out << "... ";
+  *out << "</td></tr>";
+}
+
+void Mismatch::LabelSourceMatches(int *source_annotation, int *target_annotation, int source_id, int label )
+{
+  for(INDEX ap=0; ap<m_num_alignment_points; ap++) {
+    if (m_alignment->GetSourceWord( m_sentence_id, ap ) == source_id) {
+      source_annotation[ source_id ] = label;
+      target_annotation[ m_alignment->GetTargetWord( m_sentence_id, ap ) ] = label;
+    }
+  }
+}

mosesdecoder/biconcor/Mismatch.h

@@ -0,0 +1,42 @@
+#pragma once
+
+#include <iosfwd>
+
+class Alignment;
+class SuffixArray;
+class TargetCorpus;
+
+class Mismatch
+{
+public:
+  typedef unsigned int INDEX;
+
+private:
+  SuffixArray *m_suffixArray;
+  TargetCorpus *m_targetCorpus;
+  Alignment *m_alignment;
+  INDEX m_sentence_id;
+  INDEX m_num_alignment_points;
+  int m_source_length;
+  int m_target_length;
+  INDEX m_source_position;
+  int m_source_start;
+  int m_source_end;
+  bool m_source_unaligned[ 256 ];
+  bool m_target_unaligned[ 256 ];
+  bool m_unaligned;
+
+  // No copying allowed.
+  Mismatch(const Mismatch&);
+  void operator=(const Mismatch&);
+
+public:
+  Mismatch( SuffixArray *sa, TargetCorpus *tc, Alignment *a, INDEX sentence_id, INDEX position, int source_length, int target_length, int source_start, int source_end );
+  ~Mismatch();
+
+  bool Unaligned() const {
+    return m_unaligned;
+  }
+  void PrintClippedHTML(std::ostream* out, int width );
+  void LabelSourceMatches(int *source_annotation, int *target_annotation, int source_id, int label );
+};