Add files using upload-large-folder tool

.gitattributes

@@ -39,3 +39,5 @@ fairseq-0.10.2/fairseq/data/token_block_utils_fast.cpython-310-x86_64-linux-gnu.
 fairseq-0.10.2/docs/fairseq.gif filter=lfs diff=lfs merge=lfs -text
 mosesdecoder/moses/TranslationModel/UG/util/ibm1-align filter=lfs diff=lfs merge=lfs -text
 mosesdecoder/contrib/iSenWeb/Introduction/iSenWeb[[:space:]]A[[:space:]]Web-based[[:space:]]Machine[[:space:]]Translation[[:space:]]System[[:space:]]to[[:space:]]Translate[[:space:]]Sentences.docx filter=lfs diff=lfs merge=lfs -text
+FacebookAI/xlm-roberta-large/onnx/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/contrib/promix/test_data/esen.nc.model.filtered/phrase-table.0-0.1.1.binphr.tgtdata.wa filter=lfs diff=lfs merge=lfs -text

FacebookAI/xlm-roberta-large/onnx/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:62c24cdc13d4c9952d63718d6c9fa4c287974249e16b7ade6d5a85e7bbb75626
+size 17082660

data/test/trainable_data/en2de/preprocess.log

@@ -0,0 +1,12 @@
+Namespace(no_progress_bar=False, log_interval=100, log_format=None, tensorboard_logdir=None, seed=42, cpu=False, tpu=False, bf16=False, memory_efficient_bf16=False, fp16=False, memory_efficient_fp16=False, fp16_no_flatten_grads=False, fp16_init_scale=128, fp16_scale_window=None, fp16_scale_tolerance=0.0, min_loss_scale=0.0001, threshold_loss_scale=None, user_dir=None, empty_cache_freq=0, all_gather_list_size=16384, model_parallel_size=1, checkpoint_suffix='', checkpoint_shard_count=1, quantization_config_path=None, profile=False, criterion='cross_entropy', tokenizer=None, bpe=None, optimizer=None, lr_scheduler='fixed', scoring='bleu', task='translation', source_lang='en', target_lang='de', trainpref=None, validpref=None, testpref='/mnt/congmh/luoyf/xzq-fairseq/data/test/tokenized/en2de/bpe.test.en2de.wmt23', align_suffix=None, destdir='/mnt/congmh/luoyf/xzq-fairseq/data/test/trainable_data/en2de', thresholdtgt=0, thresholdsrc=0, tgtdict='/mnt/congmh/luoyf/xzq-fairseq/data/de-en/wmt23/trainable_data/dict.de.txt', srcdict='/mnt/congmh/luoyf/xzq-fairseq/data/de-en/wmt23/trainable_data/dict.en.txt', nwordstgt=-1, nwordssrc=-1, alignfile=None, dataset_impl='mmap', joined_dictionary=False, only_source=False, padding_factor=8, workers=32)
+[en] Dictionary: 47776 types
+[en] /mnt/congmh/luoyf/xzq-fairseq/data/test/tokenized/en2de/bpe.test.en2de.wmt23.en: 557 sents, 47662 tokens, 0.0% replaced by <unk>
+[de] Dictionary: 47776 types
+[de] /mnt/congmh/luoyf/xzq-fairseq/data/test/tokenized/en2de/bpe.test.en2de.wmt23.de: 557 sents, 51978 tokens, 0.0% replaced by <unk>
+Wrote preprocessed data to /mnt/congmh/luoyf/xzq-fairseq/data/test/trainable_data/en2de
+Namespace(no_progress_bar=False, log_interval=100, log_format=None, tensorboard_logdir=None, seed=42, cpu=False, tpu=False, bf16=False, memory_efficient_bf16=False, fp16=False, memory_efficient_fp16=False, fp16_no_flatten_grads=False, fp16_init_scale=128, fp16_scale_window=None, fp16_scale_tolerance=0.0, min_loss_scale=0.0001, threshold_loss_scale=None, user_dir=None, empty_cache_freq=0, all_gather_list_size=16384, model_parallel_size=1, checkpoint_suffix='', checkpoint_shard_count=1, quantization_config_path=None, profile=False, criterion='cross_entropy', tokenizer=None, bpe=None, optimizer=None, lr_scheduler='fixed', scoring='bleu', task='translation', source_lang='en', target_lang='de', trainpref=None, validpref=None, testpref='/mnt/congmh/luoyf/xzq-fairseq/data/test/tokenized/en2de/bpe.test.en2de.wmt22', align_suffix=None, destdir='/mnt/congmh/luoyf/xzq-fairseq/data/test/trainable_data/en2de', thresholdtgt=0, thresholdsrc=0, tgtdict='/mnt/congmh/luoyf/xzq-fairseq/data/de-en/wmt23/trainable_data/dict.de.txt', srcdict='/mnt/congmh/luoyf/xzq-fairseq/data/de-en/wmt23/trainable_data/dict.en.txt', nwordstgt=-1, nwordssrc=-1, alignfile=None, dataset_impl='mmap', joined_dictionary=False, only_source=False, padding_factor=8, workers=32)
+[en] Dictionary: 47776 types
+[en] /mnt/congmh/luoyf/xzq-fairseq/data/test/tokenized/en2de/bpe.test.en2de.wmt22.en: 2037 sents, 46796 tokens, 0.00641% replaced by <unk>
+[de] Dictionary: 47776 types
+[de] /mnt/congmh/luoyf/xzq-fairseq/data/test/tokenized/en2de/bpe.test.en2de.wmt22.de: 2037 sents, 51454 tokens, 0.00389% replaced by <unk>
+Wrote preprocessed data to /mnt/congmh/luoyf/xzq-fairseq/data/test/trainable_data/en2de

data/test/trainable_data/zh2en/preprocess2.log

@@ -0,0 +1,6 @@
+Namespace(no_progress_bar=False, log_interval=100, log_format=None, tensorboard_logdir=None, seed=42, cpu=False, tpu=False, bf16=False, memory_efficient_bf16=False, fp16=False, memory_efficient_fp16=False, fp16_no_flatten_grads=False, fp16_init_scale=128, fp16_scale_window=None, fp16_scale_tolerance=0.0, min_loss_scale=0.0001, threshold_loss_scale=None, user_dir=None, empty_cache_freq=0, all_gather_list_size=16384, model_parallel_size=1, checkpoint_suffix='', checkpoint_shard_count=1, quantization_config_path=None, profile=False, criterion='cross_entropy', tokenizer=None, bpe=None, optimizer=None, lr_scheduler='fixed', scoring='bleu', task='translation', source_lang='zh', target_lang='en', trainpref=None, validpref=None, testpref='/mnt/congmh/luoyf/xzq-fairseq/data/test/tokenized/zh2en/bpe.test.zh2en.wmt23', align_suffix=None, destdir='/mnt/congmh/luoyf/xzq-fairseq/data/test/trainable_data/zh2en2', thresholdtgt=0, thresholdsrc=0, tgtdict='/mnt/congmh/luoyf/xzq-fairseq/data/en-zh/wmt23/trainable_data/dict.en.txt', srcdict='/mnt/congmh/luoyf/xzq-fairseq/data/en-zh/wmt23/trainable_data/dict.zh.txt', nwordstgt=-1, nwordssrc=-1, alignfile=None, dataset_impl='mmap', joined_dictionary=False, only_source=False, padding_factor=8, workers=32)
+[zh] Dictionary: 60432 types
+[zh] /mnt/congmh/luoyf/xzq-fairseq/data/test/tokenized/zh2en/bpe.test.zh2en.wmt23.zh: 1976 sents, 52111 tokens, 0.00192% replaced by <unk>
+[en] Dictionary: 46040 types
+[en] /mnt/congmh/luoyf/xzq-fairseq/data/test/tokenized/zh2en/bpe.test.zh2en.wmt23.en: 1976 sents, 61608 tokens, 0.00162% replaced by <unk>
+Wrote preprocessed data to /mnt/congmh/luoyf/xzq-fairseq/data/test/trainable_data/zh2en2

mosesdecoder/contrib/promix/test_data/esen.nc.model.filtered/phrase-table.0-0.1.1.binphr.tgtdata.wa

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:26f35f2fa1260b517cba96ce9a3df1890286d644ab99d975d85f5b06eae6a4d5
+size 720173

mosesdecoder/cruise-control/README

@@ -0,0 +1,34 @@
+A simple regular testing of Moses codebase, aka cruise control
+--------------------------------------------------------------
+
+Features:
+
+- Checkout latest commits
+- Test configure, make, make scripts
+- Run regression tests
+- Run a sample EMS pipeline
+- Report results into logfiles
+- A simple web interface in PHP
+
+How to run cruise control:
+
+1) Download and compile SRILM
+
+2) Download, compile and install IRSTLM
+
+3) Edit the configuration file example.config. You can create as many
+   configuration files as you like.
+
+4) Execute ./test_all_new_commits.sh yourfile.config
+
+
+How to set up the web interface:
+
+1) Install Apache and PHP
+
+2) Copy files from cruise-control/web into Apache's www directory
+
+3) Point StaticData::logs_path to correct directory, e.g. /home/cruise/logs/example/
+   Default value is 'data', you might want to just create a symlink.
+
+Written by Ondrej Bojar, Ales Tamchyna, Barry Haddow, Rimas Blazaitis

mosesdecoder/cruise-control/config.ems

@@ -0,0 +1,473 @@
+################################################
+### CONFIGURATION FILE FOR AN SMT EXPERIMENT ###
+################################################
+
+[GENERAL]
+
+### directory in which experiment is run
+#
+working-dir = WORKDIR/ems_workdir
+
+# Giza and friends
+external-bin-dir = WORKDIR/giza-pp/bin/
+
+# specification of the language pair
+input-extension = fr
+output-extension = en
+pair-extension = fr-en
+
+### directories that contain tools and data
+# 
+# moses
+moses-src-dir = WORKDIR
+#
+# moses scripts
+moses-script-dir = WORKDIR/scripts
+#
+# srilm
+srilm-dir = SRILMDIR/bin/MACHINE_TYPE
+#
+# data
+toy-data = $moses-script-dir/ems/example/data
+
+### basic tools
+#
+# moses decoder
+decoder = $moses-src-dir/bin/moses
+
+# conversion of phrase table into binary on-disk format
+ttable-binarizer = $moses-src-dir/bin/processPhraseTable
+
+# conversion of rule table into binary on-disk format
+#ttable-binarizer = "$moses-src-dir/CreateOnDisk/src/CreateOnDiskPt 1 1 5 100 2"
+
+# tokenizers - comment out if all your data is already tokenized
+input-tokenizer = "$moses-script-dir/tokenizer/tokenizer.perl -a -l $input-extension"
+output-tokenizer = "$moses-script-dir/tokenizer/tokenizer.perl -a -l $output-extension"
+
+# truecasers - comment out if you do not use the truecaser
+input-truecaser = $moses-script-dir/recaser/truecase.perl
+output-truecaser = $moses-script-dir/recaser/truecase.perl
+detruecaser = $moses-script-dir/recaser/detruecase.perl
+
+### generic parallelizer for cluster and multi-core machines
+# you may specify a script that allows the parallel execution
+# parallizable steps (see meta file). you also need specify 
+# the number of jobs (cluster) or cores (multicore)
+#
+#generic-parallelizer = $moses-script-dir/ems/support/generic-parallelizer.perl
+#generic-parallelizer = $moses-script-dir/ems/support/generic-multicore-parallelizer.perl
+
+### cluster settings (if run on a cluster machine)
+# number of jobs to be submitted in parallel
+#
+#jobs = 10
+
+# arguments to qsub when scheduling a job
+#qsub-settings = ""
+
+# project for priviledges and usage accounting 
+#qsub-project = iccs_smt
+
+# memory and time 
+#qsub-memory = 4
+#qsub-hours = 48
+
+### multi-core settings
+# when the generic parallelizer is used, the number of cores
+# specified here 
+cores = 8
+
+#################################################################
+# PARALLEL CORPUS PREPARATION: 
+# create a tokenized, sentence-aligned corpus, ready for training
+
+[CORPUS]
+
+### long sentences are filtered out, since they slow down GIZA++ 
+# and are a less reliable source of data. set here the maximum
+# length of a sentence
+#
+max-sentence-length = 80
+
+[CORPUS:toy]
+
+### command to run to get raw corpus files
+#
+# get-corpus-script = 
+
+### raw corpus files (untokenized, but sentence aligned)
+# 
+raw-stem = $toy-data/nc-5k
+
+### tokenized corpus files (may contain long sentences)
+#
+#tokenized-stem =
+
+### if sentence filtering should be skipped,
+# point to the clean training data
+#
+#clean-stem = 
+
+### if corpus preparation should be skipped,
+# point to the prepared training data
+#
+#lowercased-stem = 
+
+#################################################################
+# LANGUAGE MODEL TRAINING
+
+[LM]
+
+### tool to be used for language model training
+# for instance: ngram-count (SRILM), train-lm-on-disk.perl (Edinburgh) 
+# 
+lm-training = $srilm-dir/ngram-count
+settings = "-interpolate -kndiscount -unk"
+order = 5
+
+### tool to be used for training randomized language model from scratch
+# (more commonly, a SRILM is trained)
+#
+#rlm-training = "$moses-src-dir/randlm/bin/buildlm -falsepos 8 -values 8"
+
+### script to use for binary table format for irstlm or kenlm
+# (default: no binarization)
+
+# irstlm
+#lm-binarizer = $moses-src-dir/irstlm/bin/compile-lm
+
+# kenlm, also set type to 8
+#lm-binarizer = $moses-src-dir/kenlm/build_binary
+type = 8
+
+### script to create quantized language model format (irstlm)
+# (default: no quantization)
+# 
+#lm-quantizer = $moses-src-dir/irstlm/bin/quantize-lm
+
+### script to use for converting into randomized table format
+# (default: no randomization)
+#
+#lm-randomizer = "$moses-src-dir/randlm/bin/buildlm -falsepos 8 -values 8"
+
+### each language model to be used has its own section here
+
+[LM:toy]
+
+### command to run to get raw corpus files
+#
+#get-corpus-script = ""
+
+type = 8
+
+### raw corpus (untokenized)
+#
+raw-corpus = $toy-data/nc-5k.$output-extension
+
+### tokenized corpus files (may contain long sentences)
+#
+#tokenized-corpus = 
+
+### if corpus preparation should be skipped, 
+# point to the prepared language model
+#
+#lm = 
+
+
+[TRAINING]
+
+### training script to be used: either a legacy script or 
+# current moses training script (default) 
+# 
+script = $moses-script-dir/training/train-model.perl
+
+### general options
+#
+#training-options = ""
+
+### factored training: specify here which factors used
+# if none specified, single factor training is assumed
+# (one translation step, surface to surface)
+#
+#input-factors = word lemma pos morph
+#output-factors = word lemma pos
+#alignment-factors = "word -> word"
+#translation-factors = "word -> word"
+#reordering-factors = "word -> word"
+#generation-factors = "word -> pos"
+#decoding-steps = "t0, g0"
+
+### pre-computation for giza++
+# giza++ has a more efficient data structure that needs to be
+# initialized with snt2cooc. if run in parallel, this may reduces
+# memory requirements. set here the number of parts
+#
+run-giza-in-parts = 5
+
+### symmetrization method to obtain word alignments from giza output
+# (commonly used: grow-diag-final-and)
+#
+alignment-symmetrization-method = grow-diag-final-and
+
+### use of berkeley aligner for word alignment
+#
+#use-berkeley = true
+#alignment-symmetrization-method = berkeley
+#berkeley-train = $moses-script-dir/ems/support/berkeley-train.sh
+#berkeley-process =  $moses-script-dir/ems/support/berkeley-process.sh
+#berkeley-jar = /your/path/to/berkeleyaligner-1.1/berkeleyaligner.jar
+#berkeley-java-options = "-server -mx30000m -ea"
+#berkeley-training-options = "-Main.iters 5 5 -EMWordAligner.numThreads 8"
+#berkeley-process-options = "-EMWordAligner.numThreads 8"
+#berkeley-posterior = 0.5
+
+### if word alignment should be skipped,
+# point to word alignment files
+#
+#word-alignment = $working-dir/model/aligned.1
+
+### create a bilingual concordancer for the model
+#
+#biconcor = $moses-script-dir/ems/biconcor/biconcor
+
+### lexicalized reordering: specify orientation type
+# (default: only distance-based reordering model)
+#
+lexicalized-reordering = msd-bidirectional-fe
+
+### hierarchical rule set
+#
+#hierarchical-rule-set = true
+
+### settings for rule extraction
+#
+#extract-settings = ""
+
+### unknown word labels (target syntax only)
+# enables use of unknown word labels during decoding
+# label file is generated during rule extraction
+#
+#use-unknown-word-labels = true
+
+### if phrase extraction should be skipped,
+# point to stem for extract files
+#
+# extracted-phrases = 
+
+### settings for rule scoring
+#
+score-settings = "--GoodTuring"
+
+### include word alignment in phrase table
+#
+#include-word-alignment-in-rules = yes
+
+### if phrase table training should be skipped,
+# point to phrase translation table
+#
+# phrase-translation-table = 
+
+### if reordering table training should be skipped,
+# point to reordering table
+#
+# reordering-table = 
+
+### if training should be skipped, 
+# point to a configuration file that contains
+# pointers to all relevant model files
+#
+#config = 
+
+#####################################################
+### TUNING: finding good weights for model components
+
+[TUNING]
+
+### instead of tuning with this setting, old weights may be recycled
+# specify here an old configuration file with matching weights
+#
+weight-config = $toy-data/weight.ini
+
+### tuning script to be used
+#
+tuning-script = $moses-script-dir/training/mert-moses.pl
+tuning-settings = "-mertdir $moses-src-dir/mert"
+
+### specify the corpus used for tuning 
+# it should contain 1000s of sentences
+#
+#input-sgm = 
+#raw-input = 
+#tokenized-input = 
+#factorized-input = 
+#input =
+# 
+#reference-sgm = 
+#raw-reference = 
+#tokenized-reference = 
+#factorized-reference = 
+#reference = 
+
+### size of n-best list used (typically 100)
+#
+nbest = 100
+
+### ranges for weights for random initialization
+# if not specified, the tuning script will use generic ranges
+# it is not clear, if this matters
+#
+# lambda = 
+
+### additional flags for the filter script
+#
+filter-settings = ""
+
+### additional flags for the decoder
+#
+decoder-settings = ""
+
+### if tuning should be skipped, specify this here
+# and also point to a configuration file that contains
+# pointers to all relevant model files
+#
+#config = 
+
+#########################################################
+## RECASER: restore case, this part only trains the model
+
+[RECASING]
+
+#decoder = $moses-src-dir/moses-cmd/src/moses.1521.srilm
+
+### training data
+# raw input needs to be still tokenized,
+# also also tokenized input may be specified
+#
+#tokenized = [LM:europarl:tokenized-corpus]
+
+# recase-config = 
+
+#lm-training = $srilm-dir/ngram-count
+
+#######################################################
+## TRUECASER: train model to truecase corpora and input
+
+[TRUECASER]
+
+### script to train truecaser models
+#
+trainer = $moses-script-dir/recaser/train-truecaser.perl
+
+### training data
+# data on which truecaser is trained
+# if no training data is specified, parallel corpus is used
+#
+# raw-stem = 
+# tokenized-stem =
+
+### trained model
+#
+# truecase-model = 
+
+######################################################################
+## EVALUATION: translating a test set using the tuned system and score it
+
+[EVALUATION]
+
+### additional flags for the filter script
+#
+#filter-settings = ""
+
+### additional decoder settings
+# switches for the Moses decoder
+#
+decoder-settings = "-search-algorithm 1 -cube-pruning-pop-limit 5000 -s 5000"
+
+### specify size of n-best list, if produced
+#
+#nbest = 100
+
+### multiple reference translations
+#
+#multiref = yes
+
+### prepare system output for scoring 
+# this may include detokenization and wrapping output in sgm 
+# (needed for nist-bleu, ter, meteor)
+#
+detokenizer = "$moses-script-dir/tokenizer/detokenizer.perl -l $output-extension"
+#recaser = $moses-script-dir/recaser/recase.perl
+wrapping-script = "$moses-script-dir/ems/support/wrap-xml.perl $output-extension"
+#output-sgm = 
+
+### BLEU
+#
+nist-bleu = $moses-script-dir/generic/mteval-v12.pl
+nist-bleu-c = "$moses-script-dir/generic/mteval-v12.pl -c"
+#multi-bleu = $moses-script-dir/generic/multi-bleu.perl
+#ibm-bleu =
+
+### TER: translation error rate (BBN metric) based on edit distance
+# not yet integrated
+#
+# ter = 
+
+### METEOR: gives credit to stem / worknet synonym matches
+# not yet integrated
+#
+# meteor = 
+
+### Analysis: carry out various forms of analysis on the output
+#
+analysis = $moses-script-dir/ems/support/analysis.perl
+#
+# also report on input coverage
+analyze-coverage = yes
+#
+# also report on phrase mappings used
+report-segmentation = yes
+#
+# report precision of translations for each input word, broken down by
+# count of input word in corpus and model
+#report-precision-by-coverage = yes
+#
+# further precision breakdown by factor
+#precision-by-coverage-factor = pos
+
+[EVALUATION:test]
+
+### input data
+#
+input-sgm = $toy-data/test-src.$input-extension.sgm
+# raw-input = 
+# tokenized-input = 
+# factorized-input =
+# input = 
+
+### reference data
+#
+reference-sgm = $toy-data/test-ref.$output-extension.sgm
+# raw-reference =
+# tokenized-reference = 
+# reference = 
+
+### analysis settings 
+# may contain any of the general evaluation analysis settings
+# specific setting: base coverage statistics on earlier run
+#
+#precision-by-coverage-base = $working-dir/evaluation/test.analysis.5
+
+### wrapping frame
+# for nist-bleu and other scoring scripts, the output needs to be wrapped 
+# in sgm markup (typically like the input sgm)
+#
+wrapping-frame = $input-sgm
+
+##########################################
+### REPORTING: summarize evaluation scores
+
+[REPORTING]
+
+### currently no parameters for reporting section
+

mosesdecoder/cruise-control/create-binary.perl

@@ -0,0 +1,55 @@
+#!/usr/bin/env perl
+
+use strict;
+
+my $HOME = $ENV{"HOME"};
+my $HOSTNAME = "s0565741\@thor.inf.ed.ac.uk";
+
+my $sriPath = $ARGV[0];
+
+my $cmd;
+
+# what machine
+my $machine = `uname`;
+chomp($machine);
+
+# COMPILE
+$cmd = "git checkout master && git pull";
+print STDERR "Executing: $cmd \n";
+system($cmd);
+
+$cmd = "make -f contrib/Makefiles/install-dependencies.gmake && ./compile.sh --without-tcmalloc";
+print STDERR "Executing: $cmd \n";
+system($cmd);
+
+#ZIP
+if ($machine eq "Darwin") {
+  $machine = "mac";
+}
+
+$cmd = "mkdir -p mt-tools/moses && mv bin lib mt-tools/moses";
+print STDERR "Executing: $cmd \n";
+system($cmd);
+
+$cmd = "tar -zcvf $machine.tgz mt-tools";
+print STDERR "Executing: $cmd \n";
+system($cmd);
+
+# UPLOAD
+my $date = `date "+%F"`;
+chomp($date);
+
+my $targetDir = "/fs/thor1/hieu/binaries/$date/";
+print STDERR "Directory=$targetDir\n";
+
+$cmd = "ssh $HOSTNAME mkdir -p $targetDir";
+print STDERR "Executing: $cmd \n";
+system($cmd);
+
+$cmd = "rsync -rv --delete $machine.tgz $HOSTNAME:$targetDir";
+print STDERR "Executing: $cmd \n";
+system($cmd);
+
+$cmd = "rm $machine.tgz";
+print STDERR "Executing: $cmd \n";
+system($cmd);

mosesdecoder/cruise-control/example.config

@@ -0,0 +1,25 @@
+# the tmp directory
+MCC_TEMPDIR="/disk4/cruise-control/cruise-control/tmp/"
+
+# where logs are saved
+MCC_LOGDIR="/disk4/cruise-control/cruise-control/"
+
+# repository that will be checked out
+MCC_GITREPO="https://github.com/moses-smt/mosesdecoder.git"
+
+# arguments passed to Moses configure
+MCC_CONFIGURE_ARGS=" --with-srilm=/disk4/cruise-control/srilm --with-irstlm=/disk4/cruise-control/irstlm --with-dalm=/disk4/cruise-control/DALM --with-cmph=/disk4/cruise-control/cmph-2.0 --with-boost=/disk4/cruise-control/boost_1_55_0 --with-xmlrpc-c=/disk4/cruise-control/xmlrpc-c -j8 "
+
+ALTERNATIVE_CONFIGURE_ARGS=(
+        " --with-irstlm=/disk4/cruise-control/irstlm --with-dalm=/disk4/cruise-control/DALM --with-cmph=/disk4/cruise-control/cmph-2.0 --with-boost=/disk4/cruise-control/boost_1_55_0 --with-xmlrpc-c=/disk4/cruise-control/xmlrpc-c -j8 "
+        " --with-srilm=/disk4/cruise-control/srilm --with-dalm=/disk4/cruise-control/DALM --with-cmph=/disk4/cruise-control/cmph-2.0 --with-boost=/disk4/cruise-control/boost_1_55_0 --with-xmlrpc-c=/disk4/cruise-control/xmlrpc-c -j8 "
+        " --with-srilm=/disk4/cruise-control/srilm --with-irstlm=/disk4/cruise-control/irstlm --with-cmph=/disk4/cruise-control/cmph-2.0 --with-boost=/disk4/cruise-control/boost_1_55_0 --with-xmlrpc-c=/disk4/cruise-control/xmlrpc-c -j8 "
+        " --with-srilm=/disk4/cruise-control/srilm --with-irstlm=/disk4/cruise-control/irstlm --with-dalm=/disk4/cruise-control/DALM  --with-boost=/disk4/cruise-control/boost_1_55_0 --with-xmlrpc-c=/disk4/cruise-control/xmlrpc-c -j8 "
+        " --with-srilm=/disk4/cruise-control/srilm --with-irstlm=/disk4/cruise-control/irstlm --with-dalm=/disk4/cruise-control/DALM --with-cmph=/disk4/cruise-control/cmph-2.0 --with-boost=/disk4/cruise-control/boost_1_55_0 --no-xmlrpc-c -j8 "
+        )
+
+# list of branches to be checked
+MCC_SCAN_BRANCHES="origin/master"
+
+# run full training/eval pipeline using EMS?
+MCC_RUN_EMS="yes"

mosesdecoder/cruise-control/shorten_info.pl

@@ -0,0 +1,9 @@
+#!/usr/bin/perl
+
+use strict;
+use warnings;
+
+while (<>) {
+  last if $_ =~ m/^diff --git/;
+  print $_;
+}

mosesdecoder/cruise-control/test_all_new_commits.sh

@@ -0,0 +1,220 @@
+#!/bin/bash
+# given a config file runs tests on all untested commits of the scanned branches
+# storing detailed logs to logs/CONFIGNAME/commit
+# and extending the file brief.log
+#
+# A commit is assumed to be tested, if logs/CONFIGNAME/commit exists
+#
+# Ondrej Bojar, Ales Tamchyna, 2011
+
+function warn() { echo "$@" >&2; }
+function die() { echo "$@" >&2; exit 1; }
+set -o pipefail  # safer pipes
+
+configf="$1"
+[ -e "$configf" ] || die "usage: $0 configfile"
+configname=$(basename $configf | sed 's/\.config$//')
+
+source "$configf"
+
+# beautifier
+git clone git@github.com:moses-smt/mosesdecoder.git /tmp/moses
+cd /tmp/moses
+./scripts/other/beautify.py --format --skip-perltidy
+git commit -am "daily automatic beautifier"
+git push
+rm -rf /tmp/moses
+cd -
+
+[ -z "$MCC_SCAN_BRANCHES" ] \
+  && die "Bad config $configf; does not define MCC_SCAN_BRANCHES"
+
+# use the given tempdir or make subdir tmp here
+USE_TEMPDIR=$MCC_TEMPDIR
+[ -d "$USE_TEMPDIR" ] || USE_TEMPDIR=./tmp
+
+LOGDIR=$MCC_LOGDIR
+[ -d "$LOGDIR" ] || LOGDIR=.
+
+# ensure full path for logdir
+LOGDIR=$(readlink -f "$LOGDIR")
+[ -d "$LOGDIR" ] || die "Fatal: confusing readlink for $LOGDIR"
+
+# this is where moses is cloned into
+WORKDIR=$MCC_WORKDIR
+[ -d "$WORKDIR" ] || WORKDIR=$USE_TEMPDIR/workdir
+
+MYDIR=$(pwd)
+
+# this is where moses is taken from
+GITREPO="$MCC_GITREPO"
+[ -n "$GITREPO" ] || GITREPO=/home/obo/moses-at-google-code
+
+# location of moses regression test data archive (assumes url at the moment)
+REGTEST_ARCHIVE="$MCC_REGTEST_ARCHIVE"
+[ -n "$REGTEST_ARCHIVE" ] \
+  || REGTEST_ARCHIVE="git://github.com/moses-smt/moses-regression-tests.git"
+
+if [ ! -d "$WORKDIR" ]; then
+  mkdir "$WORKDIR" || die "Failed to create workdir $WORKDIR"
+  warn "Cloning $GITREPO into $WORKDIR"
+  git clone $GITREPO $WORKDIR \
+    || die "Failed to git clone into workdir $WORKDIR"
+else
+  ( cd "$WORKDIR" && git fetch ) \
+    || die "Failed to update our clone at $WORKDIR"
+fi
+
+mkdir -p $LOGDIR/logs/$configname \
+  || die "Failed to create dir $LOGDIR/logs/$configname"
+
+#### How is one test performed
+function run_single_test () {
+  commit=$1
+  first_char=$(echo $commit | grep -o '^.')
+  longlog="$LOGDIR/logs/$configname/$first_char/$commit.log"
+  warn "Testing commit $commit"
+
+  # Get the version of this script
+  ccversion=$(svnversion 2>/dev/null)
+  [ ! -z "$ccversion" ] || ccversion=$(git show 2>&1 | head -n 1)
+  [ ! -z "$ccversion" ] || ccversion="unknown"
+
+  # Create log header with computer details:
+  echo "#### Moses Cruise Control Log for commit $commit" > $longlog
+  date >> $longlog
+  echo "## Cruise Control version" >> $longlog
+  echo $ccversion >> $longlog
+  echo "## Parameters" >> $longlog
+  cat $MYDIR/$configf >> $longlog
+  echo "## Envinronment" >> $longlog
+  uname -a >> $longlog
+  env >> $longlog
+
+  git checkout --force $commit 2>/dev/null || die "Failed to checkout commit $commit"
+
+  err=""
+
+   cd regression-testing
+  #regtest_file=$(echo "$REGTEST_ARCHIVE" | sed 's/^.*\///')
+
+  # download data for regression tests if necessary
+  regtest_dir=$PWD/moses-reg-test-data
+  if [ -e $regtest_dir ]; then
+    (cd $regtest_dir; git pull)  &> /dev/null ||
+      die "Failed to update regression testing data"
+  else
+    git clone $REGTEST_ARCHIVE $regtest_dir &> /dev/null ||
+      die "Failed to clone regression testing data"
+  fi
+  #if [ ! -f $regtest_file.ok ]; then
+  #  wget $REGTEST_ARCHIVE &> /dev/null \
+  #    || die "Failed to download data for regression tests"
+  #  tar xzf $regtest_file
+  #  touch $regtest_file.ok
+  #fi
+  #regtest_dir=$PWD/$(basename $regtest_file .tgz)
+  cd ..
+
+  # test build with different configurations
+  echo "## test build with different configurations" >> $longlog
+  for configArgs in "${ALTERNATIVE_CONFIGURE_ARGS[@]}"
+  do
+   echo "building with args: $configArgs" >> $longlog
+   ./bjam clean -a $configArgs >> $longlog 2>&1 || warn "bjam clean failed, suspicious"
+  done
+
+  echo "## ./bjam clean" >> $longlog
+  ./bjam clean -a $MCC_CONFIGURE_ARGS --with-regtest=$regtest_dir >> $longlog 2>&1 || warn "bjam clean failed, suspicious"
+
+  echo "## ./bjam $MCC_CONFIGURE_ARGS" >> $longlog
+  if [ -z "$err" ]; then
+    ./bjam $MCC_CONFIGURE_ARGS >> $longlog 2>&1 || err="bjam"
+  fi
+
+  echo "## regression tests" >> $longlog
+  if [ -z "$err" ]; then
+    ./bjam $MCC_CONFIGURE_ARGS --with-regtest=$regtest_dir >> $longlog 2>&1 || err="regression tests"
+  fi
+
+  if [ -z "$err" ] && [ "$MCC_RUN_EMS" = "yes" ]; then
+    echo "## EMS" >> $longlog
+    if [ ! -f "giza-pp.ok" ]; then # fetch & compile Giza++
+      svn checkout http://giza-pp.googlecode.com/svn/trunk/ giza-pp \
+      || die "Failed to fetch Giza++"
+      cd giza-pp && make || die "Failed to compile Giza++"
+      mkdir -p bin
+      ln -s ../GIZA++-v2/GIZA++ ../GIZA++-v2/snt2cooc.out ../mkcls-v2/mkcls bin/
+      cd ..
+      touch giza-pp.ok
+    fi
+    ./bjam $MCC_CONFIGURE_ARGS" || err="bjam"
+    srilm_dir=$(echo $MCC_CONFIGURE_ARGS | sed -r 's/.*--with-srilm=([^ ]+) .*/\1/')
+    mach_type=$($srilm_dir/sbin/machine-type)
+    mkdir -p "$WORKDIR/ems_workdir"
+    rm -rf "$WORKDIR/ems_workdir/"* # clean any previous experiments
+    cat $MYDIR/config.ems \
+    | sed \
+      -e "s#WORKDIR#$WORKDIR#" \
+      -e "s#SRILMDIR#$srilm_dir#" \
+      -e "s#MACHINE_TYPE#$mach_type#" \
+    > ./config.ems
+    scripts/ems/experiment.perl \
+      -no-graph -exec -config $(pwd)/config.ems &>> $longlog \
+      || die "Running EMS failed"
+    [ -f $WORKDIR/ems_workdir/steps/1/REPORTING_report.1.DONE ] || err="ems"
+  fi
+
+  echo "## Finished" >> $longlog
+  date >> $longlog
+
+  if [ -z "$err" ]; then
+    status="OK"
+  else
+    git reset --hard HEAD
+    status="FAIL:$err"
+  fi
+  echo "## Status: $status" >> $longlog
+
+  nicedate=$(date +"%Y%m%d-%H%M%S")
+  echo "$commit$status$configname$ccversion$nicedate" \
+    >> "$LOGDIR/brief.log"
+
+  if [ -z "$err" ]; then
+    touch "$LOGDIR/logs/$configname/$first_char/$commit.OK"
+  else
+    return 1;
+  fi
+}
+
+cd $WORKDIR || die "Failed to chdir to $WORKDIR"
+
+# update the revision lists for all watched branches
+for i in $MCC_SCAN_BRANCHES; do
+  git rev-list $i > "$LOGDIR/logs/$configname/$(echo -n $i | sed 's/^.*\///').revlist"
+done
+
+# create info files for new commits
+for i in $(git rev-list $MCC_SCAN_BRANCHES); do
+  first_char=$(echo $i | grep -o '^.')
+  mkdir -p "$LOGDIR/logs/$configname/$first_char"
+  [ -f "$LOGDIR/logs/$configname/$first_char/$i.info" ] && break;
+  git show $i | $MYDIR/shorten_info.pl > "$LOGDIR/logs/$configname/$first_char/$i.info"
+done
+
+#### Main loop over all commits
+for i in $MCC_SCAN_BRANCHES; do
+  warn "On branch $i"
+  git rev-list $i \
+  | while read commit; do
+    first_char=$(echo $commit | grep -o '^.')
+    test_done="$LOGDIR/logs/$configname/$first_char/$commit.log"
+    if [ ! -e "$test_done" ]; then
+      run_single_test $commit && warn "Commit $commit test ok, stopping" && break
+      warn "Commit $commit test failed, continuing"
+    else
+      warn "Reached a previously tested commit ($commit), stopping"
+      break
+    fi
+  done
+done

mosesdecoder/cruise-control/web/html_templates.php

@@ -0,0 +1,112 @@
+<?php
+
+function show_header($title)
+{
+  echo "
+<html>
+  <head>
+    <META HTTP-EQUIV=\"Content-Type\" CONTENT=\"text/html;charset=utf-8\">
+    <title>$title</title>
+  </head><body>";
+}
+
+function show_heading($text, $size = 1)
+{
+  echo "
+    <h$size>$text</h$size>";
+}
+
+function show_footer()
+{
+  echo "
+  </body>
+<html>";
+}
+
+function end_table()
+{
+  echo "
+    </table>";
+}
+
+function array_to_table_row($odd = true, $data)
+{
+  $bgcolor = $odd ? " bgcolor=\"#ccccdd\"" : "";
+  echo "
+    <tr$bgcolor>";
+  foreach ($data as &$item) {
+    echo "
+      <td style=\"padding-left:8px; padding-right:8px\">$item</td>";
+  }
+  echo "
+    </tr>";
+}
+
+function start_table()
+{
+  echo '
+    <table rules="cols" frame="vsides">';
+}
+
+function start_form($action, $method = "get")
+{
+  echo "
+    <form action=\"$action\" method=\"$method\">";
+}
+
+function end_form()
+{
+  echo "
+    </form>";
+}
+
+function show_select_box($items, $name, $selected = "", $onchange_hdl = "")
+{
+  $onchange = $onchange_hdl ? " onchange=\"$onchange_hdl\"" : "";
+  echo "
+    <select name=\"$name\"$onchange>";
+  foreach ($items as &$item) {
+    $item_selected = $selected == $item ? " selected=\"yes\"" : "";
+    echo "
+      <option value=\"$item\"$item_selected>$item</option>";
+  }
+  echo "
+    </select>";
+}
+
+function get_href($label, $url, $new_window = false)
+{
+  $target = $new_window ? " target=\"_blank\"" : "";
+  return "<a href=\"$url\"$target>$label</a>";
+}
+
+function warn($msg)
+{
+  echo "<p><font color=\"red\"><b>$msg</b></font>";
+}
+
+function get_current_url()
+{
+  return $_SERVER["REQUEST_URI"];
+}
+
+function set_var($url, $var, $value)
+{
+  $url = cut_var($url, $var);
+  if ($url[strlen($url) - 1] == "?") {
+    $url .= "$var=$value";
+  } elseif (strpos($url, "?") !== false) {
+    $url .= "&$var=$value";
+  } else {
+    $url .= "?$var=$value";
+  }
+  return $url;
+}
+
+function cut_var($url, $var)
+{
+  // XXX there is probably a cleaner solution for this
+  return preg_replace('/&?' . $var . '=[^&]+/', '', $url);
+}
+
+?>

mosesdecoder/cruise-control/web/index.php

@@ -0,0 +1,101 @@
+<?php
+
+include("html_templates.php");
+include("log_wrapper.php");
+
+const SHOW_ITEMS = 50;
+const GITHUB_LINK = "https://github.com/moses-smt/mosesdecoder/commit/";
+
+show_header("Moses Cruise Control");
+echo "\n<center>\n";
+
+show_heading("Moses Cruise Control");
+echo "\n</center>\n";
+
+// show current status of 'master' branch
+$master_branch = new Branch("master");
+$last_commit = $master_branch->get_next_commit();
+$last_commit->read_log();
+show_heading("Current status of master: " . colorize_status($last_commit->get_status()), 3);
+$branch_name = ! empty($_GET["branch"]) ? $_GET["branch"] : "master";
+
+// check that user wants to see a valid branch
+$all_branches = get_all_branch_names();
+if (! in_array($branch_name, $all_branches)) {
+  warn("Branch '$branch_name' not found (only branches with some tests done can be viewed)");
+  $branch_name = "master";
+}
+
+// branch select box
+start_form("", "get");
+echo "<p>Showing log of branch: ";
+show_select_box($all_branches, "branch", $branch_name, "submit()");
+end_form();
+
+$branch = new Branch("$branch_name");
+$start_with = ! empty($_GET["start"]) ? $_GET["start"] : 0;
+$branch->set_line($start_with);
+
+show_navigation($start_with);
+
+// table of commits
+start_table();
+array_to_table_row(true, array("<b>Commit Link</b>", "<b>Status</b>", "<b>Full Log</b>",
+  "<b>Timestamp</b>", "<b>Author</b>", "<b>Commit Message</b>" ));
+for ($i = 0; $i < SHOW_ITEMS; $i++) {
+  $last_commit = $branch->get_next_commit();
+
+  if ( $last_commit->get_name() == "" ) {
+    array_to_table_row(array("=== End of log ==="));
+    break;
+  }
+  $last_commit->read_log();
+  $last_commit->read_info();
+
+  array_to_table_row(($i % 2 == 1),
+    array( get_href(substr($last_commit->get_name(), 0, 10) . "...", GITHUB_LINK . $last_commit->get_name(), true),
+           colorize_status($last_commit->get_status()),
+           $last_commit->was_tested() ? get_href("Log", $last_commit->get_log_file(), true) : "N/A",
+           $last_commit->get_timestamp(),
+           $last_commit->get_author(),
+           substr($last_commit->get_message(), 0, 30) . (strlen($last_commit->get_message()) > 30 ? "..." : "")));
+}
+
+end_table();
+
+show_navigation($start_with);
+show_footer();
+
+// HTML ends here
+
+function colorize_status($status)
+{
+  switch ( substr(strtolower($status), 0, 1) ) {
+    case "o":
+      $color = "green";
+      break;
+    case "f":
+      $color = "red";
+      break;
+    default:
+      $color = "#FFDD00";
+  }
+  return "<font color=\"$color\"><b>$status</b></font>";
+}
+
+function show_navigation($start_with)
+{
+  start_form("", "get");
+  if ($start_with > 0) {
+    echo get_href("<p>Previous",
+      set_var(get_current_url(), "start", max(0, $start_with - SHOW_ITEMS)));
+  } else {
+    echo "Previous";
+  }
+  echo " ";
+
+  echo get_href("Next", set_var(get_current_url(), "start", $start_with + SHOW_ITEMS));
+  end_form();
+}
+
+?>